2023-09-23T21:42:56.409 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T21:42:56.409 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873 --verbose --name teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873/orig.config.yaml 2023-09-23T21:42:56.471 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T21:42:56.561 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7401873' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4997 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi026.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi070.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T21:42:56.562 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T21:42:56.563 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T21:42:56.563 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T21:42:56.565 INFO:teuthology.task.internal:Checking packages... 2023-09-23T21:42:56.588 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T21:42:56.589 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T21:42:56.589 INFO:teuthology.packaging:ref: None 2023-09-23T21:42:56.589 INFO:teuthology.packaging:tag: None 2023-09-23T21:42:56.589 INFO:teuthology.packaging:branch: reef 2023-09-23T21:42:56.589 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T21:42:56.589 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-09-23T21:42:56.832 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-460.gbb0daeff 2023-09-23T21:42:56.833 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T21:42:56.841 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T21:42:56.841 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T21:42:56.892 INFO:teuthology.task.internal:Saving configuration 2023-09-23T21:42:56.917 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T21:42:56.926 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T21:42:56.951 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 21:37:51.359124', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T21:42:56.972 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi070.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 21:37:51.360798', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T21:42:56.972 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T21:42:56.979 INFO:teuthology.task.internal:roles: ubuntu@smithi026.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T21:42:56.979 INFO:teuthology.task.internal:roles: ubuntu@smithi070.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T21:42:56.980 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T21:42:57.069 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f972d064af0>, signals=[15]) 2023-09-23T21:42:57.071 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T21:42:57.078 INFO:teuthology.task.internal:Opening connections... 2023-09-23T21:42:57.078 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T21:42:57.080 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T21:42:57.148 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi070.front.sepia.ceph.com 2023-09-23T21:42:57.149 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T21:42:57.215 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T21:42:57.222 DEBUG:teuthology.orchestra.run.smithi026:> uname -m 2023-09-23T21:42:57.243 INFO:teuthology.orchestra.run.smithi026.stdout:x86_64 2023-09-23T21:42:57.243 DEBUG:teuthology.orchestra.run.smithi026:> cat /etc/os-release 2023-09-23T21:42:57.304 INFO:teuthology.orchestra.run.smithi026.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T21:42:57.304 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T21:42:57.304 INFO:teuthology.orchestra.run.smithi026.stdout:ID="rhel" 2023-09-23T21:42:57.304 INFO:teuthology.orchestra.run.smithi026.stdout:ID_LIKE="fedora" 2023-09-23T21:42:57.305 INFO:teuthology.orchestra.run.smithi026.stdout:VERSION_ID="8.6" 2023-09-23T21:42:57.305 INFO:teuthology.orchestra.run.smithi026.stdout:PLATFORM_ID="platform:el8" 2023-09-23T21:42:57.305 INFO:teuthology.orchestra.run.smithi026.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T21:42:57.305 INFO:teuthology.orchestra.run.smithi026.stdout:ANSI_COLOR="0;31" 2023-09-23T21:42:57.305 INFO:teuthology.orchestra.run.smithi026.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T21:42:57.305 INFO:teuthology.orchestra.run.smithi026.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T21:42:57.305 INFO:teuthology.orchestra.run.smithi026.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T21:42:57.305 INFO:teuthology.orchestra.run.smithi026.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T21:42:57.305 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:42:57.306 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T21:42:57.306 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T21:42:57.306 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T21:42:57.306 INFO:teuthology.orchestra.run.smithi026.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T21:42:57.306 INFO:teuthology.lock.ops:Updating smithi026.front.sepia.ceph.com on lock server 2023-09-23T21:42:57.329 DEBUG:teuthology.orchestra.run.smithi070:> uname -m 2023-09-23T21:42:57.348 INFO:teuthology.orchestra.run.smithi070.stdout:x86_64 2023-09-23T21:42:57.349 DEBUG:teuthology.orchestra.run.smithi070:> cat /etc/os-release 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:ID="rhel" 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:ID_LIKE="fedora" 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:VERSION_ID="8.6" 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:PLATFORM_ID="platform:el8" 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:ANSI_COLOR="0;31" 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T21:42:57.409 INFO:teuthology.orchestra.run.smithi070.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T21:42:57.410 INFO:teuthology.orchestra.run.smithi070.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T21:42:57.410 INFO:teuthology.orchestra.run.smithi070.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T21:42:57.410 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:42:57.410 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T21:42:57.410 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T21:42:57.410 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T21:42:57.410 INFO:teuthology.orchestra.run.smithi070.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T21:42:57.410 INFO:teuthology.lock.ops:Updating smithi070.front.sepia.ceph.com on lock server 2023-09-23T21:42:57.428 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T21:42:57.439 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T21:42:57.462 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T21:42:57.462 DEBUG:teuthology.orchestra.run.smithi026:> test '!' -e /home/ubuntu/cephtest 2023-09-23T21:42:57.464 DEBUG:teuthology.orchestra.run.smithi070:> test '!' -e /home/ubuntu/cephtest 2023-09-23T21:42:57.483 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T21:42:57.491 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T21:42:57.491 DEBUG:teuthology.orchestra.run.smithi026:> test -z $(ls -A /var/lib/ceph) 2023-09-23T21:42:57.533 DEBUG:teuthology.orchestra.run.smithi070:> test -z $(ls -A /var/lib/ceph) 2023-09-23T21:42:57.558 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T21:42:57.618 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T21:42:57.640 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T21:42:57.640 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-23T21:42:57.640 DEBUG:teuthology.orchestra.run.smithi026:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T21:42:57.641 DEBUG:teuthology.orchestra.run.smithi070:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T21:42:57.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:42:57.660 DEBUG:teuthology.orchestra.run.smithi070:> uname -r 2023-09-23T21:42:57.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:42:57.662 DEBUG:teuthology.orchestra.run.smithi026:> uname -r 2023-09-23T21:42:57.719 INFO:teuthology.orchestra.run.smithi070.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T21:42:57.720 INFO:teuthology.task.kernel:Running kernel on smithi070: 4.18.0-372.9.1.el8.x86_64 2023-09-23T21:42:57.720 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum install -y kernel 2023-09-23T21:42:57.722 INFO:teuthology.orchestra.run.smithi026.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T21:42:57.723 INFO:teuthology.task.kernel:Running kernel on smithi026: 4.18.0-372.9.1.el8.x86_64 2023-09-23T21:42:57.723 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum install -y kernel 2023-09-23T21:42:58.937 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:42:58.938 INFO:teuthology.orchestra.run.smithi026.stdout:Unable to read consumer identity 2023-09-23T21:42:58.944 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:42:58.944 INFO:teuthology.orchestra.run.smithi070.stdout:Unable to read consumer identity 2023-09-23T21:43:00.344 INFO:teuthology.orchestra.run.smithi070.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-09-23T21:43:00.472 INFO:teuthology.orchestra.run.smithi026.stdout:Copr repo for python3-asyncssh owned by ceph 11 kB/s | 3.5 kB 00:00 2023-09-23T21:43:01.070 INFO:teuthology.orchestra.run.smithi070.stdout:Extra Packages for Enterprise Linux 22 MB/s | 16 MB 00:00 2023-09-23T21:43:01.365 INFO:teuthology.orchestra.run.smithi026.stdout:Extra Packages for Enterprise Linux 18 MB/s | 16 MB 00:00 2023-09-23T21:43:04.595 INFO:teuthology.orchestra.run.smithi070.stdout:lab-extras 409 kB/s | 24 kB 00:00 2023-09-23T21:43:05.098 INFO:teuthology.orchestra.run.smithi026.stdout:lab-extras 443 kB/s | 24 kB 00:00 2023-09-23T21:43:05.256 INFO:teuthology.orchestra.run.smithi070.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T21:43:05.303 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:43:05.304 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:43:05.304 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:43:05.418 DEBUG:teuthology.orchestra.run.smithi070:> echo no | sudo yum reinstall kernel || true 2023-09-23T21:43:05.748 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:43:05.748 INFO:teuthology.orchestra.run.smithi070.stdout:Unable to read consumer identity 2023-09-23T21:43:06.008 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 09:43:04 PM UTC. 2023-09-23T21:43:06.017 INFO:teuthology.orchestra.run.smithi026.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T21:43:06.050 INFO:teuthology.orchestra.run.smithi070.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T21:43:06.075 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:43:06.076 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:43:06.076 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:43:06.087 INFO:teuthology.orchestra.run.smithi070.stderr:Error: No packages marked for reinstall. 2023-09-23T21:43:06.132 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum reinstall -y kernel || true 2023-09-23T21:43:06.183 DEBUG:teuthology.orchestra.run.smithi026:> echo no | sudo yum reinstall kernel || true 2023-09-23T21:43:06.460 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:43:06.460 INFO:teuthology.orchestra.run.smithi070.stdout:Unable to read consumer identity 2023-09-23T21:43:06.577 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:43:06.577 INFO:teuthology.orchestra.run.smithi026.stdout:Unable to read consumer identity 2023-09-23T21:43:06.721 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 09:43:04 PM UTC. 2023-09-23T21:43:06.763 INFO:teuthology.orchestra.run.smithi070.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T21:43:06.793 INFO:teuthology.orchestra.run.smithi070.stderr:Error: No packages marked for reinstall. 2023-09-23T21:43:06.839 DEBUG:teuthology.orchestra.run.smithi070:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T21:43:06.854 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:01 ago on Sat 23 Sep 2023 09:43:05 PM UTC. 2023-09-23T21:43:06.900 INFO:teuthology.orchestra.run.smithi026.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T21:43:06.918 INFO:teuthology.orchestra.run.smithi070.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T21:43:06.918 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T21:43:06.918 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T21:43:06.919 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T21:43:06.919 DEBUG:teuthology.orchestra.run.smithi070:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T21:43:06.939 INFO:teuthology.orchestra.run.smithi026.stderr:Error: No packages marked for reinstall. 2023-09-23T21:43:06.951 INFO:teuthology.orchestra.run.smithi070.stdout:ttyS1 2023-09-23T21:43:06.967 DEBUG:teuthology.parallel:result is None 2023-09-23T21:43:06.987 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum reinstall -y kernel || true 2023-09-23T21:43:07.335 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:43:07.336 INFO:teuthology.orchestra.run.smithi026.stdout:Unable to read consumer identity 2023-09-23T21:43:07.605 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 09:43:05 PM UTC. 2023-09-23T21:43:07.649 INFO:teuthology.orchestra.run.smithi026.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T21:43:07.677 INFO:teuthology.orchestra.run.smithi026.stderr:Error: No packages marked for reinstall. 2023-09-23T21:43:07.724 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T21:43:07.807 INFO:teuthology.orchestra.run.smithi026.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T21:43:07.807 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T21:43:07.807 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T21:43:07.807 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T21:43:07.808 DEBUG:teuthology.orchestra.run.smithi026:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T21:43:07.840 INFO:teuthology.orchestra.run.smithi026.stdout:ttyS1 2023-09-23T21:43:07.853 DEBUG:teuthology.parallel:result is None 2023-09-23T21:43:07.853 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T21:43:07.862 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T21:43:07.863 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T21:43:07.895 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T21:43:07.915 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T21:43:07.926 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T21:43:07.950 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T21:43:07.950 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T21:43:07.953 DEBUG:teuthology.orchestra.run.smithi070:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T21:43:07.980 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T21:43:07.991 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T21:43:07.991 DEBUG:teuthology.orchestra.run.smithi026:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T21:43:08.010 DEBUG:teuthology.orchestra.run.smithi070:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T21:43:08.047 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T21:43:08.058 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T21:43:08.061 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T21:43:08.072 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T21:43:08.073 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T21:43:08.082 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T21:43:08.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T21:43:08.102 DEBUG:teuthology.orchestra.run.smithi070:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T21:43:08.130 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T21:43:08.140 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T21:43:08.140 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T21:43:08.169 DEBUG:teuthology.orchestra.run.smithi070:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T21:43:08.187 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T21:43:08.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T21:43:08.325 DEBUG:teuthology.orchestra.run.smithi026:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T21:43:08.408 DEBUG:teuthology.orchestra.run.smithi026:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T21:43:08.452 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:43:08.452 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T21:43:08.544 DEBUG:teuthology.orchestra.run.smithi070:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T21:43:08.591 DEBUG:teuthology.orchestra.run.smithi070:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T21:43:08.645 DEBUG:teuthology.orchestra.run.smithi070:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T21:43:08.730 DEBUG:teuthology.orchestra.run.smithi070:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T21:43:08.774 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T21:43:08.774 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T21:43:08.843 DEBUG:teuthology.orchestra.run.smithi026:> sudo service rsyslog restart 2023-09-23T21:43:08.845 DEBUG:teuthology.orchestra.run.smithi070:> sudo service rsyslog restart 2023-09-23T21:43:08.901 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T21:43:08.952 INFO:teuthology.orchestra.run.smithi070.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T21:43:09.386 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T21:43:09.394 INFO:teuthology.task.internal:Starting timer... 2023-09-23T21:43:09.394 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T21:43:09.419 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T21:43:09.448 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-23T21:43:09.448 DEBUG:teuthology.orchestra.run.smithi026:> sudo service auditd rotate 2023-09-23T21:43:09.508 INFO:teuthology.orchestra.run.smithi026.stdout:Rotating logs: 2023-09-23T21:43:09.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo service auditd rotate 2023-09-23T21:43:09.574 INFO:teuthology.orchestra.run.smithi070.stdout:Rotating logs: 2023-09-23T21:43:09.575 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T21:43:09.576 DEBUG:teuthology.orchestra.run.smithi026:> /usr/sbin/getenforce 2023-09-23T21:43:09.603 INFO:teuthology.orchestra.run.smithi026.stdout:Permissive 2023-09-23T21:43:09.603 DEBUG:teuthology.orchestra.run.smithi070:> /usr/sbin/getenforce 2023-09-23T21:43:09.625 INFO:teuthology.orchestra.run.smithi070.stdout:Permissive 2023-09-23T21:43:09.625 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi026.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi070.front.sepia.ceph.com': 'permissive'} 2023-09-23T21:43:09.625 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T21:43:09.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:43:09.671 DEBUG:teuthology.orchestra.run.smithi070:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T21:43:09.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:43:09.698 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T21:43:09.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo /usr/sbin/setenforce permissive 2023-09-23T21:43:09.740 DEBUG:teuthology.orchestra.run.smithi070:> sudo /usr/sbin/setenforce permissive 2023-09-23T21:43:09.767 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T21:43:09.777 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T21:43:09.782 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T21:43:09.920 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T21:43:09.935 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T21:43:09.936 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi026.front.sepia.ceph.com,smithi070.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T21:51:57.121 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi026.front.sepia.ceph.com'), Remote(name='ubuntu@smithi070.front.sepia.ceph.com')] 2023-09-23T21:51:57.122 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi026.front.sepia.ceph.com' 2023-09-23T21:51:57.123 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi026.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T21:51:57.193 DEBUG:teuthology.orchestra.run.smithi026:> true 2023-09-23T21:51:57.260 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi026.front.sepia.ceph.com' 2023-09-23T21:51:57.260 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi070.front.sepia.ceph.com' 2023-09-23T21:51:57.261 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi070.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T21:51:57.328 DEBUG:teuthology.orchestra.run.smithi070:> true 2023-09-23T21:51:57.399 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi070.front.sepia.ceph.com' 2023-09-23T21:51:57.399 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T21:51:57.411 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T21:51:57.411 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T21:51:57.412 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T21:51:57.414 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T21:51:57.414 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T21:51:57.446 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T21:51:57.462 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T21:51:57.470 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T21:51:57.484 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T21:51:57.543 INFO:teuthology.orchestra.run.smithi026.stderr:sudo: ntpd: command not found 2023-09-23T21:51:57.550 INFO:teuthology.orchestra.run.smithi070.stderr:sudo: ntpd: command not found 2023-09-23T21:51:57.555 INFO:teuthology.orchestra.run.smithi026.stdout:506 Cannot talk to daemon 2023-09-23T21:51:57.561 INFO:teuthology.orchestra.run.smithi070.stdout:506 Cannot talk to daemon 2023-09-23T21:51:57.571 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T21:51:57.575 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T21:51:57.586 INFO:teuthology.orchestra.run.smithi026.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T21:51:57.589 INFO:teuthology.orchestra.run.smithi070.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T21:51:57.665 INFO:teuthology.orchestra.run.smithi070.stderr:bash: ntpq: command not found 2023-09-23T21:51:57.668 INFO:teuthology.orchestra.run.smithi070.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T21:51:57.668 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================== 2023-09-23T21:51:57.668 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:51:57.669 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:51:57.669 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:51:57.669 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:51:57.674 INFO:teuthology.orchestra.run.smithi026.stderr:bash: ntpq: command not found 2023-09-23T21:51:57.678 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T21:51:57.678 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2023-09-23T21:51:57.678 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:51:57.679 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:51:57.679 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:51:57.679 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:51:57.680 INFO:teuthology.run_tasks:Running task pexec... 2023-09-23T21:51:57.690 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-23T21:51:57.690 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T21:51:57.691 DEBUG:teuthology.orchestra.run.smithi026:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T21:51:57.691 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi070.front.sepia.ceph.com 2023-09-23T21:51:57.691 DEBUG:teuthology.orchestra.run.smithi070:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T21:51:58.081 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:51:58.113 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:51:58.881 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:41 ago on Sat 23 Sep 2023 09:50:17 PM UTC. 2023-09-23T21:51:58.959 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:27 ago on Sat 23 Sep 2023 09:50:31 PM UTC. 2023-09-23T21:52:00.607 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:52:00.608 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:52:00.608 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2023-09-23T21:52:00.608 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:52:00.608 INFO:teuthology.orchestra.run.smithi070.stdout:Resetting modules: 2023-09-23T21:52:00.609 INFO:teuthology.orchestra.run.smithi070.stdout: container-tools 2023-09-23T21:52:00.609 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:52:00.609 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:52:00.609 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:52:00.609 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:52:00.927 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:52:00.928 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:52:00.928 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2023-09-23T21:52:00.928 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:52:00.928 INFO:teuthology.orchestra.run.smithi026.stdout:Resetting modules: 2023-09-23T21:52:00.928 INFO:teuthology.orchestra.run.smithi026.stdout: container-tools 2023-09-23T21:52:00.928 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:52:00.928 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:52:00.928 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:52:00.929 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:52:01.583 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:44 ago on Sat 23 Sep 2023 09:50:17 PM UTC. 2023-09-23T21:52:01.927 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:30 ago on Sat 23 Sep 2023 09:50:31 PM UTC. 2023-09-23T21:52:03.916 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:52:04.378 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:52:04.694 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:52:05.168 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:52:05.196 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:47 ago on Sat 23 Sep 2023 09:50:17 PM UTC. 2023-09-23T21:52:06.009 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:34 ago on Sat 23 Sep 2023 09:50:31 PM UTC. 2023-09-23T21:52:06.899 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:52:06.990 INFO:teuthology.orchestra.run.smithi070.stdout:========================================================================================================================== 2023-09-23T21:52:06.990 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:52:06.990 INFO:teuthology.orchestra.run.smithi070.stdout:========================================================================================================================== 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout:Installing group/module packages: 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout:Downgrading: 2023-09-23T21:52:06.991 INFO:teuthology.orchestra.run.smithi070.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T21:52:06.992 INFO:teuthology.orchestra.run.smithi070.stdout:Installing module profiles: 2023-09-23T21:52:06.993 INFO:teuthology.orchestra.run.smithi070.stdout: container-tools/common 2023-09-23T21:52:06.993 INFO:teuthology.orchestra.run.smithi070.stdout:Enabling module streams: 2023-09-23T21:52:06.993 INFO:teuthology.orchestra.run.smithi070.stdout: container-tools 3.0 2023-09-23T21:52:06.993 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:52:06.993 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:52:06.993 INFO:teuthology.orchestra.run.smithi070.stdout:========================================================================================================================== 2023-09-23T21:52:06.993 INFO:teuthology.orchestra.run.smithi070.stdout:Install 7 Packages 2023-09-23T21:52:06.993 INFO:teuthology.orchestra.run.smithi070.stdout:Downgrade 12 Packages 2023-09-23T21:52:06.993 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:52:06.994 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 51 M 2023-09-23T21:52:06.994 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:52:07.058 INFO:teuthology.orchestra.run.smithi070.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 1.4 MB/s | 91 kB 00:00 2023-09-23T21:52:07.108 INFO:teuthology.orchestra.run.smithi070.stdout:(2/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 452 kB/s | 51 kB 00:00 2023-09-23T21:52:07.133 INFO:teuthology.orchestra.run.smithi070.stdout:(3/19): container-selinux-2.178.0-2.module+el8. 380 kB/s | 53 kB 00:00 2023-09-23T21:52:07.159 INFO:teuthology.orchestra.run.smithi070.stdout:(4/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 1.3 MB/s | 69 kB 00:00 2023-09-23T21:52:07.192 INFO:teuthology.orchestra.run.smithi070.stdout:(5/19): criu-3.15-1.module+el8.6.0+14874+644362 3.7 MB/s | 511 kB 00:00 2023-09-23T21:52:07.217 INFO:teuthology.orchestra.run.smithi070.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.2 MB/s | 72 kB 00:00 2023-09-23T21:52:07.251 INFO:teuthology.orchestra.run.smithi070.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.6 MB/s | 56 kB 00:00 2023-09-23T21:52:07.310 INFO:teuthology.orchestra.run.smithi070.stdout:(8/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 16 MB/s | 2.9 MB 00:00 2023-09-23T21:52:07.327 INFO:teuthology.orchestra.run.smithi070.stdout:(9/19): conmon-2.0.26-1.module+el8.6.0+14874+64 676 kB/s | 51 kB 00:00 2023-09-23T21:52:07.369 INFO:teuthology.orchestra.run.smithi070.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 7.5 MB/s | 322 kB 00:00 2023-09-23T21:52:07.402 INFO:teuthology.orchestra.run.smithi070.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 1.2 MB/s | 41 kB 00:00 2023-09-23T21:52:07.511 INFO:teuthology.orchestra.run.smithi070.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 20 MB/s | 2.2 MB 00:00 2023-09-23T21:52:07.636 INFO:teuthology.orchestra.run.smithi070.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 25 MB/s | 11 MB 00:00 2023-09-23T21:52:07.795 INFO:teuthology.orchestra.run.smithi070.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 23 MB/s | 6.4 MB 00:00 2023-09-23T21:52:07.920 INFO:teuthology.orchestra.run.smithi070.stdout:(15/19): buildah-1.19.9-3.module+el8.6.0+14874+ 24 MB/s | 6.8 MB 00:00 2023-09-23T21:52:07.962 INFO:teuthology.orchestra.run.smithi070.stdout:(16/19): cockpit-podman-29-2.module+el8.6.0+148 6.4 MB/s | 1.1 MB 00:00 2023-09-23T21:52:07.969 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:52:07.987 INFO:teuthology.orchestra.run.smithi070.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 2.7 MB/s | 185 kB 00:00 2023-09-23T21:52:08.012 INFO:teuthology.orchestra.run.smithi070.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.0 MB/s | 51 kB 00:00 2023-09-23T21:52:08.072 INFO:teuthology.orchestra.run.smithi026.stdout:========================================================================================================================== 2023-09-23T21:52:08.072 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:52:08.072 INFO:teuthology.orchestra.run.smithi026.stdout:========================================================================================================================== 2023-09-23T21:52:08.072 INFO:teuthology.orchestra.run.smithi026.stdout:Installing group/module packages: 2023-09-23T21:52:08.072 INFO:teuthology.orchestra.run.smithi026.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T21:52:08.072 INFO:teuthology.orchestra.run.smithi026.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T21:52:08.072 INFO:teuthology.orchestra.run.smithi026.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout:Downgrading: 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T21:52:08.073 INFO:teuthology.orchestra.run.smithi026.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout:Installing module profiles: 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout: container-tools/common 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling module streams: 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout: container-tools 3.0 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:52:08.074 INFO:teuthology.orchestra.run.smithi026.stdout:========================================================================================================================== 2023-09-23T21:52:08.075 INFO:teuthology.orchestra.run.smithi026.stdout:Install 7 Packages 2023-09-23T21:52:08.075 INFO:teuthology.orchestra.run.smithi026.stdout:Downgrade 12 Packages 2023-09-23T21:52:08.075 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:52:08.075 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 51 M 2023-09-23T21:52:08.075 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:52:08.146 INFO:teuthology.orchestra.run.smithi026.stdout:(1/19): containers-common-1.2.4-1.module+el8.6. 1.3 MB/s | 91 kB 00:00 2023-09-23T21:52:08.172 INFO:teuthology.orchestra.run.smithi026.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 555 kB/s | 53 kB 00:00 2023-09-23T21:52:08.187 INFO:teuthology.orchestra.run.smithi070.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 22 MB/s | 19 MB 00:00 2023-09-23T21:52:08.188 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:52:08.188 INFO:teuthology.orchestra.run.smithi070.stdout:Total 42 MB/s | 51 MB 00:01 2023-09-23T21:52:08.197 INFO:teuthology.orchestra.run.smithi026.stdout:(3/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 426 kB/s | 51 kB 00:00 2023-09-23T21:52:08.262 INFO:teuthology.orchestra.run.smithi026.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 4.3 MB/s | 511 kB 00:00 2023-09-23T21:52:08.281 INFO:teuthology.orchestra.run.smithi026.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 827 kB/s | 69 kB 00:00 2023-09-23T21:52:08.306 INFO:teuthology.orchestra.run.smithi026.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 1.6 MB/s | 72 kB 00:00 2023-09-23T21:52:08.348 INFO:teuthology.orchestra.run.smithi026.stdout:(7/19): podman-docker-3.0.1-9.module+el8.6.0+14 1.3 MB/s | 56 kB 00:00 2023-09-23T21:52:08.390 INFO:teuthology.orchestra.run.smithi026.stdout:(8/19): conmon-2.0.26-1.module+el8.6.0+14874+64 1.2 MB/s | 51 kB 00:00 2023-09-23T21:52:08.448 INFO:teuthology.orchestra.run.smithi026.stdout:(9/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 12 MB/s | 2.9 MB 00:00 2023-09-23T21:52:08.499 INFO:teuthology.orchestra.run.smithi026.stdout:(10/19): podman-catatonit-3.0.1-9.module+el8.6. 6.3 MB/s | 322 kB 00:00 2023-09-23T21:52:08.540 INFO:teuthology.orchestra.run.smithi026.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 982 kB/s | 41 kB 00:00 2023-09-23T21:52:08.641 INFO:teuthology.orchestra.run.smithi026.stdout:(12/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 22 MB/s | 2.2 MB 00:00 2023-09-23T21:52:08.663 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:52:08.706 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:52:08.706 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:52:08.791 INFO:teuthology.orchestra.run.smithi026.stdout:(13/19): podman-3.0.1-9.module+el8.6.0+14874+64 21 MB/s | 11 MB 00:00 2023-09-23T21:52:08.950 INFO:teuthology.orchestra.run.smithi026.stdout:(14/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 21 MB/s | 6.4 MB 00:00 2023-09-23T21:52:09.042 INFO:teuthology.orchestra.run.smithi026.stdout:(15/19): cockpit-podman-29-2.module+el8.6.0+148 12 MB/s | 1.1 MB 00:00 2023-09-23T21:52:09.126 INFO:teuthology.orchestra.run.smithi026.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 20 MB/s | 6.8 MB 00:00 2023-09-23T21:52:09.133 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:52:09.137 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:52:09.151 INFO:teuthology.orchestra.run.smithi026.stdout:(17/19): crun-0.18-3.module+el8.6.0+14874+64436 1.7 MB/s | 185 kB 00:00 2023-09-23T21:52:09.176 INFO:teuthology.orchestra.run.smithi026.stdout:(18/19): udica-0.2.4-1.module+el8.6.0+14874+644 1.0 MB/s | 51 kB 00:00 2023-09-23T21:52:09.376 INFO:teuthology.orchestra.run.smithi026.stdout:(19/19): containernetworking-plugins-0.9.1-1.mo 19 MB/s | 19 MB 00:00 2023-09-23T21:52:09.377 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:52:09.377 INFO:teuthology.orchestra.run.smithi026.stdout:Total 39 MB/s | 51 MB 00:01 2023-09-23T21:52:09.867 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:52:09.913 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:52:09.913 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:52:10.278 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:52:10.332 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T21:52:10.363 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:52:10.367 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:52:10.522 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T21:52:10.531 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T21:52:10.550 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T21:52:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T21:52:11.536 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:52:11.585 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T21:52:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T21:52:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T21:52:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T21:52:11.956 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T21:52:27.864 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T21:52:29.640 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T21:52:29.722 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T21:52:29.875 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T21:52:30.118 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T21:52:30.343 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T21:52:30.568 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T21:52:30.568 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T21:52:30.568 INFO:teuthology.orchestra.run.smithi070.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T21:52:30.568 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:52:31.412 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T21:52:31.590 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T21:52:31.617 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T21:52:31.776 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T21:52:31.973 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T21:52:32.134 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T21:52:32.381 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T21:52:32.381 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T21:52:32.382 INFO:teuthology.orchestra.run.smithi026.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T21:52:32.382 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:52:33.052 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T21:52:33.313 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T21:52:33.353 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T21:52:33.530 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T21:52:33.756 INFO:teuthology.orchestra.run.smithi070.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T21:52:34.015 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T21:52:34.759 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T21:52:35.033 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T21:52:35.327 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T21:52:35.515 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T21:52:35.723 INFO:teuthology.orchestra.run.smithi026.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T21:52:35.774 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T21:52:35.949 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T21:52:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T21:52:36.141 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T21:52:36.272 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T21:52:36.273 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T21:52:36.603 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T21:52:36.734 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T21:52:36.794 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T21:52:36.794 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T21:52:36.976 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T21:52:37.125 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T21:52:37.296 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T21:52:37.459 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T21:52:37.505 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T21:52:37.627 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T21:52:37.727 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T21:52:37.767 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T21:52:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T21:52:37.933 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T21:52:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T21:52:38.107 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T21:52:38.108 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T21:52:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T21:52:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T21:52:38.275 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T21:52:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T21:52:38.711 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T21:52:38.711 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T21:52:38.793 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T21:52:38.870 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T21:52:38.994 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T21:52:39.145 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T21:52:39.200 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T21:52:39.200 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T21:52:39.201 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T21:52:39.202 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T21:52:39.203 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T21:52:39.203 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T21:52:39.203 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T21:52:39.203 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T21:52:39.203 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T21:52:39.204 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T21:52:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T21:52:39.470 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T21:52:39.603 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T21:52:39.727 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T21:52:39.882 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T21:52:39.883 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T21:52:40.077 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T21:52:40.631 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T21:52:40.639 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T21:52:40.639 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:23 ago on Sat 23 Sep 2023 09:50:17 PM UTC. 2023-09-23T21:52:41.022 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T21:52:41.022 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T21:52:41.022 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T21:52:41.022 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T21:52:41.022 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T21:52:41.022 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T21:52:41.022 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T21:52:41.022 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T21:52:41.022 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T21:52:41.023 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T21:52:41.024 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T21:52:42.442 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T21:52:42.442 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:10 ago on Sat 23 Sep 2023 09:50:31 PM UTC. 2023-09-23T21:52:43.215 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:52:43.400 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:52:43.400 INFO:teuthology.orchestra.run.smithi070.stdout:Downgraded: 2023-09-23T21:52:43.400 INFO:teuthology.orchestra.run.smithi070.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.400 INFO:teuthology.orchestra.run.smithi070.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T21:52:43.400 INFO:teuthology.orchestra.run.smithi070.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.400 INFO:teuthology.orchestra.run.smithi070.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T21:52:43.401 INFO:teuthology.orchestra.run.smithi070.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.402 INFO:teuthology.orchestra.run.smithi070.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.402 INFO:teuthology.orchestra.run.smithi070.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:43.402 INFO:teuthology.orchestra.run.smithi070.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T21:52:43.402 INFO:teuthology.orchestra.run.smithi070.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T21:52:43.402 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:52:43.402 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:52:43.620 DEBUG:teuthology.parallel:result is None 2023-09-23T21:52:45.057 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:52:45.273 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:52:45.273 INFO:teuthology.orchestra.run.smithi026.stdout:Downgraded: 2023-09-23T21:52:45.273 INFO:teuthology.orchestra.run.smithi026.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.273 INFO:teuthology.orchestra.run.smithi026.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T21:52:45.273 INFO:teuthology.orchestra.run.smithi026.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.274 INFO:teuthology.orchestra.run.smithi026.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T21:52:45.275 INFO:teuthology.orchestra.run.smithi026.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.275 INFO:teuthology.orchestra.run.smithi026.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.275 INFO:teuthology.orchestra.run.smithi026.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T21:52:45.275 INFO:teuthology.orchestra.run.smithi026.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T21:52:45.275 INFO:teuthology.orchestra.run.smithi026.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T21:52:45.275 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:52:45.275 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:52:45.511 DEBUG:teuthology.parallel:result is None 2023-09-23T21:52:45.511 INFO:teuthology.run_tasks:Running task install... 2023-09-23T21:52:45.519 DEBUG:teuthology.task.install:project ceph 2023-09-23T21:52:45.519 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T21:52:45.519 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T21:52:45.520 INFO:teuthology.task.install:Using flavor: default 2023-09-23T21:52:45.526 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T21:52:45.526 INFO:teuthology.task.install:extra packages: [] 2023-09-23T21:52:45.526 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T21:52:45.526 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T21:52:45.527 INFO:teuthology.packaging:ref: None 2023-09-23T21:52:45.527 INFO:teuthology.packaging:tag: None 2023-09-23T21:52:45.527 INFO:teuthology.packaging:branch: quincy 2023-09-23T21:52:45.527 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T21:52:45.527 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T21:52:45.529 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T21:52:45.529 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T21:52:45.529 INFO:teuthology.packaging:ref: None 2023-09-23T21:52:45.529 INFO:teuthology.packaging:tag: None 2023-09-23T21:52:45.529 INFO:teuthology.packaging:branch: quincy 2023-09-23T21:52:45.530 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T21:52:45.530 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T21:52:45.763 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/ 2023-09-23T21:52:45.763 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1125.gc9bb1bcf 2023-09-23T21:52:46.011 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/ 2023-09-23T21:52:46.012 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1125.gc9bb1bcf 2023-09-23T21:52:46.017 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T21:52:46.017 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T21:52:46.017 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T21:52:46.050 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T21:52:46.050 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T21:52:46.050 INFO:teuthology.packaging:ref: None 2023-09-23T21:52:46.050 INFO:teuthology.packaging:tag: None 2023-09-23T21:52:46.050 INFO:teuthology.packaging:branch: quincy 2023-09-23T21:52:46.050 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T21:52:46.051 DEBUG:teuthology.orchestra.run.smithi070:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T21:52:46.117 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T21:52:46.117 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:52:46.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T21:52:46.121 DEBUG:teuthology.orchestra.run.smithi070:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T21:52:46.148 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T21:52:46.148 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T21:52:46.148 INFO:teuthology.packaging:ref: None 2023-09-23T21:52:46.148 INFO:teuthology.packaging:tag: None 2023-09-23T21:52:46.148 INFO:teuthology.packaging:branch: quincy 2023-09-23T21:52:46.148 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T21:52:46.148 DEBUG:teuthology.orchestra.run.smithi026:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T21:52:46.201 DEBUG:teuthology.orchestra.run.smithi070:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T21:52:46.218 DEBUG:teuthology.orchestra.run.smithi026:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T21:52:46.269 INFO:teuthology.orchestra.run.smithi070.stdout:check_obsoletes = 1 2023-09-23T21:52:46.271 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum clean all 2023-09-23T21:52:46.298 DEBUG:teuthology.orchestra.run.smithi026:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T21:52:46.367 INFO:teuthology.orchestra.run.smithi026.stdout:check_obsoletes = 1 2023-09-23T21:52:46.369 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum clean all 2023-09-23T21:52:46.643 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:52:46.762 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:52:47.030 INFO:teuthology.orchestra.run.smithi070.stdout:44 files removed 2023-09-23T21:52:47.075 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-radosgw 2023-09-23T21:52:47.100 INFO:teuthology.orchestra.run.smithi026.stdout:44 files removed 2023-09-23T21:52:47.147 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-radosgw 2023-09-23T21:52:47.405 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:52:47.503 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:52:48.426 INFO:teuthology.orchestra.run.smithi070.stdout:ceph packages for x86_64 126 kB/s | 78 kB 00:00 2023-09-23T21:52:48.479 INFO:teuthology.orchestra.run.smithi026.stdout:ceph packages for x86_64 142 kB/s | 78 kB 00:00 2023-09-23T21:52:48.880 INFO:teuthology.orchestra.run.smithi026.stdout:ceph noarch packages 25 kB/s | 9.1 kB 00:00 2023-09-23T21:52:48.902 INFO:teuthology.orchestra.run.smithi070.stdout:ceph noarch packages 21 kB/s | 9.1 kB 00:00 2023-09-23T21:52:49.252 INFO:teuthology.orchestra.run.smithi070.stdout:ceph source packages 4.4 kB/s | 1.4 kB 00:00 2023-09-23T21:52:49.256 INFO:teuthology.orchestra.run.smithi026.stdout:ceph source packages 4.1 kB/s | 1.4 kB 00:00 2023-09-23T21:52:49.445 INFO:teuthology.orchestra.run.smithi070.stdout:Copr repo for python3-asyncssh owned by ceph 20 kB/s | 3.5 kB 00:00 2023-09-23T21:52:49.473 INFO:teuthology.orchestra.run.smithi026.stdout:Copr repo for python3-asyncssh owned by ceph 18 kB/s | 3.5 kB 00:00 2023-09-23T21:52:50.071 INFO:teuthology.orchestra.run.smithi070.stdout:Extra Packages for Enterprise Linux 26 MB/s | 16 MB 00:00 2023-09-23T21:52:50.741 INFO:teuthology.orchestra.run.smithi026.stdout:Extra Packages for Enterprise Linux 13 MB/s | 16 MB 00:01 2023-09-23T21:52:53.504 INFO:teuthology.orchestra.run.smithi070.stdout:lab-extras 411 kB/s | 24 kB 00:00 2023-09-23T21:52:54.381 INFO:teuthology.orchestra.run.smithi070.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 66 MB/s | 53 MB 00:00 2023-09-23T21:52:54.399 INFO:teuthology.orchestra.run.smithi026.stdout:lab-extras 555 kB/s | 24 kB 00:00 2023-09-23T21:52:55.376 INFO:teuthology.orchestra.run.smithi026.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 53 MB/s | 47 MB 00:00 2023-09-23T21:53:02.456 INFO:teuthology.orchestra.run.smithi070.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 60 MB/s | 47 MB 00:00 2023-09-23T21:53:03.885 INFO:teuthology.orchestra.run.smithi026.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 55 MB/s | 53 MB 00:00 2023-09-23T21:53:13.384 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:53:13.387 INFO:teuthology.orchestra.run.smithi070.stdout:==================================================================================================== 2023-09-23T21:53:13.387 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:53:13.387 INFO:teuthology.orchestra.run.smithi070.stdout:==================================================================================================== 2023-09-23T21:53:13.387 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:53:13.387 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-radosgw x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 17 M 2023-09-23T21:53:13.387 INFO:teuthology.orchestra.run.smithi070.stdout:Upgrading: 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout: librados2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.5 M 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout: librbd1 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.6 M 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-base x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 5.2 M 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-common x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 23 M 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-selinux x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 24 k 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 795 k 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T21:53:13.388 INFO:teuthology.orchestra.run.smithi070.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T21:53:13.389 INFO:teuthology.orchestra.run.smithi070.stdout: libradosstriper1 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 506 k 2023-09-23T21:53:13.389 INFO:teuthology.orchestra.run.smithi070.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T21:53:13.389 INFO:teuthology.orchestra.run.smithi070.stdout: librgw2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 5.6 M 2023-09-23T21:53:13.389 INFO:teuthology.orchestra.run.smithi070.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T21:53:13.389 INFO:teuthology.orchestra.run.smithi070.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T21:53:13.389 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-argparse x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 45 k 2023-09-23T21:53:13.389 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-common x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 115 k 2023-09-23T21:53:13.389 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cephfs x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 190 k 2023-09-23T21:53:13.389 INFO:teuthology.orchestra.run.smithi070.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rados x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 372 k 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rbd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 358 k 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rgw x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 106 k 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout:==================================================================================================== 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout:Install 21 Packages 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout:Upgrade 2 Packages 2023-09-23T21:53:13.390 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:53:13.391 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 63 M 2023-09-23T21:53:13.391 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:53:14.247 INFO:teuthology.orchestra.run.smithi070.stdout:(1/23): ceph-base-17.2.6-1125.gc9bb1bcf.el8.x86 6.2 MB/s | 5.2 MB 00:00 2023-09-23T21:53:14.322 INFO:teuthology.orchestra.run.smithi070.stdout:(2/23): ceph-selinux-17.2.6-1125.gc9bb1bcf.el8. 324 kB/s | 24 kB 00:00 2023-09-23T21:53:14.498 INFO:teuthology.orchestra.run.smithi070.stdout:(3/23): libcephfs2-17.2.6-1125.gc9bb1bcf.el8.x8 4.4 MB/s | 795 kB 00:00 2023-09-23T21:53:14.606 INFO:teuthology.orchestra.run.smithi070.stdout:(4/23): libradosstriper1-17.2.6-1125.gc9bb1bcf. 4.6 MB/s | 506 kB 00:00 2023-09-23T21:53:15.415 INFO:teuthology.orchestra.run.smithi070.stdout:(5/23): ceph-radosgw-17.2.6-1125.gc9bb1bcf.el8. 8.4 MB/s | 17 MB 00:01 2023-09-23T21:53:15.499 INFO:teuthology.orchestra.run.smithi070.stdout:(6/23): librgw2-17.2.6-1125.gc9bb1bcf.el8.x86_6 6.2 MB/s | 5.6 MB 00:00 2023-09-23T21:53:15.524 INFO:teuthology.orchestra.run.smithi070.stdout:(7/23): python3-ceph-argparse-17.2.6-1125.gc9bb 412 kB/s | 45 kB 00:00 2023-09-23T21:53:15.549 INFO:teuthology.orchestra.run.smithi070.stdout:(8/23): python3-ceph-common-17.2.6-1125.gc9bb1b 2.2 MB/s | 115 kB 00:00 2023-09-23T21:53:15.583 INFO:teuthology.orchestra.run.smithi070.stdout:(9/23): python3-cephfs-17.2.6-1125.gc9bb1bcf.el 3.2 MB/s | 190 kB 00:00 2023-09-23T21:53:15.633 INFO:teuthology.orchestra.run.smithi070.stdout:(10/23): python3-rados-17.2.6-1125.gc9bb1bcf.el 4.4 MB/s | 372 kB 00:00 2023-09-23T21:53:15.658 INFO:teuthology.orchestra.run.smithi070.stdout:(11/23): python3-rbd-17.2.6-1125.gc9bb1bcf.el8. 4.7 MB/s | 358 kB 00:00 2023-09-23T21:53:15.716 INFO:teuthology.orchestra.run.smithi070.stdout:(12/23): python3-rgw-17.2.6-1125.gc9bb1bcf.el8. 1.3 MB/s | 106 kB 00:00 2023-09-23T21:53:15.885 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:53:15.887 INFO:teuthology.orchestra.run.smithi026.stdout:==================================================================================================== 2023-09-23T21:53:15.887 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:53:15.887 INFO:teuthology.orchestra.run.smithi026.stdout:==================================================================================================== 2023-09-23T21:53:15.888 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:53:15.888 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-radosgw x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 17 M 2023-09-23T21:53:15.888 INFO:teuthology.orchestra.run.smithi026.stdout:Upgrading: 2023-09-23T21:53:15.888 INFO:teuthology.orchestra.run.smithi026.stdout: librados2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.5 M 2023-09-23T21:53:15.888 INFO:teuthology.orchestra.run.smithi026.stdout: librbd1 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.6 M 2023-09-23T21:53:15.888 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T21:53:15.888 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-base x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 5.2 M 2023-09-23T21:53:15.888 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-common x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 23 M 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-selinux x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 24 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 795 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: libradosstriper1 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 506 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: librgw2 x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 5.6 M 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-argparse x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 45 k 2023-09-23T21:53:15.889 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-common x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 115 k 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cephfs x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 190 k 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rados x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 372 k 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rbd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 358 k 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rgw x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 106 k 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout:==================================================================================================== 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout:Install 21 Packages 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout:Upgrade 2 Packages 2023-09-23T21:53:15.890 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:53:15.891 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 63 M 2023-09-23T21:53:15.891 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:53:15.900 INFO:teuthology.orchestra.run.smithi070.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 319 kB/s | 59 kB 00:00 2023-09-23T21:53:16.142 INFO:teuthology.orchestra.run.smithi070.stdout:(14/23): ceph-common-17.2.6-1125.gc9bb1bcf.el8. 8.6 MB/s | 23 MB 00:02 2023-09-23T21:53:16.168 INFO:teuthology.orchestra.run.smithi070.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 281 kB/s | 75 kB 00:00 2023-09-23T21:53:16.193 INFO:teuthology.orchestra.run.smithi070.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 573 kB/s | 306 kB 00:00 2023-09-23T21:53:16.253 INFO:teuthology.orchestra.run.smithi070.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 562 kB/s | 47 kB 00:00 2023-09-23T21:53:16.277 INFO:teuthology.orchestra.run.smithi070.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 3.2 MB/s | 259 kB 00:00 2023-09-23T21:53:16.302 INFO:teuthology.orchestra.run.smithi070.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 875 kB/s | 44 kB 00:00 2023-09-23T21:53:16.327 INFO:teuthology.orchestra.run.smithi070.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 6.9 MB/s | 354 kB 00:00 2023-09-23T21:53:16.603 INFO:teuthology.orchestra.run.smithi070.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 3.8 MB/s | 1.7 MB 00:00 2023-09-23T21:53:16.720 INFO:teuthology.orchestra.run.smithi070.stdout:(22/23): librados2-17.2.6-1125.gc9bb1bcf.el8.x8 8.3 MB/s | 3.5 MB 00:00 2023-09-23T21:53:16.795 INFO:teuthology.orchestra.run.smithi070.stdout:(23/23): librbd1-17.2.6-1125.gc9bb1bcf.el8.x86_ 7.7 MB/s | 3.6 MB 00:00 2023-09-23T21:53:16.797 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:53:16.797 INFO:teuthology.orchestra.run.smithi070.stdout:Total 19 MB/s | 63 MB 00:03 2023-09-23T21:53:16.843 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:53:16.887 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:53:16.887 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:53:17.243 INFO:teuthology.orchestra.run.smithi026.stdout:(1/23): ceph-base-17.2.6-1125.gc9bb1bcf.el8.x86 4.0 MB/s | 5.2 MB 00:01 2023-09-23T21:53:17.251 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:53:17.254 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:53:17.302 INFO:teuthology.orchestra.run.smithi026.stdout:(2/23): ceph-selinux-17.2.6-1125.gc9bb1bcf.el8. 418 kB/s | 24 kB 00:00 2023-09-23T21:53:17.477 INFO:teuthology.orchestra.run.smithi026.stdout:(3/23): libcephfs2-17.2.6-1125.gc9bb1bcf.el8.x8 4.4 MB/s | 795 kB 00:00 2023-09-23T21:53:17.619 INFO:teuthology.orchestra.run.smithi026.stdout:(4/23): libradosstriper1-17.2.6-1125.gc9bb1bcf. 3.5 MB/s | 506 kB 00:00 2023-09-23T21:53:18.086 INFO:teuthology.orchestra.run.smithi026.stdout:(5/23): ceph-radosgw-17.2.6-1125.gc9bb1bcf.el8. 7.8 MB/s | 17 MB 00:02 2023-09-23T21:53:18.161 INFO:teuthology.orchestra.run.smithi026.stdout:(6/23): python3-ceph-argparse-17.2.6-1125.gc9bb 597 kB/s | 45 kB 00:00 2023-09-23T21:53:18.220 INFO:teuthology.orchestra.run.smithi026.stdout:(7/23): python3-ceph-common-17.2.6-1125.gc9bb1b 1.9 MB/s | 115 kB 00:00 2023-09-23T21:53:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:53:18.312 INFO:teuthology.orchestra.run.smithi026.stdout:(8/23): python3-cephfs-17.2.6-1125.gc9bb1bcf.el 2.0 MB/s | 190 kB 00:00 2023-09-23T21:53:18.466 INFO:teuthology.orchestra.run.smithi026.stdout:(9/23): librgw2-17.2.6-1125.gc9bb1bcf.el8.x86_6 6.6 MB/s | 5.6 MB 00:00 2023-09-23T21:53:18.481 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T21:53:18.496 INFO:teuthology.orchestra.run.smithi026.stdout:(10/23): python3-rados-17.2.6-1125.gc9bb1bcf.el 2.0 MB/s | 372 kB 00:00 2023-09-23T21:53:18.563 INFO:teuthology.orchestra.run.smithi026.stdout:(11/23): python3-rbd-17.2.6-1125.gc9bb1bcf.el8. 3.6 MB/s | 358 kB 00:00 2023-09-23T21:53:18.580 INFO:teuthology.orchestra.run.smithi026.stdout:(12/23): python3-rgw-17.2.6-1125.gc9bb1bcf.el8. 1.2 MB/s | 106 kB 00:00 2023-09-23T21:53:18.847 INFO:teuthology.orchestra.run.smithi026.stdout:(13/23): ceph-common-17.2.6-1125.gc9bb1bcf.el8. 8.1 MB/s | 23 MB 00:02 2023-09-23T21:53:18.864 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T21:53:19.032 INFO:teuthology.orchestra.run.smithi070.stdout: Upgrading : librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T21:53:19.114 INFO:teuthology.orchestra.run.smithi026.stdout:(14/23): liboath-2.6.2-3.el8.x86_64.rpm 110 kB/s | 59 kB 00:00 2023-09-23T21:53:19.127 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T21:53:19.172 INFO:teuthology.orchestra.run.smithi026.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 235 kB/s | 75 kB 00:00 2023-09-23T21:53:19.198 INFO:teuthology.orchestra.run.smithi026.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 482 kB/s | 306 kB 00:00 2023-09-23T21:53:19.265 INFO:teuthology.orchestra.run.smithi026.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.8 MB/s | 259 kB 00:00 2023-09-23T21:53:19.290 INFO:teuthology.orchestra.run.smithi026.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 479 kB/s | 44 kB 00:00 2023-09-23T21:53:19.312 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/25 2023-09-23T21:53:19.323 INFO:teuthology.orchestra.run.smithi026.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.9 MB/s | 354 kB 00:00 2023-09-23T21:53:19.349 INFO:teuthology.orchestra.run.smithi026.stdout:(20/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 799 kB/s | 47 kB 00:00 2023-09-23T21:53:19.516 INFO:teuthology.orchestra.run.smithi026.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 4.3 MB/s | 1.7 MB 00:00 2023-09-23T21:53:19.531 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T21:53:19.690 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T21:53:19.749 INFO:teuthology.orchestra.run.smithi026.stdout:(22/23): librbd1-17.2.6-1125.gc9bb1bcf.el8.x86_ 9.0 MB/s | 3.6 MB 00:00 2023-09-23T21:53:19.841 INFO:teuthology.orchestra.run.smithi026.stdout:(23/23): librados2-17.2.6-1125.gc9bb1bcf.el8.x8 6.7 MB/s | 3.5 MB 00:00 2023-09-23T21:53:19.847 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:53:19.847 INFO:teuthology.orchestra.run.smithi026.stdout:Total 16 MB/s | 63 MB 00:03 2023-09-23T21:53:19.897 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:53:19.947 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:53:19.947 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:53:20.086 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T21:53:20.265 INFO:teuthology.orchestra.run.smithi070.stdout: Upgrading : librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/25 2023-09-23T21:53:20.314 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:53:20.318 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:53:20.389 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/25 2023-09-23T21:53:20.523 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T21:53:20.576 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T21:53:20.900 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T21:53:21.401 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:53:21.580 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T21:53:21.603 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T21:53:21.748 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T21:53:21.813 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T21:53:21.958 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8. 11/25 2023-09-23T21:53:22.006 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T21:53:22.117 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 12/25 2023-09-23T21:53:22.311 INFO:teuthology.orchestra.run.smithi026.stdout: Upgrading : librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T21:53:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 13/25 2023-09-23T21:53:22.401 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T21:53:22.551 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 14/25 2023-09-23T21:53:22.588 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/25 2023-09-23T21:53:22.730 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 15/25 2023-09-23T21:53:22.801 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 15/25 2023-09-23T21:53:22.815 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T21:53:22.931 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T21:53:22.994 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T21:53:23.122 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T21:53:23.348 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T21:53:23.436 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T21:53:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x8 19/25 2023-09-23T21:53:23.620 INFO:teuthology.orchestra.run.smithi026.stdout: Upgrading : librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/25 2023-09-23T21:53:23.689 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/25 2023-09-23T21:53:23.912 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 7/25 2023-09-23T21:53:24.045 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-23T21:53:24.339 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/25 2023-09-23T21:53:25.036 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T21:53:25.170 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T21:53:25.219 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T21:53:25.348 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8. 11/25 2023-09-23T21:53:25.531 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 12/25 2023-09-23T21:53:25.740 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 13/25 2023-09-23T21:53:25.942 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 14/25 2023-09-23T21:53:26.077 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 15/25 2023-09-23T21:53:26.182 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 15/25 2023-09-23T21:53:27.038 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T21:53:27.225 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T21:53:27.452 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T21:53:27.492 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T21:53:27.611 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x8 19/25 2023-09-23T21:53:27.738 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T21:53:28.325 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T21:53:28.632 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 21/25 2023-09-23T21:53:28.708 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 21/25 2023-09-23T21:53:28.896 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T21:53:31.599 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T21:53:31.844 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T21:53:32.458 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 20/25 2023-09-23T21:53:32.794 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 21/25 2023-09-23T21:53:32.883 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 21/25 2023-09-23T21:53:33.043 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T21:53:47.635 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T21:53:47.635 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /sys 2023-09-23T21:53:47.635 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /proc 2023-09-23T21:53:47.635 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /dev 2023-09-23T21:53:47.635 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /run 2023-09-23T21:53:47.636 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /mnt 2023-09-23T21:53:47.636 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /var/tmp 2023-09-23T21:53:47.636 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /home 2023-09-23T21:53:47.636 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /tmp 2023-09-23T21:53:47.636 INFO:teuthology.orchestra.run.smithi070.stdout:skipping the directory /dev 2023-09-23T21:53:47.636 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:53:49.318 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 23/25 2023-09-23T21:53:49.390 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 23/25 2023-09-23T21:53:49.390 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T21:53:49.594 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T21:53:49.594 INFO:teuthology.orchestra.run.smithi070.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T21:53:50.267 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T21:53:50.267 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 4/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 6/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 7/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8. 8/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x8 9/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T21:53:50.268 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 11/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 12/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 13/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T21:53:50.269 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T21:53:50.270 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T21:53:50.270 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 24/25 2023-09-23T21:53:51.628 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T21:53:51.628 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:50 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:53:52.445 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T21:53:52.446 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /sys 2023-09-23T21:53:52.446 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /proc 2023-09-23T21:53:52.446 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /dev 2023-09-23T21:53:52.446 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /run 2023-09-23T21:53:52.446 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /mnt 2023-09-23T21:53:52.446 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /var/tmp 2023-09-23T21:53:52.446 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /home 2023-09-23T21:53:52.447 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /tmp 2023-09-23T21:53:52.447 INFO:teuthology.orchestra.run.smithi026.stdout:skipping the directory /dev 2023-09-23T21:53:52.447 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:53:54.185 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:53:54.224 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 23/25 2023-09-23T21:53:54.288 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 23/25 2023-09-23T21:53:54.288 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T21:53:54.368 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:53:54.369 INFO:teuthology.orchestra.run.smithi070.stdout:Upgraded: 2023-09-23T21:53:54.369 INFO:teuthology.orchestra.run.smithi070.stdout: librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.369 INFO:teuthology.orchestra.run.smithi070.stdout: librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.369 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:53:54.369 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.369 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.369 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.369 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.369 INFO:teuthology.orchestra.run.smithi070.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.370 INFO:teuthology.orchestra.run.smithi070.stdout: python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.371 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.371 INFO:teuthology.orchestra.run.smithi070.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T21:53:54.371 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.371 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.371 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:54.371 INFO:teuthology.orchestra.run.smithi070.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T21:53:54.371 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:53:54.371 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:53:54.509 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T21:53:54.509 INFO:teuthology.orchestra.run.smithi026.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T21:53:54.566 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-test 2023-09-23T21:53:54.911 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:53:55.119 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T21:53:55.119 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/25 2023-09-23T21:53:55.119 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/25 2023-09-23T21:53:55.119 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/25 2023-09-23T21:53:55.119 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 4/25 2023-09-23T21:53:55.119 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/25 2023-09-23T21:53:55.119 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_6 6/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 7/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8. 8/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x8 9/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 10/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 11/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 12/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 13/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T21:53:55.120 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 18/25 2023-09-23T21:53:55.121 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 19/25 2023-09-23T21:53:55.121 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 20/25 2023-09-23T21:53:55.121 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 21/25 2023-09-23T21:53:55.121 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 22/25 2023-09-23T21:53:55.121 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T21:53:55.121 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 24/25 2023-09-23T21:53:55.691 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:00:54 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:53:56.523 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T21:53:56.523 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:53 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:53:57.049 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:53:57.050 INFO:teuthology.orchestra.run.smithi070.stdout:============================================================================================ 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout:============================================================================================ 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-test x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 52 M 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout: libcephsqlite x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 174 k 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T21:53:57.051 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:53:57.052 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:53:57.052 INFO:teuthology.orchestra.run.smithi070.stdout:============================================================================================ 2023-09-23T21:53:57.052 INFO:teuthology.orchestra.run.smithi070.stdout:Install 6 Packages 2023-09-23T21:53:57.052 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:53:57.053 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 53 M 2023-09-23T21:53:57.053 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 233 M 2023-09-23T21:53:57.053 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:53:57.098 INFO:teuthology.orchestra.run.smithi070.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.5 MB/s | 70 kB 00:00 2023-09-23T21:53:57.165 INFO:teuthology.orchestra.run.smithi070.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.7 MB/s | 323 kB 00:00 2023-09-23T21:53:57.207 INFO:teuthology.orchestra.run.smithi070.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-09-23T21:53:57.249 INFO:teuthology.orchestra.run.smithi070.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.8 MB/s | 202 kB 00:00 2023-09-23T21:53:57.559 INFO:teuthology.orchestra.run.smithi070.stdout:(5/6): libcephsqlite-17.2.6-1125.gc9bb1bcf.el8. 345 kB/s | 174 kB 00:00 2023-09-23T21:53:59.167 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:53:59.374 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:53:59.374 INFO:teuthology.orchestra.run.smithi026.stdout:Upgraded: 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-radosgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-selinux-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T21:53:59.375 INFO:teuthology.orchestra.run.smithi026.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: libradosstriper1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: librgw2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-argparse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: python3-ceph-common-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.376 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:53:59.377 INFO:teuthology.orchestra.run.smithi026.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T21:53:59.377 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:53:59.377 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:53:59.531 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-test 2023-09-23T21:53:59.901 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:00.862 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:00:57 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:54:01.690 INFO:teuthology.orchestra.run.smithi070.stdout:(6/6): ceph-test-17.2.6-1125.gc9bb1bcf.el8.x86_ 11 MB/s | 52 MB 00:04 2023-09-23T21:54:02.201 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:54:02.201 INFO:teuthology.orchestra.run.smithi070.stdout:Total 11 MB/s | 53 MB 00:04 2023-09-23T21:54:02.201 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:54:02.201 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:54:02.201 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:54:02.201 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:54:02.201 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:54:02.377 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:54:02.378 INFO:teuthology.orchestra.run.smithi026.stdout:============================================================================================ 2023-09-23T21:54:02.378 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:54:02.378 INFO:teuthology.orchestra.run.smithi026.stdout:============================================================================================ 2023-09-23T21:54:02.378 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:54:02.378 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-test x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 52 M 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout: libcephsqlite x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 174 k 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout:============================================================================================ 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout:Install 6 Packages 2023-09-23T21:54:02.379 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:02.380 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 53 M 2023-09-23T21:54:02.380 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 233 M 2023-09-23T21:54:02.380 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:54:02.421 INFO:teuthology.orchestra.run.smithi026.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.7 MB/s | 70 kB 00:00 2023-09-23T21:54:02.484 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:54:02.521 INFO:teuthology.orchestra.run.smithi026.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 3.5 MB/s | 323 kB 00:00 2023-09-23T21:54:02.571 INFO:teuthology.orchestra.run.smithi026.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 3.7 MB/s | 187 kB 00:00 2023-09-23T21:54:02.613 INFO:teuthology.orchestra.run.smithi026.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-09-23T21:54:02.622 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T21:54:02.695 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T21:54:02.730 INFO:teuthology.orchestra.run.smithi026.stdout:(5/6): libcephsqlite-17.2.6-1125.gc9bb1bcf.el8. 499 kB/s | 174 kB 00:00 2023-09-23T21:54:02.900 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T21:54:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T21:54:03.220 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T21:54:03.380 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/6 2023-09-23T21:54:04.830 INFO:teuthology.orchestra.run.smithi026.stdout:(6/6): ceph-test-17.2.6-1125.gc9bb1bcf.el8.x86_ 21 MB/s | 52 MB 00:02 2023-09-23T21:54:04.832 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:54:04.832 INFO:teuthology.orchestra.run.smithi026.stdout:Total 21 MB/s | 53 MB 00:02 2023-09-23T21:54:04.872 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:54:04.894 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:54:04.895 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:54:05.191 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:54:05.195 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:54:05.674 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:54:05.813 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T21:54:05.892 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T21:54:06.111 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T21:54:06.273 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T21:54:06.421 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T21:54:06.562 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/6 2023-09-23T21:54:08.305 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/6 2023-09-23T21:54:08.575 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/6 2023-09-23T21:54:08.801 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/6 2023-09-23T21:54:08.801 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/6 2023-09-23T21:54:08.801 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/6 2023-09-23T21:54:08.801 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T21:54:08.801 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T21:54:08.802 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T21:54:09.900 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T21:54:09.900 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:08 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/6 2023-09-23T21:54:12.242 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/6 2023-09-23T21:54:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 6/6 2023-09-23T21:54:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/6 2023-09-23T21:54:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/6 2023-09-23T21:54:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T21:54:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T21:54:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T21:54:12.583 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:54:12.762 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:12.762 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:54:12.762 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:12.762 INFO:teuthology.orchestra.run.smithi070.stdout: jq-1.6-3.el8.x86_64 2023-09-23T21:54:12.762 INFO:teuthology.orchestra.run.smithi070.stdout: libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:12.762 INFO:teuthology.orchestra.run.smithi070.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T21:54:12.762 INFO:teuthology.orchestra.run.smithi070.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T21:54:12.762 INFO:teuthology.orchestra.run.smithi070.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T21:54:12.762 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:12.763 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:54:12.907 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph 2023-09-23T21:54:13.241 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:13.601 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T21:54:13.601 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:10 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:54:14.088 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:13 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:15.443 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:54:15.446 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================= 2023-09-23T21:54:15.446 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:54:15.446 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================= 2023-09-23T21:54:15.447 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:54:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: ceph x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 6.4 k 2023-09-23T21:54:15.447 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T21:54:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mds x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 2.3 M 2023-09-23T21:54:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 1.6 M 2023-09-23T21:54:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 238 k 2023-09-23T21:54:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mon x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.9 M 2023-09-23T21:54:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-osd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 17 M 2023-09-23T21:54:15.447 INFO:teuthology.orchestra.run.smithi070.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T21:54:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T21:54:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T21:54:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T21:54:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T21:54:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T21:54:15.448 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T21:54:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T21:54:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T21:54:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T21:54:15.449 INFO:teuthology.orchestra.run.smithi070.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T21:54:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T21:54:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T21:54:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T21:54:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T21:54:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T21:54:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T21:54:15.450 INFO:teuthology.orchestra.run.smithi070.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:15.451 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:54:15.452 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================= 2023-09-23T21:54:15.452 INFO:teuthology.orchestra.run.smithi070.stdout:Install 32 Packages 2023-09-23T21:54:15.452 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:15.452 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 28 M 2023-09-23T21:54:15.452 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 110 M 2023-09-23T21:54:15.452 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:54:15.881 INFO:teuthology.orchestra.run.smithi070.stdout:(1/32): ceph-17.2.6-1125.gc9bb1bcf.el8.x86_64.r 57 kB/s | 6.4 kB 00:00 2023-09-23T21:54:16.106 INFO:teuthology.orchestra.run.smithi070.stdout:(2/32): ceph-mds-17.2.6-1125.gc9bb1bcf.el8.x86_ 6.8 MB/s | 2.3 MB 00:00 2023-09-23T21:54:16.598 INFO:teuthology.orchestra.run.smithi070.stdout:(3/32): ceph-mon-17.2.6-1125.gc9bb1bcf.el8.x86_ 5.5 MB/s | 3.9 MB 00:00 2023-09-23T21:54:16.682 INFO:teuthology.orchestra.run.smithi070.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1125.gc9bb 2.8 MB/s | 238 kB 00:00 2023-09-23T21:54:16.999 INFO:teuthology.orchestra.run.smithi070.stdout:(5/32): ceph-osd-17.2.6-1125.gc9bb1bcf.el8.x86_ 19 MB/s | 17 MB 00:00 2023-09-23T21:54:17.015 INFO:teuthology.orchestra.run.smithi070.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 131 kB/s | 44 kB 00:00 2023-09-23T21:54:17.258 INFO:teuthology.orchestra.run.smithi070.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 713 kB/s | 173 kB 00:00 2023-09-23T21:54:17.341 INFO:teuthology.orchestra.run.smithi070.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 541 kB/s | 185 kB 00:00 2023-09-23T21:54:17.400 INFO:teuthology.orchestra.run.smithi070.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 191 kB/s | 11 kB 00:00 2023-09-23T21:54:17.425 INFO:teuthology.orchestra.run.smithi070.stdout:(10/32): python3-cherrypy-18.4.0-1.el8.noarch.r 2.2 MB/s | 384 kB 00:00 2023-09-23T21:54:17.467 INFO:teuthology.orchestra.run.smithi070.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 263 kB/s | 18 kB 00:00 2023-09-23T21:54:17.517 INFO:teuthology.orchestra.run.smithi070.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 531 kB/s | 49 kB 00:00 2023-09-23T21:54:17.559 INFO:teuthology.orchestra.run.smithi070.stdout:(13/32): ceph-mgr-17.2.6-1125.gc9bb1bcf.el8.x86 897 kB/s | 1.6 MB 00:01 2023-09-23T21:54:17.585 INFO:teuthology.orchestra.run.smithi070.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 501 kB/s | 59 kB 00:00 2023-09-23T21:54:17.610 INFO:teuthology.orchestra.run.smithi070.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 3.0 MB/s | 283 kB 00:00 2023-09-23T21:54:17.660 INFO:teuthology.orchestra.run.smithi070.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 249 kB/s | 19 kB 00:00 2023-09-23T21:54:17.685 INFO:teuthology.orchestra.run.smithi070.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 338 kB/s | 24 kB 00:00 2023-09-23T21:54:17.736 INFO:teuthology.orchestra.run.smithi070.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 388 kB/s | 29 kB 00:00 2023-09-23T21:54:17.786 INFO:teuthology.orchestra.run.smithi070.stdout:(19/32): python3-portend-2.6-1.el8.noarch.rpm 69 kB/s | 16 kB 00:00 2023-09-23T21:54:17.812 INFO:teuthology.orchestra.run.smithi070.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 219 kB/s | 27 kB 00:00 2023-09-23T21:54:17.862 INFO:teuthology.orchestra.run.smithi070.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.0 MB/s | 255 kB 00:00 2023-09-23T21:54:17.887 INFO:teuthology.orchestra.run.smithi070.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.4 MB/s | 251 kB 00:00 2023-09-23T21:54:17.929 INFO:teuthology.orchestra.run.smithi070.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 343 kB/s | 23 kB 00:00 2023-09-23T21:54:17.979 INFO:teuthology.orchestra.run.smithi070.stdout:(24/32): python3-webtest-2.0.33-1.el8.noarch.rp 510 kB/s | 85 kB 00:00 2023-09-23T21:54:18.004 INFO:teuthology.orchestra.run.smithi070.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 932 kB/s | 109 kB 00:00 2023-09-23T21:54:18.055 INFO:teuthology.orchestra.run.smithi070.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 4.5 MB/s | 559 kB 00:00 2023-09-23T21:54:18.080 INFO:teuthology.orchestra.run.smithi070.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 775 kB/s | 54 kB 00:00 2023-09-23T21:54:18.105 INFO:teuthology.orchestra.run.smithi070.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 777 kB/s | 39 kB 00:00 2023-09-23T21:54:18.117 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:54:18.131 INFO:teuthology.orchestra.run.smithi070.stdout:(29/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.5 MB/s | 238 kB 00:00 2023-09-23T21:54:18.156 INFO:teuthology.orchestra.run.smithi070.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.0 MB/s | 157 kB 00:00 2023-09-23T21:54:18.181 INFO:teuthology.orchestra.run.smithi070.stdout:(31/32): python3-werkzeug-0.12.2-4.el8.noarch.r 5.9 MB/s | 457 kB 00:00 2023-09-23T21:54:18.215 INFO:teuthology.orchestra.run.smithi070.stdout:(32/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.2 MB/s | 103 kB 00:00 2023-09-23T21:54:18.219 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:54:18.219 INFO:teuthology.orchestra.run.smithi070.stdout:Total 10 MB/s | 28 MB 00:02 2023-09-23T21:54:18.312 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:54:18.315 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:18.315 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:54:18.315 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-test-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:18.315 INFO:teuthology.orchestra.run.smithi026.stdout: jq-1.6-3.el8.x86_64 2023-09-23T21:54:18.315 INFO:teuthology.orchestra.run.smithi026.stdout: libcephsqlite-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:18.315 INFO:teuthology.orchestra.run.smithi026.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T21:54:18.315 INFO:teuthology.orchestra.run.smithi026.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T21:54:18.316 INFO:teuthology.orchestra.run.smithi026.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T21:54:18.316 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:18.316 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:54:18.349 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:54:18.349 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:54:18.447 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph 2023-09-23T21:54:18.554 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:54:18.558 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:54:18.802 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:19.649 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:54:19.840 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T21:54:20.137 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T21:54:20.363 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T21:54:20.481 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:17 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:54:20.603 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T21:54:20.816 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T21:54:21.021 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T21:54:21.260 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T21:54:21.597 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T21:54:21.878 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T21:54:21.988 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:54:21.992 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================= 2023-09-23T21:54:21.992 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:54:21.992 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================= 2023-09-23T21:54:21.992 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:54:21.992 INFO:teuthology.orchestra.run.smithi026.stdout: ceph x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 6.4 k 2023-09-23T21:54:21.992 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T21:54:21.992 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mds x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 2.3 M 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 1.6 M 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 238 k 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mon x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.9 M 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-osd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 17 M 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T21:54:21.993 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T21:54:21.994 INFO:teuthology.orchestra.run.smithi026.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================= 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout:Install 32 Packages 2023-09-23T21:54:21.995 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:21.996 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 28 M 2023-09-23T21:54:21.996 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 110 M 2023-09-23T21:54:21.996 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:54:22.050 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T21:54:22.247 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T21:54:22.398 INFO:teuthology.orchestra.run.smithi026.stdout:(1/32): ceph-17.2.6-1125.gc9bb1bcf.el8.x86_64.r 29 kB/s | 6.4 kB 00:00 2023-09-23T21:54:22.489 INFO:teuthology.orchestra.run.smithi026.stdout:(2/32): ceph-mds-17.2.6-1125.gc9bb1bcf.el8.x86_ 7.3 MB/s | 2.3 MB 00:00 2023-09-23T21:54:22.948 INFO:teuthology.orchestra.run.smithi026.stdout:(3/32): ceph-mgr-17.2.6-1125.gc9bb1bcf.el8.x86_ 2.0 MB/s | 1.6 MB 00:00 2023-09-23T21:54:23.056 INFO:teuthology.orchestra.run.smithi026.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1125.gc9bb 2.2 MB/s | 238 kB 00:00 2023-09-23T21:54:23.224 INFO:teuthology.orchestra.run.smithi026.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 262 kB/s | 44 kB 00:00 2023-09-23T21:54:23.285 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T21:54:23.399 INFO:teuthology.orchestra.run.smithi026.stdout:(6/32): ceph-osd-17.2.6-1125.gc9bb1bcf.el8.x86_ 18 MB/s | 17 MB 00:00 2023-09-23T21:54:23.424 INFO:teuthology.orchestra.run.smithi026.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 925 kB/s | 185 kB 00:00 2023-09-23T21:54:23.524 INFO:teuthology.orchestra.run.smithi026.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 4.0 MB/s | 384 kB 00:00 2023-09-23T21:54:23.558 INFO:teuthology.orchestra.run.smithi026.stdout:(9/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 1.1 MB/s | 173 kB 00:00 2023-09-23T21:54:23.583 INFO:teuthology.orchestra.run.smithi026.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 189 kB/s | 11 kB 00:00 2023-09-23T21:54:23.608 INFO:teuthology.orchestra.run.smithi026.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 353 kB/s | 18 kB 00:00 2023-09-23T21:54:23.622 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T21:54:23.634 INFO:teuthology.orchestra.run.smithi026.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 972 kB/s | 49 kB 00:00 2023-09-23T21:54:23.690 INFO:teuthology.orchestra.run.smithi026.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 723 kB/s | 59 kB 00:00 2023-09-23T21:54:23.759 INFO:teuthology.orchestra.run.smithi026.stdout:(14/32): ceph-mon-17.2.6-1125.gc9bb1bcf.el8.x86 2.9 MB/s | 3.9 MB 00:01 2023-09-23T21:54:23.785 INFO:teuthology.orchestra.run.smithi026.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 166 kB/s | 16 kB 00:00 2023-09-23T21:54:23.810 INFO:teuthology.orchestra.run.smithi026.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 373 kB/s | 19 kB 00:00 2023-09-23T21:54:23.815 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T21:54:23.844 INFO:teuthology.orchestra.run.smithi026.stdout:(17/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.3 MB/s | 283 kB 00:00 2023-09-23T21:54:23.869 INFO:teuthology.orchestra.run.smithi026.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 544 kB/s | 29 kB 00:00 2023-09-23T21:54:23.894 INFO:teuthology.orchestra.run.smithi026.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 558 kB/s | 27 kB 00:00 2023-09-23T21:54:23.920 INFO:teuthology.orchestra.run.smithi026.stdout:(20/32): python3-singledispatch-3.4.0.3-18.el8. 177 kB/s | 24 kB 00:00 2023-09-23T21:54:23.961 INFO:teuthology.orchestra.run.smithi026.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.7 MB/s | 255 kB 00:00 2023-09-23T21:54:23.978 INFO:teuthology.orchestra.run.smithi026.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.9 MB/s | 251 kB 00:00 2023-09-23T21:54:23.979 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T21:54:24.004 INFO:teuthology.orchestra.run.smithi026.stdout:(23/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.0 MB/s | 85 kB 00:00 2023-09-23T21:54:24.029 INFO:teuthology.orchestra.run.smithi026.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 341 kB/s | 23 kB 00:00 2023-09-23T21:54:24.104 INFO:teuthology.orchestra.run.smithi026.stdout:(25/32): python3-pytz-2017.2-9.el8.noarch.rpm 432 kB/s | 54 kB 00:00 2023-09-23T21:54:24.121 INFO:teuthology.orchestra.run.smithi026.stdout:(26/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.1 MB/s | 157 kB 00:00 2023-09-23T21:54:24.146 INFO:teuthology.orchestra.run.smithi026.stdout:(27/32): python3-markupsafe-0.23-19.el8.x86_64. 273 kB/s | 39 kB 00:00 2023-09-23T21:54:24.169 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T21:54:24.255 INFO:teuthology.orchestra.run.smithi026.stdout:(28/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 775 kB/s | 103 kB 00:00 2023-09-23T21:54:24.289 INFO:teuthology.orchestra.run.smithi026.stdout:(29/32): python3-werkzeug-0.12.2-4.el8.noarch.r 2.4 MB/s | 457 kB 00:00 2023-09-23T21:54:24.311 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T21:54:24.314 INFO:teuthology.orchestra.run.smithi026.stdout:(30/32): python3-pycparser-2.14-14.el8.noarch.r 653 kB/s | 109 kB 00:00 2023-09-23T21:54:24.347 INFO:teuthology.orchestra.run.smithi026.stdout:(31/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 4.0 MB/s | 238 kB 00:00 2023-09-23T21:54:24.373 INFO:teuthology.orchestra.run.smithi026.stdout:(32/32): python3-cryptography-3.2.1-5.el8.x86_6 4.7 MB/s | 559 kB 00:00 2023-09-23T21:54:24.377 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:54:24.378 INFO:teuthology.orchestra.run.smithi026.stdout:Total 12 MB/s | 28 MB 00:02 2023-09-23T21:54:24.439 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T21:54:24.475 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:54:24.514 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:54:24.514 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:54:24.653 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T21:54:24.732 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:54:24.736 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:54:24.904 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T21:54:25.097 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T21:54:25.361 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T21:54:25.638 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T21:54:25.846 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:54:25.893 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T21:54:26.078 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T21:54:26.197 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T21:54:26.324 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T21:54:26.506 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 3/32 2023-09-23T21:54:26.537 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T21:54:26.873 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8. 27/32 2023-09-23T21:54:26.895 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 4/32 2023-09-23T21:54:27.043 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 28/32 2023-09-23T21:54:27.488 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 5/32 2023-09-23T21:54:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 6/32 2023-09-23T21:54:28.155 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 7/32 2023-09-23T21:54:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 8/32 2023-09-23T21:54:28.519 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 28/32 2023-09-23T21:54:28.664 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 9/32 2023-09-23T21:54:28.751 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 29/32 2023-09-23T21:54:28.884 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 10/32 2023-09-23T21:54:29.100 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 11/32 2023-09-23T21:54:29.144 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 29/32 2023-09-23T21:54:29.293 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 12/32 2023-09-23T21:54:29.316 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 30/32 2023-09-23T21:54:29.519 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T21:54:29.602 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 30/32 2023-09-23T21:54:29.676 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T21:54:29.691 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 31/32 2023-09-23T21:54:29.710 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 31/32 2023-09-23T21:54:29.855 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T21:54:29.856 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 32/32 2023-09-23T21:54:30.030 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T21:54:30.188 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 32/32 2023-09-23T21:54:30.188 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/32 2023-09-23T21:54:30.188 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/32 2023-09-23T21:54:30.188 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/32 2023-09-23T21:54:30.188 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 4/32 2023-09-23T21:54:30.188 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/32 2023-09-23T21:54:30.188 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8. 6/32 2023-09-23T21:54:30.189 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T21:54:30.189 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T21:54:30.189 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T21:54:30.189 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T21:54:30.189 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T21:54:30.189 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T21:54:30.189 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T21:54:30.189 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T21:54:30.189 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T21:54:30.190 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T21:54:30.191 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T21:54:30.191 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T21:54:30.191 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T21:54:30.191 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T21:54:30.191 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T21:54:30.191 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T21:54:30.332 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T21:54:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T21:54:30.789 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T21:54:30.981 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T21:54:31.252 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T21:54:31.529 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T21:54:31.726 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T21:54:31.726 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:30 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:31.754 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T21:54:32.086 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T21:54:32.449 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T21:54:32.782 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8. 27/32 2023-09-23T21:54:32.942 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 28/32 2023-09-23T21:54:34.306 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:54:34.489 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:34.489 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:54:34.489 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:34.489 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:34.489 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:34.489 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:54:34.489 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:34.489 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T21:54:34.490 INFO:teuthology.orchestra.run.smithi070.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T21:54:34.491 INFO:teuthology.orchestra.run.smithi070.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T21:54:34.491 INFO:teuthology.orchestra.run.smithi070.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T21:54:34.491 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T21:54:34.491 INFO:teuthology.orchestra.run.smithi070.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T21:54:34.491 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T21:54:34.491 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T21:54:34.491 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T21:54:34.491 INFO:teuthology.orchestra.run.smithi070.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T21:54:34.491 INFO:teuthology.orchestra.run.smithi070.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T21:54:34.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T21:54:34.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T21:54:34.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T21:54:34.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T21:54:34.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T21:54:34.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T21:54:34.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T21:54:34.492 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:34.492 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:54:34.526 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 28/32 2023-09-23T21:54:34.633 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-base 2023-09-23T21:54:34.759 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 29/32 2023-09-23T21:54:34.988 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:35.183 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 29/32 2023-09-23T21:54:35.367 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 30/32 2023-09-23T21:54:35.602 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 30/32 2023-09-23T21:54:35.758 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 31/32 2023-09-23T21:54:35.777 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 31/32 2023-09-23T21:54:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 32/32 2023-09-23T21:54:35.967 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:34 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:36.287 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 32/32 2023-09-23T21:54:36.287 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/32 2023-09-23T21:54:36.287 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 3/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 4/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 5/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8. 6/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T21:54:36.288 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T21:54:36.289 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T21:54:36.289 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T21:54:36.289 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T21:54:36.289 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T21:54:36.289 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T21:54:36.289 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T21:54:36.289 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T21:54:36.289 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T21:54:36.289 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T21:54:36.290 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T21:54:36.290 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T21:54:36.290 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T21:54:36.290 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 25/32 2023-09-23T21:54:36.290 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 26/32 2023-09-23T21:54:36.290 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 27/32 2023-09-23T21:54:36.291 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 28/32 2023-09-23T21:54:36.291 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 29/32 2023-09-23T21:54:36.291 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 30/32 2023-09-23T21:54:36.291 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 31/32 2023-09-23T21:54:37.222 INFO:teuthology.orchestra.run.smithi070.stdout:Package ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:54:37.335 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:54:37.336 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:54:37.336 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:54:37.454 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install cephadm 2023-09-23T21:54:37.798 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:37.827 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 32/32 2023-09-23T21:54:37.828 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:34 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:54:39.459 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:38 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:40.895 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:54:40.896 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:54:40.896 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:54:40.896 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:54:40.896 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:54:40.896 INFO:teuthology.orchestra.run.smithi070.stdout: cephadm noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 80 k 2023-09-23T21:54:40.897 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:40.897 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:54:40.897 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:54:40.897 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T21:54:40.897 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:40.897 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 80 k 2023-09-23T21:54:40.898 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 363 k 2023-09-23T21:54:40.898 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:54:41.053 INFO:teuthology.orchestra.run.smithi070.stdout:cephadm-17.2.6-1125.gc9bb1bcf.el8.noarch.rpm 516 kB/s | 80 kB 00:00 2023-09-23T21:54:41.053 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:54:41.053 INFO:teuthology.orchestra.run.smithi070.stdout:Total 512 kB/s | 80 kB 00:00 2023-09-23T21:54:41.054 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:54:41.061 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:54:41.062 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:54:41.063 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:54:41.067 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:54:41.108 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:54:42.351 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:54:42.557 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:42.557 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:54:42.557 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:42.557 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mds-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:42.557 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:42.557 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-modules-core-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:54:42.557 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mon-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:42.557 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-osd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:42.557 INFO:teuthology.orchestra.run.smithi026.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T21:54:42.558 INFO:teuthology.orchestra.run.smithi026.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T21:54:42.559 INFO:teuthology.orchestra.run.smithi026.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T21:54:42.560 INFO:teuthology.orchestra.run.smithi026.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T21:54:42.560 INFO:teuthology.orchestra.run.smithi026.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T21:54:42.560 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:42.560 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:54:42.713 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-base 2023-09-23T21:54:42.811 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T21:54:42.977 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T21:54:43.082 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:43.140 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T21:54:44.066 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:41 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:54:44.192 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T21:54:44.192 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:43 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:45.475 INFO:teuthology.orchestra.run.smithi026.stdout:Package ceph-base-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:54:45.595 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:54:45.596 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:54:45.596 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:54:45.699 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install cephadm 2023-09-23T21:54:46.066 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:46.981 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:43 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:54:47.018 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:54:47.197 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:47.197 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:54:47.197 INFO:teuthology.orchestra.run.smithi070.stdout: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:54:47.198 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:47.198 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:54:47.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T21:54:47.689 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:48.455 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:54:48.456 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:54:48.456 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:54:48.456 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:54:48.456 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:54:48.456 INFO:teuthology.orchestra.run.smithi026.stdout: cephadm noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 80 k 2023-09-23T21:54:48.456 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:48.456 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:54:48.457 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:54:48.457 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T21:54:48.457 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:48.457 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 80 k 2023-09-23T21:54:48.457 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 363 k 2023-09-23T21:54:48.457 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:54:48.542 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:47 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:48.805 INFO:teuthology.orchestra.run.smithi026.stdout:cephadm-17.2.6-1125.gc9bb1bcf.el8.noarch.rpm 230 kB/s | 80 kB 00:00 2023-09-23T21:54:48.805 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:54:48.805 INFO:teuthology.orchestra.run.smithi026.stdout:Total 229 kB/s | 80 kB 00:00 2023-09-23T21:54:48.806 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:54:48.815 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:54:48.815 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:54:48.817 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:54:48.821 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:54:48.867 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:54:49.958 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:54:49.958 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:54:49.959 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repo Size 2023-09-23T21:54:49.959 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:54:49.959 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:54:49.959 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 162 k 2023-09-23T21:54:49.959 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:49.959 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:54:49.959 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:54:49.959 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T21:54:49.959 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:49.960 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 162 k 2023-09-23T21:54:49.960 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 490 k 2023-09-23T21:54:49.960 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:54:50.132 INFO:teuthology.orchestra.run.smithi070.stdout:ceph-immutable-object-cache-17.2.6-1125.gc9bb1b 942 kB/s | 162 kB 00:00 2023-09-23T21:54:50.132 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:54:50.133 INFO:teuthology.orchestra.run.smithi070.stdout:Total 936 kB/s | 162 kB 00:00 2023-09-23T21:54:50.133 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:54:50.143 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:54:50.144 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:54:50.194 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:54:50.198 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:54:50.316 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:54:50.519 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T21:54:50.641 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T21:54:50.837 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T21:54:50.883 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T21:54:51.013 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T21:54:51.964 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T21:54:51.964 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:50 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:52.115 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/1 2023-09-23T21:54:52.115 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:49 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:54:54.536 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:54:54.717 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:54.717 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:54:54.717 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:54:54.718 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:54:54.718 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:54:54.857 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr 2023-09-23T21:54:54.975 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:54:55.175 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:55.176 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:54:55.176 INFO:teuthology.orchestra.run.smithi026.stdout: cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:54:55.176 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:55.176 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:54:55.204 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:55.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T21:54:55.689 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:55.990 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:54 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:56.520 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:53 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:54:57.316 INFO:teuthology.orchestra.run.smithi070.stdout:Package ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:54:57.425 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:54:57.426 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:54:57.427 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:54:57.550 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T21:54:57.893 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:54:58.049 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:54:58.050 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:54:58.050 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repo Size 2023-09-23T21:54:58.050 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:54:58.050 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:54:58.050 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 162 k 2023-09-23T21:54:58.051 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:58.051 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:54:58.051 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:54:58.051 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T21:54:58.051 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:54:58.052 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 162 k 2023-09-23T21:54:58.052 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 490 k 2023-09-23T21:54:58.052 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:54:58.235 INFO:teuthology.orchestra.run.smithi026.stdout:ceph-immutable-object-cache-17.2.6-1125.gc9bb1b 884 kB/s | 162 kB 00:00 2023-09-23T21:54:58.235 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:54:58.235 INFO:teuthology.orchestra.run.smithi026.stdout:Total 878 kB/s | 162 kB 00:00 2023-09-23T21:54:58.236 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:54:58.247 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:54:58.248 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:54:58.302 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:54:58.306 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:54:58.422 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:54:58.638 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T21:54:58.693 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:01:57 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:54:59.002 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T21:55:00.048 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf. 1/1 2023-09-23T21:55:00.048 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:01:57 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:55:00.117 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:55:00.118 INFO:teuthology.orchestra.run.smithi070.stdout:=================================================================================================== 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout:=================================================================================================== 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 1.7 M 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 23 k 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 14 k 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T21:55:00.119 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:00.120 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:55:00.120 INFO:teuthology.orchestra.run.smithi070.stdout:=================================================================================================== 2023-09-23T21:55:00.120 INFO:teuthology.orchestra.run.smithi070.stdout:Install 6 Packages 2023-09-23T21:55:00.120 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:00.120 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 2.0 M 2023-09-23T21:55:00.120 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 8.0 M 2023-09-23T21:55:00.121 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:55:00.462 INFO:teuthology.orchestra.run.smithi070.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1125.gc9b 159 kB/s | 23 kB 00:00 2023-09-23T21:55:00.570 INFO:teuthology.orchestra.run.smithi070.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1125.gc9bb 57 kB/s | 14 kB 00:00 2023-09-23T21:55:00.629 INFO:teuthology.orchestra.run.smithi070.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 201 kB/s | 34 kB 00:00 2023-09-23T21:55:00.688 INFO:teuthology.orchestra.run.smithi070.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 733 kB/s | 43 kB 00:00 2023-09-23T21:55:00.813 INFO:teuthology.orchestra.run.smithi070.stdout:(5/6): ceph-mgr-dashboard-17.2.6-1125.gc9bb1bcf 3.4 MB/s | 1.7 MB 00:00 2023-09-23T21:55:00.871 INFO:teuthology.orchestra.run.smithi070.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 653 kB/s | 196 kB 00:00 2023-09-23T21:55:00.873 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:55:00.873 INFO:teuthology.orchestra.run.smithi070.stdout:Total 2.7 MB/s | 2.0 MB 00:00 2023-09-23T21:55:00.884 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:55:00.893 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:55:00.893 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:55:00.950 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:55:00.954 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:55:01.284 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:55:01.443 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T21:55:01.640 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T21:55:01.831 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T21:55:01.959 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.n 4/6 2023-09-23T21:55:02.277 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8. 5/6 2023-09-23T21:55:02.594 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:55:02.792 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:02.793 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:55:02.793 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-immutable-object-cache-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:55:02.793 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:02.793 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:55:02.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr 2023-09-23T21:55:03.203 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 6/6 2023-09-23T21:55:03.287 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:55:03.435 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 6/6 2023-09-23T21:55:03.435 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8. 1/6 2023-09-23T21:55:03.435 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 2/6 2023-09-23T21:55:03.435 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.n 3/6 2023-09-23T21:55:03.435 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T21:55:03.435 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T21:55:04.195 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:01 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:55:04.462 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T21:55:04.462 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:03 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:55:05.589 INFO:teuthology.orchestra.run.smithi026.stdout:Package ceph-mgr-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:55:05.708 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:55:05.709 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:55:05.710 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:55:06.018 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T21:55:06.431 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:55:06.975 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:55:07.157 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:07.157 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:55:07.157 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:07.157 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:07.157 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:07.157 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T21:55:07.158 INFO:teuthology.orchestra.run.smithi070.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T21:55:07.158 INFO:teuthology.orchestra.run.smithi070.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T21:55:07.158 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:07.158 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:55:07.291 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T21:55:07.336 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:04 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:55:07.636 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:55:08.479 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:07 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:55:08.857 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:55:08.858 INFO:teuthology.orchestra.run.smithi026.stdout:=================================================================================================== 2023-09-23T21:55:08.858 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:55:08.858 INFO:teuthology.orchestra.run.smithi026.stdout:=================================================================================================== 2023-09-23T21:55:08.858 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:55:08.858 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 1.7 M 2023-09-23T21:55:08.858 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T21:55:08.859 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 23 k 2023-09-23T21:55:08.859 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 14 k 2023-09-23T21:55:08.859 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T21:55:08.859 INFO:teuthology.orchestra.run.smithi026.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T21:55:08.859 INFO:teuthology.orchestra.run.smithi026.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T21:55:08.859 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:08.859 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:55:08.859 INFO:teuthology.orchestra.run.smithi026.stdout:=================================================================================================== 2023-09-23T21:55:08.860 INFO:teuthology.orchestra.run.smithi026.stdout:Install 6 Packages 2023-09-23T21:55:08.860 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:08.860 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 2.0 M 2023-09-23T21:55:08.860 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 8.0 M 2023-09-23T21:55:08.861 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:55:09.141 INFO:teuthology.orchestra.run.smithi026.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1125.gc9b 94 kB/s | 23 kB 00:00 2023-09-23T21:55:09.166 INFO:teuthology.orchestra.run.smithi026.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1125.gc9bb 54 kB/s | 14 kB 00:00 2023-09-23T21:55:09.491 INFO:teuthology.orchestra.run.smithi026.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1125.gc9bb1bcf 2.8 MB/s | 1.7 MB 00:00 2023-09-23T21:55:09.558 INFO:teuthology.orchestra.run.smithi026.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 643 kB/s | 43 kB 00:00 2023-09-23T21:55:09.625 INFO:teuthology.orchestra.run.smithi026.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 69 kB/s | 34 kB 00:00 2023-09-23T21:55:09.832 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:55:09.834 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================================= 2023-09-23T21:55:09.834 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:55:09.834 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================================= 2023-09-23T21:55:09.834 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:55:09.834 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 7.4 M 2023-09-23T21:55:09.834 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T21:55:09.834 INFO:teuthology.orchestra.run.smithi070.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T21:55:09.835 INFO:teuthology.orchestra.run.smithi070.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T21:55:09.835 INFO:teuthology.orchestra.run.smithi070.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T21:55:09.835 INFO:teuthology.orchestra.run.smithi070.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T21:55:09.835 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T21:55:09.835 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T21:55:09.835 INFO:teuthology.orchestra.run.smithi070.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T21:55:09.835 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:09.836 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:55:09.836 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================================= 2023-09-23T21:55:09.836 INFO:teuthology.orchestra.run.smithi070.stdout:Install 8 Packages 2023-09-23T21:55:09.836 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:09.836 INFO:teuthology.orchestra.run.smithi026.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 294 kB/s | 196 kB 00:00 2023-09-23T21:55:09.837 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 35 M 2023-09-23T21:55:09.837 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 210 M 2023-09-23T21:55:09.837 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:55:09.837 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:55:09.838 INFO:teuthology.orchestra.run.smithi026.stdout:Total 2.0 MB/s | 2.0 MB 00:00 2023-09-23T21:55:09.852 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:55:09.863 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:55:09.863 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:55:09.918 INFO:teuthology.orchestra.run.smithi070.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 7.7 MB/s | 644 kB 00:00 2023-09-23T21:55:09.928 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:55:09.932 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:55:09.943 INFO:teuthology.orchestra.run.smithi070.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.6 MB/s | 171 kB 00:00 2023-09-23T21:55:10.194 INFO:teuthology.orchestra.run.smithi070.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 50 MB/s | 14 MB 00:00 2023-09-23T21:55:10.230 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:55:10.244 INFO:teuthology.orchestra.run.smithi070.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 4.4 MB/s | 225 kB 00:00 2023-09-23T21:55:10.311 INFO:teuthology.orchestra.run.smithi070.stdout:(5/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 10 MB/s | 3.7 MB 00:00 2023-09-23T21:55:10.404 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T21:55:10.455 INFO:teuthology.orchestra.run.smithi070.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 23 MB/s | 4.9 MB 00:00 2023-09-23T21:55:10.536 INFO:teuthology.orchestra.run.smithi070.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 21 MB/s | 4.7 MB 00:00 2023-09-23T21:55:10.595 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T21:55:10.781 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T21:55:10.870 INFO:teuthology.orchestra.run.smithi070.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 7.1 MB/s | 7.4 MB 00:01 2023-09-23T21:55:10.871 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:55:10.871 INFO:teuthology.orchestra.run.smithi070.stdout:Total 34 MB/s | 35 MB 00:01 2023-09-23T21:55:10.960 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.n 4/6 2023-09-23T21:55:11.109 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:55:11.127 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:55:11.128 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:55:11.318 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8. 5/6 2023-09-23T21:55:11.421 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:55:11.424 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:55:11.520 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 6/6 2023-09-23T21:55:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 6/6 2023-09-23T21:55:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8. 1/6 2023-09-23T21:55:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarc 2/6 2023-09-23T21:55:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.n 3/6 2023-09-23T21:55:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T21:55:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T21:55:11.960 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:55:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T21:55:12.262 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T21:55:12.406 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T21:55:12.849 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T21:55:12.850 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:09 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:55:13.011 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T21:55:13.189 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T21:55:13.784 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T21:55:14.080 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T21:55:14.754 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T21:55:15.049 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T21:55:15.264 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T21:55:16.108 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:55:16.307 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:16.308 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:55:16.308 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-grafana-dashboards-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:16.308 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-dashboard-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:16.308 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-prometheus-alerts-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:16.308 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T21:55:16.308 INFO:teuthology.orchestra.run.smithi026.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T21:55:16.309 INFO:teuthology.orchestra.run.smithi026.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T21:55:16.309 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:16.309 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:55:16.459 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T21:55:16.825 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:55:17.002 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T21:55:17.770 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:14 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:55:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T21:55:18.380 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 8/8 2023-09-23T21:55:18.637 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 8/8 2023-09-23T21:55:18.637 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 1/8 2023-09-23T21:55:18.637 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T21:55:18.637 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T21:55:18.637 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T21:55:18.637 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T21:55:18.638 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T21:55:18.638 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T21:55:19.302 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:55:19.303 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================================= 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================================= 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 7.4 M 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T21:55:19.304 INFO:teuthology.orchestra.run.smithi026.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T21:55:19.305 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:19.305 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:55:19.305 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================================= 2023-09-23T21:55:19.305 INFO:teuthology.orchestra.run.smithi026.stdout:Install 8 Packages 2023-09-23T21:55:19.305 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:19.305 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 35 M 2023-09-23T21:55:19.305 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 210 M 2023-09-23T21:55:19.305 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:55:19.606 INFO:teuthology.orchestra.run.smithi026.stdout:(1/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 12 MB/s | 3.7 MB 00:00 2023-09-23T21:55:19.666 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T21:55:19.666 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:18 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:55:19.839 INFO:teuthology.orchestra.run.smithi026.stdout:(2/8): python3-scipy-1.0.0-21.module+el8.5.0+10 26 MB/s | 14 MB 00:00 2023-09-23T21:55:19.864 INFO:teuthology.orchestra.run.smithi026.stdout:(3/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 872 kB/s | 225 kB 00:00 2023-09-23T21:55:20.031 INFO:teuthology.orchestra.run.smithi026.stdout:(4/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 25 MB/s | 4.9 MB 00:00 2023-09-23T21:55:20.140 INFO:teuthology.orchestra.run.smithi026.stdout:(5/8): ceph-mgr-diskprediction-local-17.2.6-112 8.8 MB/s | 7.4 MB 00:00 2023-09-23T21:55:20.215 INFO:teuthology.orchestra.run.smithi026.stdout:(6/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 3.4 MB/s | 644 kB 00:00 2023-09-23T21:55:20.240 INFO:teuthology.orchestra.run.smithi026.stdout:(7/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.7 MB/s | 171 kB 00:00 2023-09-23T21:55:20.316 INFO:teuthology.orchestra.run.smithi026.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 10 MB/s | 4.7 MB 00:00 2023-09-23T21:55:20.317 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:55:20.317 INFO:teuthology.orchestra.run.smithi026.stdout:Total 35 MB/s | 35 MB 00:01 2023-09-23T21:55:20.563 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:55:20.582 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:55:20.583 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:55:20.893 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:55:20.897 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:55:21.489 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:55:21.633 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T21:55:21.762 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T21:55:22.275 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T21:55:22.948 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T21:55:23.108 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T21:55:23.313 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:55:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:23.494 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:55:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T21:55:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T21:55:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T21:55:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T21:55:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T21:55:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T21:55:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T21:55:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:23.495 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:55:23.682 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr-rook 2023-09-23T21:55:23.736 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T21:55:23.908 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T21:55:24.028 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:55:24.647 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T21:55:24.999 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T21:55:25.175 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T21:55:25.355 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:24 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:55:26.712 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:55:26.714 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================== 2023-09-23T21:55:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:55:26.714 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================== 2023-09-23T21:55:26.714 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:55:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-rook noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 47 k 2023-09-23T21:55:26.714 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T21:55:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T21:55:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:26.716 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:55:26.716 INFO:teuthology.orchestra.run.smithi070.stdout:======================================================================================================== 2023-09-23T21:55:26.716 INFO:teuthology.orchestra.run.smithi070.stdout:Install 13 Packages 2023-09-23T21:55:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:26.716 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 1.9 M 2023-09-23T21:55:26.716 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 25 M 2023-09-23T21:55:26.717 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:55:27.048 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T21:55:27.299 INFO:teuthology.orchestra.run.smithi070.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 66 kB/s | 16 kB 00:00 2023-09-23T21:55:27.368 INFO:teuthology.orchestra.run.smithi070.stdout:(2/13): ceph-mgr-rook-17.2.6-1125.gc9bb1bcf.el8 173 kB/s | 47 kB 00:00 2023-09-23T21:55:27.368 INFO:teuthology.orchestra.run.smithi070.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 110 kB/s | 33 kB 00:00 2023-09-23T21:55:27.474 INFO:teuthology.orchestra.run.smithi070.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 463 kB/s | 81 kB 00:00 2023-09-23T21:55:27.583 INFO:teuthology.orchestra.run.smithi070.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 567 kB/s | 61 kB 00:00 2023-09-23T21:55:27.675 INFO:teuthology.orchestra.run.smithi070.stdout:(6/13): python3-rsa-4.9-2.el8.noarch.rpm 197 kB/s | 64 kB 00:00 2023-09-23T21:55:27.733 INFO:teuthology.orchestra.run.smithi070.stdout:(7/13): python3-kubernetes-11.0.0-6.el8.noarch. 2.8 MB/s | 1.2 MB 00:00 2023-09-23T21:55:27.767 INFO:teuthology.orchestra.run.smithi070.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 232 kB/s | 43 kB 00:00 2023-09-23T21:55:27.834 INFO:teuthology.orchestra.run.smithi070.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 975 kB/s | 155 kB 00:00 2023-09-23T21:55:27.885 INFO:teuthology.orchestra.run.smithi070.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 731 kB/s | 110 kB 00:00 2023-09-23T21:55:27.910 INFO:teuthology.orchestra.run.smithi070.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 194 kB/s | 27 kB 00:00 2023-09-23T21:55:27.960 INFO:teuthology.orchestra.run.smithi070.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.0 MB/s | 126 kB 00:00 2023-09-23T21:55:28.010 INFO:teuthology.orchestra.run.smithi070.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 160 kB/s | 20 kB 00:00 2023-09-23T21:55:28.013 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:55:28.013 INFO:teuthology.orchestra.run.smithi070.stdout:Total 1.5 MB/s | 1.9 MB 00:01 2023-09-23T21:55:28.077 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:55:28.093 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:55:28.093 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:55:28.164 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:55:28.168 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:55:28.346 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T21:55:28.499 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 8/8 2023-09-23T21:55:28.645 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:55:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 8/8 2023-09-23T21:55:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bc 1/8 2023-09-23T21:55:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 2/8 2023-09-23T21:55:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 3/8 2023-09-23T21:55:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 4/8 2023-09-23T21:55:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 5/8 2023-09-23T21:55:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 6/8 2023-09-23T21:55:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 7/8 2023-09-23T21:55:28.852 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T21:55:29.104 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T21:55:29.292 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T21:55:29.475 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T21:55:29.688 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T21:55:29.869 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 8/8 2023-09-23T21:55:29.869 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:26 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:55:29.939 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T21:55:30.137 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T21:55:30.350 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T21:55:30.524 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T21:55:30.774 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T21:55:31.847 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T21:55:32.076 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T21:55:32.251 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 13/13 2023-09-23T21:55:32.488 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 13/13 2023-09-23T21:55:32.488 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/13 2023-09-23T21:55:32.488 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T21:55:32.488 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T21:55:32.488 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T21:55:32.489 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T21:55:32.489 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T21:55:32.489 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T21:55:32.489 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T21:55:32.489 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T21:55:32.489 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T21:55:32.489 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T21:55:32.489 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T21:55:33.554 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T21:55:33.555 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:32 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:55:35.577 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:55:35.778 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:35.778 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:55:35.779 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:35.779 INFO:teuthology.orchestra.run.smithi026.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T21:55:35.779 INFO:teuthology.orchestra.run.smithi026.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T21:55:35.779 INFO:teuthology.orchestra.run.smithi026.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T21:55:35.779 INFO:teuthology.orchestra.run.smithi026.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T21:55:35.779 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T21:55:35.779 INFO:teuthology.orchestra.run.smithi026.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T21:55:35.779 INFO:teuthology.orchestra.run.smithi026.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T21:55:35.779 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:35.780 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:55:35.920 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr-rook 2023-09-23T21:55:36.286 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:55:37.810 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:34 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:55:39.334 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:55:39.335 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================== 2023-09-23T21:55:39.336 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:55:39.336 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================== 2023-09-23T21:55:39.336 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:55:39.336 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-rook noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 47 k 2023-09-23T21:55:39.336 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T21:55:39.336 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T21:55:39.336 INFO:teuthology.orchestra.run.smithi026.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T21:55:39.336 INFO:teuthology.orchestra.run.smithi026.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T21:55:39.336 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout:======================================================================================================== 2023-09-23T21:55:39.337 INFO:teuthology.orchestra.run.smithi026.stdout:Install 13 Packages 2023-09-23T21:55:39.338 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:39.338 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 1.9 M 2023-09-23T21:55:39.338 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 25 M 2023-09-23T21:55:39.338 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:55:39.871 INFO:teuthology.orchestra.run.smithi026.stdout:(1/13): ceph-mgr-rook-17.2.6-1125.gc9bb1bcf.el8 157 kB/s | 47 kB 00:00 2023-09-23T21:55:39.962 INFO:teuthology.orchestra.run.smithi026.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 41 kB/s | 16 kB 00:00 2023-09-23T21:55:39.980 INFO:teuthology.orchestra.run.smithi026.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 80 kB/s | 33 kB 00:00 2023-09-23T21:55:40.220 INFO:teuthology.orchestra.run.smithi026.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 272 kB/s | 64 kB 00:00 2023-09-23T21:55:40.338 INFO:teuthology.orchestra.run.smithi026.stdout:(5/13): python3-websocket-client-0.56.0-5.el8.n 521 kB/s | 61 kB 00:00 2023-09-23T21:55:40.372 INFO:teuthology.orchestra.run.smithi026.stdout:(6/13): python3-google-auth-1.1.1-10.el8.noarch 162 kB/s | 81 kB 00:00 2023-09-23T21:55:40.556 INFO:teuthology.orchestra.run.smithi026.stdout:(7/13): python3-pyasn1-modules-0.3.7-6.el8.noar 508 kB/s | 110 kB 00:00 2023-09-23T21:55:40.606 INFO:teuthology.orchestra.run.smithi026.stdout:(8/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 118 kB/s | 27 kB 00:00 2023-09-23T21:55:40.648 INFO:teuthology.orchestra.run.smithi026.stdout:(9/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.3 MB/s | 126 kB 00:00 2023-09-23T21:55:40.690 INFO:teuthology.orchestra.run.smithi026.stdout:(10/13): python3-jsonpointer-1.10-11.el8.noarch 239 kB/s | 20 kB 00:00 2023-09-23T21:55:40.748 INFO:teuthology.orchestra.run.smithi026.stdout:(11/13): python3-requests-oauthlib-1.0.0-1.el8. 426 kB/s | 43 kB 00:00 2023-09-23T21:55:40.782 INFO:teuthology.orchestra.run.smithi026.stdout:(12/13): python3-kubernetes-11.0.0-6.el8.noarch 1.4 MB/s | 1.2 MB 00:00 2023-09-23T21:55:40.807 INFO:teuthology.orchestra.run.smithi026.stdout:(13/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 1.3 MB/s | 155 kB 00:00 2023-09-23T21:55:40.810 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:55:40.810 INFO:teuthology.orchestra.run.smithi026.stdout:Total 1.3 MB/s | 1.9 MB 00:01 2023-09-23T21:55:40.880 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:55:40.897 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:55:40.898 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:55:40.976 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:55:40.981 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:55:41.459 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:55:41.649 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T21:55:41.920 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T21:55:42.004 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:55:42.137 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T21:55:42.187 INFO:teuthology.orchestra.run.smithi070.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T21:55:42.188 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T21:55:42.188 INFO:teuthology.orchestra.run.smithi070.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T21:55:42.188 INFO:teuthology.orchestra.run.smithi070.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T21:55:42.188 INFO:teuthology.orchestra.run.smithi070.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T21:55:42.188 INFO:teuthology.orchestra.run.smithi070.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T21:55:42.188 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:42.188 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:55:42.329 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T21:55:42.344 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 4/13 2023-09-23T21:55:42.512 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 5/13 2023-09-23T21:55:42.681 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:55:42.688 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 6/13 2023-09-23T21:55:42.843 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 7/13 2023-09-23T21:55:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T21:55:43.647 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T21:55:43.849 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T21:55:44.133 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:43 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:55:44.957 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T21:55:45.182 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T21:55:45.330 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 13/13 2023-09-23T21:55:45.490 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:55:45.491 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================================== 2023-09-23T21:55:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:55:45.491 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================================== 2023-09-23T21:55:45.491 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:55:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 121 k 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================================== 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout:Install 5 Packages 2023-09-23T21:55:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:45.493 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 6.0 M 2023-09-23T21:55:45.493 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 26 M 2023-09-23T21:55:45.493 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:55:45.574 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 13/13 2023-09-23T21:55:45.574 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 8/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 9/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 10/13 2023-09-23T21:55:45.575 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-09-23T21:55:45.576 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 12/13 2023-09-23T21:55:46.168 INFO:teuthology.orchestra.run.smithi070.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1125.gc9bb1bcf.e 210 kB/s | 121 kB 00:00 2023-09-23T21:55:46.402 INFO:teuthology.orchestra.run.smithi070.stdout:(2/5): python3-natsort-7.1.1-2.el8.noarch.rpm 74 kB/s | 60 kB 00:00 2023-09-23T21:55:46.619 INFO:teuthology.orchestra.run.smithi070.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 463 kB/s | 476 kB 00:01 2023-09-23T21:55:46.653 INFO:teuthology.orchestra.run.smithi070.stdout:(4/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 1.1 MB/s | 538 kB 00:00 2023-09-23T21:55:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 13/13 2023-09-23T21:55:46.718 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:43 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:55:46.795 INFO:teuthology.orchestra.run.smithi070.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 12 MB/s | 4.8 MB 00:00 2023-09-23T21:55:46.797 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:55:46.797 INFO:teuthology.orchestra.run.smithi070.stdout:Total 4.6 MB/s | 6.0 MB 00:01 2023-09-23T21:55:46.854 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:55:46.866 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:55:46.866 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:55:46.946 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:55:46.950 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:55:47.923 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:55:48.285 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T21:55:48.516 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T21:55:48.868 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T21:55:49.133 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T21:55:49.339 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 5/5 2023-09-23T21:55:49.571 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 5/5 2023-09-23T21:55:49.571 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/5 2023-09-23T21:55:49.571 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T21:55:49.571 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T21:55:49.571 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T21:55:50.591 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T21:55:50.591 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:49 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:55:52.628 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:55:52.829 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:52.830 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:55:52.830 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-rook-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:52.830 INFO:teuthology.orchestra.run.smithi026.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T21:55:52.830 INFO:teuthology.orchestra.run.smithi026.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T21:55:52.830 INFO:teuthology.orchestra.run.smithi026.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T21:55:52.830 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T21:55:52.830 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T21:55:52.830 INFO:teuthology.orchestra.run.smithi026.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T21:55:52.830 INFO:teuthology.orchestra.run.smithi026.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T21:55:52.831 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T21:55:52.831 INFO:teuthology.orchestra.run.smithi026.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T21:55:52.831 INFO:teuthology.orchestra.run.smithi026.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T21:55:52.831 INFO:teuthology.orchestra.run.smithi026.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T21:55:52.831 INFO:teuthology.orchestra.run.smithi026.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T21:55:52.831 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:52.831 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:55:52.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T21:55:53.334 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:55:54.946 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:51 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:55:56.474 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================================== 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================================== 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1125.gc9bb1bcf.el8 ceph-noarch 121 k 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T21:55:56.476 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:56.477 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:55:56.477 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================================== 2023-09-23T21:55:56.477 INFO:teuthology.orchestra.run.smithi026.stdout:Install 5 Packages 2023-09-23T21:55:56.477 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:55:56.477 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 6.0 M 2023-09-23T21:55:56.477 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 26 M 2023-09-23T21:55:56.478 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:55:56.719 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:55:56.902 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:56.903 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:55:56.903 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:55:56.903 INFO:teuthology.orchestra.run.smithi070.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T21:55:56.903 INFO:teuthology.orchestra.run.smithi070.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T21:55:56.903 INFO:teuthology.orchestra.run.smithi070.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T21:55:56.903 INFO:teuthology.orchestra.run.smithi070.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T21:55:56.903 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:55:56.903 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:55:56.941 INFO:teuthology.orchestra.run.smithi026.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 229 kB/s | 60 kB 00:00 2023-09-23T21:55:57.016 INFO:teuthology.orchestra.run.smithi026.stdout:(2/5): ceph-mgr-cephadm-17.2.6-1125.gc9bb1bcf.e 357 kB/s | 121 kB 00:00 2023-09-23T21:55:57.085 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install ceph-fuse 2023-09-23T21:55:57.100 INFO:teuthology.orchestra.run.smithi026.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.3 MB/s | 538 kB 00:00 2023-09-23T21:55:57.133 INFO:teuthology.orchestra.run.smithi026.stdout:(4/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.0 MB/s | 476 kB 00:00 2023-09-23T21:55:57.225 INFO:teuthology.orchestra.run.smithi026.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 23 MB/s | 4.8 MB 00:00 2023-09-23T21:55:57.227 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:55:57.227 INFO:teuthology.orchestra.run.smithi026.stdout:Total 7.9 MB/s | 6.0 MB 00:00 2023-09-23T21:55:57.282 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:55:57.295 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:55:57.295 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:55:57.378 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:55:57.382 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:55:57.437 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:55:58.507 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:55:58.858 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T21:55:58.893 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:02:57 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:55:59.097 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T21:55:59.723 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T21:56:00.326 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repository Size 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-fuse 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout: x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 874 k 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout:Installing dependencies: 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:56:00.327 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:00.328 INFO:teuthology.orchestra.run.smithi070.stdout:Install 2 Packages 2023-09-23T21:56:00.328 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:00.328 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 957 k 2023-09-23T21:56:00.328 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 3.0 M 2023-09-23T21:56:00.328 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:56:00.402 INFO:teuthology.orchestra.run.smithi070.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 1.1 MB/s | 83 kB 00:00 2023-09-23T21:56:00.760 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T21:56:00.961 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 5/5 2023-09-23T21:56:01.208 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 5/5 2023-09-23T21:56:01.208 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 1/5 2023-09-23T21:56:01.208 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T21:56:01.208 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T21:56:01.209 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T21:56:01.311 INFO:teuthology.orchestra.run.smithi070.stdout:(2/2): ceph-fuse-17.2.6-1125.gc9bb1bcf.el8.x86_ 890 kB/s | 874 kB 00:00 2023-09-23T21:56:01.311 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:56:01.311 INFO:teuthology.orchestra.run.smithi070.stdout:Total 973 kB/s | 957 kB 00:00 2023-09-23T21:56:01.324 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:56:01.346 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:56:01.346 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:56:01.425 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:56:01.429 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:56:01.594 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:56:01.862 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T21:56:02.035 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/2 2023-09-23T21:56:02.286 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T21:56:02.286 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:02:59 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:02.367 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/2 2023-09-23T21:56:02.367 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/2 2023-09-23T21:56:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T21:56:03.457 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:02 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:05.209 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:56:05.411 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:05.411 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:56:05.411 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-mgr-cephadm-2:17.2.6-1125.gc9bb1bcf.el8.noarch 2023-09-23T21:56:05.411 INFO:teuthology.orchestra.run.smithi026.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T21:56:05.411 INFO:teuthology.orchestra.run.smithi026.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T21:56:05.411 INFO:teuthology.orchestra.run.smithi026.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T21:56:05.411 INFO:teuthology.orchestra.run.smithi026.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T21:56:05.412 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:05.412 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:05.554 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install ceph-fuse 2023-09-23T21:56:05.930 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:06.842 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:56:06.999 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:03 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:07.025 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:07.025 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:56:07.025 INFO:teuthology.orchestra.run.smithi070.stdout: ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T21:56:07.025 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:07.025 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:07.163 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install librados-devel 2023-09-23T21:56:07.518 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:08.519 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:08.520 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:08.520 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repository Size 2023-09-23T21:56:08.520 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:08.520 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:56:08.520 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-fuse 2023-09-23T21:56:08.520 INFO:teuthology.orchestra.run.smithi026.stdout: x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 874 k 2023-09-23T21:56:08.520 INFO:teuthology.orchestra.run.smithi026.stdout:Installing dependencies: 2023-09-23T21:56:08.520 INFO:teuthology.orchestra.run.smithi026.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T21:56:08.521 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:08.521 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:56:08.521 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:08.521 INFO:teuthology.orchestra.run.smithi026.stdout:Install 2 Packages 2023-09-23T21:56:08.521 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:08.521 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 957 k 2023-09-23T21:56:08.521 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 3.0 M 2023-09-23T21:56:08.522 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:56:08.712 INFO:teuthology.orchestra.run.smithi026.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 435 kB/s | 83 kB 00:00 2023-09-23T21:56:09.094 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:08 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:09.545 INFO:teuthology.orchestra.run.smithi026.stdout:(2/2): ceph-fuse-17.2.6-1125.gc9bb1bcf.el8.x86_ 854 kB/s | 874 kB 00:01 2023-09-23T21:56:09.546 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:56:09.546 INFO:teuthology.orchestra.run.smithi026.stdout:Total 933 kB/s | 957 kB 00:01 2023-09-23T21:56:09.561 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:56:09.584 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:56:09.585 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:56:09.670 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:56:09.674 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:56:09.840 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:56:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T21:56:10.275 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/2 2023-09-23T21:56:10.529 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:10.529 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:10.530 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repo Size 2023-09-23T21:56:10.530 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:10.530 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:56:10.530 INFO:teuthology.orchestra.run.smithi070.stdout: librados-devel x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 130 k 2023-09-23T21:56:10.530 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:10.530 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:56:10.530 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:10.530 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T21:56:10.530 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:10.531 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 130 k 2023-09-23T21:56:10.531 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 464 k 2023-09-23T21:56:10.531 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:56:10.617 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2/2 2023-09-23T21:56:10.617 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/2 2023-09-23T21:56:10.741 INFO:teuthology.orchestra.run.smithi070.stdout:librados-devel-17.2.6-1125.gc9bb1bcf.el8.x86_64 618 kB/s | 130 kB 00:00 2023-09-23T21:56:10.742 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:56:10.742 INFO:teuthology.orchestra.run.smithi070.stdout:Total 615 kB/s | 130 kB 00:00 2023-09-23T21:56:10.742 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:56:10.747 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:56:10.747 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:56:10.801 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:56:10.805 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:56:10.920 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:56:11.107 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:11.326 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T21:56:11.753 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:08 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:12.410 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:12.410 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:11 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:15.907 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:56:15.907 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:56:16.086 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:16.086 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:56:16.086 INFO:teuthology.orchestra.run.smithi070.stdout: librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:56:16.086 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:16.087 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:16.103 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:56:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: ceph-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T21:56:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:16.103 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:16.233 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install libcephfs2 2023-09-23T21:56:16.239 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install librados-devel 2023-09-23T21:56:16.588 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:16.618 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:17.763 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:16 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:17.962 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:14 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:19.017 INFO:teuthology.orchestra.run.smithi070.stdout:Package libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:19.125 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:19.126 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:56:19.127 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:19.243 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install libcephfs-devel 2023-09-23T21:56:19.496 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:19.496 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:19.497 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repo Size 2023-09-23T21:56:19.497 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:19.497 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:56:19.497 INFO:teuthology.orchestra.run.smithi026.stdout: librados-devel x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 130 k 2023-09-23T21:56:19.497 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:19.497 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:56:19.497 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:19.497 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T21:56:19.497 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:19.498 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 130 k 2023-09-23T21:56:19.498 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 464 k 2023-09-23T21:56:19.498 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:56:19.596 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:19.710 INFO:teuthology.orchestra.run.smithi026.stdout:librados-devel-17.2.6-1125.gc9bb1bcf.el8.x86_64 612 kB/s | 130 kB 00:00 2023-09-23T21:56:19.711 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:56:19.711 INFO:teuthology.orchestra.run.smithi026.stdout:Total 608 kB/s | 130 kB 00:00 2023-09-23T21:56:19.711 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:56:19.717 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:56:19.718 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:56:19.775 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:56:19.779 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:56:19.905 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:56:20.097 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:20.341 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:20.463 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:19 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:21.459 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:21.459 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:18 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:21.889 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:21.889 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:21.890 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repo Size 2023-09-23T21:56:21.890 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:21.890 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:56:21.890 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs-devel x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 25 k 2023-09-23T21:56:21.890 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:21.890 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:56:21.890 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:21.890 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T21:56:21.890 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:21.891 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 25 k 2023-09-23T21:56:21.891 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 109 k 2023-09-23T21:56:21.891 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:56:22.161 INFO:teuthology.orchestra.run.smithi070.stdout:libcephfs-devel-17.2.6-1125.gc9bb1bcf.el8.x86_6 92 kB/s | 25 kB 00:00 2023-09-23T21:56:22.161 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:56:22.162 INFO:teuthology.orchestra.run.smithi070.stdout:Total 91 kB/s | 25 kB 00:00 2023-09-23T21:56:22.162 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:56:22.166 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:56:22.166 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:56:22.167 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:56:22.171 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:56:22.222 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:56:22.412 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:22.630 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:23.605 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:23.605 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:22 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:24.219 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:56:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:24.420 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:56:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: librados-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:56:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:24.420 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:24.546 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install libcephfs2 2023-09-23T21:56:24.923 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:25.833 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:22 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:26.131 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:56:26.313 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:26.313 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:56:26.313 INFO:teuthology.orchestra.run.smithi070.stdout: libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:56:26.313 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:26.313 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:26.461 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install librados2 2023-09-23T21:56:26.814 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:27.249 INFO:teuthology.orchestra.run.smithi026.stdout:Package libcephfs2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:27.368 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:27.369 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:56:27.369 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:27.479 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install libcephfs-devel 2023-09-23T21:56:27.634 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:26 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:27.857 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:28.728 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:25 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:28.880 INFO:teuthology.orchestra.run.smithi070.stdout:Package librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:28.988 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:28.989 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:56:28.989 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:29.116 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install librbd1 2023-09-23T21:56:29.469 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:30.220 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:30.220 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:30.220 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repo Size 2023-09-23T21:56:30.220 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:30.221 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:56:30.221 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs-devel x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 25 k 2023-09-23T21:56:30.221 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:30.221 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:56:30.221 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:30.221 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T21:56:30.221 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:30.222 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 25 k 2023-09-23T21:56:30.222 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 109 k 2023-09-23T21:56:30.222 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:56:30.300 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:29 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:30.488 INFO:teuthology.orchestra.run.smithi026.stdout:libcephfs-devel-17.2.6-1125.gc9bb1bcf.el8.x86_6 93 kB/s | 25 kB 00:00 2023-09-23T21:56:30.489 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:56:30.489 INFO:teuthology.orchestra.run.smithi026.stdout:Total 93 kB/s | 25 kB 00:00 2023-09-23T21:56:30.489 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:56:30.493 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:56:30.493 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:56:30.496 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:56:30.500 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:56:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:56:30.745 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:30.984 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:31.619 INFO:teuthology.orchestra.run.smithi070.stdout:Package librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:31.729 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:31.730 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:56:31.730 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:31.849 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install python3-rados 2023-09-23T21:56:32.028 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:32.029 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:29 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:32.201 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:33.105 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:32 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:34.373 INFO:teuthology.orchestra.run.smithi070.stdout:Package python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:34.481 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:34.482 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:56:34.482 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:34.496 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:56:34.634 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install python3-rgw 2023-09-23T21:56:34.697 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:34.697 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:56:34.697 INFO:teuthology.orchestra.run.smithi026.stdout: libcephfs-devel-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:56:34.697 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:34.698 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:34.825 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install librados2 2023-09-23T21:56:34.987 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:35.205 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:35.818 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:34 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:36.072 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:33 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:37.135 INFO:teuthology.orchestra.run.smithi070.stdout:Package python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:37.244 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:37.245 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:56:37.246 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:37.372 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install python3-cephfs 2023-09-23T21:56:37.491 INFO:teuthology.orchestra.run.smithi026.stdout:Package librados2-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:37.611 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:37.612 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:56:37.612 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:37.716 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install librbd1 2023-09-23T21:56:37.725 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:38.091 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:38.527 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:37 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:38.997 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:35 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:39.776 INFO:teuthology.orchestra.run.smithi070.stdout:Package python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:39.886 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:39.887 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:56:39.888 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:40.017 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install python3-rbd 2023-09-23T21:56:40.369 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:40.416 INFO:teuthology.orchestra.run.smithi026.stdout:Package librbd1-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:40.536 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:40.537 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:56:40.537 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:40.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install python3-rados 2023-09-23T21:56:41.029 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:41.268 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:40 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:41.939 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:38 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:42.513 INFO:teuthology.orchestra.run.smithi070.stdout:Package python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:42.620 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:42.621 INFO:teuthology.orchestra.run.smithi070.stdout:Nothing to do. 2023-09-23T21:56:42.621 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:42.743 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install rbd-fuse 2023-09-23T21:56:43.093 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:43.352 INFO:teuthology.orchestra.run.smithi026.stdout:Package python3-rados-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:43.470 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:43.471 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:56:43.472 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:43.585 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install python3-rgw 2023-09-23T21:56:43.961 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:43.992 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:42 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:44.853 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:41 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:45.423 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:45.424 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2023-09-23T21:56:45.424 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:45.424 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:56:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-fuse x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 90 k 2023-09-23T21:56:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:45.424 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:56:45.424 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:45.424 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T21:56:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:45.425 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 90 k 2023-09-23T21:56:45.425 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 264 k 2023-09-23T21:56:45.425 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:56:45.620 INFO:teuthology.orchestra.run.smithi070.stdout:rbd-fuse-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 464 kB/s | 90 kB 00:00 2023-09-23T21:56:45.621 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:56:45.621 INFO:teuthology.orchestra.run.smithi070.stdout:Total 461 kB/s | 90 kB 00:00 2023-09-23T21:56:45.621 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:56:45.630 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:56:45.630 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:56:45.684 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:56:45.688 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:56:45.802 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:56:45.940 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:46.267 INFO:teuthology.orchestra.run.smithi026.stdout:Package python3-rgw-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:46.387 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:46.388 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:56:46.389 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:46.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install python3-cephfs 2023-09-23T21:56:46.872 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:47.244 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:47.245 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:46 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:47.707 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:44 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:49.116 INFO:teuthology.orchestra.run.smithi026.stdout:Package python3-cephfs-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:49.235 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:49.236 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:56:49.236 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:49.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install python3-rbd 2023-09-23T21:56:49.726 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:50.200 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:56:50.381 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:50.381 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:56:50.382 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:56:50.382 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:50.382 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:56:50.518 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install rbd-mirror 2023-09-23T21:56:50.624 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:47 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:50.874 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:51.708 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:50 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:56:52.034 INFO:teuthology.orchestra.run.smithi026.stdout:Package python3-rbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 is already installed. 2023-09-23T21:56:52.152 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:52.154 INFO:teuthology.orchestra.run.smithi026.stdout:Nothing to do. 2023-09-23T21:56:52.154 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:56:52.267 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install rbd-fuse 2023-09-23T21:56:52.645 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:56:53.091 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout: Package Arch Version Repo Size 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-mirror x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.5 M 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T21:56:53.092 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:56:53.093 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 3.5 M 2023-09-23T21:56:53.093 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 16 M 2023-09-23T21:56:53.093 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:56:53.560 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:50 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:54.913 INFO:teuthology.orchestra.run.smithi070.stdout:rbd-mirror-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 1.9 MB/s | 3.5 MB 00:01 2023-09-23T21:56:54.914 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:56:54.914 INFO:teuthology.orchestra.run.smithi070.stdout:Total 1.9 MB/s | 3.5 MB 00:01 2023-09-23T21:56:54.915 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:56:54.927 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:56:54.928 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:56:55.001 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:56:55.005 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:56:55.075 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:56:55.075 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:55.076 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2023-09-23T21:56:55.076 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:55.076 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:56:55.076 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-fuse x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 90 k 2023-09-23T21:56:55.076 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:55.076 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:56:55.076 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:56:55.076 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T21:56:55.076 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:56:55.077 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 90 k 2023-09-23T21:56:55.077 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 264 k 2023-09-23T21:56:55.077 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:56:55.368 INFO:teuthology.orchestra.run.smithi026.stdout:rbd-fuse-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 310 kB/s | 90 kB 00:00 2023-09-23T21:56:55.369 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:56:55.369 INFO:teuthology.orchestra.run.smithi026.stdout:Total 308 kB/s | 90 kB 00:00 2023-09-23T21:56:55.369 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:56:55.380 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:56:55.381 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:56:55.440 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:56:55.444 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:56:55.449 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:56:55.558 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:56:55.633 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:55.708 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:55.944 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:55.976 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:57.004 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:57.004 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:54 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:56:57.025 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:56:57.025 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:03:56 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:57:00.265 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:57:00.289 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:57:00.447 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:57:00.447 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:57:00.448 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:57:00.448 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:57:00.448 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:57:00.489 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:00.489 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:57:00.489 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-fuse-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:57:00.489 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:00.489 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:57:00.598 DEBUG:teuthology.orchestra.run.smithi070:> sudo yum -y install rbd-nbd 2023-09-23T21:57:00.620 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install rbd-mirror 2023-09-23T21:57:00.960 INFO:teuthology.orchestra.run.smithi070.stdout:Updating Subscription Management repositories. 2023-09-23T21:57:00.997 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:57:01.816 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:04:00 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:57:01.936 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:03:58 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:57:03.175 INFO:teuthology.orchestra.run.smithi070.stdout:Dependencies resolved. 2023-09-23T21:57:03.176 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:57:03.176 INFO:teuthology.orchestra.run.smithi070.stdout: Package Architecture Version Repository Size 2023-09-23T21:57:03.176 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:57:03.176 INFO:teuthology.orchestra.run.smithi070.stdout:Installing: 2023-09-23T21:57:03.176 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-nbd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 173 k 2023-09-23T21:57:03.177 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:57:03.177 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction Summary 2023-09-23T21:57:03.177 INFO:teuthology.orchestra.run.smithi070.stdout:================================================================================ 2023-09-23T21:57:03.177 INFO:teuthology.orchestra.run.smithi070.stdout:Install 1 Package 2023-09-23T21:57:03.177 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:57:03.177 INFO:teuthology.orchestra.run.smithi070.stdout:Total download size: 173 k 2023-09-23T21:57:03.178 INFO:teuthology.orchestra.run.smithi070.stdout:Installed size: 552 k 2023-09-23T21:57:03.178 INFO:teuthology.orchestra.run.smithi070.stdout:Downloading Packages: 2023-09-23T21:57:03.468 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:57:03.469 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:57:03.469 INFO:teuthology.orchestra.run.smithi026.stdout: Package Arch Version Repo Size 2023-09-23T21:57:03.469 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:57:03.469 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:57:03.469 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-mirror x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 3.5 M 2023-09-23T21:57:03.469 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:03.469 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:57:03.469 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:57:03.470 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T21:57:03.470 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:03.470 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 3.5 M 2023-09-23T21:57:03.470 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 16 M 2023-09-23T21:57:03.470 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:57:03.502 INFO:teuthology.orchestra.run.smithi070.stdout:rbd-nbd-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 532 kB/s | 173 kB 00:00 2023-09-23T21:57:03.502 INFO:teuthology.orchestra.run.smithi070.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:57:03.503 INFO:teuthology.orchestra.run.smithi070.stdout:Total 530 kB/s | 173 kB 00:00 2023-09-23T21:57:03.503 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction check 2023-09-23T21:57:03.514 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction check succeeded. 2023-09-23T21:57:03.514 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction test 2023-09-23T21:57:03.568 INFO:teuthology.orchestra.run.smithi070.stdout:Transaction test succeeded. 2023-09-23T21:57:03.571 INFO:teuthology.orchestra.run.smithi070.stdout:Running transaction 2023-09-23T21:57:03.687 INFO:teuthology.orchestra.run.smithi070.stdout: Preparing : 1/1 2023-09-23T21:57:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: Installing : rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:57:04.072 INFO:teuthology.orchestra.run.smithi070.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:57:04.180 INFO:teuthology.orchestra.run.smithi026.stdout:rbd-mirror-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 4.9 MB/s | 3.5 MB 00:00 2023-09-23T21:57:04.180 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:57:04.181 INFO:teuthology.orchestra.run.smithi026.stdout:Total 4.9 MB/s | 3.5 MB 00:00 2023-09-23T21:57:04.181 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:57:04.195 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:57:04.195 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:57:04.272 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:57:04.276 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:57:04.768 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:57:04.943 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:57:05.112 INFO:teuthology.orchestra.run.smithi070.stdout: Verifying : rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:57:05.112 INFO:teuthology.orchestra.run.smithi070.stdout:Last metadata expiration check: 0:04:04 ago on Sat 23 Sep 2023 09:53:01 PM UTC. 2023-09-23T21:57:05.326 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:57:06.404 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:57:06.404 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:04:03 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:57:07.436 INFO:teuthology.orchestra.run.smithi070.stdout:Installed products updated. 2023-09-23T21:57:07.616 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:57:07.616 INFO:teuthology.orchestra.run.smithi070.stdout:Installed: 2023-09-23T21:57:07.616 INFO:teuthology.orchestra.run.smithi070.stdout: rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:57:07.616 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:57:07.616 INFO:teuthology.orchestra.run.smithi070.stdout:Complete! 2023-09-23T21:57:07.780 DEBUG:teuthology.parallel:result is None 2023-09-23T21:57:08.904 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:57:09.102 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:09.103 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:57:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-mirror-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:57:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:09.103 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:57:09.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo yum -y install rbd-nbd 2023-09-23T21:57:09.616 INFO:teuthology.orchestra.run.smithi026.stdout:Updating Subscription Management repositories. 2023-09-23T21:57:10.539 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:04:07 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:57:12.034 INFO:teuthology.orchestra.run.smithi026.stdout:Dependencies resolved. 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout: Package Architecture Version Repository Size 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout:Installing: 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-nbd x86_64 2:17.2.6-1125.gc9bb1bcf.el8 ceph 173 k 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction Summary 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout:================================================================================ 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout:Install 1 Package 2023-09-23T21:57:12.035 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:12.036 INFO:teuthology.orchestra.run.smithi026.stdout:Total download size: 173 k 2023-09-23T21:57:12.036 INFO:teuthology.orchestra.run.smithi026.stdout:Installed size: 552 k 2023-09-23T21:57:12.036 INFO:teuthology.orchestra.run.smithi026.stdout:Downloading Packages: 2023-09-23T21:57:12.366 INFO:teuthology.orchestra.run.smithi026.stdout:rbd-nbd-17.2.6-1125.gc9bb1bcf.el8.x86_64.rpm 524 kB/s | 173 kB 00:00 2023-09-23T21:57:12.367 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:57:12.367 INFO:teuthology.orchestra.run.smithi026.stdout:Total 522 kB/s | 173 kB 00:00 2023-09-23T21:57:12.367 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction check 2023-09-23T21:57:12.379 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction check succeeded. 2023-09-23T21:57:12.379 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction test 2023-09-23T21:57:12.437 INFO:teuthology.orchestra.run.smithi026.stdout:Transaction test succeeded. 2023-09-23T21:57:12.441 INFO:teuthology.orchestra.run.smithi026.stdout:Running transaction 2023-09-23T21:57:12.561 INFO:teuthology.orchestra.run.smithi026.stdout: Preparing : 1/1 2023-09-23T21:57:12.706 INFO:teuthology.orchestra.run.smithi026.stdout: Installing : rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:57:12.974 INFO:teuthology.orchestra.run.smithi026.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:57:14.076 INFO:teuthology.orchestra.run.smithi026.stdout: Verifying : rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 1/1 2023-09-23T21:57:14.076 INFO:teuthology.orchestra.run.smithi026.stdout:Last metadata expiration check: 0:04:11 ago on Sat 23 Sep 2023 09:53:03 PM UTC. 2023-09-23T21:57:16.687 INFO:teuthology.orchestra.run.smithi026.stdout:Installed products updated. 2023-09-23T21:57:16.888 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:16.888 INFO:teuthology.orchestra.run.smithi026.stdout:Installed: 2023-09-23T21:57:16.888 INFO:teuthology.orchestra.run.smithi026.stdout: rbd-nbd-2:17.2.6-1125.gc9bb1bcf.el8.x86_64 2023-09-23T21:57:16.888 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:57:16.888 INFO:teuthology.orchestra.run.smithi026.stdout:Complete! 2023-09-23T21:57:17.014 DEBUG:teuthology.parallel:result is None 2023-09-23T21:57:17.015 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T21:57:17.015 INFO:teuthology.packaging:ref: None 2023-09-23T21:57:17.015 INFO:teuthology.packaging:tag: None 2023-09-23T21:57:17.015 INFO:teuthology.packaging:branch: quincy 2023-09-23T21:57:17.015 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T21:57:17.015 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T21:57:17.426 DEBUG:teuthology.orchestra.run.smithi026:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T21:57:17.450 INFO:teuthology.orchestra.run.smithi026.stdout:17.2.6-1125.gc9bb1bcf.el8 2023-09-23T21:57:17.450 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1125.gc9bb1bcf.el8 2023-09-23T21:57:17.450 INFO:teuthology.task.install:The correct ceph version 17.2.6-1125.gc9bb1bcf is installed. 2023-09-23T21:57:17.451 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T21:57:17.451 INFO:teuthology.packaging:ref: None 2023-09-23T21:57:17.451 INFO:teuthology.packaging:tag: None 2023-09-23T21:57:17.451 INFO:teuthology.packaging:branch: quincy 2023-09-23T21:57:17.451 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T21:57:17.451 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T21:57:17.852 DEBUG:teuthology.orchestra.run.smithi070:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T21:57:17.880 INFO:teuthology.orchestra.run.smithi070.stdout:17.2.6-1125.gc9bb1bcf.el8 2023-09-23T21:57:17.880 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1125.gc9bb1bcf.el8 2023-09-23T21:57:17.880 INFO:teuthology.task.install:The correct ceph version 17.2.6-1125.gc9bb1bcf is installed. 2023-09-23T21:57:17.880 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T21:57:17.881 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:57:17.881 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T21:57:17.913 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T21:57:17.913 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T21:57:17.955 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T21:57:17.955 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:57:17.955 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T21:57:17.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T21:57:18.057 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T21:57:18.058 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T21:57:18.088 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T21:57:18.158 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T21:57:18.158 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:57:18.159 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T21:57:18.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T21:57:18.260 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T21:57:18.260 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T21:57:18.290 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T21:57:18.359 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T21:57:18.359 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:57:18.359 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T21:57:18.390 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T21:57:18.460 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T21:57:18.460 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T21:57:18.489 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T21:57:18.558 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T21:57:18.635 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T21:57:18.635 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-09-23T21:57:18.636 INFO:tasks.cephadm:Cluster fsid is 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:57:18.636 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T21:57:18.636 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.26', 'mon.c': '[v2:172.21.15.26:3301,v1:172.21.15.26:6790]', 'mon.b': '172.21.15.70'} 2023-09-23T21:57:18.636 INFO:tasks.cephadm:First mon is mon.a on smithi026 2023-09-23T21:57:18.636 INFO:tasks.cephadm:First mgr is y 2023-09-23T21:57:18.637 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T21:57:18.637 DEBUG:teuthology.orchestra.run.smithi026:> sudo hostname $(hostname -s) 2023-09-23T21:57:18.668 DEBUG:teuthology.orchestra.run.smithi070:> sudo hostname $(hostname -s) 2023-09-23T21:57:18.699 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-09-23T21:57:18.699 DEBUG:teuthology.orchestra.run.smithi026:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T21:57:18.931 INFO:teuthology.orchestra.run.smithi026.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 21:57 /home/ubuntu/cephtest/cephadm 2023-09-23T21:57:18.931 DEBUG:teuthology.orchestra.run.smithi070:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T21:57:19.072 INFO:teuthology.orchestra.run.smithi070.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 21:57 /home/ubuntu/cephtest/cephadm 2023-09-23T21:57:19.072 DEBUG:teuthology.orchestra.run.smithi026:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T21:57:19.095 DEBUG:teuthology.orchestra.run.smithi070:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T21:57:19.127 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-09-23T21:57:19.128 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T21:57:19.138 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T21:57:20.287 INFO:teuthology.orchestra.run.smithi070.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T21:57:20.340 INFO:teuthology.orchestra.run.smithi026.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T21:58:10.412 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T21:58:10.412 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph_version": "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)", 2023-09-23T21:58:10.412 INFO:teuthology.orchestra.run.smithi026.stdout: "image_id": "444539aec5bf9e20782edb464496ac68eeb42e917d810f31b62a4e32fd3e4cab", 2023-09-23T21:58:10.413 INFO:teuthology.orchestra.run.smithi026.stdout: "repo_digests": [ 2023-09-23T21:58:10.413 INFO:teuthology.orchestra.run.smithi026.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:3bb80add7917e9e7cd2f8b41f950baba963e834e98b4566c8403c083fc4f1c17" 2023-09-23T21:58:10.413 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T21:58:10.413 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T21:58:11.094 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T21:58:11.095 INFO:teuthology.orchestra.run.smithi070.stdout: "ceph_version": "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)", 2023-09-23T21:58:11.095 INFO:teuthology.orchestra.run.smithi070.stdout: "image_id": "444539aec5bf9e20782edb464496ac68eeb42e917d810f31b62a4e32fd3e4cab", 2023-09-23T21:58:11.095 INFO:teuthology.orchestra.run.smithi070.stdout: "repo_digests": [ 2023-09-23T21:58:11.095 INFO:teuthology.orchestra.run.smithi070.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:3bb80add7917e9e7cd2f8b41f950baba963e834e98b4566c8403c083fc4f1c17" 2023-09-23T21:58:11.095 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T21:58:11.095 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T21:58:11.120 DEBUG:teuthology.orchestra.run.smithi026:> sudo mkdir -p /etc/ceph 2023-09-23T21:58:11.158 DEBUG:teuthology.orchestra.run.smithi070:> sudo mkdir -p /etc/ceph 2023-09-23T21:58:11.191 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 777 /etc/ceph 2023-09-23T21:58:11.231 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod 777 /etc/ceph 2023-09-23T21:58:11.259 INFO:tasks.cephadm:Writing seed config... 2023-09-23T21:58:11.260 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T21:58:11.260 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T21:58:11.260 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T21:58:11.260 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T21:58:11.260 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T21:58:11.260 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-09-23T21:58:11.261 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T21:58:11.261 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T21:58:11.261 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T21:58:11.261 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T21:58:11.261 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T21:58:11.261 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:58:11.261 DEBUG:teuthology.orchestra.run.smithi026:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T21:58:11.290 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T21:58:11.291 DEBUG:teuthology.orchestra.run.smithi026:mon.a> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T21:58:11.333 DEBUG:teuthology.orchestra.run.smithi026:mgr.y> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T21:58:11.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T21:58:11.365 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T21:58:11.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.26 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:58:11.395 INFO:journalctl@ceph.mgr.y.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T21:58:11.581 INFO:teuthology.orchestra.run.smithi026.stdout:-------------------------------------------------------------------------------- 2023-09-23T21:58:11.581 INFO:teuthology.orchestra.run.smithi026.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '2a3049e2-5a5c-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.26', '--skip-admin-label'] 2023-09-23T21:58:11.644 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 3.0.1 2023-09-23T21:58:11.645 INFO:teuthology.orchestra.run.smithi026.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-09-23T21:58:11.645 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying podman|docker is present... 2023-09-23T21:58:11.702 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 3.0.1 2023-09-23T21:58:11.702 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying lvm2 is present... 2023-09-23T21:58:11.702 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying time synchronization is in place... 2023-09-23T21:58:11.709 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T21:58:11.709 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T21:58:11.715 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T21:58:11.715 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T21:58:11.721 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout enabled 2023-09-23T21:58:11.728 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout active 2023-09-23T21:58:11.728 INFO:teuthology.orchestra.run.smithi026.stdout:Unit chronyd.service is enabled and running 2023-09-23T21:58:11.728 INFO:teuthology.orchestra.run.smithi026.stdout:Repeating the final host check... 2023-09-23T21:58:11.789 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 3.0.1 2023-09-23T21:58:11.789 INFO:teuthology.orchestra.run.smithi026.stdout:podman (/bin/podman) version 3.0.1 is present 2023-09-23T21:58:11.789 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl is present 2023-09-23T21:58:11.789 INFO:teuthology.orchestra.run.smithi026.stdout:lvcreate is present 2023-09-23T21:58:11.795 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T21:58:11.795 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T21:58:11.801 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T21:58:11.801 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T21:58:11.807 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout enabled 2023-09-23T21:58:11.813 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout active 2023-09-23T21:58:11.813 INFO:teuthology.orchestra.run.smithi026.stdout:Unit chronyd.service is enabled and running 2023-09-23T21:58:11.813 INFO:teuthology.orchestra.run.smithi026.stdout:Host looks OK 2023-09-23T21:58:11.813 INFO:teuthology.orchestra.run.smithi026.stdout:Cluster fsid: 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:11.814 INFO:teuthology.orchestra.run.smithi026.stdout:Acquiring lock 140173041527272 on /run/cephadm/2a3049e2-5a5c-11ee-8db4-212e2dc638e7.lock 2023-09-23T21:58:11.814 INFO:teuthology.orchestra.run.smithi026.stdout:Lock 140173041527272 acquired on /run/cephadm/2a3049e2-5a5c-11ee-8db4-212e2dc638e7.lock 2023-09-23T21:58:11.814 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying IP 172.21.15.26 port 3300 ... 2023-09-23T21:58:11.815 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying IP 172.21.15.26 port 6789 ... 2023-09-23T21:58:11.815 INFO:teuthology.orchestra.run.smithi026.stdout:Base mon IP(s) is [172.21.15.26:3300, 172.21.15.26:6789], mon addrv is [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2023-09-23T21:58:11.818 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-09-23T21:58:11.818 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.26 metric 100 2023-09-23T21:58:11.821 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T21:58:11.821 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-09-23T21:58:11.821 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1773sec hoplimit 64 pref medium 2023-09-23T21:58:11.824 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T21:58:11.824 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-09-23T21:58:11.824 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T21:58:11.824 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2023-09-23T21:58:11.824 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fea5:cac0/64 scope link 2023-09-23T21:58:11.824 INFO:teuthology.orchestra.run.smithi026.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T21:58:11.825 INFO:teuthology.orchestra.run.smithi026.stdout:Mon IP `172.21.15.26` is in CIDR network `172.21.0.0/20` 2023-09-23T21:58:11.825 INFO:teuthology.orchestra.run.smithi026.stdout:Mon IP `172.21.15.26` is in CIDR network `172.21.0.0/20` 2023-09-23T21:58:11.825 INFO:teuthology.orchestra.run.smithi026.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T21:58:11.825 INFO:teuthology.orchestra.run.smithi026.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T21:58:11.826 INFO:teuthology.orchestra.run.smithi026.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T21:58:12.197 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stdout 444539aec5bf9e20782edb464496ac68eeb42e917d810f31b62a4e32fd3e4cab 2023-09-23T21:58:12.197 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T21:58:12.197 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Getting image source signatures 2023-09-23T21:58:12.197 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Copying blob sha256:ce16db1fb95f1913bf433ce46f6a040afd03f13a267a85c9c5302717cf75a00a 2023-09-23T21:58:12.197 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-09-23T21:58:12.198 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Copying config sha256:444539aec5bf9e20782edb464496ac68eeb42e917d810f31b62a4e32fd3e4cab 2023-09-23T21:58:12.198 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Writing manifest to image destination 2023-09-23T21:58:12.198 INFO:teuthology.orchestra.run.smithi026.stdout:/bin/podman: stderr Storing signatures 2023-09-23T21:58:13.515 INFO:teuthology.orchestra.run.smithi026.stdout:ceph: stdout ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable) 2023-09-23T21:58:13.515 INFO:teuthology.orchestra.run.smithi026.stdout:Ceph version: ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable) 2023-09-23T21:58:13.515 INFO:teuthology.orchestra.run.smithi026.stdout:Extracting ceph user uid/gid from container image... 2023-09-23T21:58:14.470 INFO:teuthology.orchestra.run.smithi026.stdout:stat: stdout 167 167 2023-09-23T21:58:14.470 INFO:teuthology.orchestra.run.smithi026.stdout:Creating initial keys... 2023-09-23T21:58:15.390 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph-authtool: stdout AQB2Xw9lQ5r3MRAA/k56Yo1PzXqhG2cY24Mnuw== 2023-09-23T21:58:16.351 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph-authtool: stdout AQB3Xw9li5DCLBAAUPz05d1cNQ7IKfEsUdnIwg== 2023-09-23T21:58:17.292 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph-authtool: stdout AQB4Xw9lkBBJKRAAVaSXQo4t58zuRS47V5hFig== 2023-09-23T21:58:17.293 INFO:teuthology.orchestra.run.smithi026.stdout:Creating initial monmap... 2023-09-23T21:58:18.245 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T21:58:18.245 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-09-23T21:58:18.245 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:18.245 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T21:58:18.245 INFO:teuthology.orchestra.run.smithi026.stdout:monmaptool for a [v2:172.21.15.26:3300,v1:172.21.15.26:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T21:58:18.246 INFO:teuthology.orchestra.run.smithi026.stdout:setting min_mon_release = octopus 2023-09-23T21:58:18.246 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: set fsid to 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:18.246 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T21:58:18.246 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:58:18.246 INFO:teuthology.orchestra.run.smithi026.stdout:Creating mon... 2023-09-23T21:58:19.184 INFO:teuthology.orchestra.run.smithi026.stdout:create mon.a on 2023-09-23T21:58:19.290 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T21:58:19.383 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-23T21:58:19.481 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7.target. 2023-09-23T21:58:19.481 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7.target. 2023-09-23T21:58:19.622 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T21:58:19.622 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T21:58:19.712 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7.target.wants/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@.service. 2023-09-23T21:58:19.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:19 smithi026 systemd[1]: Starting Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T21:58:20.272 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T21:58:20.273 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2023-09-23T21:58:20.280 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T21:58:20.280 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T21:58:20.280 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2023-09-23T21:58:20.280 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T21:58:20.281 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mon to start... 2023-09-23T21:58:20.281 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mon... 2023-09-23T21:58:20.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:20 smithi026 bash[99133]: e1139a1b458acf3e06b98884b38344722f4843f860aff511ffe9000e12bd3045 2023-09-23T21:58:20.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:20 smithi026 systemd[1]: Started Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T21:58:20.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:20 smithi026 ceph-mon[99198]: mkfs 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:20.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:20 smithi026 ceph-mon[99198]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T21:58:21.726 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout cluster: 2023-09-23T21:58:21.726 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout id: 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:21.726 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-09-23T21:58:21.726 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout services: 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.637627s) 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout data: 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout pgs: 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:mon is available 2023-09-23T21:58:21.727 INFO:teuthology.orchestra.run.smithi026.stdout:Assimilating anything we can from ceph.conf... 2023-09-23T21:58:21.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:21 smithi026 ceph-mon[99198]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T21:58:21.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:21 smithi026 ceph-mon[99198]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} removed_ranks: {} 2023-09-23T21:58:21.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:21 smithi026 ceph-mon[99198]: fsmap 2023-09-23T21:58:21.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:21 smithi026 ceph-mon[99198]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T21:58:21.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:21 smithi026 ceph-mon[99198]: mgrmap e1: no daemons active 2023-09-23T21:58:21.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:21 smithi026 ceph-mon[99198]: from='client.? 172.21.15.26:0/2137313964' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T21:58:22.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:22 smithi026 ceph-mon[99198]: from='client.? 172.21.15.26:0/927086004' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T21:58:22.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:22 smithi026 ceph-mon[99198]: from='client.? 172.21.15.26:0/927086004' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T21:58:22.900 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:22.900 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T21:58:22.900 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout fsid = 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:22.900 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.26:3300,v1:172.21.15.26:6789] 2023-09-23T21:58:22.900 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T21:58:22.900 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T21:58:22.901 INFO:teuthology.orchestra.run.smithi026.stdout:Generating new minimal ceph.conf... 2023-09-23T21:58:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:23 smithi026 ceph-mon[99198]: from='client.? 172.21.15.26:0/528595576' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:58:24.140 INFO:teuthology.orchestra.run.smithi026.stdout:Restarting the monitor... 2023-09-23T21:58:24.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:24 smithi026 systemd[1]: Stopping Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T21:58:24.714 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:24 smithi026 conmon[99174]: 2023-09-23T21:58:24.406+0000 7fee37da3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T21:58:24.715 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:24 smithi026 conmon[99174]: 2023-09-23T21:58:24.406+0000 7fee37da3700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T21:58:24.715 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:24 smithi026 bash[99762]: e1139a1b458acf3e06b98884b38344722f4843f860aff511ffe9000e12bd3045 2023-09-23T21:58:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T21:58:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 systemd[1]: Stopped Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T21:58:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 systemd[1]: Starting Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T21:58:25.586 INFO:teuthology.orchestra.run.smithi026.stdout:Setting mon public_network to 172.21.0.0/20 2023-09-23T21:58:25.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable), process ceph-mon, pid 7 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: pidfile_write: ignore empty --pid-file 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: load: jerasure load: lrc 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: RocksDB version: 6.15.5 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Compile date Sep 23 2023 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: DB SUMMARY 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: DB Session ID: U6EW3X7J1FK6XZSFCWVT 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: CURRENT file: CURRENT 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T21:58:25.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83577 ; 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.error_if_exists: 0 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.create_if_missing: 0 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.paranoid_checks: 1 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.env: 0x55fb78005a80 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.fs: Posix File System 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.info_log: 0x55fb79fd45a0 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.statistics: (nil) 2023-09-23T21:58:25.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.use_fsync: 0 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_log_file_size: 0 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.allow_fallocate: 1 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.use_direct_reads: 0 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.db_log_dir: 2023-09-23T21:58:25.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T21:58:25.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T21:58:25.839 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T21:58:25.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T21:58:25.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T21:58:25.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T21:58:25.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T21:58:25.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T21:58:25.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.write_buffer_manager: 0x55fb7ad46960 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.unordered_write: 0 2023-09-23T21:58:25.841 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.row_cache: None 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.wal_filter: None 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.preserve_deletes: 0 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.two_write_queues: 0 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.atomic_flush: 0 2023-09-23T21:58:25.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.log_readahead_size: 0 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_background_jobs: 2 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_background_compactions: -1 2023-09-23T21:58:25.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_subcompactions: 1 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_open_files: -1 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T21:58:25.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_background_flushes: -1 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Compression algorithms supported: 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: kZSTD supported: 0 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: kXpressCompression supported: 0 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: kZlibCompression supported: 1 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: kSnappyCompression supported: 1 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: kLZ4Compression supported: 1 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: kBZip2Compression supported: 0 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T21:58:25.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.merge_operator: 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_filter: None 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55fb7a026fb8) 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T21:58:25.846 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: hash_index_allow_collision: 1 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 1 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x55fb7a059090 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2023-09-23T21:58:25.847 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2023-09-23T21:58:25.848 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 4 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compression: NoCompression 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T21:58:25.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.num_levels: 7 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T21:58:25.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T21:58:25.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T21:58:25.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T21:58:25.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.table_properties_collectors: 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.inplace_update_support: 0 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.bloom_locality: 0 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.max_successive_merges: 0 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T21:58:25.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.ttl: 2592000 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.enable_blob_files: false 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.min_blob_size: 0 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-09-23T21:58:25.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506305574429, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506305575163, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80124, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78392, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72600, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695506305, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "323a2072-3ea7-4e7c-808e-252eb8138c13", "db_session_id": "U6EW3X7J1FK6XZSFCWVT"}} 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506305575512, "job": 1, "event": "recovery_finished"} 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55fb79fed180 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: DB pointer 0x55fb7a0c2000 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2023-09-23T21:58:25.856 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 80.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 168.7 0.00 0.00 1 0.000 0 0 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 80.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 168.7 0.00 0.00 1 0.000 0 0 2023-09-23T21:58:25.857 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 168.7 0.00 0.00 1 0.000 0 0 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 168.7 0.00 0.00 1 0.000 0 0 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T21:58:25.858 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 25.09 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 25.09 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 2/0 80.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 168.7 0.00 0.00 1 0.000 0 0 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 80.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 168.7 0.00 0.00 1 0.000 0 0 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T21:58:25.859 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 168.7 0.00 0.00 1 0.000 0 0 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 24.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T21:58:25.860 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mon.a@-1(???) e1 preinit fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mon.a@-1(???).mds e1 new map 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mon.a@-1(???).mds e1 print_map 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout: e1 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: -1 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout: No filesystems configured 2023-09-23T21:58:25.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 bash[100092]: feb74e5008f79cab8ca4326132018939160eb19ea214a55769fe24434c065172 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 systemd[1]: Started Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: monmap e1: 1 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0]} removed_ranks: {} 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: fsmap 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T21:58:25.862 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:25 smithi026 ceph-mon[100157]: mgrmap e1: no daemons active 2023-09-23T21:58:26.840 INFO:teuthology.orchestra.run.smithi026.stdout:Wrote config to /etc/ceph/ceph.conf 2023-09-23T21:58:26.840 INFO:teuthology.orchestra.run.smithi026.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:58:26.841 INFO:teuthology.orchestra.run.smithi026.stdout:Creating mgr... 2023-09-23T21:58:26.841 INFO:teuthology.orchestra.run.smithi026.stdout:Verifying port 9283 ... 2023-09-23T21:58:26.975 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T21:58:26.976 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T21:58:27.068 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7.target.wants/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@.service. 2023-09-23T21:58:27.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:27 smithi026 systemd[1]: Starting Ceph mgr.y for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T21:58:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:27 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/4171036317' entity='client.admin' 2023-09-23T21:58:27.694 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T21:58:27.694 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2023-09-23T21:58:27.702 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T21:58:27.702 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T21:58:27.702 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2023-09-23T21:58:27.702 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T21:58:27.710 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T21:58:27.711 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2023-09-23T21:58:27.719 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T21:58:27.719 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T21:58:27.719 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2023-09-23T21:58:27.719 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T21:58:27.719 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr to start... 2023-09-23T21:58:27.719 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr... 2023-09-23T21:58:27.932 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:27 smithi026 bash[100545]: d8b44817b175e7fcfe51739e71b21a1a33127806be22c3c15c0f3c4c8fbec879 2023-09-23T21:58:27.932 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:27 smithi026 systemd[1]: Started Ceph mgr.y for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T21:58:27.932 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:27 smithi026 conmon[100586]: 2023-09-23T21:58:27.823+0000 7fac3e24e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T21:58:27.932 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:27 smithi026 conmon[100586]: 2023-09-23T21:58:27.917+0000 7fac3e24e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T21:58:28.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:28 smithi026 conmon[100586]: 2023-09-23T21:58:28.029+0000 7fac3e24e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T21:58:28.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:28 smithi026 conmon[100586]: 2023-09-23T21:58:28.138+0000 7fac3e24e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T21:58:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:28 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/535832133' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T21:58:28.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:28 smithi026 conmon[100586]: 2023-09-23T21:58:28.345+0000 7fac3e24e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T21:58:29.008 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:28 smithi026 conmon[100586]: 2023-09-23T21:58:28.752+0000 7fac3e24e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T21:58:29.009 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:28 smithi026 conmon[100586]: 2023-09-23T21:58:28.858+0000 7fac3e24e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T21:58:29.022 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:29.023 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T21:58:29.023 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "2a3049e2-5a5c-11ee-8db4-212e2dc638e7", 2023-09-23T21:58:29.023 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T21:58:29.023 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T21:58:29.023 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T21:58:29.023 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T21:58:29.023 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:29.023 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T21:58:29.024 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T21:58:29.024 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2023-09-23T21:58:29.024 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:29.024 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T21:58:29.024 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T21:58:29.024 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:29.024 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-09-23T21:58:29.024 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T21:58:29.025 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:29.025 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T21:58:29.025 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T21:58:29.025 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:29.025 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T21:58:29.025 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:29.025 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T21:58:29.026 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T21:58:29.026 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T21:58:29.026 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T21:58:29.026 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T21:58:29.026 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T21:58:29.026 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:29.026 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T21:58:29.026 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T21:58:29.027 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T21:58:29.027 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T21:58:29.027 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T21:58:29.027 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T21:58:29.027 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T21:58:29.027 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T21:58:29.027 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T21:58:29.027 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:29.028 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T21:58:29.028 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:29.028 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T21:58:29.028 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T21:58:29.028 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:29.028 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T21:58:29.028 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T21:58:29.028 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T21:58:29.029 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T21:58:29.029 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T21:58:29.029 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T21:58:29.029 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T21:58:29.029 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:29.029 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T21:58:29.029 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:29.029 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T21:58:29.030 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:29.030 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T21:58:20.282890+0000", 2023-09-23T21:58:29.030 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T21:58:29.030 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:29.030 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T21:58:29.030 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T21:58:29.030 INFO:teuthology.orchestra.run.smithi026.stdout:mgr not available, waiting (1/15)... 2023-09-23T21:58:29.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:29 smithi026 conmon[100586]: 2023-09-23T21:58:29.007+0000 7fac3e24e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T21:58:29.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:29 smithi026 conmon[100586]: 2023-09-23T21:58:29.121+0000 7fac3e24e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T21:58:29.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:29 smithi026 conmon[100586]: 2023-09-23T21:58:29.626+0000 7fac3e24e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T21:58:30.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:30 smithi026 conmon[100586]: 2023-09-23T21:58:30.315+0000 7fac3e24e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T21:58:30.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:30 smithi026 conmon[100586]: 2023-09-23T21:58:30.505+0000 7fac3e24e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T21:58:31.733 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:31 smithi026 conmon[100586]: 2023-09-23T21:58:31.473+0000 7fac3e24e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T21:58:31.733 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:31 smithi026 conmon[100586]: 2023-09-23T21:58:31.584+0000 7fac3e24e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T21:58:31.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:31 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/4202727402' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T21:58:31.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:31 smithi026 conmon[100586]: 2023-09-23T21:58:31.810+0000 7fac3e24e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T21:58:32.255 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:32.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T21:58:32.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "2a3049e2-5a5c-11ee-8db4-212e2dc638e7", 2023-09-23T21:58:32.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T21:58:32.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T21:58:32.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T21:58:32.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T21:58:32.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:32.259 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T21:58:32.260 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T21:58:32.263 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:32.263 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T21:58:32.263 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T21:58:32.264 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T21:58:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:32.266 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T21:58:20.282890+0000", 2023-09-23T21:58:32.266 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T21:58:32.266 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:32.266 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T21:58:32.266 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T21:58:32.266 INFO:teuthology.orchestra.run.smithi026.stdout:mgr not available, waiting (2/15)... 2023-09-23T21:58:32.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:32 smithi026 conmon[100586]: 2023-09-23T21:58:32.155+0000 7fac3e24e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T21:58:32.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:32 smithi026 conmon[100586]: 2023-09-23T21:58:32.260+0000 7fac3e24e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T21:58:32.908 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:32 smithi026 conmon[100586]: 2023-09-23T21:58:32.588+0000 7fac3e24e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T21:58:33.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:32 smithi026 conmon[100586]: 2023-09-23T21:58:32.907+0000 7fac3e24e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T21:58:33.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:33 smithi026 conmon[100586]: 2023-09-23T21:58:33.729+0000 7fac3e24e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T21:58:33.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:33 smithi026 conmon[100586]: 2023-09-23T21:58:33.884+0000 7fac3e24e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T21:58:34.463 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:34 smithi026 conmon[100586]: 2023-09-23T21:58:34.120+0000 7fac3e24e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T21:58:34.463 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:34 smithi026 conmon[100586]: 2023-09-23T21:58:34.255+0000 7fac3e24e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T21:58:34.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:34 smithi026 conmon[100586]: 2023-09-23T21:58:34.538+0000 7fac3e24e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T21:58:35.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:34 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3219553221' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T21:58:35.443 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:35.443 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T21:58:35.443 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "2a3049e2-5a5c-11ee-8db4-212e2dc638e7", 2023-09-23T21:58:35.443 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T21:58:35.443 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T21:58:35.443 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T21:58:35.443 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T21:58:35.444 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T21:58:35.445 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T21:58:35.446 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T21:58:35.447 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:35.448 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T21:58:20.282890+0000", 2023-09-23T21:58:35.448 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T21:58:35.448 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:35.448 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T21:58:35.448 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T21:58:35.448 INFO:teuthology.orchestra.run.smithi026.stdout:mgr not available, waiting (3/15)... 2023-09-23T21:58:35.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:35 smithi026 conmon[100586]: 2023-09-23T21:58:35.590+0000 7fac3e24e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T21:58:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: Activating manager daemon y 2023-09-23T21:58:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: mgrmap e2: y(active, starting, since 0.00299674s) 2023-09-23T21:58:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T21:58:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T21:58:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T21:58:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T21:58:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T21:58:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: Manager daemon y is now available 2023-09-23T21:58:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T21:58:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T21:58:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' 2023-09-23T21:58:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' 2023-09-23T21:58:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:36 smithi026 ceph-mon[100157]: from='mgr.14100 172.21.15.26:0/3082786655' entity='mgr.y' 2023-09-23T21:58:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:37 smithi026 ceph-mon[100157]: mgrmap e3: y(active, since 1.00742s) 2023-09-23T21:58:38.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:38 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1579881472' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T21:58:39.048 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:39.048 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T21:58:39.048 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsid": "2a3049e2-5a5c-11ee-8db4-212e2dc638e7", 2023-09-23T21:58:39.048 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 0 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2023-09-23T21:58:39.049 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T21:58:39.050 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T21:58:39.051 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T21:58:39.053 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T21:58:39.054 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T21:58:39.055 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ], 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T21:58:20.282890+0000", 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout }, 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T21:58:39.056 INFO:teuthology.orchestra.run.smithi026.stdout:mgr is available 2023-09-23T21:58:39.911 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:39 smithi026 ceph-mon[100157]: mgrmap e4: y(active, since 3s) 2023-09-23T21:58:40.471 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout fsid = 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T21:58:40.472 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 2023-09-23T21:58:40.473 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T21:58:40.473 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T21:58:40.473 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T21:58:40.473 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling cephadm module... 2023-09-23T21:58:40.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:40 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/392203618' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T21:58:41.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:41 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1083285597' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T21:58:41.886 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:41 smithi026 conmon[100586]: ignoring --setuser ceph since I am not root 2023-09-23T21:58:41.887 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:41 smithi026 conmon[100586]: ignoring --setgroup ceph since I am not root 2023-09-23T21:58:41.887 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:41 smithi026 conmon[100586]: 2023-09-23T21:58:41.789+0000 7f2f495be1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T21:58:41.887 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:41 smithi026 conmon[100586]: 2023-09-23T21:58:41.885+0000 7f2f495be1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T21:58:42.153 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:41 smithi026 conmon[100586]: 2023-09-23T21:58:41.991+0000 7f2f495be1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T21:58:42.153 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:42 smithi026 conmon[100586]: 2023-09-23T21:58:42.099+0000 7f2f495be1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T21:58:42.422 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:42 smithi026 conmon[100586]: 2023-09-23T21:58:42.310+0000 7f2f495be1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T21:58:42.733 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:42 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1083285597' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T21:58:42.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:42 smithi026 ceph-mon[100157]: mgrmap e5: y(active, since 6s) 2023-09-23T21:58:42.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:42 smithi026 conmon[100586]: 2023-09-23T21:58:42.731+0000 7f2f495be1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T21:58:42.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:42 smithi026 conmon[100586]: 2023-09-23T21:58:42.838+0000 7f2f495be1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T21:58:43.335 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:42 smithi026 conmon[100586]: 2023-09-23T21:58:42.987+0000 7f2f495be1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T21:58:43.336 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:43 smithi026 conmon[100586]: 2023-09-23T21:58:43.100+0000 7f2f495be1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T21:58:43.679 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T21:58:43.680 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-09-23T21:58:43.680 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T21:58:43.680 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T21:58:43.680 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T21:58:43.680 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T21:58:43.680 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for the mgr to restart... 2023-09-23T21:58:43.680 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr epoch 5... 2023-09-23T21:58:43.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:43 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2110234700' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T21:58:43.912 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:43 smithi026 conmon[100586]: 2023-09-23T21:58:43.613+0000 7f2f495be1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T21:58:44.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:44 smithi026 conmon[100586]: 2023-09-23T21:58:44.313+0000 7f2f495be1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T21:58:44.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:44 smithi026 conmon[100586]: 2023-09-23T21:58:44.503+0000 7f2f495be1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T21:58:45.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:45 smithi026 conmon[100586]: 2023-09-23T21:58:45.470+0000 7f2f495be1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T21:58:45.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:45 smithi026 conmon[100586]: 2023-09-23T21:58:45.578+0000 7f2f495be1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T21:58:46.131 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:45 smithi026 conmon[100586]: 2023-09-23T21:58:45.792+0000 7f2f495be1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T21:58:46.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:46 smithi026 conmon[100586]: 2023-09-23T21:58:46.131+0000 7f2f495be1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T21:58:46.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:46 smithi026 conmon[100586]: 2023-09-23T21:58:46.234+0000 7f2f495be1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T21:58:46.908 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:46 smithi026 conmon[100586]: 2023-09-23T21:58:46.588+0000 7f2f495be1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T21:58:47.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:46 smithi026 conmon[100586]: 2023-09-23T21:58:46.908+0000 7f2f495be1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T21:58:47.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:47 smithi026 conmon[100586]: 2023-09-23T21:58:47.727+0000 7f2f495be1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T21:58:47.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:47 smithi026 conmon[100586]: 2023-09-23T21:58:47.883+0000 7f2f495be1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T21:58:48.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:48 smithi026 conmon[100586]: 2023-09-23T21:58:48.119+0000 7f2f495be1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T21:58:48.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:48 smithi026 conmon[100586]: 2023-09-23T21:58:48.254+0000 7f2f495be1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T21:58:48.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:48 smithi026 conmon[100586]: 2023-09-23T21:58:48.520+0000 7f2f495be1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T21:58:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:49 smithi026 conmon[100586]: 2023-09-23T21:58:49.546+0000 7f2f495be1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T21:58:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: Active manager daemon y restarted 2023-09-23T21:58:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: Activating manager daemon y 2023-09-23T21:58:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: osdmap e2: 0 total, 0 up, 0 in 2023-09-23T21:58:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: mgrmap e6: y(active, starting, since 0.00317151s) 2023-09-23T21:58:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T21:58:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T21:58:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T21:58:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T21:58:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T21:58:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: Manager daemon y is now available 2023-09-23T21:58:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:58:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:58:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:58:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:49 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:58:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:50 smithi026 ceph-mon[100157]: Found migration_current of "None". Setting to last migration. 2023-09-23T21:58:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:50 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:58:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:50 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T21:58:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:50 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T21:58:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:50 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:58:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:50 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:58:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:50 smithi026 ceph-mon[100157]: mgrmap e7: y(active, since 1.00524s) 2023-09-23T21:58:51.152 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T21:58:51.153 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-09-23T21:58:51.153 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T21:58:51.153 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T21:58:51.153 INFO:teuthology.orchestra.run.smithi026.stdout:mgr epoch 5 is available 2023-09-23T21:58:51.153 INFO:teuthology.orchestra.run.smithi026.stdout:Setting orchestrator backend to cephadm... 2023-09-23T21:58:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:51 smithi026 ceph-mon[100157]: [23/Sep/2023:21:58:50] ENGINE Bus STARTING 2023-09-23T21:58:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:51 smithi026 ceph-mon[100157]: [23/Sep/2023:21:58:50] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T21:58:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:51 smithi026 ceph-mon[100157]: [23/Sep/2023:21:58:50] ENGINE Bus STARTED 2023-09-23T21:58:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:51 smithi026 ceph-mon[100157]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T21:58:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:51 smithi026 ceph-mon[100157]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T21:58:52.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:52 smithi026 ceph-mon[100157]: mgrmap e8: y(active, since 2s) 2023-09-23T21:58:52.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:52 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:58:52.803 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:52 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:58:53.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:53 smithi026 ceph-mon[100157]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:58:53.927 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout value unchanged 2023-09-23T21:58:53.927 INFO:teuthology.orchestra.run.smithi026.stdout:Generating ssh key... 2023-09-23T21:58:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:54 smithi026 ceph-mon[100157]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:58:55.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: Generating public/private rsa key pair. 2023-09-23T21:58:55.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: Your identification has been saved in /tmp/tmpvxy_8bq5/key. 2023-09-23T21:58:55.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: Your public key has been saved in /tmp/tmpvxy_8bq5/key.pub. 2023-09-23T21:58:55.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: The key fingerprint is: 2023-09-23T21:58:55.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: SHA256:QqSITXy7l7W71H1qXnrOsnDnkducc3yUJ8ZA1qaskTY ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:55.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: The key's randomart image is: 2023-09-23T21:58:55.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: +---[RSA 3072]----+ 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: | .. . . | 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: | +...o o o | 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: |. o.... = o | 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: | .. . E + | 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: | ..oSo + o .| 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: | . o..... +oo| 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: | . ..o o.*+.| 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: | .. ooO.*=| 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: | .. o===.*| 2023-09-23T21:58:55.486 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:58:55 smithi026 conmon[100586]: +----[SHA256]-----+ 2023-09-23T21:58:56.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:56 smithi026 ceph-mon[100157]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:58:56.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:56 smithi026 ceph-mon[100157]: Generating ssh key... 2023-09-23T21:58:56.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:56 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:58:56.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:56 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:58:57.268 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAS1p2judXZ/UT5VbXBx/f+FB4JHHd9IoQ3Mxx9rOvRW3At3n5XUHkV15bNoxOnajFpM0vsatyyKeZ6HCbP0wK0vJH0Caw76lcSShulwdurhoBDj/VqEHK0BsbDZw06Z+1z1qlTc3GAfG4fEO7YtYZDO0cPIkQQ1IQ2mCVff08BCoWwgUPe7WVHly7ZlpfH9reRD5HvVfCuETt8jWSAnkzwERAyTcGscXIqGrsdSm5642mVWN4A82mE+FJ5f8GWMZJMDmPGZ7jKrM8//U8Dq988+5ITcjEKjfA9zcyiXdkSmTrysWe+4UuBNQXjMwo+7KlJqVbzgxhJ3thMwEdx1B3uNxzwxNrszbLwX45Aa2eIpKn2B5zKS/AGcu8vHxpyxrDkUy/m/LQxKC5xIbXH7g8dVdVxB8sqqSVY9/it4sdepHsz9gR5aNbjh6spGCrYPU4pVsI+OJHyUpbJNd66T3d6I49HYsc/G8Eof7sr30PCDN5bvWkPATURB2nS3RFapU= ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:58:57.269 INFO:teuthology.orchestra.run.smithi026.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T21:58:57.269 INFO:teuthology.orchestra.run.smithi026.stdout:Adding key to root@localhost authorized_keys... 2023-09-23T21:58:57.270 INFO:teuthology.orchestra.run.smithi026.stdout:Adding host smithi026... 2023-09-23T21:58:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:57 smithi026 ceph-mon[100157]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:58:58.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:58 smithi026 ceph-mon[100157]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi026", "addr": "172.21.15.26", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:58:58.944 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:58:58 smithi026 ceph-mon[100157]: Deploying cephadm binary to smithi026 2023-09-23T21:59:00.835 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Added host 'smithi026' with addr '172.21.15.26' 2023-09-23T21:59:00.835 INFO:teuthology.orchestra.run.smithi026.stdout:Deploying unmanaged mon service... 2023-09-23T21:59:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:01 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:59:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:01 smithi026 ceph-mon[100157]: Added host smithi026 2023-09-23T21:59:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:01 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:59:02.586 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-09-23T21:59:02.586 INFO:teuthology.orchestra.run.smithi026.stdout:Deploying unmanaged mgr service... 2023-09-23T21:59:03.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:02 smithi026 ceph-mon[100157]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:59:03.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:02 smithi026 ceph-mon[100157]: Saving service mon spec with placement count:5 2023-09-23T21:59:03.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:02 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:59:03.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:03 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:59:03.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:03 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:59:03.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:03 smithi026 ceph-mon[100157]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:59:03.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:03 smithi026 ceph-mon[100157]: Saving service mgr spec with placement count:2 2023-09-23T21:59:03.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:03 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:59:04.217 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-09-23T21:59:06.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:05 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:59:06.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:05 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1920628097' entity='client.admin' 2023-09-23T21:59:06.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:05 smithi026 ceph-mon[100157]: from='mgr.14120 172.21.15.26:0/3709438833' entity='mgr.y' 2023-09-23T21:59:07.558 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling the dashboard module... 2023-09-23T21:59:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:07 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3922123972' entity='client.admin' 2023-09-23T21:59:08.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:08 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3020220509' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T21:59:08.960 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:08 smithi026 conmon[100586]: ignoring --setuser ceph since I am not root 2023-09-23T21:59:08.960 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:08 smithi026 conmon[100586]: ignoring --setgroup ceph since I am not root 2023-09-23T21:59:08.961 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:08 smithi026 conmon[100586]: 2023-09-23T21:59:08.864+0000 7f8dd9afd1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T21:59:08.961 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:08 smithi026 conmon[100586]: 2023-09-23T21:59:08.960+0000 7f8dd9afd1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T21:59:09.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:09 smithi026 conmon[100586]: 2023-09-23T21:59:09.066+0000 7f8dd9afd1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T21:59:09.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:09 smithi026 conmon[100586]: 2023-09-23T21:59:09.175+0000 7f8dd9afd1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T21:59:09.668 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:09 smithi026 conmon[100586]: 2023-09-23T21:59:09.385+0000 7f8dd9afd1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T21:59:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:09 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3020220509' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T21:59:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:09 smithi026 ceph-mon[100157]: mgrmap e9: y(active, since 19s) 2023-09-23T21:59:09.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:09 smithi026 conmon[100586]: 2023-09-23T21:59:09.792+0000 7f8dd9afd1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T21:59:09.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:09 smithi026 conmon[100586]: 2023-09-23T21:59:09.899+0000 7f8dd9afd1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T21:59:10.389 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:10 smithi026 conmon[100586]: 2023-09-23T21:59:10.048+0000 7f8dd9afd1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T21:59:10.390 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:10 smithi026 conmon[100586]: 2023-09-23T21:59:10.161+0000 7f8dd9afd1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T21:59:10.715 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T21:59:10.718 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-09-23T21:59:10.718 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T21:59:10.718 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T21:59:10.718 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T21:59:10.718 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T21:59:10.718 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for the mgr to restart... 2023-09-23T21:59:10.718 INFO:teuthology.orchestra.run.smithi026.stdout:Waiting for mgr epoch 9... 2023-09-23T21:59:10.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:10 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2164461247' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T21:59:10.939 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:10 smithi026 conmon[100586]: 2023-09-23T21:59:10.678+0000 7f8dd9afd1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T21:59:11.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:11 smithi026 conmon[100586]: 2023-09-23T21:59:11.389+0000 7f8dd9afd1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T21:59:11.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:11 smithi026 conmon[100586]: 2023-09-23T21:59:11.580+0000 7f8dd9afd1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T21:59:12.858 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:12 smithi026 conmon[100586]: 2023-09-23T21:59:12.534+0000 7f8dd9afd1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T21:59:12.858 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:12 smithi026 conmon[100586]: 2023-09-23T21:59:12.643+0000 7f8dd9afd1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T21:59:13.200 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:12 smithi026 conmon[100586]: 2023-09-23T21:59:12.858+0000 7f8dd9afd1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T21:59:13.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:13 smithi026 conmon[100586]: 2023-09-23T21:59:13.199+0000 7f8dd9afd1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T21:59:13.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:13 smithi026 conmon[100586]: 2023-09-23T21:59:13.303+0000 7f8dd9afd1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T21:59:13.963 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:13 smithi026 conmon[100586]: 2023-09-23T21:59:13.639+0000 7f8dd9afd1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T21:59:14.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:13 smithi026 conmon[100586]: 2023-09-23T21:59:13.963+0000 7f8dd9afd1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T21:59:15.180 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:14 smithi026 conmon[100586]: 2023-09-23T21:59:14.786+0000 7f8dd9afd1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T21:59:15.180 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:14 smithi026 conmon[100586]: 2023-09-23T21:59:14.942+0000 7f8dd9afd1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T21:59:15.484 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:15 smithi026 conmon[100586]: 2023-09-23T21:59:15.179+0000 7f8dd9afd1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T21:59:15.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:15 smithi026 conmon[100586]: 2023-09-23T21:59:15.315+0000 7f8dd9afd1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T21:59:15.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:15 smithi026 conmon[100586]: 2023-09-23T21:59:15.584+0000 7f8dd9afd1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T21:59:16.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 21:59:16 smithi026 conmon[100586]: 2023-09-23T21:59:16.615+0000 7f8dd9afd1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T21:59:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: Active manager daemon y restarted 2023-09-23T21:59:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: Activating manager daemon y 2023-09-23T21:59:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: osdmap e3: 0 total, 0 up, 0 in 2023-09-23T21:59:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: mgrmap e10: y(active, starting, since 0.00351297s) 2023-09-23T21:59:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T21:59:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T21:59:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T21:59:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T21:59:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T21:59:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: Manager daemon y is now available 2023-09-23T21:59:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:59:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:59:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:17 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T21:59:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:17 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T21:59:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:17 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:17 smithi026 ceph-mon[100157]: mgrmap e11: y(active, since 1.0062s) 2023-09-23T21:59:18.257 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout { 2023-09-23T21:59:18.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-09-23T21:59:18.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T21:59:18.258 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout } 2023-09-23T21:59:18.258 INFO:teuthology.orchestra.run.smithi026.stdout:mgr epoch 9 is available 2023-09-23T21:59:18.258 INFO:teuthology.orchestra.run.smithi026.stdout:Generating a dashboard self-signed certificate... 2023-09-23T21:59:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:18 smithi026 ceph-mon[100157]: [23/Sep/2023:21:59:17] ENGINE Bus STARTING 2023-09-23T21:59:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:18 smithi026 ceph-mon[100157]: [23/Sep/2023:21:59:17] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T21:59:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:18 smithi026 ceph-mon[100157]: [23/Sep/2023:21:59:17] ENGINE Bus STARTED 2023-09-23T21:59:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:18 smithi026 ceph-mon[100157]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T21:59:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:18 smithi026 ceph-mon[100157]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T21:59:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:19.789 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-09-23T21:59:19.790 INFO:teuthology.orchestra.run.smithi026.stdout:Creating initial admin user... 2023-09-23T21:59:20.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:20.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:20 smithi026 ceph-mon[100157]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:59:20.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:20.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:20.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:20 smithi026 ceph-mon[100157]: mgrmap e12: y(active, since 2s) 2023-09-23T21:59:21.464 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$dykvPg0Xc1YvpWhdBhL1fudSAieUJNoZs3.KAIDYoEaSNkIgV.2ly", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695506360, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T21:59:21.465 INFO:teuthology.orchestra.run.smithi026.stdout:Fetching dashboard port number... 2023-09-23T21:59:21.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:21 smithi026 ceph-mon[100157]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:59:21.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:21 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:22.806 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stdout 8443 2023-09-23T21:59:22.817 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T21:59:22.817 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout disabled 2023-09-23T21:59:22.825 INFO:teuthology.orchestra.run.smithi026.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T21:59:22.826 INFO:teuthology.orchestra.run.smithi026.stdout:systemctl: stdout inactive 2023-09-23T21:59:22.826 INFO:teuthology.orchestra.run.smithi026.stdout:firewalld.service is not enabled 2023-09-23T21:59:22.826 INFO:teuthology.orchestra.run.smithi026.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T21:59:22.828 INFO:teuthology.orchestra.run.smithi026.stdout:Ceph Dashboard is now available at: 2023-09-23T21:59:22.828 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:22.828 INFO:teuthology.orchestra.run.smithi026.stdout: URL: https://smithi026:8443/ 2023-09-23T21:59:22.828 INFO:teuthology.orchestra.run.smithi026.stdout: User: admin 2023-09-23T21:59:22.828 INFO:teuthology.orchestra.run.smithi026.stdout: Password: v12rlyu1kp 2023-09-23T21:59:22.828 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:22.829 INFO:teuthology.orchestra.run.smithi026.stdout:Saving cluster configuration to /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config directory 2023-09-23T21:59:22.829 INFO:teuthology.orchestra.run.smithi026.stdout:Enabling autotune for osd_memory_target 2023-09-23T21:59:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:22 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2589029380' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T21:59:24.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:24 smithi026 ceph-mon[100157]: mgrmap e13: y(active, since 6s) 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout:Or, if you are only running a single cluster on this host: 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout:Please consider enabling telemetry to help improve Ceph: 2023-09-23T21:59:25.828 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:25.829 INFO:teuthology.orchestra.run.smithi026.stdout: ceph telemetry on 2023-09-23T21:59:25.829 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:25.829 INFO:teuthology.orchestra.run.smithi026.stdout:For more information see: 2023-09-23T21:59:25.829 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:25.829 INFO:teuthology.orchestra.run.smithi026.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-09-23T21:59:25.829 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:25.829 INFO:teuthology.orchestra.run.smithi026.stdout:Bootstrap complete. 2023-09-23T21:59:25.871 INFO:tasks.cephadm:Fetching config... 2023-09-23T21:59:25.871 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:59:25.871 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T21:59:25.889 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T21:59:25.889 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:59:25.889 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T21:59:25.948 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T21:59:25.948 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:59:25.948 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T21:59:26.017 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T21:59:26.017 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T21:59:26.018 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T21:59:26.075 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T21:59:26.075 DEBUG:teuthology.orchestra.run.smithi026:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAS1p2judXZ/UT5VbXBx/f+FB4JHHd9IoQ3Mxx9rOvRW3At3n5XUHkV15bNoxOnajFpM0vsatyyKeZ6HCbP0wK0vJH0Caw76lcSShulwdurhoBDj/VqEHK0BsbDZw06Z+1z1qlTc3GAfG4fEO7YtYZDO0cPIkQQ1IQ2mCVff08BCoWwgUPe7WVHly7ZlpfH9reRD5HvVfCuETt8jWSAnkzwERAyTcGscXIqGrsdSm5642mVWN4A82mE+FJ5f8GWMZJMDmPGZ7jKrM8//U8Dq988+5ITcjEKjfA9zcyiXdkSmTrysWe+4UuBNQXjMwo+7KlJqVbzgxhJ3thMwEdx1B3uNxzwxNrszbLwX45Aa2eIpKn2B5zKS/AGcu8vHxpyxrDkUy/m/LQxKC5xIbXH7g8dVdVxB8sqqSVY9/it4sdepHsz9gR5aNbjh6spGCrYPU4pVsI+OJHyUpbJNd66T3d6I49HYsc/G8Eof7sr30PCDN5bvWkPATURB2nS3RFapU= ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T21:59:26.158 INFO:teuthology.orchestra.run.smithi026.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAS1p2judXZ/UT5VbXBx/f+FB4JHHd9IoQ3Mxx9rOvRW3At3n5XUHkV15bNoxOnajFpM0vsatyyKeZ6HCbP0wK0vJH0Caw76lcSShulwdurhoBDj/VqEHK0BsbDZw06Z+1z1qlTc3GAfG4fEO7YtYZDO0cPIkQQ1IQ2mCVff08BCoWwgUPe7WVHly7ZlpfH9reRD5HvVfCuETt8jWSAnkzwERAyTcGscXIqGrsdSm5642mVWN4A82mE+FJ5f8GWMZJMDmPGZ7jKrM8//U8Dq988+5ITcjEKjfA9zcyiXdkSmTrysWe+4UuBNQXjMwo+7KlJqVbzgxhJ3thMwEdx1B3uNxzwxNrszbLwX45Aa2eIpKn2B5zKS/AGcu8vHxpyxrDkUy/m/LQxKC5xIbXH7g8dVdVxB8sqqSVY9/it4sdepHsz9gR5aNbjh6spGCrYPU4pVsI+OJHyUpbJNd66T3d6I49HYsc/G8Eof7sr30PCDN5bvWkPATURB2nS3RFapU= ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:59:26.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAS1p2judXZ/UT5VbXBx/f+FB4JHHd9IoQ3Mxx9rOvRW3At3n5XUHkV15bNoxOnajFpM0vsatyyKeZ6HCbP0wK0vJH0Caw76lcSShulwdurhoBDj/VqEHK0BsbDZw06Z+1z1qlTc3GAfG4fEO7YtYZDO0cPIkQQ1IQ2mCVff08BCoWwgUPe7WVHly7ZlpfH9reRD5HvVfCuETt8jWSAnkzwERAyTcGscXIqGrsdSm5642mVWN4A82mE+FJ5f8GWMZJMDmPGZ7jKrM8//U8Dq988+5ITcjEKjfA9zcyiXdkSmTrysWe+4UuBNQXjMwo+7KlJqVbzgxhJ3thMwEdx1B3uNxzwxNrszbLwX45Aa2eIpKn2B5zKS/AGcu8vHxpyxrDkUy/m/LQxKC5xIbXH7g8dVdVxB8sqqSVY9/it4sdepHsz9gR5aNbjh6spGCrYPU4pVsI+OJHyUpbJNd66T3d6I49HYsc/G8Eof7sr30PCDN5bvWkPATURB2nS3RFapU= ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T21:59:26.215 INFO:teuthology.orchestra.run.smithi070.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDAS1p2judXZ/UT5VbXBx/f+FB4JHHd9IoQ3Mxx9rOvRW3At3n5XUHkV15bNoxOnajFpM0vsatyyKeZ6HCbP0wK0vJH0Caw76lcSShulwdurhoBDj/VqEHK0BsbDZw06Z+1z1qlTc3GAfG4fEO7YtYZDO0cPIkQQ1IQ2mCVff08BCoWwgUPe7WVHly7ZlpfH9reRD5HvVfCuETt8jWSAnkzwERAyTcGscXIqGrsdSm5642mVWN4A82mE+FJ5f8GWMZJMDmPGZ7jKrM8//U8Dq988+5ITcjEKjfA9zcyiXdkSmTrysWe+4UuBNQXjMwo+7KlJqVbzgxhJ3thMwEdx1B3uNxzwxNrszbLwX45Aa2eIpKn2B5zKS/AGcu8vHxpyxrDkUy/m/LQxKC5xIbXH7g8dVdVxB8sqqSVY9/it4sdepHsz9gR5aNbjh6spGCrYPU4pVsI+OJHyUpbJNd66T3d6I49HYsc/G8Eof7sr30PCDN5bvWkPATURB2nS3RFapU= ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:59:26.228 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T21:59:26.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:26 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/261806260' entity='client.admin' 2023-09-23T21:59:26.573 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:59:29.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:28 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:29.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:28 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:29.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:28 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi026", "name": "osd_memory_target"}]: dispatch 2023-09-23T21:59:29.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:28 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:29.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:28 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:59:29.211 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:28 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:29.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:28 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1897568561' entity='client.admin' 2023-09-23T21:59:29.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:28 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:59:29.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:28 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:29.787 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T21:59:29.787 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T21:59:30.143 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:59:32.412 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi070 2023-09-23T21:59:32.413 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T21:59:32.413 DEBUG:teuthology.orchestra.run.smithi070:> dd of=/etc/ceph/ceph.conf 2023-09-23T21:59:32.434 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T21:59:32.434 DEBUG:teuthology.orchestra.run.smithi070:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T21:59:32.494 INFO:tasks.cephadm:Adding host smithi070 to orchestrator... 2023-09-23T21:59:32.494 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi070 2023-09-23T21:59:32.857 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:59:33.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:32 smithi026 ceph-mon[100157]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:59:33.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:32 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:33.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:32 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:59:33.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:32 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:33.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:32 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:59:33.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:32 smithi026 ceph-mon[100157]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T21:59:33.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:32 smithi026 ceph-mon[100157]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T21:59:33.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:32 smithi026 ceph-mon[100157]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T21:59:33.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:32 smithi026 ceph-mon[100157]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T21:59:34.094 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:33 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:34.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:33 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:34.095 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:33 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:35.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:34 smithi026 ceph-mon[100157]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi070", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:59:36.403 INFO:teuthology.orchestra.run.smithi026.stdout:Added host 'smithi070' with addr '172.21.15.70' 2023-09-23T21:59:36.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:36 smithi026 ceph-mon[100157]: Deploying cephadm binary to smithi070 2023-09-23T21:59:36.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:36 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:36.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:36 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:59:36.984 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T21:59:37.336 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:59:37.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:37 smithi026 ceph-mon[100157]: Added host smithi070 2023-09-23T21:59:37.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:37 smithi026 ceph-mon[100157]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:37.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:37 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:37.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:37 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:38.932 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T21:59:38.933 INFO:teuthology.orchestra.run.smithi026.stdout:[{"addr": "172.21.15.26", "hostname": "smithi026", "labels": [], "status": ""}, {"addr": "172.21.15.70", "hostname": "smithi070", "labels": [], "status": ""}] 2023-09-23T21:59:39.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:38 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:39.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:38 smithi026 ceph-mon[100157]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:39.544 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T21:59:39.544 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T21:59:39.890 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T21:59:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:39 smithi026 ceph-mon[100157]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T21:59:41.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:41 smithi026 ceph-mon[100157]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2023-09-23T21:59:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:59:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:41 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2538618134' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T21:59:42.262 INFO:teuthology.orchestra.run.smithi026.stderr:adjusted tunables profile to default 2023-09-23T21:59:42.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:42 smithi026 ceph-mon[100157]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T21:59:42.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:42 smithi026 ceph-mon[100157]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T21:59:42.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:42 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:42.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:42 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:42.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:42 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:42.858 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:42 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2538618134' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T21:59:42.859 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:42 smithi026 ceph-mon[100157]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T21:59:42.859 INFO:tasks.cephadm:Adding mon.a on smithi026 2023-09-23T21:59:42.860 INFO:tasks.cephadm:Adding mon.c on smithi026 2023-09-23T21:59:42.860 INFO:tasks.cephadm:Adding mon.b on smithi070 2023-09-23T21:59:42.860 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b' 2023-09-23T21:59:43.223 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T21:59:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:43 smithi026 ceph-mon[100157]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T21:59:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:43 smithi026 ceph-mon[100157]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T21:59:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:43 smithi026 ceph-mon[100157]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:44.987 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled mon update... 2023-09-23T21:59:45.701 DEBUG:teuthology.orchestra.run.smithi026:mon.c> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T21:59:45.704 DEBUG:teuthology.orchestra.run.smithi070:mon.b> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T21:59:45.707 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T21:59:45.707 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T21:59:45.737 INFO:journalctl@ceph.mon.b.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T21:59:45.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T21:59:45.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:45 smithi026 ceph-mon[100157]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:45.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:45.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:59:45.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:45.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:59:45.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:45.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T21:59:45.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:46.782 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T21:59:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:46 smithi026 ceph-mon[100157]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:59:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:46 smithi026 ceph-mon[100157]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b;count:3 2023-09-23T21:59:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:46 smithi026 ceph-mon[100157]: Deploying daemon mon.b on smithi070 2023-09-23T21:59:47.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:47 smithi026 ceph-mon[100157]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:49.213 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:59:49.214 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":1,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","modified":"2023-09-23T21:58:17.635311Z","created":"2023-09-23T21:58:17.635311Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T21:59:49.216 INFO:teuthology.orchestra.run.smithi070.stderr:dumped monmap epoch 1 2023-09-23T21:59:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:49 smithi070 bash[105083]: b61f3bc30723a214f18fa899a50721599d5e4b650e73b261a08e0ed7688eec2f 2023-09-23T21:59:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:49 smithi070 systemd[1]: Started Ceph mon.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T21:59:51.101 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T21:59:51.102 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T21:59:51.456 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T21:59:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T21:59:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: mon.a calling monitor election 2023-09-23T21:59:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: mon.b calling monitor election 2023-09-23T21:59:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T21:59:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0]} removed_ranks: {} 2023-09-23T21:59:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: fsmap 2023-09-23T21:59:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T21:59:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: mgrmap e13: y(active, since 38s) 2023-09-23T21:59:54.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: overall HEALTH_OK 2023-09-23T21:59:54.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:54.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T21:59:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: mon.a calling monitor election 2023-09-23T21:59:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: mon.b calling monitor election 2023-09-23T21:59:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0]} removed_ranks: {} 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: fsmap 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: mgrmap e13: y(active, since 38s) 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: overall HEALTH_OK 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:55 smithi026 ceph-mon[100157]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:55 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:55 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T21:59:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:55 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:55 smithi026 ceph-mon[100157]: Deploying daemon mon.c on smithi026 2023-09-23T21:59:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:55 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:55 smithi070 ceph-mon[105170]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:55 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:55 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T21:59:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:55 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:56.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:55 smithi070 ceph-mon[105170]: Deploying daemon mon.c on smithi026 2023-09-23T21:59:56.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:55 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:56.212 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T21:59:56.212 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":2,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","modified":"2023-09-23T21:59:49.598614Z","created":"2023-09-23T21:58:17.635311Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T21:59:56.214 INFO:teuthology.orchestra.run.smithi070.stderr:dumped monmap epoch 2 2023-09-23T21:59:56.958 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:56 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/819822055' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T21:59:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:56 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/819822055' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T21:59:57.208 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 systemd[1]: Starting Ceph mon.c for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T21:59:57.809 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T21:59:57.809 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T21:59:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[100157]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T21:59:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable), process ceph-mon, pid 7 2023-09-23T21:59:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: pidfile_write: ignore empty --pid-file 2023-09-23T21:59:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: load: jerasure load: lrc 2023-09-23T21:59:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 bash[118814]: fce3aa9fda1a2e863b0168b7d431c2cd270a6396cd435059ac9d5aed00391b2d 2023-09-23T21:59:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: RocksDB version: 6.15.5 2023-09-23T21:59:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T21:59:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Compile date Sep 23 2023 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: DB SUMMARY 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: DB Session ID: KAAFKUCWOYR9QHCPC7EX 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: CURRENT file: CURRENT 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.error_if_exists: 0 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.create_if_missing: 0 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.paranoid_checks: 1 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T21:59:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.env: 0x55f708fc7a80 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.fs: Posix File System 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.info_log: 0x55f70a17c5a0 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.statistics: (nil) 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.use_fsync: 0 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_log_file_size: 0 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.allow_fallocate: 1 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T21:59:57.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.use_direct_reads: 0 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.db_log_dir: 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T21:59:57.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.write_buffer_manager: 0x55f70aeee930 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T21:59:57.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.unordered_write: 0 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.row_cache: None 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.wal_filter: None 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.preserve_deletes: 0 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.two_write_queues: 0 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.atomic_flush: 0 2023-09-23T21:59:57.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.log_readahead_size: 0 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_background_jobs: 2 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_background_compactions: -1 2023-09-23T21:59:57.993 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_subcompactions: 1 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_open_files: -1 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T21:59:57.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_background_flushes: -1 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Compression algorithms supported: 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: kZSTD supported: 0 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: kXpressCompression supported: 0 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: kZlibCompression supported: 1 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: kSnappyCompression supported: 1 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: kLZ4Compression supported: 1 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: kBZip2Compression supported: 0 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T21:59:57.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.merge_operator: 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_filter: None 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f70a1cefb8) 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks: 1 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T21:59:57.996 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_top_level_index_and_filter: 1 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_type: 0 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_index_type: 0 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_shortening: 1 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: hash_index_allow_collision: 1 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: checksum: 1 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: no_block_cache: 0 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache: 0x55f70a201090 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_name: BinnedLRUCache 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_options: 2023-09-23T21:59:57.997 INFO:journalctl@ceph.mon.c.smithi026.stdout: capacity : 536870912 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_shard_bits : 4 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: strict_capacity_limit : 0 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: high_pri_pool_ratio: 0.000 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_compressed: (nil) 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: persistent_cache: (nil) 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size: 4096 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size_deviation: 10 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_restart_interval: 16 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_block_restart_interval: 1 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_block_size: 4096 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: partition_filters: 0 2023-09-23T21:59:57.998 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_delta_encoding: 1 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout: whole_key_filtering: 1 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout: verify_compression: 0 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout: read_amp_bytes_per_bit: 0 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout: format_version: 4 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_index_compression: 1 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_align: 0 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compression: NoCompression 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T21:59:57.999 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.num_levels: 7 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T21:59:58.000 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T21:59:58.001 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T21:59:58.002 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T21:59:58.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.table_properties_collectors: 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.inplace_update_support: 0 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.bloom_locality: 0 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.max_successive_merges: 0 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T21:59:58.004 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.ttl: 2592000 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.enable_blob_files: false 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.min_blob_size: 0 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T21:59:58.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506397717803, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506397718200, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695506397, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "785aec0b-a8a3-4980-87ae-fdfb3b57e814", "db_session_id": "KAAFKUCWOYR9QHCPC7EX"}} 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506397718655, "job": 1, "event": "recovery_finished"} 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 systemd[1]: Started Ceph mon.c for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55f70a195180 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: DB pointer 0x55f70a26a000 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T21:59:58.006 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** DB Stats ** 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T21:59:58.007 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T21:59:58.008 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 0.41 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-09-23T21:59:58.009 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.3 0.00 0.00 1 0.000 0 0 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T21:59:58.010 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 0.40 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: using public_addrv [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: starting mon.c rank -1 at public addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] at bind addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(???) e0 preinit fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).mds e1 new map 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).mds e1 print_map 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout: e1 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T21:59:58.011 INFO:journalctl@ceph.mon.c.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout: legacy client fscid: -1 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout: No filesystems configured 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T21:59:58.012 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi070", "name": "osd_memory_target"}]: dispatch 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2538618134' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.013 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2538618134' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:58.014 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: Saving service mon spec with placement smithi026:172.21.15.26=a;smithi026:[v2:172.21.15.26:3301,v1:172.21.15.26:6790]=c;smithi070:172.21.15.70=b;count:3 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: Deploying daemon mon.b on smithi070 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.a calling monitor election 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:58.015 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.b calling monitor election 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: monmap e2: 2 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0]} removed_ranks: {} 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: fsmap 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mgrmap e13: y(active, since 38s) 2023-09-23T21:59:58.016 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: overall HEALTH_OK 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: Deploying daemon mon.c on smithi026 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/819822055' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T21:59:58.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 21:59:57 smithi026 ceph-mon[118879]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-09-23T21:59:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 21:59:57 smithi070 ceph-mon[105170]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T21:59:58.149 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:00:03.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:00:03.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: mon.b calling monitor election 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: mon.a calling monitor election 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: fsmap 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: mgrmap e13: y(active, since 46s) 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: mon.c (rank 2) addr [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] is down (out of quorum) 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:03.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:03.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:00:03.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: mon.b calling monitor election 2023-09-23T22:00:03.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: mon.a calling monitor election 2023-09-23T22:00:03.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:00:03.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: fsmap 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:00:03.193 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: mgrmap e13: y(active, since 46s) 2023-09-23T22:00:03.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:00:03.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:00:03.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:00:03.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: mon.c (rank 2) addr [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] is down (out of quorum) 2023-09-23T22:00:03.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:03.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:03.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:03.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:03.434 INFO:teuthology.orchestra.run.smithi070.stdout: 2023-09-23T22:00:03.434 INFO:teuthology.orchestra.run.smithi070.stdout:{"epoch":3,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","modified":"2023-09-23T21:59:57.742124Z","created":"2023-09-23T21:58:17.635311Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3300","nonce":0},{"type":"v1","addr":"172.21.15.26:6789","nonce":0}]},"addr":"172.21.15.26:6789/0","public_addr":"172.21.15.26:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:3300","nonce":0},{"type":"v1","addr":"172.21.15.70:6789","nonce":0}]},"addr":"172.21.15.70:6789/0","public_addr":"172.21.15.70:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:3301","nonce":0},{"type":"v1","addr":"172.21.15.26:6790","nonce":0}]},"addr":"172.21.15.26:6790/0","public_addr":"172.21.15.26:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T22:00:03.438 INFO:teuthology.orchestra.run.smithi070.stderr:dumped monmap epoch 3 2023-09-23T22:00:04.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:03 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:04.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:03 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/957322604' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:00:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:03 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:04.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:03 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:04.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:03 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/957322604' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:00:04.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:03 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:04.298 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T22:00:04.298 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T22:00:04.697 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:00:05.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: mon.b calling monitor election 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: mon.a calling monitor election 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:05.094 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: fsmap 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: mgrmap e13: y(active, since 46s) 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: mon.c (rank 2) addr [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] is down (out of quorum) 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.095 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:05.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/957322604' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:00:05.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: mon.c calling monitor election 2023-09-23T22:00:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: mon.c calling monitor election 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: mon.b calling monitor election 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: mon.a calling monitor election 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: fsmap 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: mgrmap e13: y(active, since 48s) 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: Cluster is now healthy 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: overall HEALTH_OK 2023-09-23T22:00:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:00:05.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:00:05.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:05.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: mon.c calling monitor election 2023-09-23T22:00:05.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: mon.c calling monitor election 2023-09-23T22:00:05.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: mon.b calling monitor election 2023-09-23T22:00:05.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: mon.a calling monitor election 2023-09-23T22:00:05.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:00:05.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:00:05.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: fsmap 2023-09-23T22:00:05.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: mgrmap e13: y(active, since 48s) 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: Cluster is now healthy 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: overall HEALTH_OK 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:00:05.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: mon.c calling monitor election 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: mon.c calling monitor election 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: mon.b calling monitor election 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: mon.a calling monitor election 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: fsmap 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T22:00:05.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: mgrmap e13: y(active, since 48s) 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: Cluster is now healthy 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: overall HEALTH_OK 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:00:05.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:06.594 INFO:teuthology.orchestra.run.smithi026.stdout:# minimal ceph.conf for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T22:00:06.594 INFO:teuthology.orchestra.run.smithi026.stdout:[global] 2023-09-23T22:00:06.594 INFO:teuthology.orchestra.run.smithi026.stdout: fsid = 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T22:00:06.594 INFO:teuthology.orchestra.run.smithi026.stdout: mon_host = [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] 2023-09-23T22:00:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:06 smithi070 ceph-mon[105170]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T22:00:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:06 smithi070 ceph-mon[105170]: Reconfiguring daemon mon.a on smithi026 2023-09-23T22:00:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:06 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3620346563' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:06 smithi026 ceph-mon[118879]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T22:00:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:06 smithi026 ceph-mon[118879]: Reconfiguring daemon mon.a on smithi026 2023-09-23T22:00:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:06 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3620346563' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:06 smithi026 ceph-mon[100157]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T22:00:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:06 smithi026 ceph-mon[100157]: Reconfiguring daemon mon.a on smithi026 2023-09-23T22:00:06.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:00:06.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:06 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3620346563' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:07.553 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T22:00:07.553 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T22:00:07.553 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T22:00:07.586 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T22:00:07.586 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:00:07.658 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T22:00:07.658 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T22:00:07.692 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T22:00:07.692 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:00:07.763 INFO:tasks.cephadm:Adding mgr.y on smithi026 2023-09-23T22:00:07.764 INFO:tasks.cephadm:Adding mgr.x on smithi070 2023-09-23T22:00:07.764 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi026=y;smithi070=x' 2023-09-23T22:00:07.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[118879]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:07.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:07.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:07.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:07.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:00:07.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:00:07.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:07.914 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[100157]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:07.914 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:07.914 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:07.914 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:07.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:00:07.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:00:07.915 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:07 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:07 smithi070 ceph-mon[105170]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:07 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:07 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:07 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:07 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:00:08.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:07 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:00:08.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:07 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:08.133 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:00:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:08 smithi026 ceph-mon[100157]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:00:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:08 smithi026 ceph-mon[100157]: Reconfiguring daemon mon.c on smithi026 2023-09-23T22:00:08.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:08 smithi026 ceph-mon[118879]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:00:08.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:08 smithi026 ceph-mon[118879]: Reconfiguring daemon mon.c on smithi026 2023-09-23T22:00:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:08 smithi070 ceph-mon[105170]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:00:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:08 smithi070 ceph-mon[105170]: Reconfiguring daemon mon.c on smithi026 2023-09-23T22:00:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:09 smithi070 ceph-mon[105170]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:09 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:09 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:09 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:00:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:09 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:00:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:09 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:09.979 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled mgr update... 2023-09-23T22:00:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[100157]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:00:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:00:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[118879]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:00:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:00:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:09 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:10.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:00:10.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: Reconfiguring daemon mon.b on smithi070 2023-09-23T22:00:10.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:10.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:10.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:10.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:00:10.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T22:00:10.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:00:10.857 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:10.879 DEBUG:teuthology.orchestra.run.smithi070:mgr.x> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T22:00:10.881 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T22:00:10.881 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T22:00:10.881 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T22:00:10.903 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T22:00:10.903 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_1 2023-09-23T22:00:10.909 INFO:journalctl@ceph.mgr.x.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:00:10.965 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T22:00:10.965 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:00:10.965 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 6h/6d Inode: 376719 Links: 1 2023-09-23T22:00:10.965 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:00:10.965 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:00:10.965 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2023-09-23 21:59:57.155555198 +0000 2023-09-23T22:00:10.965 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2023-09-23 21:51:28.599347724 +0000 2023-09-23T22:00:10.966 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2023-09-23 21:51:28.599347724 +0000 2023-09-23T22:00:10.966 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: - 2023-09-23T22:00:10.966 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: Reconfiguring daemon mon.b on smithi070 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:00:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T22:00:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:00:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:00:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: Reconfiguring daemon mon.b on smithi070 2023-09-23T22:00:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:00:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T22:00:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:00:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:11.017 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2023-09-23T22:00:11.017 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2023-09-23T22:00:11.018 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000127529 s, 4.0 MB/s 2023-09-23T22:00:11.019 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T22:00:11.079 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_2 2023-09-23T22:00:11.137 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T22:00:11.138 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:00:11.138 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 6h/6d Inode: 387895 Links: 1 2023-09-23T22:00:11.138 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:00:11.138 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:00:11.138 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2023-09-23 21:59:57.155555198 +0000 2023-09-23T22:00:11.138 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2023-09-23 21:51:29.102339905 +0000 2023-09-23T22:00:11.138 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2023-09-23 21:51:29.102339905 +0000 2023-09-23T22:00:11.138 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: - 2023-09-23T22:00:11.138 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T22:00:11.206 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2023-09-23T22:00:11.206 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2023-09-23T22:00:11.206 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000120429 s, 4.3 MB/s 2023-09-23T22:00:11.207 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T22:00:11.267 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_3 2023-09-23T22:00:11.327 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T22:00:11.327 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:00:11.327 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 6h/6d Inode: 389556 Links: 1 2023-09-23T22:00:11.327 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:00:11.327 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:00:11.327 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2023-09-23 21:59:57.156555183 +0000 2023-09-23T22:00:11.327 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2023-09-23 21:51:29.680330921 +0000 2023-09-23T22:00:11.327 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2023-09-23 21:51:29.680330921 +0000 2023-09-23T22:00:11.327 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: - 2023-09-23T22:00:11.328 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T22:00:11.394 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2023-09-23T22:00:11.394 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2023-09-23T22:00:11.395 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000111024 s, 4.6 MB/s 2023-09-23T22:00:11.396 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T22:00:11.457 DEBUG:teuthology.orchestra.run.smithi026:> stat /dev/vg_nvme/lv_4 2023-09-23T22:00:11.515 INFO:teuthology.orchestra.run.smithi026.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T22:00:11.515 INFO:teuthology.orchestra.run.smithi026.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:00:11.515 INFO:teuthology.orchestra.run.smithi026.stdout:Device: 6h/6d Inode: 394480 Links: 1 2023-09-23T22:00:11.515 INFO:teuthology.orchestra.run.smithi026.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:00:11.516 INFO:teuthology.orchestra.run.smithi026.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:00:11.516 INFO:teuthology.orchestra.run.smithi026.stdout:Access: 2023-09-23 21:59:57.156555183 +0000 2023-09-23T22:00:11.516 INFO:teuthology.orchestra.run.smithi026.stdout:Modify: 2023-09-23 21:51:30.190322993 +0000 2023-09-23T22:00:11.516 INFO:teuthology.orchestra.run.smithi026.stdout:Change: 2023-09-23 21:51:30.190322993 +0000 2023-09-23T22:00:11.516 INFO:teuthology.orchestra.run.smithi026.stdout: Birth: - 2023-09-23T22:00:11.516 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T22:00:11.583 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records in 2023-09-23T22:00:11.583 INFO:teuthology.orchestra.run.smithi026.stderr:1+0 records out 2023-09-23T22:00:11.583 INFO:teuthology.orchestra.run.smithi026.stderr:512 bytes copied, 0.000119826 s, 4.3 MB/s 2023-09-23T22:00:11.584 DEBUG:teuthology.orchestra.run.smithi026:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T22:00:11.644 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T22:00:11.644 DEBUG:teuthology.orchestra.run.smithi070:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T22:00:11.662 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T22:00:11.662 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_1 2023-09-23T22:00:11.719 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T22:00:11.719 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:00:11.719 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 6h/6d Inode: 384301 Links: 1 2023-09-23T22:00:11.719 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:00:11.719 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:00:11.719 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2023-09-23 21:59:48.664243451 +0000 2023-09-23T22:00:11.719 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2023-09-23 21:51:10.691798295 +0000 2023-09-23T22:00:11.719 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2023-09-23 21:51:10.691798295 +0000 2023-09-23T22:00:11.719 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: - 2023-09-23T22:00:11.720 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T22:00:11.786 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2023-09-23T22:00:11.786 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2023-09-23T22:00:11.786 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000117572 s, 4.4 MB/s 2023-09-23T22:00:11.787 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T22:00:11.845 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_2 2023-09-23T22:00:11.902 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T22:00:11.902 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:00:11.902 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 6h/6d Inode: 384427 Links: 1 2023-09-23T22:00:11.902 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:00:11.902 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:00:11.902 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2023-09-23 21:59:48.664243451 +0000 2023-09-23T22:00:11.902 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2023-09-23 21:51:11.185783954 +0000 2023-09-23T22:00:11.902 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2023-09-23 21:51:11.185783954 +0000 2023-09-23T22:00:11.903 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: - 2023-09-23T22:00:11.903 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T22:00:11.964 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:11 smithi070 ceph-mon[105170]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi070=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:00:11.964 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:11 smithi070 ceph-mon[105170]: Saving service mgr spec with placement smithi026=y;smithi070=x;count:2 2023-09-23T22:00:11.965 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:11 smithi070 ceph-mon[105170]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:11.965 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:11 smithi070 ceph-mon[105170]: Deploying daemon mgr.x on smithi070 2023-09-23T22:00:11.966 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2023-09-23T22:00:11.966 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2023-09-23T22:00:11.967 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000103526 s, 4.9 MB/s 2023-09-23T22:00:11.967 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T22:00:11.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:11 smithi026 ceph-mon[118879]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi070=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:00:11.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:11 smithi026 ceph-mon[118879]: Saving service mgr spec with placement smithi026=y;smithi070=x;count:2 2023-09-23T22:00:11.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:11 smithi026 ceph-mon[118879]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:11.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:11 smithi026 ceph-mon[118879]: Deploying daemon mgr.x on smithi070 2023-09-23T22:00:11.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:11 smithi026 ceph-mon[100157]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi026=y;smithi070=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:00:11.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:11 smithi026 ceph-mon[100157]: Saving service mgr spec with placement smithi026=y;smithi070=x;count:2 2023-09-23T22:00:11.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:11 smithi026 ceph-mon[100157]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:11.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:11 smithi026 ceph-mon[100157]: Deploying daemon mgr.x on smithi070 2023-09-23T22:00:12.024 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_3 2023-09-23T22:00:12.081 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T22:00:12.081 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:00:12.081 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 6h/6d Inode: 390541 Links: 1 2023-09-23T22:00:12.082 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:00:12.082 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:00:12.082 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2023-09-23 21:59:48.664243451 +0000 2023-09-23T22:00:12.082 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2023-09-23 21:51:11.663770078 +0000 2023-09-23T22:00:12.082 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2023-09-23 21:51:11.663770078 +0000 2023-09-23T22:00:12.082 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: - 2023-09-23T22:00:12.082 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T22:00:12.176 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2023-09-23T22:00:12.177 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2023-09-23T22:00:12.177 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.000145814 s, 3.5 MB/s 2023-09-23T22:00:12.178 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T22:00:12.196 DEBUG:teuthology.orchestra.run.smithi070:> stat /dev/vg_nvme/lv_4 2023-09-23T22:00:12.253 INFO:teuthology.orchestra.run.smithi070.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T22:00:12.253 INFO:teuthology.orchestra.run.smithi070.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T22:00:12.253 INFO:teuthology.orchestra.run.smithi070.stdout:Device: 6h/6d Inode: 387902 Links: 1 2023-09-23T22:00:12.253 INFO:teuthology.orchestra.run.smithi070.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T22:00:12.253 INFO:teuthology.orchestra.run.smithi070.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T22:00:12.253 INFO:teuthology.orchestra.run.smithi070.stdout:Access: 2023-09-23 22:00:12.128566788 +0000 2023-09-23T22:00:12.253 INFO:teuthology.orchestra.run.smithi070.stdout:Modify: 2023-09-23 21:51:12.157755738 +0000 2023-09-23T22:00:12.254 INFO:teuthology.orchestra.run.smithi070.stdout:Change: 2023-09-23 21:51:12.157755738 +0000 2023-09-23T22:00:12.254 INFO:teuthology.orchestra.run.smithi070.stdout: Birth: - 2023-09-23T22:00:12.254 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T22:00:12.321 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records in 2023-09-23T22:00:12.321 INFO:teuthology.orchestra.run.smithi070.stderr:1+0 records out 2023-09-23T22:00:12.321 INFO:teuthology.orchestra.run.smithi070.stderr:512 bytes copied, 0.00010311 s, 5.0 MB/s 2023-09-23T22:00:12.322 DEBUG:teuthology.orchestra.run.smithi070:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T22:00:12.381 INFO:tasks.cephadm:Deploying osd.0 on smithi026 with /dev/vg_nvme/lv_4... 2023-09-23T22:00:12.382 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T22:00:12.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:12 smithi070 systemd[1]: Starting Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:00:12.742 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:00:13.051 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:12 smithi070 ceph-mgr[108391]: -- 172.21.15.70:0/2014964468 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557b442dc9c0 con 0x557b442b4800 2023-09-23T22:00:13.051 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:12 smithi070 bash[108326]: bc3ea1afcb46b6b2a5f2767a25b6486ace36227d59db4ca75fe111e1fa87ae0b 2023-09-23T22:00:13.051 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:12 smithi070 systemd[1]: Started Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:00:13.320 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:13 smithi070 conmon[108367]: 2023-09-23T22:00:13.050+0000 7fa645d2e1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:00:13.320 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:13 smithi070 conmon[108367]: 2023-09-23T22:00:13.130+0000 7fa645d2e1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:00:13.320 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:13 smithi070 conmon[108367]: 2023-09-23T22:00:13.225+0000 7fa645d2e1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:00:13.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:13 smithi070 conmon[108367]: 2023-09-23T22:00:13.319+0000 7fa645d2e1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:00:13.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:13 smithi070 conmon[108367]: 2023-09-23T22:00:13.506+0000 7fa645d2e1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:00:13.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[100157]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:13.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:13.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:13.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:13.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:13.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:13.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[118879]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:13.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:13.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:13.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:13.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:13.963 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:14.088 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:13 smithi070 conmon[108367]: 2023-09-23T22:00:13.847+0000 7fa645d2e1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:00:14.088 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:13 smithi070 conmon[108367]: 2023-09-23T22:00:13.936+0000 7fa645d2e1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:00:14.089 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:13 smithi070 ceph-mon[105170]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:14.089 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:14.089 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:14.089 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:14.089 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:14.089 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:14.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:14 smithi070 conmon[108367]: 2023-09-23T22:00:14.088+0000 7fa645d2e1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:00:14.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:14 smithi070 conmon[108367]: 2023-09-23T22:00:14.181+0000 7fa645d2e1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:00:14.454 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T22:00:14.454 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T22:00:14.454 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2023-09-23T22:00:14.454 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2023-09-23T22:00:14.454 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00922745 s, 1.1 GB/s 2023-09-23T22:00:14.454 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2023-09-23T22:00:14.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:14 smithi070 conmon[108367]: 2023-09-23T22:00:14.596+0000 7fa645d2e1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:00:15.249 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi026:vg_nvme/lv_4 2023-09-23T22:00:15.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:15 smithi070 conmon[108367]: 2023-09-23T22:00:15.161+0000 7fa645d2e1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:00:15.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:15 smithi070 conmon[108367]: 2023-09-23T22:00:15.326+0000 7fa645d2e1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:00:15.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T22:00:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:15 smithi070 ceph-mon[105170]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:15.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:15.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:15.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:15.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:15.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:15.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T22:00:15.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:00:15.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:00:15.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[100157]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:00:15.394 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:00:15.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:15.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T22:00:15.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:15 smithi026 ceph-mon[118879]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:16.034 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:00:16.333 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:16 smithi070 conmon[108367]: 2023-09-23T22:00:16.072+0000 7fa645d2e1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:00:16.333 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:16 smithi070 conmon[108367]: 2023-09-23T22:00:16.167+0000 7fa645d2e1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:00:16.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:16 smithi070 conmon[108367]: 2023-09-23T22:00:16.332+0000 7fa645d2e1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:00:16.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:16 smithi070 conmon[108367]: 2023-09-23T22:00:16.599+0000 7fa645d2e1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:00:16.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:16 smithi070 conmon[108367]: 2023-09-23T22:00:16.684+0000 7fa645d2e1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:00:17.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:17.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:17.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:17.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[100157]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:17.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:17.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:17.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:17.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.204 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[118879]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:17.205 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:16 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.220 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:16 smithi070 conmon[108367]: 2023-09-23T22:00:16.967+0000 7fa645d2e1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:00:17.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:16 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:16 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:16 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:17.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:16 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:17.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:16 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:17.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:16 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:16 smithi070 ceph-mon[105170]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:17.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:16 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:17.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:17 smithi070 conmon[108367]: 2023-09-23T22:00:17.219+0000 7fa645d2e1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:00:18.220 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:17 smithi070 conmon[108367]: 2023-09-23T22:00:17.907+0000 7fa645d2e1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:00:18.220 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:18 smithi070 conmon[108367]: 2023-09-23T22:00:18.036+0000 7fa645d2e1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:00:18.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:18 smithi070 ceph-mon[105170]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:00:18.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:00:18.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:00:18.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:18.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:18 smithi026 ceph-mon[100157]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:00:18.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:00:18.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:00:18.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:18.368 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:18 smithi026 ceph-mon[118879]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:00:18.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:00:18.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:00:18.369 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:18.540 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:18 smithi070 conmon[108367]: 2023-09-23T22:00:18.219+0000 7fa645d2e1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:00:18.540 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:18 smithi070 conmon[108367]: 2023-09-23T22:00:18.330+0000 7fa645d2e1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:00:18.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:18 smithi070 conmon[108367]: 2023-09-23T22:00:18.539+0000 7fa645d2e1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:00:19.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:19 smithi026 ceph-mon[100157]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:19.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:19 smithi026 ceph-mon[118879]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:19.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:19 smithi070 ceph-mon[105170]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:19.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:00:19 smithi070 conmon[108367]: 2023-09-23T22:00:19.391+0000 7fa645d2e1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:00:20.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: Standby manager daemon x started 2023-09-23T22:00:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:00:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:00:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:00:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:00:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3090318312' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0a195-9329-458c-b103-38db9a4408a8"}]: dispatch 2023-09-23T22:00:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0a195-9329-458c-b103-38db9a4408a8"}]: dispatch 2023-09-23T22:00:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fdf0a195-9329-458c-b103-38db9a4408a8"}]': finished 2023-09-23T22:00:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T22:00:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:20 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: Standby manager daemon x started 2023-09-23T22:00:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:00:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:00:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:00:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:00:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3090318312' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0a195-9329-458c-b103-38db9a4408a8"}]: dispatch 2023-09-23T22:00:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0a195-9329-458c-b103-38db9a4408a8"}]: dispatch 2023-09-23T22:00:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fdf0a195-9329-458c-b103-38db9a4408a8"}]': finished 2023-09-23T22:00:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T22:00:20.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: Standby manager daemon x started 2023-09-23T22:00:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:00:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:00:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:00:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/4087601615' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:00:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3090318312' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0a195-9329-458c-b103-38db9a4408a8"}]: dispatch 2023-09-23T22:00:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "fdf0a195-9329-458c-b103-38db9a4408a8"}]: dispatch 2023-09-23T22:00:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "fdf0a195-9329-458c-b103-38db9a4408a8"}]': finished 2023-09-23T22:00:20.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T22:00:20.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:21.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:21 smithi070 ceph-mon[105170]: mgrmap e14: y(active, since 63s), standbys: x 2023-09-23T22:00:21.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:21 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:00:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:21 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2026031980' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:00:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:21 smithi070 ceph-mon[105170]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:21 smithi026 ceph-mon[100157]: mgrmap e14: y(active, since 63s), standbys: x 2023-09-23T22:00:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:21 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:00:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:21 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2026031980' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:00:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:21 smithi026 ceph-mon[100157]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:21.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:21 smithi026 ceph-mon[118879]: mgrmap e14: y(active, since 63s), standbys: x 2023-09-23T22:00:21.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:21 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:00:21.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:21 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2026031980' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:00:21.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:21 smithi026 ceph-mon[118879]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:23 smithi026 ceph-mon[100157]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:23.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:23 smithi026 ceph-mon[118879]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:24.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:23 smithi070 ceph-mon[105170]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:25.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:25 smithi026 ceph-mon[118879]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:25.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:25 smithi026 ceph-mon[100157]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:25 smithi070 ceph-mon[105170]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:26.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:00:26.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:00:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:00:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:27.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:27 smithi026 ceph-mon[100157]: Deploying daemon osd.0 on smithi026 2023-09-23T22:00:27.797 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:27 smithi026 ceph-mon[100157]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:27.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:27 smithi026 ceph-mon[118879]: Deploying daemon osd.0 on smithi026 2023-09-23T22:00:27.797 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:27 smithi026 ceph-mon[118879]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:27 smithi070 ceph-mon[105170]: Deploying daemon osd.0 on smithi026 2023-09-23T22:00:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:27 smithi070 ceph-mon[105170]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:29.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:29 smithi026 ceph-mon[100157]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:29.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:29 smithi026 ceph-mon[118879]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:29 smithi070 ceph-mon[105170]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:31.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:31 smithi026 ceph-mon[100157]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:31.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:31 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:31.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:31 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:31.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:31 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:31.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:31 smithi026 ceph-mon[118879]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:31.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:31 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:31.868 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:31 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:31.868 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:31 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:31 smithi070 ceph-mon[105170]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:31 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:31 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:31 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:33 smithi026 ceph-mon[100157]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:33 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:33 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:33 smithi026 ceph-mon[118879]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:33 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:33 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:34.081 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 0 on host 'smithi026' 2023-09-23T22:00:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:33 smithi070 ceph-mon[105170]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:33 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:33 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:34.683 DEBUG:teuthology.orchestra.run.smithi026:osd.0> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T22:00:34.685 INFO:tasks.cephadm:Deploying osd.1 on smithi026 with /dev/vg_nvme/lv_3... 2023-09-23T22:00:34.685 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T22:00:34.718 INFO:journalctl@ceph.osd.0.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T22:00:34.961 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:34.961 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:34.961 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:34.961 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:34.961 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:34.961 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:34.961 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[100157]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:00:34.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:34.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:34.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:34.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:34.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:34.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:34.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:34 smithi026 ceph-mon[118879]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:00:35.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:35.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:35.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:35.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:35.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:35.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:35.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:34 smithi070 ceph-mon[105170]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:00:35.789 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:00:35.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[100157]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:35.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[100157]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:00:35.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[100157]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T22:00:35.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:35.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[100157]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:00:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[118879]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:35.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[118879]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:00:35.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[118879]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T22:00:35.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:35.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:35 smithi026 ceph-mon[118879]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:35 smithi070 ceph-mon[105170]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:35 smithi070 ceph-mon[105170]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:00:36.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:35 smithi070 ceph-mon[105170]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:35 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:36.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:35 smithi070 ceph-mon[105170]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:00:36.393 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:00:36 smithi026 conmon[125164]: 2023-09-23T22:00:36.050+0000 7f64cd978700 -1 osd.0 0 waiting for initial osdmap 2023-09-23T22:00:36.393 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:00:36 smithi026 conmon[125164]: 2023-09-23T22:00:36.053+0000 7f64c9f76700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:00:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[100157]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:00:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[100157]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T22:00:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[100157]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[118879]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:00:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[118879]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T22:00:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:37 smithi026 ceph-mon[118879]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:37.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:37 smithi070 ceph-mon[105170]: from='osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:00:37.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:37 smithi070 ceph-mon[105170]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T22:00:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:37 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:37 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:37 smithi070 ceph-mon[105170]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T22:00:37.839 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T22:00:37.840 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T22:00:37.840 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2023-09-23T22:00:37.840 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2023-09-23T22:00:37.840 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00829863 s, 1.3 GB/s 2023-09-23T22:00:37.840 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2023-09-23T22:00:38.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:38 smithi070 ceph-mon[105170]: purged_snaps scrub starts 2023-09-23T22:00:38.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:38 smithi070 ceph-mon[105170]: purged_snaps scrub ok 2023-09-23T22:00:38.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:38 smithi070 ceph-mon[105170]: osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623] boot 2023-09-23T22:00:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:38 smithi070 ceph-mon[105170]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T22:00:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:38 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:38.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[118879]: purged_snaps scrub starts 2023-09-23T22:00:38.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[118879]: purged_snaps scrub ok 2023-09-23T22:00:38.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[118879]: osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623] boot 2023-09-23T22:00:38.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[118879]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T22:00:38.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:38.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[100157]: purged_snaps scrub starts 2023-09-23T22:00:38.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[100157]: purged_snaps scrub ok 2023-09-23T22:00:38.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[100157]: osd.0 [v2:172.21.15.26:6802/3939081623,v1:172.21.15.26:6803/3939081623] boot 2023-09-23T22:00:38.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[100157]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T22:00:38.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:38 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:00:38.875 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi026:vg_nvme/lv_3 2023-09-23T22:00:39.229 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: Detected new or changed devices on smithi026 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi026 to 16149M 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:39.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: Detected new or changed devices on smithi026 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi026 to 16149M 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:39.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:39.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:39 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T22:00:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: Detected new or changed devices on smithi026 2023-09-23T22:00:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:00:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi026 to 16149M 2023-09-23T22:00:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:39 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:41 smithi070 ceph-mon[105170]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:41 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:00:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:41 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:00:41.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:41 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:41.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:41 smithi026 ceph-mon[118879]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:41.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:41 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:00:41.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:41 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:00:41.884 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:41 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:41.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:41 smithi026 ceph-mon[100157]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:41.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:00:41.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:00:41.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:42.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:42 smithi026 ceph-mon[100157]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:00:42.644 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:42 smithi026 ceph-mon[118879]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:00:42.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:42 smithi070 ceph-mon[105170]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:00:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:43 smithi070 ceph-mon[105170]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:43 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/668009054' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c0507d8-f63e-47b2-841d-394196258d6b"}]: dispatch 2023-09-23T22:00:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:43 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c0507d8-f63e-47b2-841d-394196258d6b"}]: dispatch 2023-09-23T22:00:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:43 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9c0507d8-f63e-47b2-841d-394196258d6b"}]': finished 2023-09-23T22:00:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:43 smithi070 ceph-mon[105170]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T22:00:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:43 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:00:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[100157]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/668009054' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c0507d8-f63e-47b2-841d-394196258d6b"}]: dispatch 2023-09-23T22:00:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c0507d8-f63e-47b2-841d-394196258d6b"}]: dispatch 2023-09-23T22:00:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9c0507d8-f63e-47b2-841d-394196258d6b"}]': finished 2023-09-23T22:00:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[100157]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T22:00:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:00:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[118879]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/668009054' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c0507d8-f63e-47b2-841d-394196258d6b"}]: dispatch 2023-09-23T22:00:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9c0507d8-f63e-47b2-841d-394196258d6b"}]: dispatch 2023-09-23T22:00:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9c0507d8-f63e-47b2-841d-394196258d6b"}]': finished 2023-09-23T22:00:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[118879]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T22:00:43.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:43 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:00:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:44 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2926425152' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:00:44.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:44 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2926425152' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:00:44.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:44 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2926425152' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:00:45.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:45 smithi070 ceph-mon[105170]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:45.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:45 smithi026 ceph-mon[100157]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:45.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:45 smithi026 ceph-mon[118879]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:47.900 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:47 smithi026 ceph-mon[100157]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:47.900 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:47 smithi026 ceph-mon[118879]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:48.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:47 smithi070 ceph-mon[105170]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:49.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:49 smithi026 ceph-mon[118879]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:49.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:00:49.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:49 smithi026 ceph-mon[100157]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:00:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:50.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:49 smithi070 ceph-mon[105170]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:50.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:00:50.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:50 smithi026 ceph-mon[100157]: Deploying daemon osd.1 on smithi026 2023-09-23T22:00:50.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:50 smithi026 ceph-mon[118879]: Deploying daemon osd.1 on smithi026 2023-09-23T22:00:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:50 smithi070 ceph-mon[105170]: Deploying daemon osd.1 on smithi026 2023-09-23T22:00:51.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:51 smithi026 ceph-mon[100157]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:51.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:51 smithi026 ceph-mon[118879]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:52.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:51 smithi070 ceph-mon[105170]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:53.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:53 smithi026 ceph-mon[100157]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:53.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:53 smithi026 ceph-mon[118879]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:54.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:53 smithi070 ceph-mon[105170]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:55 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:55 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:55 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:55 smithi026 ceph-mon[100157]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:55.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:55 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:55.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:55 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:55.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:55 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:55.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:55 smithi026 ceph-mon[118879]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:55.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:55 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:55.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:55 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:55.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:55 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:55 smithi070 ceph-mon[105170]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:57.544 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 1 on host 'smithi026' 2023-09-23T22:00:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[100157]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:00:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:57 smithi026 ceph-mon[118879]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:00:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:00:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:00:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:00:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:00:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:57 smithi070 ceph-mon[105170]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:00:58.137 DEBUG:teuthology.orchestra.run.smithi026:osd.1> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T22:00:58.139 INFO:tasks.cephadm:Deploying osd.2 on smithi026 with /dev/vg_nvme/lv_2... 2023-09-23T22:00:58.139 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T22:00:58.174 INFO:journalctl@ceph.osd.1.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T22:00:59.212 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:00:59.735 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:00:59 smithi026 conmon[134913]: 2023-09-23T22:00:59.544+0000 7fdeab854700 -1 osd.1 0 waiting for initial osdmap 2023-09-23T22:00:59.735 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:00:59 smithi026 conmon[134913]: 2023-09-23T22:00:59.547+0000 7fdea564d700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:00:59.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[100157]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:00:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[100157]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T22:00:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:00:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[100157]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:00:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[100157]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[118879]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:00:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[118879]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T22:00:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:00:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[118879]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:00:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:00:59 smithi026 ceph-mon[118879]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:00:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:59 smithi070 ceph-mon[105170]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:00:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:59 smithi070 ceph-mon[105170]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T22:00:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:59 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:00:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:59 smithi070 ceph-mon[105170]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:00:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:00:59 smithi070 ceph-mon[105170]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:01:00.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:00 smithi026 ceph-mon[118879]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:01:00.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:00 smithi026 ceph-mon[118879]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T22:01:00.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:00 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:01:00.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:00 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:01:00.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:00 smithi026 ceph-mon[100157]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:01:00.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:00 smithi026 ceph-mon[100157]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T22:01:00.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:00 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:01:00.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:00 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:01:00.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:00 smithi070 ceph-mon[105170]: from='osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:01:00.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:00 smithi070 ceph-mon[105170]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T22:01:00.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:00 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:01:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:00 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:01:01.331 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T22:01:01.331 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T22:01:01.331 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2023-09-23T22:01:01.331 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2023-09-23T22:01:01.331 INFO:teuthology.orchestra.run.smithi026.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0078889 s, 1.3 GB/s 2023-09-23T22:01:01.331 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 2023-09-23T22:01:01.331 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2023-09-23T22:01:01.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[100157]: purged_snaps scrub starts 2023-09-23T22:01:01.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[100157]: purged_snaps scrub ok 2023-09-23T22:01:01.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[100157]: osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047] boot 2023-09-23T22:01:01.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[100157]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T22:01:01.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:01:01.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[100157]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:01:01.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[118879]: purged_snaps scrub starts 2023-09-23T22:01:01.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[118879]: purged_snaps scrub ok 2023-09-23T22:01:01.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[118879]: osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047] boot 2023-09-23T22:01:01.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[118879]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T22:01:01.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:01:01.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:01 smithi026 ceph-mon[118879]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:01:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:01 smithi070 ceph-mon[105170]: purged_snaps scrub starts 2023-09-23T22:01:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:01 smithi070 ceph-mon[105170]: purged_snaps scrub ok 2023-09-23T22:01:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:01 smithi070 ceph-mon[105170]: osd.1 [v2:172.21.15.26:6810/2870847047,v1:172.21.15.26:6811/2870847047] boot 2023-09-23T22:01:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:01 smithi070 ceph-mon[105170]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T22:01:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:01 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:01:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:01 smithi070 ceph-mon[105170]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T22:01:02.137 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi026:vg_nvme/lv_2 2023-09-23T22:01:02.492 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:01:02.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:02 smithi070 ceph-mon[105170]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T22:01:02.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:02 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[118879]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T22:01:02.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:02.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:02.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[100157]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:02.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:02.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:02 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:03.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:03 smithi026 ceph-mon[100157]: Detected new or changed devices on smithi026 2023-09-23T22:01:03.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:03 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi026 to 8074M 2023-09-23T22:01:03.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:03 smithi026 ceph-mon[100157]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:03.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:03 smithi026 ceph-mon[118879]: Detected new or changed devices on smithi026 2023-09-23T22:01:03.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:03 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi026 to 8074M 2023-09-23T22:01:03.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:03 smithi026 ceph-mon[118879]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:03.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:03 smithi070 ceph-mon[105170]: Detected new or changed devices on smithi026 2023-09-23T22:01:03.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:03 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi026 to 8074M 2023-09-23T22:01:03.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:03 smithi070 ceph-mon[105170]: pgmap v57: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:04 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:01:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:04 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:01:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:04 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:04.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:04 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:01:04.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:04 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:01:04.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:04 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:04.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:01:04.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:01:04.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:04 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:05 smithi070 ceph-mon[105170]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:01:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:05 smithi070 ceph-mon[105170]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:05.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:05 smithi026 ceph-mon[100157]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:01:05.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:05 smithi026 ceph-mon[100157]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:05.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:05 smithi026 ceph-mon[118879]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:01:05.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:05 smithi026 ceph-mon[118879]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:06 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1799043019' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4b241a88-8860-4616-8af5-383823ce290f"}]: dispatch 2023-09-23T22:01:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:06 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1799043019' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4b241a88-8860-4616-8af5-383823ce290f"}]': finished 2023-09-23T22:01:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:06 smithi026 ceph-mon[100157]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T22:01:06.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:06 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1799043019' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4b241a88-8860-4616-8af5-383823ce290f"}]: dispatch 2023-09-23T22:01:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:06 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1799043019' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4b241a88-8860-4616-8af5-383823ce290f"}]': finished 2023-09-23T22:01:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:06 smithi026 ceph-mon[118879]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T22:01:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:06 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1799043019' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "4b241a88-8860-4616-8af5-383823ce290f"}]: dispatch 2023-09-23T22:01:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:06 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1799043019' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "4b241a88-8860-4616-8af5-383823ce290f"}]': finished 2023-09-23T22:01:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:06 smithi070 ceph-mon[105170]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T22:01:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:07 smithi026 ceph-mon[100157]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:07 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2350273687' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:01:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:07 smithi026 ceph-mon[118879]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:07 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2350273687' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:01:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:07 smithi070 ceph-mon[105170]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:07 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2350273687' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:01:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:09 smithi026 ceph-mon[100157]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:09.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:09 smithi026 ceph-mon[118879]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:10.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:09 smithi070 ceph-mon[105170]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:11.781 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:11 smithi026 ceph-mon[118879]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:11.781 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:11 smithi026 ceph-mon[100157]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:11.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:11 smithi070 ceph-mon[105170]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:13 smithi026 ceph-mon[100157]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:01:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:13 smithi026 ceph-mon[118879]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:01:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:14.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:13 smithi070 ceph-mon[105170]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:14.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:01:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:14.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:14 smithi026 ceph-mon[100157]: Deploying daemon osd.2 on smithi026 2023-09-23T22:01:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:14 smithi026 ceph-mon[118879]: Deploying daemon osd.2 on smithi026 2023-09-23T22:01:15.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:14 smithi070 ceph-mon[105170]: Deploying daemon osd.2 on smithi026 2023-09-23T22:01:15.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:15 smithi026 ceph-mon[100157]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:15.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:15 smithi026 ceph-mon[118879]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:16.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:15 smithi070 ceph-mon[105170]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:17 smithi026 ceph-mon[100157]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:17 smithi026 ceph-mon[118879]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:17 smithi070 ceph-mon[105170]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:18 smithi026 ceph-mon[100157]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:18 smithi026 ceph-mon[118879]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:18 smithi070 ceph-mon[105170]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:20.657 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 2 on host 'smithi026' 2023-09-23T22:01:21.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:21.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:21.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[100157]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:21.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:21.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[118879]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:20 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:21.285 DEBUG:teuthology.orchestra.run.smithi026:osd.2> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T22:01:21.295 INFO:tasks.cephadm:Deploying osd.3 on smithi026 with /dev/vg_nvme/lv_1... 2023-09-23T22:01:21.296 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T22:01:21.331 INFO:journalctl@ceph.osd.2.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T22:01:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:20 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:20 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:20 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:20 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:20 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:20 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:20 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:21.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:20 smithi070 ceph-mon[105170]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:21.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:20 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:22.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:21 smithi026 ceph-mon[100157]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:01:22.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:21 smithi026 ceph-mon[118879]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:01:22.363 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:01:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:21 smithi070 ceph-mon[105170]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:01:22.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[100157]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:01:22.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[100157]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T22:01:22.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:22.973 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[100157]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:01:22.973 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[100157]: pgmap v69: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:23.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[118879]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:01:23.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[118879]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T22:01:23.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:23.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[118879]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:01:23.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:22 smithi026 ceph-mon[118879]: pgmap v69: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:23.236 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:22 smithi026 conmon[144599]: 2023-09-23T22:01:22.973+0000 7f454f995700 -1 osd.2 0 waiting for initial osdmap 2023-09-23T22:01:23.236 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:22 smithi026 conmon[144599]: 2023-09-23T22:01:22.978+0000 7f454c794700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:01:23.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:22 smithi070 ceph-mon[105170]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:01:23.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:22 smithi070 ceph-mon[105170]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T22:01:23.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:22 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:23.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:22 smithi070 ceph-mon[105170]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:01:23.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:22 smithi070 ceph-mon[105170]: pgmap v69: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T22:01:24.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:23 smithi026 ceph-mon[118879]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:01:24.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:23 smithi026 ceph-mon[118879]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T22:01:24.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:23 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:24.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:23 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:24.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:23 smithi026 ceph-mon[100157]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:01:24.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:23 smithi026 ceph-mon[100157]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T22:01:24.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:23 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:24.070 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:23 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:24.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:23 smithi070 ceph-mon[105170]: from='osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:01:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:23 smithi070 ceph-mon[105170]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T22:01:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:23 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:23 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:24.393 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T22:01:24.393 INFO:teuthology.orchestra.run.smithi026.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T22:01:24.393 INFO:teuthology.orchestra.run.smithi026.stderr: stderr: 10+0 records in 2023-09-23T22:01:24.393 INFO:teuthology.orchestra.run.smithi026.stderr:10+0 records out 2023-09-23T22:01:24.393 INFO:teuthology.orchestra.run.smithi026.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00816722 s, 1.3 GB/s 2023-09-23T22:01:24.393 INFO:teuthology.orchestra.run.smithi026.stderr:--> Zapping successful for: 2023-09-23T22:01:24.942 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi026:vg_nvme/lv_1 2023-09-23T22:01:25.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[118879]: purged_snaps scrub starts 2023-09-23T22:01:25.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[118879]: purged_snaps scrub ok 2023-09-23T22:01:25.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[118879]: osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178] boot 2023-09-23T22:01:25.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[118879]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T22:01:25.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[100157]: purged_snaps scrub starts 2023-09-23T22:01:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[100157]: purged_snaps scrub ok 2023-09-23T22:01:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[100157]: osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178] boot 2023-09-23T22:01:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[100157]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T22:01:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:25 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:25.312 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:01:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:25 smithi070 ceph-mon[105170]: purged_snaps scrub starts 2023-09-23T22:01:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:25 smithi070 ceph-mon[105170]: purged_snaps scrub ok 2023-09-23T22:01:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:25 smithi070 ceph-mon[105170]: osd.2 [v2:172.21.15.26:6818/3354962178,v1:172.21.15.26:6819/3354962178] boot 2023-09-23T22:01:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:25 smithi070 ceph-mon[105170]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T22:01:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:25 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:01:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:01:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: Detected new or changed devices on smithi026 2023-09-23T22:01:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T22:01:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T22:01:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:01:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi026 to 5383M 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:01:26.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: Detected new or changed devices on smithi026 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi026 to 5383M 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:26.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:26.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:26 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: pgmap v72: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:26.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T22:01:26.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: Detected new or changed devices on smithi026 2023-09-23T22:01:26.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi026 to 5383M 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:26 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:27.176 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:01:26 smithi026 sudo[150738]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:01:27.176 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:01:26 smithi026 sudo[150738]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:27.176 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:01:26 smithi026 sudo[150738]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:27.176 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:01:26 smithi026 sudo[150738]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:27.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:01:27.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[118879]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T22:01:27.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[118879]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:27.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:01:27.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:01:27.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:27.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:01:27.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[100157]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T22:01:27.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[100157]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:27.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:01:27.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:01:27.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:27 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:27.485 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:01:27 smithi026 sudo[150892]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:01:27.485 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:01:27 smithi026 sudo[150892]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:27.485 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:01:27 smithi026 sudo[150892]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:27.485 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:01:27 smithi026 sudo[150892]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:27.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:27 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:01:27.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:27 smithi070 ceph-mon[105170]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T22:01:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:27 smithi070 ceph-mon[105170]: pgmap v75: 1 pgs: 1 active+clean; 449 KiB data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:27 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:01:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:27 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:01:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:27 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:27.968 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:01:27 smithi026 sudo[150990]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:01:27.968 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:01:27 smithi026 sudo[150990]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:27.968 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:01:27 smithi026 sudo[150990]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:27.968 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:01:27 smithi026 sudo[150990]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:28.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:28 smithi026 ceph-mon[118879]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:01:28.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:28 smithi026 ceph-mon[118879]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T22:01:28.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:28 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:28.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:28 smithi026 ceph-mon[100157]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:01:28.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:28 smithi026 ceph-mon[100157]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T22:01:28.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:28 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:28.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:28 smithi070 ceph-mon[105170]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi026:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:01:28.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:28 smithi070 ceph-mon[105170]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T22:01:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:28 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:28.717 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:01:28 smithi026 sudo[151038]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:01:28.717 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:01:28 smithi026 sudo[151038]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:28.717 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:01:28 smithi026 sudo[151038]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:28.717 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:01:28 smithi026 sudo[151038]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:28.985 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:28 smithi026 sudo[151109]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:01:28.985 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:28 smithi026 sudo[151109]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:28.985 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:28 smithi026 sudo[151109]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:28.985 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:28 smithi026 sudo[151109]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:29.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:29 smithi026 ceph-mon[118879]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T22:01:29.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:29 smithi026 ceph-mon[118879]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:29.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:29 smithi026 ceph-mon[100157]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T22:01:29.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:29 smithi026 ceph-mon[100157]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:29.486 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:29 smithi026 sudo[151141]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:01:29.486 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:29 smithi026 sudo[151141]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:29.486 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:29 smithi026 sudo[151141]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:29.486 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:01:29 smithi026 sudo[151141]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:29 smithi070 ceph-mon[105170]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T22:01:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:29 smithi070 ceph-mon[105170]: pgmap v77: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:29.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:29 smithi026 sudo[151165]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:01:29.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:29 smithi026 sudo[151165]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:29.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:29 smithi026 sudo[151165]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:29.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:29 smithi026 sudo[151165]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:30.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 sudo[151168]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:01:30.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 sudo[151168]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:30.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 sudo[151168]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:30.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 sudo[151168]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:30.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 sudo[109277]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:01:30.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 sudo[109277]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:30.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 sudo[109277]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:30.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 sudo[109277]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:30.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 sudo[109280]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:01:30.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 sudo[109280]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:30.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 sudo[109280]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:30.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 sudo[109280]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:01:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2258872934' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea"}]: dispatch 2023-09-23T22:01:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:01:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea"}]: dispatch 2023-09-23T22:01:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:01:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:01:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:01:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea"}]': finished 2023-09-23T22:01:30.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T22:01:30.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:30.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:01:30.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:01:30.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:01:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:01:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2258872934' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea"}]: dispatch 2023-09-23T22:01:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:01:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea"}]: dispatch 2023-09-23T22:01:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:01:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:01:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:01:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea"}]': finished 2023-09-23T22:01:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T22:01:30.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:30.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:01:30.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:01:30.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:01:30.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:01:30.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2258872934' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea"}]: dispatch 2023-09-23T22:01:30.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:01:30.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea"}]: dispatch 2023-09-23T22:01:30.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:01:30.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:01:30.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:01:30.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea"}]': finished 2023-09-23T22:01:30.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T22:01:30.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:30.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:01:30.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:01:30.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:01:31.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 sudo[151206]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T22:01:31.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 sudo[151206]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:31.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 sudo[151206]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:31.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 sudo[151206]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:01:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:01:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:01:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:01:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:01:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:01:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:01:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:31 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3427567726' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:01:31.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:01:31.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:01:31.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:31.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3427567726' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 sudo[151209]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 sudo[151209]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 sudo[151209]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T22:01:31.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 sudo[151209]: pam_unix(sudo:session): session closed for user root 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: pgmap v79: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T22:01:31.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:31 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3427567726' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:01:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:33 smithi026 ceph-mon[100157]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2023-09-23T22:01:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:33 smithi026 ceph-mon[118879]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2023-09-23T22:01:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:33 smithi070 ceph-mon[105170]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 218 KiB/s wr, 14 op/s 2023-09-23T22:01:35.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:35 smithi026 ceph-mon[100157]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 20 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2023-09-23T22:01:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:35 smithi026 ceph-mon[118879]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 20 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2023-09-23T22:01:36.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:35 smithi070 ceph-mon[105170]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 20 MiB used, 268 GiB / 268 GiB avail; 3.4 KiB/s rd, 163 KiB/s wr, 11 op/s 2023-09-23T22:01:37.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:37 smithi026 ceph-mon[118879]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 138 KiB/s wr, 9 op/s 2023-09-23T22:01:37.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:37 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:01:37.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:37 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:37.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:37 smithi026 ceph-mon[100157]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 138 KiB/s wr, 9 op/s 2023-09-23T22:01:37.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:37 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:01:37.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:37 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:37 smithi070 ceph-mon[105170]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.8 KiB/s rd, 138 KiB/s wr, 9 op/s 2023-09-23T22:01:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:37 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:01:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:37 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:38.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:38 smithi026 ceph-mon[100157]: Deploying daemon osd.3 on smithi026 2023-09-23T22:01:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:38 smithi026 ceph-mon[118879]: Deploying daemon osd.3 on smithi026 2023-09-23T22:01:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:38 smithi070 ceph-mon[105170]: Deploying daemon osd.3 on smithi026 2023-09-23T22:01:39.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:39 smithi026 ceph-mon[100157]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-09-23T22:01:39.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:39 smithi026 ceph-mon[118879]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-09-23T22:01:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:39 smithi070 ceph-mon[105170]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.7 KiB/s rd, 131 KiB/s wr, 8 op/s 2023-09-23T22:01:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:41 smithi070 ceph-mon[105170]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 124 KiB/s wr, 8 op/s 2023-09-23T22:01:41.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:41 smithi026 ceph-mon[100157]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 124 KiB/s wr, 8 op/s 2023-09-23T22:01:41.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:41 smithi026 ceph-mon[118879]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.6 KiB/s rd, 124 KiB/s wr, 8 op/s 2023-09-23T22:01:43.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:43 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:43.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:43 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:43.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:43 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:43.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:43 smithi026 ceph-mon[118879]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2023-09-23T22:01:43.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:43 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:43.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:43 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:43.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:43 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:43.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:43 smithi026 ceph-mon[100157]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2023-09-23T22:01:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:43 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:43 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:43 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:43 smithi070 ceph-mon[105170]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2023-09-23T22:01:45.054 INFO:teuthology.orchestra.run.smithi026.stdout:Created osd(s) 3 on host 'smithi026' 2023-09-23T22:01:45.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[118879]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:45.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:45.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:45.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:45.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[100157]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:45.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:45.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:45.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:45 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:45 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:45 smithi070 ceph-mon[105170]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:45 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:45 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:45 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:45 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:45 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:45 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:01:45.643 DEBUG:teuthology.orchestra.run.smithi026:osd.3> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T22:01:45.645 INFO:tasks.cephadm:Deploying osd.4 on smithi070 with /dev/vg_nvme/lv_4... 2023-09-23T22:01:45.645 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T22:01:45.677 INFO:journalctl@ceph.osd.3.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T22:01:45.991 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:01:46.553 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:46 smithi070 ceph-mon[105170]: from='osd.3 [v2:172.21.15.26:6826/861409764,v1:172.21.15.26:6827/861409764]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:01:46.553 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:46 smithi070 ceph-mon[105170]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:01:46.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:46 smithi026 ceph-mon[100157]: from='osd.3 [v2:172.21.15.26:6826/861409764,v1:172.21.15.26:6827/861409764]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:01:46.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:46 smithi026 ceph-mon[100157]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:01:46.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:46 smithi026 ceph-mon[118879]: from='osd.3 [v2:172.21.15.26:6826/861409764,v1:172.21.15.26:6827/861409764]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:01:46.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:46 smithi026 ceph-mon[118879]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:01:47.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[100157]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:01:47.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[100157]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T22:01:47.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:47.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[100157]: from='osd.3 [v2:172.21.15.26:6826/861409764,v1:172.21.15.26:6827/861409764]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:01:47.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[100157]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:01:47.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[100157]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:47.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[118879]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:01:47.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[118879]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T22:01:47.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:47.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[118879]: from='osd.3 [v2:172.21.15.26:6826/861409764,v1:172.21.15.26:6827/861409764]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:01:47.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[118879]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:01:47.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:47 smithi026 ceph-mon[118879]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:47.487 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:01:47 smithi026 conmon[153746]: 2023-09-23T22:01:47.261+0000 7f738ad2a700 -1 osd.3 0 waiting for initial osdmap 2023-09-23T22:01:47.487 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:01:47 smithi026 conmon[153746]: 2023-09-23T22:01:47.266+0000 7f7383b21700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:01:47.616 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T22:01:47.617 INFO:teuthology.orchestra.run.smithi070.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T22:01:47.617 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 10+0 records in 2023-09-23T22:01:47.617 INFO:teuthology.orchestra.run.smithi070.stderr:10+0 records out 2023-09-23T22:01:47.617 INFO:teuthology.orchestra.run.smithi070.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00873762 s, 1.2 GB/s 2023-09-23T22:01:47.617 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 2023-09-23T22:01:47.617 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping successful for: 2023-09-23T22:01:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:47 smithi070 ceph-mon[105170]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:01:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:47 smithi070 ceph-mon[105170]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T22:01:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:47 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:47 smithi070 ceph-mon[105170]: from='osd.3 [v2:172.21.15.26:6826/861409764,v1:172.21.15.26:6827/861409764]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:01:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:47 smithi070 ceph-mon[105170]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:01:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:47 smithi070 ceph-mon[105170]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:48.191 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi070:vg_nvme/lv_4 2023-09-23T22:01:48.528 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:01:48.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:48 smithi070 ceph-mon[105170]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:01:48.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:48 smithi070 ceph-mon[105170]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T22:01:48.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:48 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:48 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:48.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:48 smithi026 ceph-mon[100157]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:01:48.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:48 smithi026 ceph-mon[100157]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T22:01:48.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:48 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:48.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:48 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:48.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:48 smithi026 ceph-mon[118879]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]': finished 2023-09-23T22:01:48.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:48 smithi026 ceph-mon[118879]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T22:01:48.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:48 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:48.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:48 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:49.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: purged_snaps scrub starts 2023-09-23T22:01:49.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: purged_snaps scrub ok 2023-09-23T22:01:49.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: osd.3 [v2:172.21.15.26:6826/861409764,v1:172.21.15.26:6827/861409764] boot 2023-09-23T22:01:49.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: Detected new or changed devices on smithi026 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi026 to 4037M 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:49.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:49.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: purged_snaps scrub starts 2023-09-23T22:01:49.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: purged_snaps scrub ok 2023-09-23T22:01:49.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: osd.3 [v2:172.21.15.26:6826/861409764,v1:172.21.15.26:6827/861409764] boot 2023-09-23T22:01:49.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T22:01:49.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:49.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: Detected new or changed devices on smithi026 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi026 to 4037M 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: purged_snaps scrub starts 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: purged_snaps scrub ok 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: osd.3 [v2:172.21.15.26:6826/861409764,v1:172.21.15.26:6827/861409764] boot 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: Detected new or changed devices on smithi026 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:01:49.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi026 to 4037M 2023-09-23T22:01:49.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:49.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:49.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:01:49.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:01:50.609 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:50 smithi070 ceph-mon[105170]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T22:01:50.609 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:50 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:01:50.609 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:50 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:01:50.609 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:50 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:50 smithi026 ceph-mon[100157]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T22:01:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:50 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:01:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:50 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:01:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:50 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:50 smithi026 ceph-mon[118879]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T22:01:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:50 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:01:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:50 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:01:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:50 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:51.737 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:51 smithi070 ceph-mon[105170]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:01:51.737 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:51 smithi070 ceph-mon[105170]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:51 smithi026 ceph-mon[100157]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:01:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:51 smithi026 ceph-mon[100157]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:51.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:51 smithi026 ceph-mon[118879]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:01:51.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:51 smithi026 ceph-mon[118879]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:52 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/3096914058' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d4a4d17-075d-4538-9109-7a1dde1b422d"}]: dispatch 2023-09-23T22:01:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:52 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d4a4d17-075d-4538-9109-7a1dde1b422d"}]: dispatch 2023-09-23T22:01:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:52 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d4a4d17-075d-4538-9109-7a1dde1b422d"}]': finished 2023-09-23T22:01:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:52 smithi070 ceph-mon[105170]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T22:01:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:52 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:01:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/3096914058' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d4a4d17-075d-4538-9109-7a1dde1b422d"}]: dispatch 2023-09-23T22:01:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d4a4d17-075d-4538-9109-7a1dde1b422d"}]: dispatch 2023-09-23T22:01:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d4a4d17-075d-4538-9109-7a1dde1b422d"}]': finished 2023-09-23T22:01:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[118879]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T22:01:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:01:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/3096914058' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d4a4d17-075d-4538-9109-7a1dde1b422d"}]: dispatch 2023-09-23T22:01:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d4a4d17-075d-4538-9109-7a1dde1b422d"}]: dispatch 2023-09-23T22:01:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d4a4d17-075d-4538-9109-7a1dde1b422d"}]': finished 2023-09-23T22:01:52.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[100157]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T22:01:52.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:52 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:01:53.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:53 smithi026 ceph-mon[100157]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:53.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:53 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/377364787' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:01:53.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:53 smithi026 ceph-mon[118879]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:53.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:53 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/377364787' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:01:54.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:53 smithi070 ceph-mon[105170]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:53 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/377364787' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:01:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:55 smithi026 ceph-mon[100157]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:55 smithi026 ceph-mon[118879]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:55.988 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:55 smithi070 ceph-mon[105170]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:57.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:57 smithi070 ceph-mon[105170]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:57 smithi026 ceph-mon[100157]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:57 smithi026 ceph-mon[118879]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:58.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:58 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:01:58.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:58 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:58.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:58 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:01:58.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:58 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:58.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:58 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:01:58.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:58 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:01:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:59 smithi070 ceph-mon[105170]: Deploying daemon osd.4 on smithi070 2023-09-23T22:01:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:01:59 smithi070 ceph-mon[105170]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:59.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:59 smithi026 ceph-mon[100157]: Deploying daemon osd.4 on smithi070 2023-09-23T22:01:59.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:01:59 smithi026 ceph-mon[100157]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:01:59.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:59 smithi026 ceph-mon[118879]: Deploying daemon osd.4 on smithi070 2023-09-23T22:01:59.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:01:59 smithi026 ceph-mon[118879]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:01.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:01 smithi070 ceph-mon[105170]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:01.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:01 smithi026 ceph-mon[100157]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:01 smithi026 ceph-mon[118879]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:03.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:03 smithi070 ceph-mon[105170]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:03.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:03 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:03 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:03 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:03.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:03 smithi026 ceph-mon[100157]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:03.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:03 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:03.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:03 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:03.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:03 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:03.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:03 smithi026 ceph-mon[118879]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:03.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:03 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:03.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:03 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:03.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:03 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:05 smithi070 ceph-mon[105170]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:05 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:05.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:05 smithi026 ceph-mon[100157]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:05.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:05.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:05 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:05.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:05 smithi026 ceph-mon[118879]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:05.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:05.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:05 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.586 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 4 on host 'smithi070' 2023-09-23T22:02:06.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:06.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:06.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:07.210 DEBUG:teuthology.orchestra.run.smithi070:osd.4> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T22:02:07.212 INFO:tasks.cephadm:Deploying osd.5 on smithi070 with /dev/vg_nvme/lv_3... 2023-09-23T22:02:07.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T22:02:07.241 INFO:journalctl@ceph.osd.4.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:02:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:07 smithi070 ceph-mon[105170]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:07 smithi070 ceph-mon[105170]: from='osd.4 [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:02:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:07 smithi070 ceph-mon[105170]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:02:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:07 smithi026 ceph-mon[100157]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:07 smithi026 ceph-mon[100157]: from='osd.4 [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:02:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:07 smithi026 ceph-mon[100157]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:02:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:07 smithi026 ceph-mon[118879]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:07 smithi026 ceph-mon[118879]: from='osd.4 [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:02:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:07 smithi026 ceph-mon[118879]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:02:08.368 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:02:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[100157]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:02:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[100157]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T22:02:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[100157]: from='osd.4 [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[100157]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:08.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[118879]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:02:08.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[118879]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T22:02:08.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:08.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[118879]: from='osd.4 [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:08.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:08 smithi026 ceph-mon[118879]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:09.006 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:02:08 smithi070 conmon[113214]: 2023-09-23T22:02:08.733+0000 7f7499448700 -1 osd.4 0 waiting for initial osdmap 2023-09-23T22:02:09.006 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:02:08 smithi070 conmon[113214]: 2023-09-23T22:02:08.738+0000 7f7495245700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:02:09.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:08 smithi070 ceph-mon[105170]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:02:09.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:08 smithi070 ceph-mon[105170]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T22:02:09.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:08 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:09.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:08 smithi070 ceph-mon[105170]: from='osd.4 [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:09.007 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:08 smithi070 ceph-mon[105170]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:09.785 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:09 smithi070 ceph-mon[105170]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:09.785 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:09 smithi070 ceph-mon[105170]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:02:09.786 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:09 smithi070 ceph-mon[105170]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T22:02:09.786 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:09 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:09.786 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:09 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[100157]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[100157]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:02:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[100157]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T22:02:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[118879]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T22:02:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[118879]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:02:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[118879]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T22:02:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:09 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:10.339 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T22:02:10.340 INFO:teuthology.orchestra.run.smithi070.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T22:02:10.340 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 10+0 records in 2023-09-23T22:02:10.340 INFO:teuthology.orchestra.run.smithi070.stderr:10+0 records out 2023-09-23T22:02:10.340 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00820237 s, 1.3 GB/s 2023-09-23T22:02:10.340 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping successful for: 2023-09-23T22:02:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: purged_snaps scrub starts 2023-09-23T22:02:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: purged_snaps scrub ok 2023-09-23T22:02:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: osd.4 [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884] boot 2023-09-23T22:02:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T22:02:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T22:02:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: purged_snaps scrub starts 2023-09-23T22:02:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: purged_snaps scrub ok 2023-09-23T22:02:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: osd.4 [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884] boot 2023-09-23T22:02:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T22:02:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T22:02:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:10.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:10 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:11.111 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: purged_snaps scrub starts 2023-09-23T22:02:11.111 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: purged_snaps scrub ok 2023-09-23T22:02:11.111 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: osd.4 [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884] boot 2023-09-23T22:02:11.112 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T22:02:11.112 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:02:11.112 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T22:02:11.112 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:11.112 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:11.112 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:11.112 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:11.112 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:11.112 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:10 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:11.128 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi070:vg_nvme/lv_3 2023-09-23T22:02:11.460 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:02:12.080 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:11 smithi070 ceph-mon[105170]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:12.080 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:11 smithi070 ceph-mon[105170]: Detected new or changed devices on smithi070 2023-09-23T22:02:12.080 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:11 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi070 to 17168M 2023-09-23T22:02:12.080 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:11 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:12.080 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:11 smithi070 ceph-mon[105170]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[100157]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[100157]: Detected new or changed devices on smithi070 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi070 to 17168M 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[100157]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[118879]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[118879]: Detected new or changed devices on smithi070 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi070 to 17168M 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:11 smithi026 ceph-mon[118879]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T22:02:13.958 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:13 smithi070 ceph-mon[105170]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T22:02:13.959 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:13 smithi070 ceph-mon[105170]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:13.959 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:02:13.959 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:02:13.959 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[100157]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T22:02:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[100157]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:02:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:02:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[118879]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T22:02:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[118879]: pgmap v111: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:02:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:02:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:14.922 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:14 smithi070 ceph-mon[105170]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:02:14.922 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:14 smithi070 ceph-mon[105170]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T22:02:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:14 smithi026 ceph-mon[100157]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:02:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:14 smithi026 ceph-mon[100157]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T22:02:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:14 smithi026 ceph-mon[118879]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:02:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:14 smithi026 ceph-mon[118879]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T22:02:15.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:15 smithi026 ceph-mon[100157]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:15.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:15 smithi026 ceph-mon[118879]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:16.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:15 smithi070 ceph-mon[105170]: pgmap v113: 1 pgs: 1 remapped+peering; 577 KiB data, 28 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/4117545230' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b25ee343-9def-42dc-8b45-a3c85bf14805"}]: dispatch 2023-09-23T22:02:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b25ee343-9def-42dc-8b45-a3c85bf14805"}]: dispatch 2023-09-23T22:02:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b25ee343-9def-42dc-8b45-a3c85bf14805"}]': finished 2023-09-23T22:02:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[100157]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T22:02:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/1260343868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:02:16.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/4117545230' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b25ee343-9def-42dc-8b45-a3c85bf14805"}]: dispatch 2023-09-23T22:02:16.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b25ee343-9def-42dc-8b45-a3c85bf14805"}]: dispatch 2023-09-23T22:02:16.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b25ee343-9def-42dc-8b45-a3c85bf14805"}]': finished 2023-09-23T22:02:16.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[118879]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T22:02:16.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:16.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:16 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/1260343868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:02:17.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:16 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/4117545230' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b25ee343-9def-42dc-8b45-a3c85bf14805"}]: dispatch 2023-09-23T22:02:17.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:16 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b25ee343-9def-42dc-8b45-a3c85bf14805"}]: dispatch 2023-09-23T22:02:17.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:16 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b25ee343-9def-42dc-8b45-a3c85bf14805"}]': finished 2023-09-23T22:02:17.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:16 smithi070 ceph-mon[105170]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T22:02:17.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:16 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:17.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:16 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/1260343868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:02:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:17 smithi026 ceph-mon[100157]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2023-09-23T22:02:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:17 smithi026 ceph-mon[118879]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2023-09-23T22:02:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:17 smithi070 ceph-mon[105170]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 115 KiB/s, 0 objects/s recovering 2023-09-23T22:02:19.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:19 smithi070 ceph-mon[105170]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:02:19.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:19 smithi026 ceph-mon[100157]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:02:19.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:19 smithi026 ceph-mon[118879]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:02:21.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:21 smithi070 ceph-mon[105170]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:02:21.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:21 smithi026 ceph-mon[100157]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:02:21.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:21 smithi026 ceph-mon[118879]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:02:22.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:22 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:02:22.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:22 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:22.984 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:22 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:02:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:22 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:22 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:02:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:22 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:23.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:23 smithi070 ceph-mon[105170]: Deploying daemon osd.5 on smithi070 2023-09-23T22:02:23.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:23 smithi070 ceph-mon[105170]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T22:02:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:23 smithi026 ceph-mon[100157]: Deploying daemon osd.5 on smithi070 2023-09-23T22:02:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:23 smithi026 ceph-mon[100157]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T22:02:23.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:23 smithi026 ceph-mon[118879]: Deploying daemon osd.5 on smithi070 2023-09-23T22:02:23.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:23 smithi026 ceph-mon[118879]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T22:02:25.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:25 smithi026 ceph-mon[100157]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:02:25.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:25 smithi026 ceph-mon[118879]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:02:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:25 smithi070 ceph-mon[105170]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:02:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:27 smithi026 ceph-mon[100157]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T22:02:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:27 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:27 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:27 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:27.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:27 smithi026 ceph-mon[118879]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T22:02:27.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:27 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:27.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:27 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:27.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:27 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:27 smithi070 ceph-mon[105170]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T22:02:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:27 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:27 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:27 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:29.912 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 5 on host 'smithi070' 2023-09-23T22:02:29.936 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:29.936 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:29.936 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:29.936 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:29.936 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:29.936 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:29 smithi070 ceph-mon[105170]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:29.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:29.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:29.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:29.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:29.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:29.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[100157]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:29.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:29 smithi026 ceph-mon[118879]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:30.486 DEBUG:teuthology.orchestra.run.smithi070:osd.5> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T22:02:30.488 INFO:tasks.cephadm:Deploying osd.6 on smithi070 with /dev/vg_nvme/lv_2... 2023-09-23T22:02:30.488 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T22:02:30.521 INFO:journalctl@ceph.osd.5.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:02:31.092 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:31.092 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:31.092 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:30 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:31.092 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:30 smithi070 ceph-mon[105170]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:02:31.093 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:30 smithi070 ceph-mon[105170]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[100157]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:02:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[100157]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[118879]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T22:02:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:30 smithi026 ceph-mon[118879]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:31.487 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:02:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:31 smithi070 ceph-mon[105170]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T22:02:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:31 smithi070 ceph-mon[105170]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T22:02:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:31 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:31 smithi070 ceph-mon[105170]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:32.124 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:02:31 smithi070 conmon[122257]: 2023-09-23T22:02:31.883+0000 7fe385e71700 -1 osd.5 0 waiting for initial osdmap 2023-09-23T22:02:32.125 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:02:31 smithi070 conmon[122257]: 2023-09-23T22:02:31.888+0000 7fe37ec68700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:02:32.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:31 smithi026 ceph-mon[100157]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T22:02:32.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:31 smithi026 ceph-mon[100157]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T22:02:32.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:31 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:32.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:31 smithi026 ceph-mon[100157]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:32.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:31 smithi026 ceph-mon[118879]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T22:02:32.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:31 smithi026 ceph-mon[118879]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T22:02:32.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:31 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:32.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:31 smithi026 ceph-mon[118879]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:32 smithi070 ceph-mon[105170]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:02:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:32 smithi070 ceph-mon[105170]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T22:02:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:32 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:32 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:32 smithi070 ceph-mon[105170]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:33.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[118879]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:02:33.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[118879]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T22:02:33.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:33.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:33.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[118879]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[100157]: from='osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994]' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:02:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[100157]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T22:02:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:32 smithi026 ceph-mon[100157]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T22:02:33.513 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T22:02:33.513 INFO:teuthology.orchestra.run.smithi070.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T22:02:33.513 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 10+0 records in 2023-09-23T22:02:33.513 INFO:teuthology.orchestra.run.smithi070.stderr:10+0 records out 2023-09-23T22:02:33.513 INFO:teuthology.orchestra.run.smithi070.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00827436 s, 1.3 GB/s 2023-09-23T22:02:33.514 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 2023-09-23T22:02:33.514 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping successful for: 2023-09-23T22:02:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:33 smithi070 ceph-mon[105170]: purged_snaps scrub starts 2023-09-23T22:02:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:33 smithi070 ceph-mon[105170]: purged_snaps scrub ok 2023-09-23T22:02:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:33 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:33 smithi070 ceph-mon[105170]: osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994] boot 2023-09-23T22:02:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:33 smithi070 ceph-mon[105170]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T22:02:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:33 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:34.180 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi070:vg_nvme/lv_2 2023-09-23T22:02:34.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[100157]: purged_snaps scrub starts 2023-09-23T22:02:34.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[100157]: purged_snaps scrub ok 2023-09-23T22:02:34.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:34.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[100157]: osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994] boot 2023-09-23T22:02:34.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[100157]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T22:02:34.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[118879]: purged_snaps scrub starts 2023-09-23T22:02:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[118879]: purged_snaps scrub ok 2023-09-23T22:02:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[118879]: osd.5 [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994] boot 2023-09-23T22:02:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[118879]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T22:02:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:33 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:02:34.515 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:02:35.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T22:02:35.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: Detected new or changed devices on smithi070 2023-09-23T22:02:35.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:35.033 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:35.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi070 to 8584M 2023-09-23T22:02:35.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:35.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:35.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:34 smithi070 ceph-mon[105170]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:35.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T22:02:35.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: Detected new or changed devices on smithi070 2023-09-23T22:02:35.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi070 to 8584M 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[100157]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: Detected new or changed devices on smithi070 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi070 to 8584M 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:35.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:34 smithi026 ceph-mon[118879]: pgmap v128: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:36.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:35 smithi070 ceph-mon[105170]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T22:02:36.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:35 smithi070 ceph-mon[105170]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:02:36.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:35 smithi070 ceph-mon[105170]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T22:02:36.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:35 smithi026 ceph-mon[100157]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T22:02:36.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:35 smithi026 ceph-mon[100157]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:02:36.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:35 smithi026 ceph-mon[100157]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T22:02:36.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:35 smithi026 ceph-mon[118879]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T22:02:36.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:35 smithi026 ceph-mon[118879]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:02:36.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:35 smithi026 ceph-mon[118879]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T22:02:37.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[118879]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:02:37.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:02:37.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:02:37.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:37.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[118879]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T22:02:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[118879]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[100157]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:02:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:02:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:02:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[100157]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T22:02:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:36 smithi026 ceph-mon[100157]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:37.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:36 smithi070 ceph-mon[105170]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:02:37.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:36 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:02:37.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:36 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:02:37.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:36 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:37.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:36 smithi070 ceph-mon[105170]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T22:02:37.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:36 smithi070 ceph-mon[105170]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/2424747821' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7ecfbb9-ea1a-480b-b31e-114e2d7b1721"}]: dispatch 2023-09-23T22:02:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7ecfbb9-ea1a-480b-b31e-114e2d7b1721"}]: dispatch 2023-09-23T22:02:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f7ecfbb9-ea1a-480b-b31e-114e2d7b1721"}]': finished 2023-09-23T22:02:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[118879]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T22:02:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:38.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/2424747821' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7ecfbb9-ea1a-480b-b31e-114e2d7b1721"}]: dispatch 2023-09-23T22:02:38.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7ecfbb9-ea1a-480b-b31e-114e2d7b1721"}]: dispatch 2023-09-23T22:02:38.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f7ecfbb9-ea1a-480b-b31e-114e2d7b1721"}]': finished 2023-09-23T22:02:38.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[100157]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T22:02:38.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:38 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:38 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/2424747821' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7ecfbb9-ea1a-480b-b31e-114e2d7b1721"}]: dispatch 2023-09-23T22:02:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:38 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f7ecfbb9-ea1a-480b-b31e-114e2d7b1721"}]: dispatch 2023-09-23T22:02:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:38 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f7ecfbb9-ea1a-480b-b31e-114e2d7b1721"}]': finished 2023-09-23T22:02:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:38 smithi070 ceph-mon[105170]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T22:02:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:38 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:39.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:39 smithi026 ceph-mon[100157]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:39.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:39 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/3411038443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:02:39.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:39 smithi026 ceph-mon[118879]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:39 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/3411038443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:02:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:39 smithi070 ceph-mon[105170]: pgmap v134: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:39 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/3411038443' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:02:41.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:41 smithi026 ceph-mon[100157]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-09-23T22:02:41.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:41 smithi026 ceph-mon[118879]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-09-23T22:02:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:41 smithi070 ceph-mon[105170]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 100 KiB/s, 0 objects/s recovering 2023-09-23T22:02:42.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:42 smithi070 ceph-mon[105170]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:02:42.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:42 smithi070 ceph-mon[105170]: Cluster is now healthy 2023-09-23T22:02:42.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:42 smithi026 ceph-mon[100157]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:02:42.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:42 smithi026 ceph-mon[100157]: Cluster is now healthy 2023-09-23T22:02:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:42 smithi026 ceph-mon[118879]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:02:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:42 smithi026 ceph-mon[118879]: Cluster is now healthy 2023-09-23T22:02:43.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:43 smithi026 ceph-mon[118879]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-09-23T22:02:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:43 smithi026 ceph-mon[100157]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-09-23T22:02:44.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:43 smithi070 ceph-mon[105170]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-09-23T22:02:44.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:44 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:02:44.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:44 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:44.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:44 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:02:44.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:44 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:44.989 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:44 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:02:44.989 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:44 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:45.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:45 smithi070 ceph-mon[105170]: Deploying daemon osd.6 on smithi070 2023-09-23T22:02:45.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:45 smithi070 ceph-mon[105170]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:02:45.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:45 smithi026 ceph-mon[100157]: Deploying daemon osd.6 on smithi070 2023-09-23T22:02:45.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:45 smithi026 ceph-mon[100157]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:02:45.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:45 smithi026 ceph-mon[118879]: Deploying daemon osd.6 on smithi070 2023-09-23T22:02:45.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:45 smithi026 ceph-mon[118879]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:02:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:47 smithi070 ceph-mon[105170]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:02:47.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:47 smithi026 ceph-mon[100157]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:02:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:47 smithi026 ceph-mon[118879]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:02:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:49 smithi070 ceph-mon[105170]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-09-23T22:02:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:49 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:49 smithi026 ceph-mon[100157]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-09-23T22:02:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:49 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:50.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:49 smithi026 ceph-mon[118879]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 56 KiB/s, 0 objects/s recovering 2023-09-23T22:02:50.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:50.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:50.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:49 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:51.725 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 6 on host 'smithi070' 2023-09-23T22:02:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:51 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:51 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:51 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:51 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:51 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:51 smithi070 ceph-mon[105170]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T22:02:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[100157]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T22:02:51.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:51.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:51.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:51.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:51.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:51.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:51 smithi026 ceph-mon[118879]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T22:02:52.366 DEBUG:teuthology.orchestra.run.smithi070:osd.6> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T22:02:52.369 INFO:tasks.cephadm:Deploying osd.7 on smithi070 with /dev/vg_nvme/lv_1... 2023-09-23T22:02:52.369 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T22:02:52.400 INFO:journalctl@ceph.osd.6.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:02:52.979 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:52 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:52.979 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:52 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:52.979 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:52 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:52.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:52 smithi070 ceph-mon[105170]: from='osd.6 [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:02:52.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:52 smithi070 ceph-mon[105170]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:02:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[100157]: from='osd.6 [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:02:52.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[100157]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:02:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:02:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[118879]: from='osd.6 [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:02:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:52 smithi026 ceph-mon[118879]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:02:53.467 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:02:53.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[100157]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:53.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[100157]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:02:53.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[100157]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T22:02:53.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:53.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[100157]: from='osd.6 [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:53.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[100157]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:53.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[118879]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:53.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[118879]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:02:53.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[118879]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T22:02:53.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:53.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[118879]: from='osd.6 [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:53.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:53 smithi026 ceph-mon[118879]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:53 smithi070 ceph-mon[105170]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:53 smithi070 ceph-mon[105170]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:02:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:53 smithi070 ceph-mon[105170]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T22:02:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:53 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:53 smithi070 ceph-mon[105170]: from='osd.6 [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:53 smithi070 ceph-mon[105170]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:02:54.125 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:02:53 smithi070 conmon[131227]: 2023-09-23T22:02:53.706+0000 7fb775f6c700 -1 osd.6 0 waiting for initial osdmap 2023-09-23T22:02:54.125 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:02:53 smithi070 conmon[131227]: 2023-09-23T22:02:53.712+0000 7fb76ed63700 -1 osd.6 45 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:02:54.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:54 smithi070 ceph-mon[105170]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:02:54.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:54 smithi070 ceph-mon[105170]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T22:02:54.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:54.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:54 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:54 smithi026 ceph-mon[100157]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:02:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:54 smithi026 ceph-mon[100157]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T22:02:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:54 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:54 smithi026 ceph-mon[118879]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:02:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:54 smithi026 ceph-mon[118879]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T22:02:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:54 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:54 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:55.490 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T22:02:55.490 INFO:teuthology.orchestra.run.smithi070.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T22:02:55.490 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 10+0 records in 2023-09-23T22:02:55.490 INFO:teuthology.orchestra.run.smithi070.stderr:10+0 records out 2023-09-23T22:02:55.490 INFO:teuthology.orchestra.run.smithi070.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00822534 s, 1.3 GB/s 2023-09-23T22:02:55.491 INFO:teuthology.orchestra.run.smithi070.stderr: stderr: 2023-09-23T22:02:55.491 INFO:teuthology.orchestra.run.smithi070.stderr:--> Zapping successful for: 2023-09-23T22:02:55.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:55 smithi070 ceph-mon[105170]: purged_snaps scrub starts 2023-09-23T22:02:55.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:55 smithi070 ceph-mon[105170]: purged_snaps scrub ok 2023-09-23T22:02:55.787 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:55 smithi070 ceph-mon[105170]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:55.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:55 smithi070 ceph-mon[105170]: osd.6 [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869] boot 2023-09-23T22:02:55.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:55 smithi070 ceph-mon[105170]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T22:02:55.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:55 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:55.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:55 smithi070 ceph-mon[105170]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T22:02:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[100157]: purged_snaps scrub starts 2023-09-23T22:02:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[100157]: purged_snaps scrub ok 2023-09-23T22:02:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[100157]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[100157]: osd.6 [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869] boot 2023-09-23T22:02:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[100157]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T22:02:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:55.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[100157]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T22:02:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[118879]: purged_snaps scrub starts 2023-09-23T22:02:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[118879]: purged_snaps scrub ok 2023-09-23T22:02:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[118879]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T22:02:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[118879]: osd.6 [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869] boot 2023-09-23T22:02:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[118879]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T22:02:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:02:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:55 smithi026 ceph-mon[118879]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T22:02:56.202 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi070:vg_nvme/lv_1 2023-09-23T22:02:56.540 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:02:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: Detected new or changed devices on smithi070 2023-09-23T22:02:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:56 smithi070 ceph-mon[105170]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T22:02:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: Detected new or changed devices on smithi070 2023-09-23T22:02:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:57.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:57.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:57.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[100157]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T22:02:57.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: Detected new or changed devices on smithi070 2023-09-23T22:02:57.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:02:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:02:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:02:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:56 smithi026 ceph-mon[118879]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T22:02:57.822 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:57 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi070 to 5722M 2023-09-23T22:02:57.822 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:57 smithi070 ceph-mon[105170]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:02:58.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:57 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi070 to 5722M 2023-09-23T22:02:58.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:57 smithi026 ceph-mon[100157]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:02:58.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:57 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi070 to 5722M 2023-09-23T22:02:58.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:57 smithi026 ceph-mon[118879]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:02:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:58 smithi070 ceph-mon[105170]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T22:02:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:58 smithi070 ceph-mon[105170]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:02:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:58 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:02:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:58 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:02:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:58 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:58 smithi070 ceph-mon[105170]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:02:59.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[100157]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T22:02:59.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[100157]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:02:59.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:02:59.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:02:59.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[100157]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:02:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[118879]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T22:02:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[118879]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi070:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:02:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T22:02:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T22:02:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:02:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:58 smithi026 ceph-mon[118879]: pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:02:59.896 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:02:59 smithi070 ceph-mon[105170]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T22:03:00.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:02:59 smithi026 ceph-mon[100157]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T22:03:00.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:02:59 smithi026 ceph-mon[118879]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T22:03:01.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:00 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/3917432944' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9057258-2f16-4805-a17d-b672eacc4cb0"}]: dispatch 2023-09-23T22:03:01.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:00 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9057258-2f16-4805-a17d-b672eacc4cb0"}]: dispatch 2023-09-23T22:03:01.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:00 smithi070 ceph-mon[105170]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e9057258-2f16-4805-a17d-b672eacc4cb0"}]': finished 2023-09-23T22:03:01.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:00 smithi070 ceph-mon[105170]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T22:03:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:00 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:01.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:00 smithi070 ceph-mon[105170]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2023-09-23T22:03:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/3917432944' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9057258-2f16-4805-a17d-b672eacc4cb0"}]: dispatch 2023-09-23T22:03:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9057258-2f16-4805-a17d-b672eacc4cb0"}]: dispatch 2023-09-23T22:03:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[100157]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e9057258-2f16-4805-a17d-b672eacc4cb0"}]': finished 2023-09-23T22:03:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[100157]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T22:03:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[100157]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2023-09-23T22:03:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/3917432944' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9057258-2f16-4805-a17d-b672eacc4cb0"}]: dispatch 2023-09-23T22:03:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e9057258-2f16-4805-a17d-b672eacc4cb0"}]: dispatch 2023-09-23T22:03:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[118879]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e9057258-2f16-4805-a17d-b672eacc4cb0"}]': finished 2023-09-23T22:03:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[118879]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T22:03:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:00 smithi026 ceph-mon[118879]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2023-09-23T22:03:02.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:01 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/3156955405' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:03:02.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:01 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/3156955405' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:03:02.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:01 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/3156955405' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T22:03:03.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:02 smithi070 ceph-mon[105170]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:03:03.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:02 smithi026 ceph-mon[100157]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:03:03.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:02 smithi026 ceph-mon[118879]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:03:05.774 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:05 smithi070 ceph-mon[105170]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2023-09-23T22:03:05.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:05 smithi026 ceph-mon[100157]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2023-09-23T22:03:05.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:05 smithi026 ceph-mon[118879]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 83 KiB/s, 0 objects/s recovering 2023-09-23T22:03:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:03:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:06 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:03:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:06 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:03:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:06 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:07 smithi026 ceph-mon[100157]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:03:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:07 smithi026 ceph-mon[100157]: Deploying daemon osd.7 on smithi070 2023-09-23T22:03:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:07 smithi026 ceph-mon[118879]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:03:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:07 smithi026 ceph-mon[118879]: Deploying daemon osd.7 on smithi070 2023-09-23T22:03:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:07 smithi070 ceph-mon[105170]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:03:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:07 smithi070 ceph-mon[105170]: Deploying daemon osd.7 on smithi070 2023-09-23T22:03:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:09 smithi070 ceph-mon[105170]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:03:10.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:09 smithi026 ceph-mon[118879]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:03:10.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:09 smithi026 ceph-mon[100157]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:03:11.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:11 smithi026 ceph-mon[100157]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:11.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:11 smithi026 ceph-mon[118879]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:12.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:11 smithi070 ceph-mon[105170]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:12.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:12 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:12 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:12 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:12 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:12 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:12 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:12 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:12 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:12 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:13.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:13 smithi070 ceph-mon[105170]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:13.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:13.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:13 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:13 smithi026 ceph-mon[100157]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:13 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:13 smithi026 ceph-mon[118879]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:13 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:14.586 INFO:teuthology.orchestra.run.smithi070.stdout:Created osd(s) 7 on host 'smithi070' 2023-09-23T22:03:14.800 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:14 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:14.800 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:14 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:14.800 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:14 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:14.800 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:14 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:14.800 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:14 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:14.800 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:14 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:14 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:15.253 DEBUG:teuthology.orchestra.run.smithi070:osd.7> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T22:03:15.255 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T22:03:15.256 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T22:03:15.284 INFO:journalctl@ceph.osd.7.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:03:15.627 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:03:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:15 smithi070 ceph-mon[105170]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:15 smithi070 ceph-mon[105170]: from='osd.7 [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:03:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:15 smithi070 ceph-mon[105170]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:03:15.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:15 smithi026 ceph-mon[100157]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:15.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:15 smithi026 ceph-mon[100157]: from='osd.7 [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:03:15.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:15 smithi026 ceph-mon[100157]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:03:15.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:15 smithi026 ceph-mon[118879]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:15.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:15 smithi026 ceph-mon[118879]: from='osd.7 [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:03:15.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:15 smithi026 ceph-mon[118879]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T22:03:16.868 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[100157]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T22:03:16.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[100157]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T22:03:16.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:16.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[100157]: from='osd.7 [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:03:16.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[100157]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:03:16.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[118879]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T22:03:16.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[118879]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T22:03:16.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:16.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[118879]: from='osd.7 [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:03:16.870 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:16 smithi026 ceph-mon[118879]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:03:16.873 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:03:16 smithi070 conmon[140233]: 2023-09-23T22:03:16.729+0000 7febf10ca700 -1 osd.7 0 waiting for initial osdmap 2023-09-23T22:03:16.873 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:03:16 smithi070 conmon[140233]: 2023-09-23T22:03:16.737+0000 7febec6c6700 -1 osd.7 53 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:03:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:16 smithi070 ceph-mon[105170]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T22:03:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:16 smithi070 ceph-mon[105170]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T22:03:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:16 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:16 smithi070 ceph-mon[105170]: from='osd.7 [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:03:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:16 smithi070 ceph-mon[105170]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:03:17.249 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:03:17.941 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":53,"num_osds":8,"num_up_osds":7,"osd_up_since":1695506574,"num_in_osds":8,"osd_in_since":1695506580,"num_remapped_pgs":0} 2023-09-23T22:03:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[118879]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[118879]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:03:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[118879]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T22:03:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/900052541' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:03:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[100157]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[100157]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:03:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[100157]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T22:03:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:17 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/900052541' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:03:18.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:17 smithi070 ceph-mon[105170]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T22:03:18.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:17 smithi070 ceph-mon[105170]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]': finished 2023-09-23T22:03:18.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:17 smithi070 ceph-mon[105170]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T22:03:18.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:17 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:18.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:17 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:18.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:17 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/900052541' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:03:18.942 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T22:03:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: purged_snaps scrub starts 2023-09-23T22:03:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: purged_snaps scrub ok 2023-09-23T22:03:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: osd.7 [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624] boot 2023-09-23T22:03:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T22:03:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: purged_snaps scrub starts 2023-09-23T22:03:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: purged_snaps scrub ok 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: osd.7 [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624] boot 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:18 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: purged_snaps scrub starts 2023-09-23T22:03:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: purged_snaps scrub ok 2023-09-23T22:03:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: osd.7 [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624] boot 2023-09-23T22:03:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T22:03:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:18 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:19.310 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:03:19.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:19 smithi026 ceph-mon[100157]: Detected new or changed devices on smithi070 2023-09-23T22:03:19.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:19 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi070 to 4292M 2023-09-23T22:03:19.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:19 smithi026 ceph-mon[100157]: pgmap v165: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:19.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:19 smithi026 ceph-mon[100157]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T22:03:19.817 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:19 smithi026 ceph-mon[118879]: Detected new or changed devices on smithi070 2023-09-23T22:03:19.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:19 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi070 to 4292M 2023-09-23T22:03:19.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:19 smithi026 ceph-mon[118879]: pgmap v165: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:19.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:19 smithi026 ceph-mon[118879]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T22:03:20.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:19 smithi070 ceph-mon[105170]: Detected new or changed devices on smithi070 2023-09-23T22:03:20.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:19 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi070 to 4292M 2023-09-23T22:03:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:19 smithi070 ceph-mon[105170]: pgmap v165: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:19 smithi070 ceph-mon[105170]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T22:03:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:20 smithi026 ceph-mon[100157]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T22:03:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:20 smithi026 ceph-mon[100157]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T22:03:20.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:20 smithi026 ceph-mon[118879]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T22:03:20.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:20 smithi026 ceph-mon[118879]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T22:03:21.064 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:03:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:20 smithi070 ceph-mon[105170]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T22:03:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:20 smithi070 ceph-mon[105170]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T22:03:21.867 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":57,"num_osds":8,"num_up_osds":8,"osd_up_since":1695506597,"num_in_osds":8,"osd_in_since":1695506580,"num_remapped_pgs":0} 2023-09-23T22:03:21.867 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:03:22.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:21 smithi070 ceph-mon[105170]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:22.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:21 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1494939148' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:03:22.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:21 smithi070 ceph-mon[105170]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T22:03:22.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:21 smithi026 ceph-mon[118879]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:22.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:21 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1494939148' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:03:22.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:21 smithi026 ceph-mon[118879]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T22:03:22.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:21 smithi026 ceph-mon[100157]: pgmap v169: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:22.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:21 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1494939148' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T22:03:22.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:21 smithi026 ceph-mon[100157]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T22:03:22.264 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:03:24.029 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:03:24.029 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":58,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:03:21.639824+0000","last_up_change":"2023-09-23T22:03:17.723571+0000","last_in_change":"2023-09-23T22:03:00.614896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6803","nonce":3939081623}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6805","nonce":3939081623}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6809","nonce":3939081623}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6807","nonce":3939081623}]},"public_addr":"172.21.15.26:6803/3939081623","cluster_addr":"172.21.15.26:6805/3939081623","heartbeat_back_addr":"172.21.15.26:6809/3939081623","heartbeat_front_addr":"172.21.15.26:6807/3939081623","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6811","nonce":2870847047}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6813","nonce":2870847047}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6817","nonce":2870847047}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6815","nonce":2870847047}]},"public_addr":"172.21.15.26:6811/2870847047","cluster_addr":"172.21.15.26:6813/2870847047","heartbeat_back_addr":"172.21.15.26:6817/2870847047","heartbeat_front_addr":"172.21.15.26:6815/2870847047","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6819","nonce":3354962178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6821","nonce":3354962178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6825","nonce":3354962178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6823","nonce":3354962178}]},"public_addr":"172.21.15.26:6819/3354962178","cluster_addr":"172.21.15.26:6821/3354962178","heartbeat_back_addr":"172.21.15.26:6825/3354962178","heartbeat_front_addr":"172.21.15.26:6823/3354962178","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6827","nonce":861409764}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6829","nonce":861409764}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6833","nonce":861409764}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6831","nonce":861409764}]},"public_addr":"172.21.15.26:6827/861409764","cluster_addr":"172.21.15.26:6829/861409764","heartbeat_back_addr":"172.21.15.26:6833/861409764","heartbeat_front_addr":"172.21.15.26:6831/861409764","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6801","nonce":1461751884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6803","nonce":1461751884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6807","nonce":1461751884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6805","nonce":1461751884}]},"public_addr":"172.21.15.70:6801/1461751884","cluster_addr":"172.21.15.70:6803/1461751884","heartbeat_back_addr":"172.21.15.70:6807/1461751884","heartbeat_front_addr":"172.21.15.70:6805/1461751884","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6817","nonce":572290869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6819","nonce":572290869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6823","nonce":572290869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6821","nonce":572290869}]},"public_addr":"172.21.15.70:6817/572290869","cluster_addr":"172.21.15.70:6819/572290869","heartbeat_back_addr":"172.21.15.70:6823/572290869","heartbeat_front_addr":"172.21.15.70:6821/572290869","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:03:24.051 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:23 smithi026 ceph-mon[100157]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:24.052 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:23 smithi026 ceph-mon[118879]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:24.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:23 smithi070 ceph-mon[105170]: pgmap v171: 1 pgs: 1 remapped+peering; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:24.648 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-09-23T22:01:25.159293+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-09-23T22:03:24.649 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-09-23T22:03:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:24 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/902858859' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:03:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:24 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/902858859' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:03:25.004 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:03:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:24 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/902858859' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:03:25.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:25 smithi026 ceph-mon[100157]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s, 0 objects/s recovering 2023-09-23T22:03:25.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:25 smithi026 ceph-mon[118879]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s, 0 objects/s recovering 2023-09-23T22:03:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:25 smithi070 ceph-mon[105170]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 97 KiB/s, 0 objects/s recovering 2023-09-23T22:03:26.583 INFO:teuthology.orchestra.run.smithi026.stdout:pg_num: 1 2023-09-23T22:03:26.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:26 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3889762545' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T22:03:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:26 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3889762545' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T22:03:27.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:26 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3889762545' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T22:03:27.266 INFO:tasks.cephadm:Adding prometheus.a on smithi070 2023-09-23T22:03:27.266 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi070=a' 2023-09-23T22:03:27.615 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:03:28.014 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:27 smithi070 ceph-mon[105170]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2023-09-23T22:03:28.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:27 smithi026 ceph-mon[100157]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2023-09-23T22:03:28.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:27 smithi026 ceph-mon[118879]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2023-09-23T22:03:29.333 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled prometheus update... 2023-09-23T22:03:30.030 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:29 smithi070 ceph-mon[105170]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:03:30.030 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:30.030 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:30.030 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:30.030 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:30.030 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:29 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:30.048 DEBUG:teuthology.orchestra.run.smithi070:prometheus.a> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T22:03:30.050 INFO:tasks.cephadm:Adding node-exporter.a on smithi026 2023-09-23T22:03:30.050 INFO:tasks.cephadm:Adding node-exporter.b on smithi070 2023-09-23T22:03:30.050 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi026=a;smithi070=b' 2023-09-23T22:03:30.080 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:03:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[100157]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:03:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:30.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[118879]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T22:03:30.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:29 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:30.409 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:03:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:30 smithi070 ceph-mon[105170]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:03:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:30 smithi070 ceph-mon[105170]: Saving service prometheus spec with placement smithi070=a;count:1 2023-09-23T22:03:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:30 smithi070 ceph-mon[105170]: Deploying daemon prometheus.a on smithi070 2023-09-23T22:03:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:30 smithi026 ceph-mon[100157]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:03:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:30 smithi026 ceph-mon[100157]: Saving service prometheus spec with placement smithi070=a;count:1 2023-09-23T22:03:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:30 smithi026 ceph-mon[100157]: Deploying daemon prometheus.a on smithi070 2023-09-23T22:03:31.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:30 smithi026 ceph-mon[118879]: from='client.24319 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi070=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:03:31.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:30 smithi026 ceph-mon[118879]: Saving service prometheus spec with placement smithi070=a;count:1 2023-09-23T22:03:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:30 smithi026 ceph-mon[118879]: Deploying daemon prometheus.a on smithi070 2023-09-23T22:03:32.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:31 smithi070 ceph-mon[105170]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:03:32.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:31 smithi026 ceph-mon[100157]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:03:32.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:31 smithi026 ceph-mon[118879]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T22:03:32.394 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled node-exporter update... 2023-09-23T22:03:33.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:33 smithi026 ceph-mon[100157]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi070=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:03:33.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:33 smithi026 ceph-mon[100157]: Saving service node-exporter spec with placement smithi026=a;smithi070=b;count:2 2023-09-23T22:03:33.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:33 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:33.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:33 smithi026 ceph-mon[100157]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T22:03:33.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:33 smithi026 ceph-mon[118879]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi070=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:03:33.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:33 smithi026 ceph-mon[118879]: Saving service node-exporter spec with placement smithi026=a;smithi070=b;count:2 2023-09-23T22:03:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:33 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:33 smithi026 ceph-mon[118879]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T22:03:33.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:33 smithi070 ceph-mon[105170]: from='client.24325 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi026=a;smithi070=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:03:33.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:33 smithi070 ceph-mon[105170]: Saving service node-exporter spec with placement smithi026=a;smithi070=b;count:2 2023-09-23T22:03:33.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:33 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:33.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:33 smithi070 ceph-mon[105170]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T22:03:33.909 DEBUG:teuthology.orchestra.run.smithi026:node-exporter.a> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T22:03:33.912 DEBUG:teuthology.orchestra.run.smithi070:node-exporter.b> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T22:03:33.914 INFO:tasks.cephadm:Adding alertmanager.a on smithi026 2023-09-23T22:03:33.914 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi026=a' 2023-09-23T22:03:33.943 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:03:33.949 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T22:03:36.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:35 smithi070 ceph-mon[105170]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T22:03:36.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:35 smithi026 ceph-mon[118879]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T22:03:36.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:35 smithi026 ceph-mon[100157]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T22:03:36.432 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:03:37.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:37 smithi070 ceph-mon[105170]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:38.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:37 smithi026 ceph-mon[118879]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:38.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:37 smithi026 ceph-mon[100157]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:39 smithi070 ceph-mon[105170]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:39 smithi026 ceph-mon[118879]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:39 smithi026 ceph-mon[100157]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:41.575 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled alertmanager update... 2023-09-23T22:03:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:41 smithi070 ceph-mon[105170]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:41 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:41.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:41 smithi026 ceph-mon[118879]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:41.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:41 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:41.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:41 smithi026 ceph-mon[100157]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:41.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:41 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:42.382 DEBUG:teuthology.orchestra.run.smithi026:alertmanager.a> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T22:03:42.385 INFO:tasks.cephadm:Adding grafana.a on smithi070 2023-09-23T22:03:42.385 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi070=a' 2023-09-23T22:03:42.419 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T22:03:42.623 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:42 smithi070 systemd[1]: Starting Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:03:42.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:42 smithi026 ceph-mon[100157]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:03:42.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:42 smithi026 ceph-mon[100157]: Saving service alertmanager spec with placement smithi026=a;count:1 2023-09-23T22:03:42.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:42 smithi026 ceph-mon[118879]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:03:42.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:42 smithi026 ceph-mon[118879]: Saving service alertmanager spec with placement smithi026=a;count:1 2023-09-23T22:03:43.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:42 smithi070 ceph-mon[105170]: from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi026=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:03:43.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:42 smithi070 ceph-mon[105170]: Saving service alertmanager spec with placement smithi026=a;count:1 2023-09-23T22:03:43.066 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:03:43.301 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.034Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T22:03:43.301 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.034Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T22:03:43.301 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.034Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi070 (none))" 2023-09-23T22:03:43.301 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.034Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:03:43.301 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.034Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:03:43.301 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.035Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:03:43.301 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.036Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.036Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.036Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.039Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.039Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.885µs 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.039Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.039Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.039Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.756µs wal_replay_duration=205.262µs wbl_replay_duration=89ns total_replay_duration=335.182µs 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.040Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.040Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T22:03:43.302 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.040Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:03:43.303 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.054Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.920081ms db_storage=766ns remote_storage=999ns web_handler=374ns query_engine=521ns scrape=303.832µs scrape_sd=18.411µs notify=659ns notify_sd=1.147µs rules=13.435815ms tracing=5.269µs 2023-09-23T22:03:43.303 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.054Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T22:03:43.303 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 conmon[147765]: ts=2023-09-23T22:03:43.054Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T22:03:43.303 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 bash[147647]: 08bee6665fe6bd6be2b00f74f1a167c76cfae22bd4fba3939fc4404c4ca826f1 2023-09-23T22:03:43.303 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:03:43 smithi070 systemd[1]: Started Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:03:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[100157]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:03:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[118879]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:43 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:03:44.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:43 smithi070 ceph-mon[105170]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:44.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:43 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:44.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:43 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:43 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' 2023-09-23T22:03:44.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:43 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T22:03:44.354 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:44 smithi070 conmon[108367]: ignoring --setuser ceph since I am not root 2023-09-23T22:03:44.354 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:44 smithi070 conmon[108367]: ignoring --setgroup ceph since I am not root 2023-09-23T22:03:44.354 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:44 smithi070 ceph-mgr[108391]: -- 172.21.15.70:0/3164219293 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564b7727e9c0 con 0x564b77256c00 2023-09-23T22:03:44.354 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:44 smithi070 conmon[108367]: 2023-09-23T22:03:44.343+0000 7f17557161c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:03:44.451 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:44 smithi026 conmon[100586]: ignoring --setuser ceph since I am not root 2023-09-23T22:03:44.451 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:44 smithi026 conmon[100586]: ignoring --setgroup ceph since I am not root 2023-09-23T22:03:44.451 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:44 smithi026 ceph-mgr[100610]: -- 172.21.15.26:0/612255248 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564f9b9609c0 con 0x564f9b939000 2023-09-23T22:03:44.451 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:44 smithi026 conmon[100586]: 2023-09-23T22:03:44.358+0000 7f18d76e01c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:03:44.451 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:44 smithi026 conmon[100586]: 2023-09-23T22:03:44.451+0000 7f18d76e01c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:03:44.622 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:44 smithi070 conmon[108367]: 2023-09-23T22:03:44.430+0000 7f17557161c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:03:44.622 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:44 smithi070 conmon[108367]: 2023-09-23T22:03:44.520+0000 7f17557161c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:03:44.622 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:44 smithi070 conmon[108367]: 2023-09-23T22:03:44.621+0000 7f17557161c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:03:44.734 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:44 smithi026 conmon[100586]: 2023-09-23T22:03:44.557+0000 7f18d76e01c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:03:44.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:44 smithi026 conmon[100586]: 2023-09-23T22:03:44.666+0000 7f18d76e01c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:03:44.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:44 smithi070 conmon[108367]: 2023-09-23T22:03:44.789+0000 7f17557161c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:03:45.166 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:44 smithi026 conmon[100586]: 2023-09-23T22:03:44.868+0000 7f18d76e01c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:03:45.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:45 smithi026 ceph-mon[100157]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:03:45.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:45 smithi026 ceph-mon[100157]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T22:03:45.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:45 smithi070 ceph-mon[105170]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:03:45.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:45 smithi070 ceph-mon[105170]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T22:03:45.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:45 smithi070 conmon[108367]: 2023-09-23T22:03:45.094+0000 7f17557161c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:03:45.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:45 smithi070 conmon[108367]: 2023-09-23T22:03:45.181+0000 7f17557161c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:03:45.374 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:45 smithi070 conmon[108367]: 2023-09-23T22:03:45.307+0000 7f17557161c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:03:45.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:45 smithi026 conmon[100586]: 2023-09-23T22:03:45.262+0000 7f18d76e01c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:03:45.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:45 smithi026 conmon[100586]: 2023-09-23T22:03:45.368+0000 7f18d76e01c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:03:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:45 smithi026 ceph-mon[118879]: from='mgr.14152 172.21.15.26:0/2157874174' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T22:03:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:45 smithi026 ceph-mon[118879]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T22:03:45.812 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:45 smithi070 conmon[108367]: 2023-09-23T22:03:45.399+0000 7f17557161c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:03:45.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:45 smithi026 conmon[100586]: 2023-09-23T22:03:45.517+0000 7f18d76e01c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:03:45.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:45 smithi026 conmon[100586]: 2023-09-23T22:03:45.630+0000 7f18d76e01c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:03:46.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:45 smithi070 conmon[108367]: 2023-09-23T22:03:45.812+0000 7f17557161c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:03:46.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:46 smithi026 conmon[100586]: 2023-09-23T22:03:46.136+0000 7f18d76e01c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:03:46.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:46 smithi070 conmon[108367]: 2023-09-23T22:03:46.368+0000 7f17557161c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:03:46.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:46 smithi070 conmon[108367]: 2023-09-23T22:03:46.534+0000 7f17557161c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:03:47.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:46 smithi026 conmon[100586]: 2023-09-23T22:03:46.815+0000 7f18d76e01c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:03:47.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:47 smithi026 conmon[100586]: 2023-09-23T22:03:47.005+0000 7f18d76e01c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:03:47.535 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:47 smithi070 conmon[108367]: 2023-09-23T22:03:47.282+0000 7f17557161c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:03:47.535 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:47 smithi070 conmon[108367]: 2023-09-23T22:03:47.369+0000 7f17557161c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:03:47.803 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:47 smithi070 conmon[108367]: 2023-09-23T22:03:47.535+0000 7f17557161c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:03:48.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:47 smithi070 conmon[108367]: 2023-09-23T22:03:47.803+0000 7f17557161c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:03:48.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:47 smithi070 conmon[108367]: 2023-09-23T22:03:47.888+0000 7f17557161c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:03:48.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:47 smithi026 conmon[100586]: 2023-09-23T22:03:47.961+0000 7f18d76e01c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:03:48.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:48 smithi026 conmon[100586]: 2023-09-23T22:03:48.070+0000 7f18d76e01c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:03:48.421 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:48 smithi070 conmon[108367]: 2023-09-23T22:03:48.170+0000 7f17557161c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:03:48.630 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:48 smithi026 conmon[100586]: 2023-09-23T22:03:48.285+0000 7f18d76e01c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:03:48.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:48 smithi070 conmon[108367]: 2023-09-23T22:03:48.421+0000 7f17557161c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:03:48.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:48 smithi026 conmon[100586]: 2023-09-23T22:03:48.629+0000 7f18d76e01c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:03:48.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:48 smithi026 conmon[100586]: 2023-09-23T22:03:48.734+0000 7f18d76e01c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:03:49.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:49 smithi070 conmon[108367]: 2023-09-23T22:03:49.112+0000 7f17557161c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:03:49.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:49 smithi070 conmon[108367]: 2023-09-23T22:03:49.239+0000 7f17557161c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:03:49.387 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:49 smithi026 conmon[100586]: 2023-09-23T22:03:49.064+0000 7f18d76e01c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:03:49.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:49 smithi026 conmon[100586]: 2023-09-23T22:03:49.386+0000 7f18d76e01c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:03:49.745 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:49 smithi070 conmon[108367]: 2023-09-23T22:03:49.423+0000 7f17557161c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:03:49.745 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:49 smithi070 conmon[108367]: 2023-09-23T22:03:49.534+0000 7f17557161c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:03:50.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:49 smithi070 conmon[108367]: 2023-09-23T22:03:49.745+0000 7f17557161c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:03:50.484 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:50 smithi026 conmon[100586]: 2023-09-23T22:03:50.213+0000 7f18d76e01c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:03:50.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:50 smithi026 conmon[100586]: 2023-09-23T22:03:50.370+0000 7f18d76e01c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:03:50.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:50 smithi070 conmon[108367]: 2023-09-23T22:03:50.586+0000 7f17557161c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:03:50.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:50 smithi070 conmon[108367]: [23/Sep/2023:22:03:50] ENGINE Bus STARTING 2023-09-23T22:03:50.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:50 smithi070 conmon[108367]: CherryPy Checker: 2023-09-23T22:03:50.874 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:50 smithi070 conmon[108367]: The Application mounted at '' has an empty config. 2023-09-23T22:03:50.874 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:50 smithi070 conmon[108367]: 2023-09-23T22:03:50.874 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:50 smithi070 conmon[108367]: [23/Sep/2023:22:03:50] ENGINE Serving on http://:::9283 2023-09-23T22:03:50.874 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:50 smithi070 conmon[108367]: [23/Sep/2023:22:03:50] ENGINE Bus STARTED 2023-09-23T22:03:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:50 smithi070 ceph-mon[105170]: Standby manager daemon x restarted 2023-09-23T22:03:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:50 smithi070 ceph-mon[105170]: Standby manager daemon x started 2023-09-23T22:03:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:50 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:03:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:50 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:03:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:50 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:03:50.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:50 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:03:50.984 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:50 smithi026 conmon[100586]: 2023-09-23T22:03:50.612+0000 7f18d76e01c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:03:50.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:50 smithi026 conmon[100586]: 2023-09-23T22:03:50.752+0000 7f18d76e01c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:03:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[100157]: Standby manager daemon x restarted 2023-09-23T22:03:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[100157]: Standby manager daemon x started 2023-09-23T22:03:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:03:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:03:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:03:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:03:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[118879]: Standby manager daemon x restarted 2023-09-23T22:03:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[118879]: Standby manager daemon x started 2023-09-23T22:03:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:03:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:03:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:03:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:50 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/1775865963' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:03:51.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:51 smithi026 conmon[100586]: 2023-09-23T22:03:51.021+0000 7f18d76e01c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:03:51.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:03:51 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:03:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:03:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:51 smithi070 ceph-mon[105170]: mgrmap e17: y(active, since 4m), standbys: x 2023-09-23T22:03:51.984 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:51 smithi026 ceph-mon[100157]: mgrmap e17: y(active, since 4m), standbys: x 2023-09-23T22:03:51.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:51 smithi026 ceph-mon[118879]: mgrmap e17: y(active, since 4m), standbys: x 2023-09-23T22:03:52.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:52 smithi026 conmon[100586]: 2023-09-23T22:03:52.056+0000 7f18d76e01c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:03:52.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:52 smithi026 conmon[100586]: [23/Sep/2023:22:03:52] ENGINE Bus STARTING 2023-09-23T22:03:52.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:52 smithi026 conmon[100586]: CherryPy Checker: 2023-09-23T22:03:52.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:52 smithi026 conmon[100586]: The Application mounted at '' has an empty config. 2023-09-23T22:03:52.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:52 smithi026 conmon[100586]: 2023-09-23T22:03:52.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:52 smithi026 conmon[100586]: [23/Sep/2023:22:03:52] ENGINE Serving on http://:::9283 2023-09-23T22:03:52.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:03:52 smithi026 conmon[100586]: [23/Sep/2023:22:03:52] ENGINE Bus STARTED 2023-09-23T22:03:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: Active manager daemon y restarted 2023-09-23T22:03:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: Activating manager daemon y 2023-09-23T22:03:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T22:03:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: mgrmap e18: y(active, starting, since 0.00779891s), standbys: x 2023-09-23T22:03:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:03:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: Manager daemon y is now available 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:03:52.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: Active manager daemon y restarted 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: Activating manager daemon y 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: mgrmap e18: y(active, starting, since 0.00779891s), standbys: x 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:03:52.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: Manager daemon y is now available 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:03:52.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:03:52.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:03:53.092 INFO:teuthology.orchestra.run.smithi070.stdout:Scheduled grafana update... 2023-09-23T22:03:53.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: Active manager daemon y restarted 2023-09-23T22:03:53.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: Activating manager daemon y 2023-09-23T22:03:53.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T22:03:53.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: mgrmap e18: y(active, starting, since 0.00779891s), standbys: x 2023-09-23T22:03:53.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:03:53.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:03:53.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:03:53.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:03:53.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:03:53.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:03:53.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:03:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: Manager daemon y is now available 2023-09-23T22:03:53.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:53.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:03:53.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:53.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:03:53.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:03:53.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:03:53.910 DEBUG:teuthology.orchestra.run.smithi070:grafana.a> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T22:03:53.912 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T22:03:53.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T22:03:53.944 INFO:journalctl@ceph.grafana.a.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:03:54.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[118879]: mgrmap e19: y(active, since 1.01145s), standbys: x 2023-09-23T22:03:54.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[118879]: Saving service grafana spec with placement smithi070=a;count:1 2023-09-23T22:03:54.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[118879]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:54.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[100157]: mgrmap e19: y(active, since 1.01145s), standbys: x 2023-09-23T22:03:54.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[100157]: Saving service grafana spec with placement smithi070=a;count:1 2023-09-23T22:03:54.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[100157]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:54.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.290 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:03:54.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:54 smithi070 ceph-mon[105170]: mgrmap e19: y(active, since 1.01145s), standbys: x 2023-09-23T22:03:54.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:54 smithi070 ceph-mon[105170]: Saving service grafana spec with placement smithi070=a;count:1 2023-09-23T22:03:54.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:54 smithi070 ceph-mon[105170]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:54.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:55 smithi070 ceph-mon[105170]: [23/Sep/2023:22:03:53] ENGINE Bus STARTING 2023-09-23T22:03:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:55 smithi070 ceph-mon[105170]: [23/Sep/2023:22:03:53] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T22:03:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:55 smithi070 ceph-mon[105170]: [23/Sep/2023:22:03:53] ENGINE Bus STARTED 2023-09-23T22:03:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:55 smithi070 ceph-mon[105170]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:55 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:55 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:55 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:55 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:55 smithi070 ceph-mon[105170]: mgrmap e20: y(active, since 3s), standbys: x 2023-09-23T22:03:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[118879]: [23/Sep/2023:22:03:53] ENGINE Bus STARTING 2023-09-23T22:03:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[118879]: [23/Sep/2023:22:03:53] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T22:03:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[118879]: [23/Sep/2023:22:03:53] ENGINE Bus STARTED 2023-09-23T22:03:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[118879]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[118879]: mgrmap e20: y(active, since 3s), standbys: x 2023-09-23T22:03:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[100157]: [23/Sep/2023:22:03:53] ENGINE Bus STARTING 2023-09-23T22:03:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[100157]: [23/Sep/2023:22:03:53] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T22:03:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[100157]: [23/Sep/2023:22:03:53] ENGINE Bus STARTED 2023-09-23T22:03:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[100157]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:55.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:55 smithi026 ceph-mon[100157]: mgrmap e20: y(active, since 3s), standbys: x 2023-09-23T22:03:56.047 INFO:teuthology.orchestra.run.smithi026.stdout:[client.0] 2023-09-23T22:03:56.047 INFO:teuthology.orchestra.run.smithi026.stdout: key = AQDMYA9lob+UAhAAfhi//a0o3R7pKK0H6cvdXw== 2023-09-23T22:03:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi070 to 4036M 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/625901928' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:03:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/625901928' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:03:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:03:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:03:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi070 to 4036M 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:03:56.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:03:56.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/625901928' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:03:56.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/625901928' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:03:56.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:56.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:03:56.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:56 smithi026 ceph-mon[100157]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:03:56.852 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T22:03:56.852 DEBUG:teuthology.orchestra.run.smithi026:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T22:03:56.852 DEBUG:teuthology.orchestra.run.smithi026:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T22:03:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi070 to 4036M 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:03:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/625901928' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:03:56.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/625901928' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:03:56.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:56.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:03:56.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:56 smithi070 ceph-mon[105170]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:03:56.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T22:03:57.231 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:03:57.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:57 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:57 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:57 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.782 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:57 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.782 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:57 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.782 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:57 smithi070 ceph-mon[105170]: Deploying daemon alertmanager.a on smithi026 2023-09-23T22:03:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[100157]: Deploying daemon alertmanager.a on smithi026 2023-09-23T22:03:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:03:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:57 smithi026 ceph-mon[118879]: Deploying daemon alertmanager.a on smithi026 2023-09-23T22:03:58.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:58 smithi070 ceph-mon[105170]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:58.922 INFO:teuthology.orchestra.run.smithi070.stdout:[client.1] 2023-09-23T22:03:58.922 INFO:teuthology.orchestra.run.smithi070.stdout: key = AQDOYA9lqk3JNhAAH0VPW+bg3WQmrEfyZxMErg== 2023-09-23T22:03:58.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:58 smithi026 ceph-mon[118879]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:58.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:58 smithi026 ceph-mon[100157]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:03:59.554 DEBUG:teuthology.orchestra.run.smithi070:> set -ex 2023-09-23T22:03:59.554 DEBUG:teuthology.orchestra.run.smithi070:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T22:03:59.554 DEBUG:teuthology.orchestra.run.smithi070:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T22:03:59.595 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T22:03:59.596 DEBUG:teuthology.orchestra.run.smithi026:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T22:03:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:59 smithi070 ceph-mon[105170]: from='client.? 172.21.15.70:0/1153344160' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:03:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:59 smithi070 ceph-mon[105170]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:03:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:03:59 smithi070 ceph-mon[105170]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:03:59.943 INFO:teuthology.misc.health.smithi026.stdout: 2023-09-23T22:03:59.943 INFO:teuthology.misc.health.smithi026.stdout:{"epoch":59,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:03:52.071232+0000","last_up_change":"2023-09-23T22:03:17.723571+0000","last_in_change":"2023-09-23T22:03:00.614896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6803","nonce":3939081623}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6805","nonce":3939081623}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6809","nonce":3939081623}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6807","nonce":3939081623}]},"public_addr":"172.21.15.26:6803/3939081623","cluster_addr":"172.21.15.26:6805/3939081623","heartbeat_back_addr":"172.21.15.26:6809/3939081623","heartbeat_front_addr":"172.21.15.26:6807/3939081623","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6811","nonce":2870847047}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6813","nonce":2870847047}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6817","nonce":2870847047}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6815","nonce":2870847047}]},"public_addr":"172.21.15.26:6811/2870847047","cluster_addr":"172.21.15.26:6813/2870847047","heartbeat_back_addr":"172.21.15.26:6817/2870847047","heartbeat_front_addr":"172.21.15.26:6815/2870847047","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6819","nonce":3354962178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6821","nonce":3354962178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6825","nonce":3354962178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6823","nonce":3354962178}]},"public_addr":"172.21.15.26:6819/3354962178","cluster_addr":"172.21.15.26:6821/3354962178","heartbeat_back_addr":"172.21.15.26:6825/3354962178","heartbeat_front_addr":"172.21.15.26:6823/3354962178","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6827","nonce":861409764}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6829","nonce":861409764}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6833","nonce":861409764}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6831","nonce":861409764}]},"public_addr":"172.21.15.26:6827/861409764","cluster_addr":"172.21.15.26:6829/861409764","heartbeat_back_addr":"172.21.15.26:6833/861409764","heartbeat_front_addr":"172.21.15.26:6831/861409764","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6801","nonce":1461751884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6803","nonce":1461751884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6807","nonce":1461751884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6805","nonce":1461751884}]},"public_addr":"172.21.15.70:6801/1461751884","cluster_addr":"172.21.15.70:6803/1461751884","heartbeat_back_addr":"172.21.15.70:6807/1461751884","heartbeat_front_addr":"172.21.15.70:6805/1461751884","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":49,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6817","nonce":572290869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6819","nonce":572290869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6823","nonce":572290869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6821","nonce":572290869}]},"public_addr":"172.21.15.70:6817/572290869","cluster_addr":"172.21.15.70:6819/572290869","heartbeat_back_addr":"172.21.15.70:6823/572290869","heartbeat_front_addr":"172.21.15.70:6821/572290869","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6800/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3350176691":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3419561437":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/577898280":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/1887156644":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:03:59.956 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T22:03:59.956 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T22:03:59.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T22:03:59.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:59 smithi026 ceph-mon[118879]: from='client.? 172.21.15.70:0/1153344160' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:03:59.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:59 smithi026 ceph-mon[118879]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:03:59.981 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:03:59 smithi026 ceph-mon[118879]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:03:59.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:59 smithi026 ceph-mon[100157]: from='client.? 172.21.15.70:0/1153344160' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:03:59.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:59 smithi026 ceph-mon[100157]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T22:03:59.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:03:59 smithi026 ceph-mon[100157]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T22:04:00.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:00 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3374653877' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:04:00.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:00 smithi070 ceph-mon[105170]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:00.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:00 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3374653877' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:04:00.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:00 smithi026 ceph-mon[118879]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:00.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:00 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3374653877' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:04:00.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:00 smithi026 ceph-mon[100157]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:01.222 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:01.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:04:01 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:04:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:04:03.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:03 smithi026 ceph-mon[118879]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:03.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:03 smithi026 ceph-mon[100157]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:03 smithi070 ceph-mon[105170]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:05.155 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'rbd' created 2023-09-23T22:04:05.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:05 smithi026 ceph-mon[100157]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:05.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:05 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2584612407' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:04:05.311 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:05 smithi026 ceph-mon[100157]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:04:05.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:05 smithi026 ceph-mon[118879]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:05.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:05 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2584612407' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:04:05.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:05 smithi026 ceph-mon[118879]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:04:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:05 smithi070 ceph-mon[105170]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:05 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2584612407' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:04:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:05 smithi070 ceph-mon[105170]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T22:04:05.750 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T22:04:06.241 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:06.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:06 smithi026 ceph-mon[118879]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T22:04:06.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:06 smithi026 ceph-mon[118879]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T22:04:06.436 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:06 smithi026 ceph-mon[100157]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T22:04:06.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:06 smithi026 ceph-mon[100157]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T22:04:06.437 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:06 smithi026 systemd[1]: Started Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:04:06.437 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:06 smithi026 conmon[164633]: ts=2023-09-23T22:04:06.163Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T22:04:06.437 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:06 smithi026 conmon[164633]: ts=2023-09-23T22:04:06.163Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T22:04:06.437 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:06 smithi026 conmon[164633]: ts=2023-09-23T22:04:06.163Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2023-09-23T22:04:06.437 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:06 smithi026 conmon[164633]: ts=2023-09-23T22:04:06.165Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T22:04:06.437 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:06 smithi026 conmon[164633]: ts=2023-09-23T22:04:06.200Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:04:06.437 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:06 smithi026 conmon[164633]: ts=2023-09-23T22:04:06.200Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:04:06.437 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:06 smithi026 conmon[164633]: ts=2023-09-23T22:04:06.203Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T22:04:06.437 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:06 smithi026 conmon[164633]: ts=2023-09-23T22:04:06.203Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T22:04:06.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:06 smithi070 ceph-mon[105170]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T22:04:06.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:06 smithi070 ceph-mon[105170]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T22:04:07.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[118879]: pgmap v11: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:07.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[118879]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T22:04:07.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.442 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[100157]: pgmap v11: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:07.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[100157]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T22:04:07.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.443 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:07 smithi070 ceph-mon[105170]: pgmap v11: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:07.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:07 smithi070 ceph-mon[105170]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T22:04:07.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:07.735 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:07 smithi026 bash[165349]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T22:04:08.167 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T22:04:08.485 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:08 smithi026 bash[165349]: Getting image source signatures 2023-09-23T22:04:08.485 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:08 smithi026 conmon[164633]: ts=2023-09-23T22:04:08.165Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000897618s 2023-09-23T22:04:08.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[118879]: Deploying daemon node-exporter.a on smithi026 2023-09-23T22:04:08.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:08.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[118879]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T22:04:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/924147569' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:04:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[100157]: Deploying daemon node-exporter.a on smithi026 2023-09-23T22:04:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[100157]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T22:04:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:08 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/924147569' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:04:08.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:08 smithi070 ceph-mon[105170]: Deploying daemon node-exporter.a on smithi026 2023-09-23T22:04:08.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:08 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:08.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:08 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:08.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:08 smithi070 ceph-mon[105170]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T22:04:08.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:08 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/924147569' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:04:08.935 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T22:04:08.936 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T22:04:08.936 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T22:04:09.162 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:08 smithi026 bash[165349]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T22:04:09.163 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:08 smithi026 bash[165349]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T22:04:09.163 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:08 smithi026 bash[165349]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T22:04:09.299 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:09.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:09 smithi026 ceph-mon[100157]: pgmap v14: 9 pgs: 5 active+clean, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:09.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:09 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/924147569' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:04:09.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:09 smithi026 ceph-mon[100157]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T22:04:09.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:09 smithi026 ceph-mon[118879]: pgmap v14: 9 pgs: 5 active+clean, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:09.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:09 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/924147569' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:04:09.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:09 smithi026 ceph-mon[118879]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T22:04:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:09 smithi070 ceph-mon[105170]: pgmap v14: 9 pgs: 5 active+clean, 4 unknown; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:09 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/924147569' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:04:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:09 smithi070 ceph-mon[105170]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T22:04:10.235 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:10 smithi026 bash[165349]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T22:04:10.735 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:10 smithi026 bash[165349]: Writing manifest to image destination 2023-09-23T22:04:10.735 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:10 smithi026 bash[165349]: Storing signatures 2023-09-23T22:04:11.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:11 smithi026 ceph-mon[118879]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:11.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:11 smithi026 ceph-mon[100157]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:11.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:11 smithi070 ceph-mon[105170]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:11.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:04:11 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:04:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:04:11.985 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.778Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T22:04:11.985 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.778Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T22:04:11.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T22:04:11.987 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T22:04:11.988 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.779Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T22:04:11.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T22:04:11.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T22:04:11.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T22:04:11.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T22:04:11.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T22:04:11.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T22:04:11.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 conmon[165825]: ts=2023-09-23T22:04:11.780Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T22:04:11.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 bash[165349]: 89b44a6ca08ed7ec41a741f146d68f6edaa46d936211c90f9d39323054583702 2023-09-23T22:04:11.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:04:11 smithi026 systemd[1]: Started Ceph node-exporter.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:04:12.236 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:04:12.624 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:12 smithi070 systemd[1]: Starting Ceph node-exporter.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:04:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[100157]: Deploying daemon node-exporter.b on smithi070 2023-09-23T22:04:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[100157]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2481486767' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:04:12.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:12.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:12.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:12.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[118879]: Deploying daemon node-exporter.b on smithi070 2023-09-23T22:04:12.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[118879]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:12.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:12 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2481486767' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:04:13.029 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":20,"active_gid":24341,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6800","nonce":3418308803},{"type":"v1","addr":"172.21.15.26:6801","nonce":3418308803}]},"active_addr":"172.21.15.26:6801/3418308803","active_change":"2023-09-23T22:03:52.071373+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24344,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":3996051568}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":771693589}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":3618866901}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":1692711239}]}]} 2023-09-23T22:04:13.033 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T22:04:13.033 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T22:04:13.033 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:04:13.123 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:12 smithi070 bash[150793]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T22:04:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:12 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:12 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:12 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:12 smithi070 ceph-mon[105170]: Deploying daemon node-exporter.b on smithi070 2023-09-23T22:04:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:12 smithi070 ceph-mon[105170]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:12 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2481486767' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:04:13.399 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:13.692 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:13 smithi070 bash[150793]: Getting image source signatures 2023-09-23T22:04:13.693 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:13 smithi070 bash[150793]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T22:04:13.693 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:13 smithi070 bash[150793]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T22:04:13.693 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:13 smithi070 bash[150793]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T22:04:14.373 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:14 smithi070 bash[150793]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T22:04:14.373 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:14 smithi070 bash[150793]: Writing manifest to image destination 2023-09-23T22:04:14.373 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:14 smithi070 bash[150793]: Storing signatures 2023-09-23T22:04:14.928 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:04:14.928 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":63,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:04:08.158529+0000","last_up_change":"2023-09-23T22:03:17.723571+0000","last_in_change":"2023-09-23T22:03:00.614896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:04:04.880309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6803","nonce":3939081623}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6805","nonce":3939081623}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6809","nonce":3939081623}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6807","nonce":3939081623}]},"public_addr":"172.21.15.26:6803/3939081623","cluster_addr":"172.21.15.26:6805/3939081623","heartbeat_back_addr":"172.21.15.26:6809/3939081623","heartbeat_front_addr":"172.21.15.26:6807/3939081623","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6811","nonce":2870847047}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6813","nonce":2870847047}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6817","nonce":2870847047}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6815","nonce":2870847047}]},"public_addr":"172.21.15.26:6811/2870847047","cluster_addr":"172.21.15.26:6813/2870847047","heartbeat_back_addr":"172.21.15.26:6817/2870847047","heartbeat_front_addr":"172.21.15.26:6815/2870847047","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6819","nonce":3354962178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6821","nonce":3354962178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6825","nonce":3354962178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6823","nonce":3354962178}]},"public_addr":"172.21.15.26:6819/3354962178","cluster_addr":"172.21.15.26:6821/3354962178","heartbeat_back_addr":"172.21.15.26:6825/3354962178","heartbeat_front_addr":"172.21.15.26:6823/3354962178","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6827","nonce":861409764}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6829","nonce":861409764}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6833","nonce":861409764}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6831","nonce":861409764}]},"public_addr":"172.21.15.26:6827/861409764","cluster_addr":"172.21.15.26:6829/861409764","heartbeat_back_addr":"172.21.15.26:6833/861409764","heartbeat_front_addr":"172.21.15.26:6831/861409764","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6801","nonce":1461751884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6803","nonce":1461751884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6807","nonce":1461751884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6805","nonce":1461751884}]},"public_addr":"172.21.15.70:6801/1461751884","cluster_addr":"172.21.15.70:6803/1461751884","heartbeat_back_addr":"172.21.15.70:6807/1461751884","heartbeat_front_addr":"172.21.15.70:6805/1461751884","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6817","nonce":572290869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6819","nonce":572290869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6823","nonce":572290869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6821","nonce":572290869}]},"public_addr":"172.21.15.70:6817/572290869","cluster_addr":"172.21.15.70:6819/572290869","heartbeat_back_addr":"172.21.15.70:6823/572290869","heartbeat_front_addr":"172.21.15.70:6821/572290869","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6800/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3350176691":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3419561437":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/577898280":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/1887156644":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:04:15.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:15 smithi026 ceph-mon[100157]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:15.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:15 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2012751443' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:04:15.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:15 smithi026 ceph-mon[118879]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:15.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:15 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2012751443' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:04:15.465 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T22:04:15.465 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:04:15.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:15 smithi070 ceph-mon[105170]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:15.487 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:15 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2012751443' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:04:15.839 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:15.874 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.487Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T22:04:15.874 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.487Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T22:04:15.874 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T22:04:15.874 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T22:04:15.874 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T22:04:15.874 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:04:15.874 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T22:04:15.874 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T22:04:15.874 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T22:04:15.875 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T22:04:15.876 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T22:04:15.877 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T22:04:15.878 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T22:04:15.878 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.488Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T22:04:15.878 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.489Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T22:04:15.878 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 conmon[150869]: ts=2023-09-23T22:04:15.489Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T22:04:15.878 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 bash[150793]: 6bac96aa7b4fc11de3ee9c9b18ae239e0aa3efc43233740cb39f8f91fe9e3b02 2023-09-23T22:04:15.878 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:04:15 smithi070 systemd[1]: Started Ceph node-exporter.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:04:15.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:15 smithi026 conmon[100586]: [23/Sep/2023:22:04:15] ENGINE Bus STOPPING 2023-09-23T22:04:15.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:15 smithi026 conmon[100586]: [23/Sep/2023:22:04:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:04:15.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:15 smithi026 conmon[100586]: [23/Sep/2023:22:04:15] ENGINE Bus STOPPED 2023-09-23T22:04:15.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:15 smithi026 conmon[100586]: [23/Sep/2023:22:04:15] ENGINE Bus STARTING 2023-09-23T22:04:15.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:15 smithi026 conmon[100586]: [23/Sep/2023:22:04:15] ENGINE Serving on http://:::9283 2023-09-23T22:04:15.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:15 smithi026 conmon[100586]: [23/Sep/2023:22:04:15] ENGINE Bus STARTED 2023-09-23T22:04:16.485 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:16 smithi026 conmon[164633]: ts=2023-09-23T22:04:16.168Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003550602s 2023-09-23T22:04:16.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: Deploying daemon grafana.a on smithi070 2023-09-23T22:04:16.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[100157]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: Deploying daemon grafana.a on smithi070 2023-09-23T22:04:16.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:16 smithi026 ceph-mon[118879]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:04:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T22:04:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: Deploying daemon grafana.a on smithi070 2023-09-23T22:04:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:16 smithi070 ceph-mon[105170]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:17.502 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:04:17.503 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":63,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:04:08.158529+0000","last_up_change":"2023-09-23T22:03:17.723571+0000","last_in_change":"2023-09-23T22:03:00.614896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:04:04.880309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6803","nonce":3939081623}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6805","nonce":3939081623}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6809","nonce":3939081623}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6807","nonce":3939081623}]},"public_addr":"172.21.15.26:6803/3939081623","cluster_addr":"172.21.15.26:6805/3939081623","heartbeat_back_addr":"172.21.15.26:6809/3939081623","heartbeat_front_addr":"172.21.15.26:6807/3939081623","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6811","nonce":2870847047}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6813","nonce":2870847047}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6817","nonce":2870847047}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6815","nonce":2870847047}]},"public_addr":"172.21.15.26:6811/2870847047","cluster_addr":"172.21.15.26:6813/2870847047","heartbeat_back_addr":"172.21.15.26:6817/2870847047","heartbeat_front_addr":"172.21.15.26:6815/2870847047","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6819","nonce":3354962178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6821","nonce":3354962178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6825","nonce":3354962178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6823","nonce":3354962178}]},"public_addr":"172.21.15.26:6819/3354962178","cluster_addr":"172.21.15.26:6821/3354962178","heartbeat_back_addr":"172.21.15.26:6825/3354962178","heartbeat_front_addr":"172.21.15.26:6823/3354962178","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6827","nonce":861409764}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6829","nonce":861409764}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6833","nonce":861409764}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6831","nonce":861409764}]},"public_addr":"172.21.15.26:6827/861409764","cluster_addr":"172.21.15.26:6829/861409764","heartbeat_back_addr":"172.21.15.26:6833/861409764","heartbeat_front_addr":"172.21.15.26:6831/861409764","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6801","nonce":1461751884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6803","nonce":1461751884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6807","nonce":1461751884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6805","nonce":1461751884}]},"public_addr":"172.21.15.70:6801/1461751884","cluster_addr":"172.21.15.70:6803/1461751884","heartbeat_back_addr":"172.21.15.70:6807/1461751884","heartbeat_front_addr":"172.21.15.70:6805/1461751884","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6817","nonce":572290869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6819","nonce":572290869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6823","nonce":572290869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6821","nonce":572290869}]},"public_addr":"172.21.15.70:6817/572290869","cluster_addr":"172.21.15.70:6819/572290869","heartbeat_back_addr":"172.21.15.70:6823/572290869","heartbeat_front_addr":"172.21.15.70:6821/572290869","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6800/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3350176691":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3419561437":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/577898280":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/1887156644":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:04:18.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T22:04:18.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T22:04:18.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T22:04:18.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T22:04:18.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T22:04:18.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T22:04:18.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T22:04:18.060 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T22:04:18.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:18 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:18.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:18 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1668416817' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:04:18.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:18 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:18.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:18 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1668416817' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:04:18.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:18 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:18.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:18 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1668416817' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:04:18.689 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:18.872 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:18.923 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:19.192 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:19 smithi026 ceph-mon[100157]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:19.217 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:19.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:19 smithi026 ceph-mon[118879]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:19.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:19 smithi070 ceph-mon[105170]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:20.861 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:20.962 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:21.042 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:21.183 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:21.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:21 smithi026 ceph-mon[118879]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:21.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:21 smithi026 ceph-mon[100157]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:21.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:21 smithi070 ceph-mon[105170]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:21.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:04:21 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:04:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:04:22.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:22 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:22.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:22 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:22.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:22 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:22.655 INFO:teuthology.orchestra.run.smithi026.stdout:77309411365 2023-09-23T22:04:22.655 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T22:04:23.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:23 smithi026 ceph-mon[100157]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:23.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:23 smithi026 ceph-mon[118879]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:23 smithi070 ceph-mon[105170]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:24.083 INFO:teuthology.orchestra.run.smithi026.stdout:107374182432 2023-09-23T22:04:24.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T22:04:24.912 INFO:teuthology.orchestra.run.smithi026.stdout:55834574889 2023-09-23T22:04:24.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T22:04:25.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:25 smithi026 ceph-mon[118879]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:25.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:25 smithi026 ceph-mon[100157]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:25 smithi070 ceph-mon[105170]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:26 smithi026 ceph-mon[118879]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:26.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:26 smithi026 ceph-mon[100157]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:27.115 INFO:teuthology.orchestra.run.smithi026.stdout:231928233999 2023-09-23T22:04:27.115 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T22:04:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:26 smithi070 ceph-mon[105170]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:27.507 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:27.538 INFO:teuthology.orchestra.run.smithi026.stdout:197568495635 2023-09-23T22:04:27.539 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T22:04:28.276 INFO:teuthology.orchestra.run.smithi026.stdout:34359738416 2023-09-23T22:04:28.277 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T22:04:28.558 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:28.619 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:29.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:29 smithi070 ceph-mon[105170]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:29.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:29 smithi026 ceph-mon[118879]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:29.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:29 smithi026 ceph-mon[100157]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:30.301 INFO:teuthology.orchestra.run.smithi026.stdout:163208757272 2023-09-23T22:04:30.301 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T22:04:30.344 INFO:teuthology.orchestra.run.smithi026.stdout:128849018909 2023-09-23T22:04:30.345 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T22:04:30.888 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:31 smithi026 ceph-mon[100157]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:31 smithi026 ceph-mon[118879]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:31.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:31 smithi070 ceph-mon[105170]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:31.804 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:31.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:04:31 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:04:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:04:31.957 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:32.157 INFO:teuthology.orchestra.run.smithi026.stdout:77309411367 2023-09-23T22:04:32.200 INFO:teuthology.orchestra.run.smithi026.stdout:107374182433 2023-09-23T22:04:33.408 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:33 smithi026 ceph-mon[118879]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:33.408 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:33 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2290011174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:04:33.408 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:33 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/395512662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:04:33.408 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:33 smithi026 ceph-mon[100157]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:33.408 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:33 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2290011174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:04:33.409 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:33 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/395512662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:04:33.506 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411365 got 77309411367 for osd.2 2023-09-23T22:04:33.506 DEBUG:teuthology.parallel:result is None 2023-09-23T22:04:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:33 smithi070 ceph-mon[105170]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:33 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2290011174' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:04:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:33 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/395512662' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:04:34.643 INFO:teuthology.orchestra.run.smithi026.stdout:55834574892 2023-09-23T22:04:35.287 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:35.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:35 smithi026 ceph-mon[118879]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:35.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:35 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1552848795' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:04:35.315 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182432 got 107374182433 for osd.3 2023-09-23T22:04:35.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:35 smithi026 ceph-mon[100157]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:35.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:35 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1552848795' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:04:35.315 DEBUG:teuthology.parallel:result is None 2023-09-23T22:04:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:35 smithi070 ceph-mon[105170]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:35 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1552848795' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:04:35.654 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574892 for osd.1 2023-09-23T22:04:35.654 DEBUG:teuthology.parallel:result is None 2023-09-23T22:04:35.876 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:36.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:35 smithi070 systemd[1]: Starting Ceph grafana.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:04:36.147 INFO:teuthology.orchestra.run.smithi026.stdout:197568495637 2023-09-23T22:04:36.461 INFO:teuthology.orchestra.run.smithi026.stdout:231928234001 2023-09-23T22:04:36.819 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:36 smithi070 ceph-mon[105170]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:36.820 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:36 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1784682775' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:04:36.820 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:36 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2626176398' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:04:36.820 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:36 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.820 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:36 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.820 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:36 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.820 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:36 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.821 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 bash[151544]: d197cdcf40c153ee431dd38f6a3aa551d7173992efec9236b006407c8c4f4169 2023-09-23T22:04:36.821 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 systemd[1]: Started Ceph grafana.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:04:36.821 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.664797177Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T22:04:36.821 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.664987294Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T22:04:36.821 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.665006089Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T22:04:36.821 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.665013088Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T22:04:36.821 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.665019094Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T22:04:36.822 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.66502499Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T22:04:36.822 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.665030899Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T22:04:36.822 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.665036719Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T22:04:36.822 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.665045176Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T22:04:36.822 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.665050929Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T22:04:36.822 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.66505638Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T22:04:36.822 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.665062299Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T22:04:36.822 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.665068006Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T22:04:36.822 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=settings t=2023-09-23T22:04:36.66507363Z level=info msg="App mode production" 2023-09-23T22:04:36.823 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=sqlstore t=2023-09-23T22:04:36.665120266Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T22:04:36.823 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=sqlstore t=2023-09-23T22:04:36.665135927Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T22:04:36.823 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:36.665980664Z level=info msg="Starting DB migrations" 2023-09-23T22:04:36.823 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:36.666053054Z level=info msg="Executing migration" id="create migration_log table" 2023-09-23T22:04:36.823 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:36.711177483Z level=info msg="Executing migration" id="create user table" 2023-09-23T22:04:36.823 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:36.75304069Z level=info msg="Executing migration" id="add unique index user.login" 2023-09-23T22:04:36.823 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:36.820064318Z level=info msg="Executing migration" id="add unique index user.email" 2023-09-23T22:04:36.870 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[118879]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:36.870 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1784682775' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:04:36.870 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2626176398' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:04:36.870 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.870 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[100157]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:36.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1784682775' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:04:36.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2626176398' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:04:36.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.871 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:36.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:36 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:37.071 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:36.870643591Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-09-23T22:04:37.072 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:36.912490002Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-09-23T22:04:37.072 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:36.937628005Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-09-23T22:04:37.072 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:36 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:36.962838506Z level=info msg="Executing migration" id="create user table v2" 2023-09-23T22:04:37.072 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.004782483Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-09-23T22:04:37.072 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.046921761Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-09-23T22:04:37.335 INFO:teuthology.orchestra.run.smithi026.stdout:34359738418 2023-09-23T22:04:37.363 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.072244725Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-09-23T22:04:37.363 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.114141969Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-09-23T22:04:37.363 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.15611503Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-09-23T22:04:37.363 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.19793232Z level=info msg="Executing migration" id="Update user table charset" 2023-09-23T22:04:37.363 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.231563551Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-09-23T22:04:37.363 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.248442834Z level=info msg="Executing migration" id="Add missing user data" 2023-09-23T22:04:37.363 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.282032279Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-09-23T22:04:37.363 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.307486297Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-09-23T22:04:37.363 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.36372631Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-09-23T22:04:37.477 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495635 got 197568495637 for osd.6 2023-09-23T22:04:37.478 DEBUG:teuthology.parallel:result is None 2023-09-23T22:04:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.439217003Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-09-23T22:04:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.48322566Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-09-23T22:04:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.525315732Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-09-23T22:04:37.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.600746866Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T22:04:37.927 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:37 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:04:37.927 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:37 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:37.928 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:37 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:37.928 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:37 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/391259475' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:04:37.928 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.680519866Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-09-23T22:04:37.928 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.738708683Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-09-23T22:04:37.928 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.793589503Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-09-23T22:04:37.928 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.835476626Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-09-23T22:04:37.928 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.860731244Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-09-23T22:04:37.928 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.885789339Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-09-23T22:04:37.928 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.927752845Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-09-23T22:04:37.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:37 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:04:37.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:37 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:37.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:37 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:37.938 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:37 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/391259475' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:04:37.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:37 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:04:37.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:37 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:37.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:37 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:37.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:37 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/391259475' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:04:38.188 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.953121979Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T22:04:38.188 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.969983507Z level=info msg="Executing migration" id="create temp_user v2" 2023-09-23T22:04:38.188 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:37 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:37.995181117Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-09-23T22:04:38.188 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.020461795Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-09-23T22:04:38.188 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.04572451Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-09-23T22:04:38.188 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.087425053Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-09-23T22:04:38.188 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.121158607Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-09-23T22:04:38.188 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.163053915Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-09-23T22:04:38.447 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.188590576Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T22:04:38.448 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.230784066Z level=info msg="Executing migration" id="create star table" 2023-09-23T22:04:38.448 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.28102553Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-09-23T22:04:38.448 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.392696993Z level=info msg="Executing migration" id="create org table v1" 2023-09-23T22:04:38.708 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:38 smithi070 ceph-mon[105170]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:38.708 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.448501347Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-09-23T22:04:38.708 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.532603104Z level=info msg="Executing migration" id="create org_user table v1" 2023-09-23T22:04:38.708 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.592471747Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-09-23T22:04:38.708 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.649844123Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T22:04:38.936 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:38 smithi026 ceph-mon[118879]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:38.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:38 smithi026 ceph-mon[100157]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:38.954 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928233999 got 231928234001 for osd.7 2023-09-23T22:04:38.954 DEBUG:teuthology.parallel:result is None 2023-09-23T22:04:38.976 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.708510249Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-09-23T22:04:38.976 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.750495052Z level=info msg="Executing migration" id="Update org table charset" 2023-09-23T22:04:38.976 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.775738187Z level=info msg="Executing migration" id="Update org_user table charset" 2023-09-23T22:04:38.976 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.817575002Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-09-23T22:04:38.976 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.84287986Z level=info msg="Executing migration" id="create dashboard table" 2023-09-23T22:04:38.976 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.859633337Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-09-23T22:04:38.976 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.913259736Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-09-23T22:04:38.997 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738418 for osd.0 2023-09-23T22:04:38.997 DEBUG:teuthology.parallel:result is None 2023-09-23T22:04:39.278 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:38 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:38.976912969Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-09-23T22:04:39.278 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.002376259Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T22:04:39.278 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.052788288Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T22:04:39.278 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.13644516Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T22:04:39.278 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.205083289Z level=info msg="Executing migration" id="create dashboard v2" 2023-09-23T22:04:39.278 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.27889326Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-09-23T22:04:39.443 INFO:teuthology.orchestra.run.smithi026.stdout:128849018911 2023-09-23T22:04:39.582 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.31272104Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T22:04:39.582 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.33791812Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-09-23T22:04:39.582 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.379733154Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-09-23T22:04:39.582 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.405205933Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-09-23T22:04:39.582 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.447079673Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-09-23T22:04:39.582 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.504923143Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-09-23T22:04:39.582 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.582841245Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-09-23T22:04:39.662 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:39 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3735312365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:04:39.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:39 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3735312365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:04:39.844 INFO:teuthology.orchestra.run.smithi026.stdout:163208757275 2023-09-23T22:04:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:39 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3735312365' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:04:39.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.598140533Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-09-23T22:04:39.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.631655059Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-09-23T22:04:39.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.715635534Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-09-23T22:04:39.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.7408951Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-09-23T22:04:39.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.766077341Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-09-23T22:04:39.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.824684659Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-09-23T22:04:40.160 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.902484143Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-09-23T22:04:40.160 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.959114543Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-09-23T22:04:40.160 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:39 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:39.992717953Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-09-23T22:04:40.160 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.009351629Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-09-23T22:04:40.161 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.04297342Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-09-23T22:04:40.161 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.060025351Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-09-23T22:04:40.161 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.076800108Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-09-23T22:04:40.161 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.093630276Z level=info msg="Executing migration" id="Update dashboard title length" 2023-09-23T22:04:40.161 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.110406088Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T22:04:40.161 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.127316298Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-09-23T22:04:40.161 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.161078166Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T22:04:40.429 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.186169479Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.20308763Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.220149771Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.236705489Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.253671675Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.2871517Z level=info msg="Executing migration" id="Add check_sum column" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.304007Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.320814625Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.337889706Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.354459954Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-09-23T22:04:40.430 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.371371299Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-09-23T22:04:40.431 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.396996895Z level=info msg="Executing migration" id="create data_source table" 2023-09-23T22:04:40.431 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.430350571Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-09-23T22:04:40.682 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:40 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2175750433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:04:40.682 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:40 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:40.682 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:40 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:40.682 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:40 smithi070 ceph-mon[105170]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:40.682 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.447471088Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-09-23T22:04:40.682 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.464294182Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-09-23T22:04:40.683 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.48113035Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T22:04:40.683 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.49797656Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-09-23T22:04:40.683 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.531467626Z level=info msg="Executing migration" id="create data_source table v2" 2023-09-23T22:04:40.683 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.565221403Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-09-23T22:04:40.683 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.58205615Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-09-23T22:04:40.683 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.649047276Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-09-23T22:04:40.683 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.682605062Z level=info msg="Executing migration" id="Add column with_credentials" 2023-09-23T22:04:40.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:40 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2175750433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:04:40.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:40 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:40.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:40 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:40.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:40 smithi026 ceph-mon[100157]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:40.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:40 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2175750433' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:04:40.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:40 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:40.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:40 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:40.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:40 smithi026 ceph-mon[118879]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:40.721 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018909 got 128849018911 for osd.4 2023-09-23T22:04:40.721 DEBUG:teuthology.parallel:result is None 2023-09-23T22:04:40.870 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757272 got 163208757275 for osd.5 2023-09-23T22:04:40.870 DEBUG:teuthology.parallel:result is None 2023-09-23T22:04:40.870 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:04:40.871 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:04:40.933 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.724542827Z level=info msg="Executing migration" id="Add secure json data column" 2023-09-23T22:04:40.933 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.749876566Z level=info msg="Executing migration" id="Update data_source table charset" 2023-09-23T22:04:40.933 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.791674127Z level=info msg="Executing migration" id="Update initial version to 1" 2023-09-23T22:04:40.933 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.833560867Z level=info msg="Executing migration" id="Add read_only data column" 2023-09-23T22:04:40.934 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.858935599Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-09-23T22:04:40.934 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.892300057Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-09-23T22:04:40.934 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.934252621Z level=info msg="Executing migration" id="Add uid column" 2023-09-23T22:04:41.193 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.96792299Z level=info msg="Executing migration" id="Update uid value" 2023-09-23T22:04:41.193 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:40 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:40.993124524Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-09-23T22:04:41.194 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.035093638Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-09-23T22:04:41.194 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.060245406Z level=info msg="Executing migration" id="create api_key table" 2023-09-23T22:04:41.194 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.085240433Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-09-23T22:04:41.194 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.110509303Z level=info msg="Executing migration" id="add index api_key.key" 2023-09-23T22:04:41.194 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.152559403Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-09-23T22:04:41.194 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.194332809Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-09-23T22:04:41.314 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:41.445 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.228001306Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-09-23T22:04:41.445 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.269779957Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T22:04:41.445 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.295161488Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-09-23T22:04:41.445 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.336969192Z level=info msg="Executing migration" id="create api_key table v2" 2023-09-23T22:04:41.445 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.362011057Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-09-23T22:04:41.713 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:04:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:04:41.714 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.445705858Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-09-23T22:04:41.714 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.479315606Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-09-23T22:04:41.714 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.521079588Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-09-23T22:04:41.714 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.546263344Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-09-23T22:04:41.714 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.588382096Z level=info msg="Executing migration" id="Update api_key table charset" 2023-09-23T22:04:41.714 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.622142875Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-09-23T22:04:41.714 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.647265481Z level=info msg="Executing migration" id="Add service account foreign key" 2023-09-23T22:04:41.714 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.68917002Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-09-23T22:04:41.714 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.714246096Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-09-23T22:04:41.973 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.756528868Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-09-23T22:04:41.974 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.78176401Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-09-23T22:04:41.974 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.823403218Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-09-23T22:04:41.974 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.848616425Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-09-23T22:04:41.974 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.873990292Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T22:04:41.974 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.899091102Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T22:04:41.974 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.92435139Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T22:04:41.974 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.949274509Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T22:04:41.974 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:41 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:41.974573752Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-09-23T22:04:42.259 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.025256702Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T22:04:42.259 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.050315739Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-09-23T22:04:42.259 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.075359669Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T22:04:42.259 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.100623887Z level=info msg="Executing migration" id="create quota table v1" 2023-09-23T22:04:42.259 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.125897699Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T22:04:42.259 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.192629644Z level=info msg="Executing migration" id="Update quota table charset" 2023-09-23T22:04:42.260 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.217880806Z level=info msg="Executing migration" id="create plugin_setting table" 2023-09-23T22:04:42.260 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.259871158Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T22:04:42.519 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.302043887Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-09-23T22:04:42.520 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.326993149Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-09-23T22:04:42.520 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.343856129Z level=info msg="Executing migration" id="create session table" 2023-09-23T22:04:42.520 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.369215373Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-09-23T22:04:42.520 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.410998146Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-09-23T22:04:42.520 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.452932667Z level=info msg="Executing migration" id="create playlist table v2" 2023-09-23T22:04:42.520 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.4697458Z level=info msg="Executing migration" id="create playlist item table v2" 2023-09-23T22:04:42.520 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.486534072Z level=info msg="Executing migration" id="Update playlist table charset" 2023-09-23T22:04:42.520 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.520260169Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-09-23T22:04:42.780 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.553913944Z level=info msg="Executing migration" id="drop preferences table v2" 2023-09-23T22:04:42.780 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.570848602Z level=info msg="Executing migration" id="drop preferences table v3" 2023-09-23T22:04:42.780 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.595959633Z level=info msg="Executing migration" id="create preferences table v3" 2023-09-23T22:04:42.781 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.621141497Z level=info msg="Executing migration" id="Update preferences table charset" 2023-09-23T22:04:42.781 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.66294247Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-09-23T22:04:42.781 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.704981845Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-09-23T22:04:42.781 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.730351494Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-09-23T22:04:42.781 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.755572048Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-09-23T22:04:43.048 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.78059868Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-09-23T22:04:43.049 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.822769068Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-09-23T22:04:43.049 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.847957461Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-09-23T22:04:43.049 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.864857829Z level=info msg="Executing migration" id="create alert table v1" 2023-09-23T22:04:43.049 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.898554039Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-09-23T22:04:43.049 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:42 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:42.973927514Z level=info msg="Executing migration" id="add index alert state" 2023-09-23T22:04:43.049 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.015607679Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-09-23T22:04:43.049 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.049223754Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-09-23T22:04:43.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[118879]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:43.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:43.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:43.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:04:43.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:04:43.148 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:43.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[100157]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:43.148 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:43.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:43.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:04:43.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:04:43.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:43 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:43.275 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:04:43.278 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:04:43.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:43 smithi070 ceph-mon[105170]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:43.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:43 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:43.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:43 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:43.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:43 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:04:43.301 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:43 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:04:43.301 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:43 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:43.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.091272706Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T22:04:43.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.116528096Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T22:04:43.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.158443591Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T22:04:43.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.183688896Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-09-23T22:04:43.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.208722135Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T22:04:43.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.250730067Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-09-23T22:04:43.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.275937183Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-09-23T22:04:43.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.300981764Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-09-23T22:04:43.577 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.343086771Z level=info msg="Executing migration" id="Add column is_default" 2023-09-23T22:04:43.577 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.359853925Z level=info msg="Executing migration" id="Add column frequency" 2023-09-23T22:04:43.577 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.401917281Z level=info msg="Executing migration" id="Add column send_reminder" 2023-09-23T22:04:43.577 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.426987444Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-09-23T22:04:43.578 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.468810894Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-09-23T22:04:43.578 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.510819815Z level=info msg="Executing migration" id="Update alert table charset" 2023-09-23T22:04:43.578 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.535919177Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-09-23T22:04:43.578 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.577719317Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-09-23T22:04:43.828 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.628164583Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T22:04:43.829 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.670109365Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-09-23T22:04:43.829 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.737215684Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-09-23T22:04:43.829 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.762323851Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T22:04:43.829 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.78750484Z level=info msg="Executing migration" id="Add for to alert table" 2023-09-23T22:04:43.829 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.829346833Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-09-23T22:04:44.043 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":32,"stamp":"2023-09-23T22:04:42.093486+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50312,"kb_used_data":6664,"kb_used_omap":0,"kb_used_meta":43584,"kb_avail":749910904,"statfs":{"total":767960285184,"available":767908765696,"internally_reserved":0,"allocated":6823936,"data_stored":3876565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44630016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002173"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.538126+0000","last_change":"2023-09-23T22:04:06.152239+0000","last_active":"2023-09-23T22:04:07.538126+0000","last_peered":"2023-09-23T22:04:07.538126+0000","last_clean":"2023-09-23T22:04:07.538126+0000","last_became_active":"2023-09-23T22:04:06.151906+0000","last_became_peered":"2023-09-23T22:04:06.151906+0000","last_unstale":"2023-09-23T22:04:07.538126+0000","last_undegraded":"2023-09-23T22:04:07.538126+0000","last_fullsized":"2023-09-23T22:04:07.538126+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:29.292018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.537480+0000","last_change":"2023-09-23T22:04:06.211004+0000","last_active":"2023-09-23T22:04:07.537480+0000","last_peered":"2023-09-23T22:04:07.537480+0000","last_clean":"2023-09-23T22:04:07.537480+0000","last_became_active":"2023-09-23T22:04:06.210652+0000","last_became_peered":"2023-09-23T22:04:06.210652+0000","last_unstale":"2023-09-23T22:04:07.537480+0000","last_undegraded":"2023-09-23T22:04:07.537480+0000","last_fullsized":"2023-09-23T22:04:07.537480+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:32.721396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.159966+0000","last_change":"2023-09-23T22:04:06.250190+0000","last_active":"2023-09-23T22:04:07.159966+0000","last_peered":"2023-09-23T22:04:07.159966+0000","last_clean":"2023-09-23T22:04:07.159966+0000","last_became_active":"2023-09-23T22:04:06.249826+0000","last_became_peered":"2023-09-23T22:04:06.249826+0000","last_unstale":"2023-09-23T22:04:07.159966+0000","last_undegraded":"2023-09-23T22:04:07.159966+0000","last_fullsized":"2023-09-23T22:04:07.159966+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:16.888898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.159839+0000","last_change":"2023-09-23T22:04:06.250172+0000","last_active":"2023-09-23T22:04:07.159839+0000","last_peered":"2023-09-23T22:04:07.159839+0000","last_clean":"2023-09-23T22:04:07.159839+0000","last_became_active":"2023-09-23T22:04:06.249764+0000","last_became_peered":"2023-09-23T22:04:06.249764+0000","last_unstale":"2023-09-23T22:04:07.159839+0000","last_undegraded":"2023-09-23T22:04:07.159839+0000","last_fullsized":"2023-09-23T22:04:07.159839+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:28:11.590738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.159860+0000","last_change":"2023-09-23T22:03:21.650107+0000","last_active":"2023-09-23T22:04:07.159860+0000","last_peered":"2023-09-23T22:04:07.159860+0000","last_clean":"2023-09-23T22:04:07.159860+0000","last_became_active":"2023-09-23T22:03:21.649660+0000","last_became_peered":"2023-09-23T22:03:21.649660+0000","last_unstale":"2023-09-23T22:04:07.159860+0000","last_undegraded":"2023-09-23T22:04:07.159860+0000","last_fullsized":"2023-09-23T22:04:07.159860+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:13.053339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.538013+0000","last_change":"2023-09-23T22:04:06.154819+0000","last_active":"2023-09-23T22:04:07.538013+0000","last_peered":"2023-09-23T22:04:07.538013+0000","last_clean":"2023-09-23T22:04:07.538013+0000","last_became_active":"2023-09-23T22:04:06.154506+0000","last_became_peered":"2023-09-23T22:04:06.154506+0000","last_unstale":"2023-09-23T22:04:07.538013+0000","last_undegraded":"2023-09-23T22:04:07.538013+0000","last_fullsized":"2023-09-23T22:04:07.538013+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:49.420305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.159599+0000","last_change":"2023-09-23T22:04:06.250115+0000","last_active":"2023-09-23T22:04:07.159599+0000","last_peered":"2023-09-23T22:04:07.159599+0000","last_clean":"2023-09-23T22:04:07.159599+0000","last_became_active":"2023-09-23T22:04:06.249760+0000","last_became_peered":"2023-09-23T22:04:06.249760+0000","last_unstale":"2023-09-23T22:04:07.159599+0000","last_undegraded":"2023-09-23T22:04:07.159599+0000","last_fullsized":"2023-09-23T22:04:07.159599+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:48.266985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.537982+0000","last_change":"2023-09-23T22:04:06.153745+0000","last_active":"2023-09-23T22:04:07.537982+0000","last_peered":"2023-09-23T22:04:07.537982+0000","last_clean":"2023-09-23T22:04:07.537982+0000","last_became_active":"2023-09-23T22:04:06.153371+0000","last_became_peered":"2023-09-23T22:04:06.153371+0000","last_unstale":"2023-09-23T22:04:07.537982+0000","last_undegraded":"2023-09-23T22:04:07.537982+0000","last_fullsized":"2023-09-23T22:04:07.537982+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:36.366505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.537480+0000","last_change":"2023-09-23T22:04:06.250575+0000","last_active":"2023-09-23T22:04:07.537480+0000","last_peered":"2023-09-23T22:04:07.537480+0000","last_clean":"2023-09-23T22:04:07.537480+0000","last_became_active":"2023-09-23T22:04:06.249857+0000","last_became_peered":"2023-09-23T22:04:06.249857+0000","last_unstale":"2023-09-23T22:04:07.537480+0000","last_undegraded":"2023-09-23T22:04:07.537480+0000","last_fullsized":"2023-09-23T22:04:07.537480+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:52.618997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234002,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6320,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738832,"statfs":{"total":95995035648,"available":95988563968,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.45500000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.56599999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.51700000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.55200000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.0459999999999998,"5min":3.0459999999999998,"15min":3.0459999999999998},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.64400000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495638,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6388,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738764,"statfs":{"total":95995035648,"available":95988494336,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.56599999999999995}]},{"osd":1,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.53400000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.6429999999999998,"5min":7.6429999999999998,"15min":7.6429999999999998},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.59599999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.497}]},{"osd":5,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.41499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:04:19 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.626}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6512,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738640,"statfs":{"total":95995035648,"available":95988367360,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.33100000000000002},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.54600000000000004,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.372},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38100000000000001,"15min":0.36099999999999999},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.64500000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:04:30 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.42999999999999999,"15min":0.41999999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.749,"5min":0.47399999999999998,"15min":0.42299999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.4450000000000003,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":0.44500000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46700000000000003,"15min":0.47099999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.63900000000000001,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72999999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.67500000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.61499999999999999,"15min":0.62},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.60599999999999998,"15min":0.61299999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.48099999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.997,"15min":1.022},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.1609999999999996,"5min":7.3479999999999999,"15min":7.3479999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.626,"5min":0.97299999999999998,"15min":1.0309999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.1850000000000001,"5min":6.9539999999999997,"15min":6.9539999999999997},"last":0.57899999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.117,"5min":6.117,"15min":6.117},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.92500000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7156,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737996,"statfs":{"total":95995035648,"available":95987707904,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:04:09 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36899999999999999,"15min":0.36099999999999999},"min":{"1min":0.24299999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.64200000000000002,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.255},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.38,"15min":0.36599999999999999},"min":{"1min":0.24399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.57599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.34899999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:04:28 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.502,"5min":0.44400000000000001,"15min":0.42099999999999999},"min":{"1min":0.29399999999999998,"5min":0.248,"15min":0.248},"max":{"1min":1.593,"5min":1.593,"15min":1.593},"last":0.29899999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:03:50 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.504,"15min":0.504},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73499999999999999,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.504,"15min":0.502},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.752,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.31}]},{"osd":4,"last update":"Sat Sep 23 22:04:13 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.58099999999999996,"15min":0.57599999999999996},"min":{"1min":0.43099999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.64700000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65300000000000002,"15min":0.65300000000000002},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.63400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.70299999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.73699999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411369,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6384,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738768,"statfs":{"total":95995035648,"available":95988498432,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.39100000000000001,"15min":0.38700000000000001},"min":{"1min":0.246,"5min":0.222,"15min":0.222},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.40699999999999997,"15min":0.40000000000000002},"min":{"1min":0.254,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.41499999999999998,"15min":0.40899999999999997},"min":{"1min":0.29199999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.67100000000000004,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.443,"5min":0.435,"15min":0.42499999999999999},"min":{"1min":0.27000000000000002,"5min":0.193,"15min":0.193},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.442}]},{"osd":3,"last update":"Sat Sep 23 22:03:54 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.47099999999999997,"15min":0.47499999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.68799999999999994,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.316},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52400000000000002,"15min":0.52500000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.374}]},{"osd":4,"last update":"Sat Sep 23 22:04:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.55900000000000005,"15min":0.55500000000000005},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.433}]},{"osd":5,"last update":"Sat Sep 23 22:04:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.069,"5min":1.069,"15min":1.069},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:04:19 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.51300000000000001}]}]},{"osd":3,"up_from":25,"seq":107374182435,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41899999999999998,"15min":0.41999999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.58899999999999997,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.373},{"interface":"front","average":{"1min":0.44,"5min":0.46000000000000002,"15min":0.46400000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.70299999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.30499999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64100000000000001,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.53200000000000003,"15min":0.53300000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.748,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.497},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88200000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.623,"15min":0.63},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84499999999999997,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:03:37 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":1.022,"5min":1.022,"15min":1.022},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":1.002,"5min":1.002,"15min":1.002},"last":0.65900000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:03:57 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.55800000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.70199999999999996}]}]},{"osd":4,"up_from":30,"seq":128849018911,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.78000000000000003,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.54800000000000004,"15min":0.53000000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.1240000000000001,"5min":3.1240000000000001,"15min":3.1240000000000001},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54100000000000004,"15min":0.54400000000000004},"min":{"1min":0.40400000000000003,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76200000000000001,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59299999999999997,"15min":0.59899999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.76700000000000002,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":0.40000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.58199999999999996,"15min":0.58499999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.93999999999999995,"5min":2.2629999999999999,"15min":2.2629999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.56299999999999994,"15min":0.54800000000000004},"min":{"1min":0.35999999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.0910000000000002,"5min":3.0910000000000002,"15min":3.0910000000000002},"last":0.50600000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.57699999999999996,"15min":0.55900000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.1499999999999999,"5min":3.1499999999999999,"15min":3.1499999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.54500000000000004,"15min":0.54100000000000004},"min":{"1min":0.40300000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.56499999999999995,"15min":0.56499999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.0470000000000002,"5min":3.0470000000000002,"15min":3.0470000000000002},"last":0.64100000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.2440000000000002,"5min":2.2440000000000002,"15min":2.2440000000000002},"last":0.65600000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":3.9609999999999999,"5min":3.9609999999999999,"15min":3.9609999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.626}]}]},{"osd":5,"up_from":38,"seq":163208757275,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5808,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739344,"statfs":{"total":95995035648,"available":95989088256,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.54600000000000004,"15min":0.55700000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.752,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.67600000000000005,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.51100000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.54900000000000004,"15min":0.56399999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.439},{"interface":"front","average":{"1min":0.504,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.55700000000000005,"15min":0.56699999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.5},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.87,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.51800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.69199999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.54400000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.41199999999999998,"15min":0.40400000000000003},"min":{"1min":0.30199999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.441,"15min":0.42899999999999999},"min":{"1min":0.34899999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.36099999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:04:21 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.48799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:04:44.045 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:04:44.080 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.862936691Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-09-23T22:04:44.081 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.888263332Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-09-23T22:04:44.081 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.913589892Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-09-23T22:04:44.081 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.938761717Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-09-23T22:04:44.081 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:43 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:43.98061966Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-09-23T22:04:44.081 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.005886582Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T22:04:44.081 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.031051221Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T22:04:44.081 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.056214822Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-09-23T22:04:44.081 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.081316801Z level=info msg="Executing migration" id="create annotation table v5" 2023-09-23T22:04:44.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:44 smithi026 ceph-mon[100157]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:04:44.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:44 smithi026 ceph-mon[100157]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T22:04:44.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:44 smithi026 ceph-mon[118879]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:04:44.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:44 smithi026 ceph-mon[118879]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T22:04:44.340 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:44 smithi070 ceph-mon[105170]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:04:44.341 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:44 smithi070 ceph-mon[105170]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T22:04:44.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.123555775Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-09-23T22:04:44.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.165491742Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-09-23T22:04:44.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.190452761Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-09-23T22:04:44.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.232597271Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-09-23T22:04:44.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.274423027Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-09-23T22:04:44.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.299448907Z level=info msg="Executing migration" id="Update annotation table charset" 2023-09-23T22:04:44.341 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.341405574Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-09-23T22:04:44.617 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.375277627Z level=info msg="Executing migration" id="Drop category_id index" 2023-09-23T22:04:44.617 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.400393669Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-09-23T22:04:44.618 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.425493434Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-09-23T22:04:44.618 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.500685866Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T22:04:44.618 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.517670059Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T22:04:44.618 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.534493474Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T22:04:44.618 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.576241824Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-09-23T22:04:44.618 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.618260596Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T22:04:44.718 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:44.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.6603031Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-09-23T22:04:44.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.701987846Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-09-23T22:04:44.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.727183903Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-09-23T22:04:44.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.752389068Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-09-23T22:04:44.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.77738437Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-09-23T22:04:44.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.819497862Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-09-23T22:04:44.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.844505331Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-09-23T22:04:44.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.861341347Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-09-23T22:04:45.155 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:45 smithi070 ceph-mon[105170]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:04:45.155 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:45 smithi070 ceph-mon[105170]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:45.155 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.903462391Z level=info msg="Executing migration" id="Add epoch_end column" 2023-09-23T22:04:45.155 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.936988043Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.970583333Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:44 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:44.987404329Z level=info msg="Executing migration" id="Move region to single row" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.004407056Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.021114893Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.038092376Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.054803809Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.071913023Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.088917517Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.105488965Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.12234661Z level=info msg="Executing migration" id="create test_data table" 2023-09-23T22:04:45.156 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.15597209Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-09-23T22:04:45.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:45 smithi026 ceph-mon[118879]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:04:45.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:45 smithi026 ceph-mon[118879]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:45.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:45 smithi026 ceph-mon[100157]: from='client.14616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:04:45.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:45 smithi026 ceph-mon[100157]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:45.415 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.206613326Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-09-23T22:04:45.415 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.273571933Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T22:04:45.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.315628076Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-09-23T22:04:45.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.340817824Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-09-23T22:04:45.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.36592817Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-09-23T22:04:45.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.391094263Z level=info msg="Executing migration" id="create team table" 2023-09-23T22:04:45.416 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.416226311Z level=info msg="Executing migration" id="add index team.org_id" 2023-09-23T22:04:45.693 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.458229862Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-09-23T22:04:45.693 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.475256663Z level=info msg="Executing migration" id="create team member table" 2023-09-23T22:04:45.693 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.491960538Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-09-23T22:04:45.693 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.508733716Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T22:04:45.693 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.525544923Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-09-23T22:04:45.693 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.542491729Z level=info msg="Executing migration" id="Add column email to team table" 2023-09-23T22:04:45.694 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.575994886Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-09-23T22:04:45.694 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.6097731Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-09-23T22:04:45.694 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.626536065Z level=info msg="Executing migration" id="create dashboard acl table" 2023-09-23T22:04:45.694 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.651671379Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-09-23T22:04:45.694 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.693577198Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T22:04:45.953 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.735770659Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T22:04:45.954 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.760868009Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-09-23T22:04:45.954 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.786193708Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-09-23T22:04:45.954 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.819896545Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-09-23T22:04:45.954 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.836754634Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-09-23T22:04:45.954 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.853349171Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-09-23T22:04:45.954 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.870433944Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-09-23T22:04:45.954 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.887158272Z level=info msg="Executing migration" id="create tag table" 2023-09-23T22:04:45.954 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.904062568Z level=info msg="Executing migration" id="add index tag.key_value" 2023-09-23T22:04:45.954 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.937712491Z level=info msg="Executing migration" id="create login attempt table" 2023-09-23T22:04:45.955 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:45 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:45.954434829Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-09-23T22:04:46.235 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:45 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:04:46.239 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.029853621Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-09-23T22:04:46.239 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.055390688Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T22:04:46.239 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.08035979Z level=info msg="Executing migration" id="create login_attempt v2" 2023-09-23T22:04:46.239 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.105614369Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-09-23T22:04:46.239 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.130969265Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-09-23T22:04:46.239 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.172900971Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-09-23T22:04:46.240 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.198054496Z level=info msg="Executing migration" id="create user auth table" 2023-09-23T22:04:46.240 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.239810394Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T22:04:46.507 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.281786462Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-09-23T22:04:46.508 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.307039474Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-09-23T22:04:46.508 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.348986458Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-09-23T22:04:46.508 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.374061548Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-09-23T22:04:46.508 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.416057936Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-09-23T22:04:46.508 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.441277315Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-09-23T22:04:46.508 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.466414013Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-09-23T22:04:46.508 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.508379405Z level=info msg="Executing migration" id="create server_lock table" 2023-09-23T22:04:46.612 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:46 smithi026 conmon[164633]: ts=2023-09-23T22:04:46.292Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:04:46.612 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:46 smithi026 bash[175346]: 745d17619eff391df7468ed691f1d36cb8e49c064f0d80c6fd325f2555252a0c 2023-09-23T22:04:46.792 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:46 smithi070 ceph-mon[105170]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:46.793 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.55862569Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-09-23T22:04:46.793 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.584058823Z level=info msg="Executing migration" id="create user auth token table" 2023-09-23T22:04:46.793 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.625742543Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-09-23T22:04:46.793 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.650850787Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-09-23T22:04:46.793 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.693017892Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-09-23T22:04:46.793 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.718131783Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-09-23T22:04:46.793 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.793252838Z level=info msg="Executing migration" id="create cache_data table" 2023-09-23T22:04:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:46 smithi026 ceph-mon[100157]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:46 smithi026 ceph-mon[118879]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:47.061 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.843817769Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-09-23T22:04:47.061 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.869050153Z level=info msg="Executing migration" id="create short_url table v1" 2023-09-23T22:04:47.061 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.894131843Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-09-23T22:04:47.061 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.919275515Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-09-23T22:04:47.061 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.961199749Z level=info msg="Executing migration" id="delete alert_definition table" 2023-09-23T22:04:47.061 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:46 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:46.986275918Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-09-23T22:04:47.062 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.011584808Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-09-23T22:04:47.062 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.036868826Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-09-23T22:04:47.062 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.061906331Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T22:04:47.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.112323891Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-09-23T22:04:47.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.154316796Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-09-23T22:04:47.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.179346365Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-09-23T22:04:47.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.204509545Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T22:04:47.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.229925466Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-09-23T22:04:47.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.255277753Z level=info msg="Executing migration" id="drop alert_definition table" 2023-09-23T22:04:47.347 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.280315221Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-09-23T22:04:47.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.305792392Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-09-23T22:04:47.348 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.347642272Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T22:04:47.485 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:47 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T22:04:47.486 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:47 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:04:47.486 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:47 smithi026 systemd[1]: Starting Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:04:47.539 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:04:47.541 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:04:47.615 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.389538727Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T22:04:47.615 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.431664412Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T22:04:47.615 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.456752534Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-09-23T22:04:47.615 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.481814526Z level=info msg="Executing migration" id="create alert_instance table" 2023-09-23T22:04:47.616 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.547902661Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T22:04:47.616 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.590524005Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T22:04:47.616 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.615917016Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-09-23T22:04:47.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.674606749Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T22:04:47.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.716505039Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-09-23T22:04:47.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.741443893Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T22:04:47.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.783400218Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-09-23T22:04:47.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.825294375Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T22:04:47.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.85054311Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-09-23T22:04:48.168 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.892302572Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-09-23T22:04:48.168 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.934277273Z level=info msg="Executing migration" id="create alert_rule table" 2023-09-23T22:04:48.168 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.959293274Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-09-23T22:04:48.169 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:47 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:47.984645059Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-09-23T22:04:48.169 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.034802736Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T22:04:48.169 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.059938401Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T22:04:48.169 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.085123509Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-09-23T22:04:48.169 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.127011185Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-09-23T22:04:48.169 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.16905094Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-09-23T22:04:48.362 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 conmon[175842]: ts=2023-09-23T22:04:48.077Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T22:04:48.362 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 conmon[175842]: ts=2023-09-23T22:04:48.077Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T22:04:48.362 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 conmon[175842]: ts=2023-09-23T22:04:48.079Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2023-09-23T22:04:48.362 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 conmon[175842]: ts=2023-09-23T22:04:48.080Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T22:04:48.362 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 conmon[175842]: ts=2023-09-23T22:04:48.110Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:04:48.362 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 conmon[175842]: ts=2023-09-23T22:04:48.110Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:04:48.362 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 conmon[175842]: ts=2023-09-23T22:04:48.113Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T22:04:48.362 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 conmon[175842]: ts=2023-09-23T22:04:48.113Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T22:04:48.362 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 bash[175785]: 7a14521d2de95daf65a67e600dbb8afc1448c76c60199f2079d3a55800eba154 2023-09-23T22:04:48.363 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:48 smithi026 systemd[1]: Started Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:04:48.378 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":34,"stamp":"2023-09-23T22:04:46.094131+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50312,"kb_used_data":6664,"kb_used_omap":0,"kb_used_meta":43584,"kb_avail":749910904,"statfs":{"total":767960285184,"available":767908765696,"internally_reserved":0,"allocated":6823936,"data_stored":3876565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44630016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002087"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.538126+0000","last_change":"2023-09-23T22:04:06.152239+0000","last_active":"2023-09-23T22:04:07.538126+0000","last_peered":"2023-09-23T22:04:07.538126+0000","last_clean":"2023-09-23T22:04:07.538126+0000","last_became_active":"2023-09-23T22:04:06.151906+0000","last_became_peered":"2023-09-23T22:04:06.151906+0000","last_unstale":"2023-09-23T22:04:07.538126+0000","last_undegraded":"2023-09-23T22:04:07.538126+0000","last_fullsized":"2023-09-23T22:04:07.538126+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:29.292018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.537480+0000","last_change":"2023-09-23T22:04:06.211004+0000","last_active":"2023-09-23T22:04:07.537480+0000","last_peered":"2023-09-23T22:04:07.537480+0000","last_clean":"2023-09-23T22:04:07.537480+0000","last_became_active":"2023-09-23T22:04:06.210652+0000","last_became_peered":"2023-09-23T22:04:06.210652+0000","last_unstale":"2023-09-23T22:04:07.537480+0000","last_undegraded":"2023-09-23T22:04:07.537480+0000","last_fullsized":"2023-09-23T22:04:07.537480+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:32.721396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.159966+0000","last_change":"2023-09-23T22:04:06.250190+0000","last_active":"2023-09-23T22:04:07.159966+0000","last_peered":"2023-09-23T22:04:07.159966+0000","last_clean":"2023-09-23T22:04:07.159966+0000","last_became_active":"2023-09-23T22:04:06.249826+0000","last_became_peered":"2023-09-23T22:04:06.249826+0000","last_unstale":"2023-09-23T22:04:07.159966+0000","last_undegraded":"2023-09-23T22:04:07.159966+0000","last_fullsized":"2023-09-23T22:04:07.159966+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:16.888898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.159839+0000","last_change":"2023-09-23T22:04:06.250172+0000","last_active":"2023-09-23T22:04:07.159839+0000","last_peered":"2023-09-23T22:04:07.159839+0000","last_clean":"2023-09-23T22:04:07.159839+0000","last_became_active":"2023-09-23T22:04:06.249764+0000","last_became_peered":"2023-09-23T22:04:06.249764+0000","last_unstale":"2023-09-23T22:04:07.159839+0000","last_undegraded":"2023-09-23T22:04:07.159839+0000","last_fullsized":"2023-09-23T22:04:07.159839+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:28:11.590738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":271,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.159860+0000","last_change":"2023-09-23T22:03:21.650107+0000","last_active":"2023-09-23T22:04:07.159860+0000","last_peered":"2023-09-23T22:04:07.159860+0000","last_clean":"2023-09-23T22:04:07.159860+0000","last_became_active":"2023-09-23T22:03:21.649660+0000","last_became_peered":"2023-09-23T22:03:21.649660+0000","last_unstale":"2023-09-23T22:04:07.159860+0000","last_undegraded":"2023-09-23T22:04:07.159860+0000","last_fullsized":"2023-09-23T22:04:07.159860+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:13.053339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.538013+0000","last_change":"2023-09-23T22:04:06.154819+0000","last_active":"2023-09-23T22:04:07.538013+0000","last_peered":"2023-09-23T22:04:07.538013+0000","last_clean":"2023-09-23T22:04:07.538013+0000","last_became_active":"2023-09-23T22:04:06.154506+0000","last_became_peered":"2023-09-23T22:04:06.154506+0000","last_unstale":"2023-09-23T22:04:07.538013+0000","last_undegraded":"2023-09-23T22:04:07.538013+0000","last_fullsized":"2023-09-23T22:04:07.538013+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:49.420305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.159599+0000","last_change":"2023-09-23T22:04:06.250115+0000","last_active":"2023-09-23T22:04:07.159599+0000","last_peered":"2023-09-23T22:04:07.159599+0000","last_clean":"2023-09-23T22:04:07.159599+0000","last_became_active":"2023-09-23T22:04:06.249760+0000","last_became_peered":"2023-09-23T22:04:06.249760+0000","last_unstale":"2023-09-23T22:04:07.159599+0000","last_undegraded":"2023-09-23T22:04:07.159599+0000","last_fullsized":"2023-09-23T22:04:07.159599+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:48.266985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.537982+0000","last_change":"2023-09-23T22:04:06.153745+0000","last_active":"2023-09-23T22:04:07.537982+0000","last_peered":"2023-09-23T22:04:07.537982+0000","last_clean":"2023-09-23T22:04:07.537982+0000","last_became_active":"2023-09-23T22:04:06.153371+0000","last_became_peered":"2023-09-23T22:04:06.153371+0000","last_unstale":"2023-09-23T22:04:07.537982+0000","last_undegraded":"2023-09-23T22:04:07.537982+0000","last_fullsized":"2023-09-23T22:04:07.537982+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:36.366505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T22:04:07.537480+0000","last_change":"2023-09-23T22:04:06.250575+0000","last_active":"2023-09-23T22:04:07.537480+0000","last_peered":"2023-09-23T22:04:07.537480+0000","last_clean":"2023-09-23T22:04:07.537480+0000","last_became_active":"2023-09-23T22:04:06.249857+0000","last_became_peered":"2023-09-23T22:04:06.249857+0000","last_unstale":"2023-09-23T22:04:07.537480+0000","last_undegraded":"2023-09-23T22:04:07.537480+0000","last_fullsized":"2023-09-23T22:04:07.537480+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:52.618997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234003,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6320,"kb_used_data":1192,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738832,"statfs":{"total":95995035648,"available":95988563968,"internally_reserved":0,"allocated":1220608,"data_stored":853268,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.35099999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.37}]},{"osd":3,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.314}]},{"osd":5,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.30299999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":3.0459999999999998,"5min":3.0459999999999998,"15min":3.0459999999999998},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.35999999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495639,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6388,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738764,"statfs":{"total":95995035648,"available":95988494336,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.55000000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.60199999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.49399999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.6429999999999998,"5min":7.6429999999999998,"15min":7.6429999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.63300000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.38,"15min":0.38},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.61299999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:04:19 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.1990000000000001,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.65400000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574894,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6512,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738640,"statfs":{"total":95995035648,"available":95988367360,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:07 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.35299999999999998,"15min":0.33100000000000002},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.54600000000000004,"5min":0.60199999999999998,"15min":0.60199999999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38100000000000001,"15min":0.36099999999999999},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.45200000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:04:30 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.42999999999999999,"15min":0.41999999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.749,"5min":0.47399999999999998,"15min":0.42299999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.4450000000000003,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":0.434}]},{"osd":3,"last update":"Sat Sep 23 22:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.46700000000000003,"15min":0.47099999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.63900000000000001,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55300000000000005,"15min":0.55500000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.72999999999999998,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.61499999999999999,"15min":0.62},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.60599999999999998,"15min":0.61299999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.74399999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.997,"15min":1.022},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.1609999999999996,"5min":7.3479999999999999,"15min":7.3479999999999999},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.626,"5min":0.97299999999999998,"15min":1.0309999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.1850000000000001,"5min":6.9539999999999997,"15min":6.9539999999999997},"last":0.78600000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:03:56 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.80600000000000005},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.69999999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.117,"5min":6.117,"15min":6.117},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.76500000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738420,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7156,"kb_used_data":1196,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737996,"statfs":{"total":95995035648,"available":95987707904,"internally_reserved":0,"allocated":1224704,"data_stored":853591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:04:09 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.36899999999999999,"15min":0.36099999999999999},"min":{"1min":0.24299999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.64200000000000002,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.497},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.38,"15min":0.36599999999999999},"min":{"1min":0.24399999999999999,"5min":0.193,"15min":0.193},"max":{"1min":0.57599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.311}]},{"osd":2,"last update":"Sat Sep 23 22:04:28 2023","interfaces":[{"interface":"back","average":{"1min":0.38100000000000001,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.76800000000000002,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.28100000000000003},{"interface":"front","average":{"1min":0.502,"5min":0.44400000000000001,"15min":0.42099999999999999},"min":{"1min":0.29399999999999998,"5min":0.248,"15min":0.248},"max":{"1min":1.593,"5min":1.593,"15min":1.593},"last":0.36799999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:03:50 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.504,"15min":0.504},"min":{"1min":0.34100000000000003,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.73499999999999999,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.437},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.504,"15min":0.502},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.752,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.44900000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:04:13 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.58099999999999996,"15min":0.57599999999999996},"min":{"1min":0.43099999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:04:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.63700000000000001,"15min":0.64700000000000002},"min":{"1min":0.44900000000000001,"5min":0.439,"15min":0.439},"max":{"1min":0.81799999999999995,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.629,"15min":0.63900000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76700000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.55700000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"max":{"1min":0.89300000000000002,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.52200000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411370,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6384,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738768,"statfs":{"total":95995035648,"available":95988498432,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.39100000000000001,"15min":0.38700000000000001},"min":{"1min":0.246,"5min":0.222,"15min":0.222},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.40699999999999997,"15min":0.40000000000000002},"min":{"1min":0.254,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.32800000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:04:27 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.41499999999999998,"15min":0.40899999999999997},"min":{"1min":0.29199999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.67100000000000004,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.317},{"interface":"front","average":{"1min":0.443,"5min":0.435,"15min":0.42499999999999999},"min":{"1min":0.27000000000000002,"5min":0.193,"15min":0.193},"max":{"1min":1.121,"5min":1.121,"15min":1.121},"last":0.39800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:03:54 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.47099999999999997,"15min":0.47499999999999998},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.68799999999999994,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52400000000000002,"15min":0.52500000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:04:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.55900000000000005,"15min":0.55500000000000005},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.65000000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:04:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.069,"5min":1.069,"15min":1.069},"last":0.495},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:03:59 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:04:19 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.63,"15min":0.63},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.627},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.61499999999999999}]}]},{"osd":3,"up_from":25,"seq":107374182436,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41899999999999998,"15min":0.41999999999999998},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.58899999999999997,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.44,"5min":0.46000000000000002,"15min":0.46400000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.70299999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":1.075}]},{"osd":1,"last update":"Sat Sep 23 22:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.64100000000000001,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.48099999999999998,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:03:48 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.53200000000000003,"15min":0.53300000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.748,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.37}]},{"osd":4,"last update":"Sat Sep 23 22:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88200000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.623,"15min":0.63},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84499999999999997,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:04:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.63,"15min":0.64200000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80100000000000005,"5min":1.022,"15min":1.022},"last":0.443},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.65500000000000003,"15min":0.66700000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85699999999999998,"5min":1.002,"15min":1.002},"last":0.39300000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:03:57 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":1.091,"5min":1.091,"15min":1.091},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.0389999999999999,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.53700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018912,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5872,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739280,"statfs":{"total":95995035648,"available":95989022720,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.52400000000000002,"15min":0.52600000000000002},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.78000000000000003,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":0.54800000000000004,"15min":0.53000000000000003},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.1240000000000001,"5min":3.1240000000000001,"15min":3.1240000000000001},"last":0.434}]},{"osd":1,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54100000000000004,"15min":0.54400000000000004},"min":{"1min":0.40400000000000003,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76200000000000001,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.498},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.59299999999999997,"15min":0.59899999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.76700000000000002,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.58199999999999996,"15min":0.58499999999999996},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.93999999999999995,"5min":2.2629999999999999,"15min":2.2629999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.56299999999999994,"15min":0.54800000000000004},"min":{"1min":0.35999999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.0910000000000002,"5min":3.0910000000000002,"15min":3.0910000000000002},"last":0.48999999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:04:14 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.57699999999999996,"15min":0.55900000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.1499999999999999,"5min":3.1499999999999999,"15min":3.1499999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.54500000000000004,"15min":0.54100000000000004},"min":{"1min":0.40300000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.69199999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:04:38 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.42099999999999999,"15min":0.41199999999999998},"min":{"1min":0.317,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60199999999999998,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.626},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.39500000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.73099999999999998,"5min":3.0470000000000002,"15min":3.0470000000000002},"last":0.52400000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.2440000000000002,"5min":2.2440000000000002,"15min":2.2440000000000002},"last":0.66900000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:04:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":3.9609999999999999,"5min":3.9609999999999999,"15min":3.9609999999999999},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.72299999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757276,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5808,"kb_used_data":616,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739344,"statfs":{"total":95995035648,"available":95989088256,"internally_reserved":0,"allocated":630784,"data_stored":263223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.54600000000000004,"15min":0.55700000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.752,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.67600000000000005,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.51100000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.54900000000000004,"15min":0.56399999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.439},{"interface":"front","average":{"1min":0.504,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.46100000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.55700000000000005,"15min":0.56699999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.5},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.87,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.51800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.69199999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.54400000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.41199999999999998,"15min":0.40400000000000003},"min":{"1min":0.30199999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.441,"15min":0.42899999999999999},"min":{"1min":0.34899999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:03:55 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.499,"15min":0.499},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.85599999999999998,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.36099999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:04:21 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.48799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:04:48.381 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:04:48.381 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T22:04:48.381 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T22:04:48.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T22:04:48.444 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.202630751Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T22:04:48.445 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.236091955Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T22:04:48.445 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.27820121Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-09-23T22:04:48.445 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.344983534Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-09-23T22:04:48.445 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.392754269Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T22:04:48.705 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.445433536Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-09-23T22:04:48.705 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.495882092Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-09-23T22:04:48.705 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.537707056Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-09-23T22:04:48.705 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.579566076Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-09-23T22:04:48.705 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.621539746Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T22:04:48.705 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.672058796Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T22:04:48.705 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.705728364Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T22:04:48.742 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:48.986 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.79004714Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-09-23T22:04:48.986 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.865935804Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-09-23T22:04:48.986 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.919350375Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-09-23T22:04:49.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:49 smithi026 ceph-mon[100157]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:04:49.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:49 smithi026 ceph-mon[100157]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:49.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:49 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:49.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:49 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:49.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:49 smithi026 ceph-mon[118879]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:04:49.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:49 smithi026 ceph-mon[118879]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:49.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:49 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:49.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:49 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:49.302 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:49 smithi070 ceph-mon[105170]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:04:49.302 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:49 smithi070 ceph-mon[105170]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:49.302 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:49 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:49.302 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:49 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:49.302 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:48 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:48.98600674Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-09-23T22:04:49.302 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.104223862Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-09-23T22:04:49.302 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.14261887Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-09-23T22:04:49.303 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.176603908Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-09-23T22:04:49.303 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.201736193Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-09-23T22:04:49.303 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.226652389Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T22:04:49.303 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.302651804Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-09-23T22:04:49.577 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.377579796Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-09-23T22:04:49.577 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.435154929Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-09-23T22:04:49.577 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.506690836Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-09-23T22:04:49.577 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.57775917Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-09-23T22:04:49.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:49 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:49.822234242Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-09-23T22:04:50.235 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:50 smithi026 conmon[175842]: ts=2023-09-23T22:04:50.081Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000657726s 2023-09-23T22:04:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:50 smithi026 ceph-mon[100157]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:04:50.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:50 smithi026 ceph-mon[100157]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T22:04:50.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:50 smithi026 ceph-mon[118879]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:04:50.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:50 smithi026 ceph-mon[118879]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T22:04:50.331 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:50 smithi070 ceph-mon[105170]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:04:50.331 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:50 smithi070 ceph-mon[105170]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T22:04:50.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.04197103Z level=info msg="Executing migration" id="create provenance_type table" 2023-09-23T22:04:50.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.180812246Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-09-23T22:04:50.331 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.256351903Z level=info msg="Executing migration" id="create alert_image table" 2023-09-23T22:04:50.361 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:04:50.361 INFO:teuthology.orchestra.run.smithi026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T22:04:50.592 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.33184275Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-09-23T22:04:50.592 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.382191254Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-09-23T22:04:50.592 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.407679853Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-09-23T22:04:50.592 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.458459242Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-09-23T22:04:50.592 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.514502772Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T22:04:50.592 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.514617219Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T22:04:50.593 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.592992829Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-09-23T22:04:50.843 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.668562907Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-09-23T22:04:50.844 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.777365801Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-09-23T22:04:50.844 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.777522219Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-09-23T22:04:50.844 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.777655928Z level=info msg="alerts found to migrate" alerts=0 2023-09-23T22:04:50.943 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T22:04:50.944 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T22:04:50.944 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T22:04:50.952 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T22:04:50.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T22:04:51.118 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.844384572Z level=info msg="Executing migration" id="create library_element table v1" 2023-09-23T22:04:51.118 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.894982836Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-09-23T22:04:51.118 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.937174574Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-09-23T22:04:51.118 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:50 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:50.978984893Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T22:04:51.119 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.045845672Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-09-23T22:04:51.119 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.11901753Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-09-23T22:04:51.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:51 smithi026 ceph-mon[100157]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:51.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:51 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3890824606' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:04:51.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:51 smithi026 ceph-mon[118879]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:51.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:51 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3890824606' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:04:51.313 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:51.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:51 smithi070 ceph-mon[105170]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:51.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:51 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3890824606' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:04:51.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.197026369Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-09-23T22:04:51.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.273007181Z level=info msg="Executing migration" id="create data_keys table" 2023-09-23T22:04:51.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.330992304Z level=info msg="Executing migration" id="create secrets table" 2023-09-23T22:04:51.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:04:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:04:51.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.597028572Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-09-23T22:04:51.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.727258372Z level=info msg="Executing migration" id="add name column into data_keys" 2023-09-23T22:04:51.874 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.807858632Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-09-23T22:04:52.151 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:51 smithi070 systemd[1]: Stopping Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:04:52.152 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.877132306Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-09-23T22:04:52.152 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:51 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:51.960435042Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-09-23T22:04:52.152 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.00927875Z level=info msg="Executing migration" id="create kv_store table v1" 2023-09-23T22:04:52.152 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.034431491Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-09-23T22:04:52.152 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.076441775Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T22:04:52.152 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.110028995Z level=info msg="Executing migration" id="create permission table" 2023-09-23T22:04:52.152 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.151830607Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-09-23T22:04:52.421 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.206279136Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-09-23T22:04:52.422 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.235612923Z level=info msg="Executing migration" id="create role table" 2023-09-23T22:04:52.422 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.277573819Z level=info msg="Executing migration" id="add column display_name" 2023-09-23T22:04:52.422 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.361189513Z level=info msg="Executing migration" id="add column group_name" 2023-09-23T22:04:52.422 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.227Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:04:52.422 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.227Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:04:52.422 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.227Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:04:52.422 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.228Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:04:52.422 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.228Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:04:52.423 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.228Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T22:04:52.423 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.228Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:04:52.423 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.228Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T22:04:52.423 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.228Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T22:04:52.423 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.228Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:04:52.423 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.228Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T22:04:52.423 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[147765]: ts=2023-09-23T22:04:52.228Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T22:04:52.797 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.446124564Z level=info msg="Executing migration" id="add index role.org_id" 2023-09-23T22:04:52.797 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.649834717Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-09-23T22:04:52.797 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.797881252Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-09-23T22:04:53.048 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:52 smithi070 bash[153129]: 08bee6665fe6bd6be2b00f74f1a167c76cfae22bd4fba3939fc4404c4ca826f1 2023-09-23T22:04:53.048 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.863978299Z level=info msg="Executing migration" id="create team role table" 2023-09-23T22:04:53.048 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.926916444Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-09-23T22:04:53.048 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:52 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:52.964769004Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T22:04:53.048 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.014994225Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-09-23T22:04:53.048 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.048559182Z level=info msg="Executing migration" id="create user role table" 2023-09-23T22:04:53.300 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T22:04:53.300 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 systemd[1]: Stopped Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:04:53.300 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 systemd[1]: Starting Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:04:53.300 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.132292716Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-09-23T22:04:53.300 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.174246954Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T22:04:53.300 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.199456672Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-09-23T22:04:53.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.224697907Z level=info msg="Executing migration" id="create builtin role table" 2023-09-23T22:04:53.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.249979255Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-09-23T22:04:53.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.275098073Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-09-23T22:04:53.301 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.300718907Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-09-23T22:04:53.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:53 smithi070 ceph-mon[105170]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:53.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:53 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:53.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:53 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T22:04:53.570 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.342263214Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-09-23T22:04:53.570 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.39300816Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T22:04:53.570 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.452064415Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-09-23T22:04:53.570 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.511014296Z level=info msg="Executing migration" id="add unique index role.uid" 2023-09-23T22:04:53.570 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.5698043Z level=info msg="Executing migration" id="create seed assignment table" 2023-09-23T22:04:53.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:53 smithi026 ceph-mon[100157]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:53.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:53 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:53.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:53 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T22:04:53.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:53 smithi026 ceph-mon[118879]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:53.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:53 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:04:53.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:53 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T22:04:53.826 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.612158089Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-09-23T22:04:53.826 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.654190436Z level=info msg="Executing migration" id="add column hidden to role table" 2023-09-23T22:04:53.826 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.696106789Z level=info msg="Executing migration" id="create query_history table v1" 2023-09-23T22:04:53.826 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.76338503Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-09-23T22:04:53.826 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.805Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T22:04:53.826 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.805Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T22:04:53.826 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.805Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi070 (none))" 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.805Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.805Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.806Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.807Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.808Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.808Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.811Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.811Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.839µs 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.811Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.811Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.811Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T22:04:53.827 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.811Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.208µs wal_replay_duration=523.348µs wbl_replay_duration=88ns total_replay_duration=562.594µs 2023-09-23T22:04:53.828 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.812Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:04:53.828 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.812Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T22:04:53.828 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.812Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:04:53.828 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.826Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.255333ms db_storage=677ns remote_storage=1.019µs web_handler=409ns query_engine=475ns scrape=180.489µs scrape_sd=23.213µs notify=22.23µs notify_sd=5.88µs rules=13.82953ms tracing=5.808µs 2023-09-23T22:04:53.828 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.826Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T22:04:54.081 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[153497]: ts=2023-09-23T22:04:53.826Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T22:04:54.081 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 bash[153456]: a37f53e6e83abe9cbf8a32f5b0f487c9ed1726b74cbae36b5a865780b6901573 2023-09-23T22:04:54.081 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:04:53 smithi070 systemd[1]: Started Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:04:54.081 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.830357803Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-09-23T22:04:54.082 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.905992778Z level=info msg="Executing migration" id="teams permissions migration" 2023-09-23T22:04:54.082 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.947616887Z level=info msg="Executing migration" id="dashboard permissions" 2023-09-23T22:04:54.082 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:53 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:53.989509763Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-09-23T22:04:54.082 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.03154851Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-09-23T22:04:54.082 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.056747698Z level=info msg="Executing migration" id="alerting notification permissions" 2023-09-23T22:04:54.082 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.081911779Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-09-23T22:04:54.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:53 smithi026 conmon[100586]: [23/Sep/2023:22:04:53] ENGINE Bus STOPPING 2023-09-23T22:04:54.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:53 smithi026 conmon[100586]: [23/Sep/2023:22:04:53] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:04:54.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:53 smithi026 conmon[100586]: [23/Sep/2023:22:04:53] ENGINE Bus STOPPED 2023-09-23T22:04:54.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:53 smithi026 conmon[100586]: [23/Sep/2023:22:04:53] ENGINE Bus STARTING 2023-09-23T22:04:54.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Serving on http://:::9283 2023-09-23T22:04:54.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Bus STARTED 2023-09-23T22:04:54.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Bus STOPPING 2023-09-23T22:04:54.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:04:54.236 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Bus STOPPED 2023-09-23T22:04:54.236 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Bus STARTING 2023-09-23T22:04:54.236 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Serving on http://:::9283 2023-09-23T22:04:54.236 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Bus STARTED 2023-09-23T22:04:54.236 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Bus STOPPING 2023-09-23T22:04:54.236 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:04:54.236 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Bus STOPPED 2023-09-23T22:04:54.236 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Bus STARTING 2023-09-23T22:04:54.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.115577603Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-09-23T22:04:54.333 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.140732925Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-09-23T22:04:54.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.182682721Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-09-23T22:04:54.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.224646441Z level=info msg="Executing migration" id="create correlation table v1" 2023-09-23T22:04:54.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.249906873Z level=info msg="Executing migration" id="add index correlations.uid" 2023-09-23T22:04:54.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.291865602Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-09-23T22:04:54.334 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.33375971Z level=info msg="Executing migration" id="add correlation config column" 2023-09-23T22:04:54.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:04:54.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T22:04:54.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T22:04:54.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:04:54.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:04:54.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:04:54.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:04:54.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:04:54.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:54 smithi070 ceph-mon[105170]: pgmap v39: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:54.577 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Serving on http://:::9283 2023-09-23T22:04:54.577 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100586]: [23/Sep/2023:22:04:54] ENGINE Bus STARTED 2023-09-23T22:04:54.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:04:54.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T22:04:54.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T22:04:54.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:04:54.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[100157]: pgmap v39: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:54 smithi026 conmon[100133]: 2023-09-23T22:04:54.390+0000 7f50842c1700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='client.24530 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T22:04:54.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026:9093"}]: dispatch 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi070.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T22:04:54.580 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:04:54.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:04:54.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:04:54.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi070.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T22:04:54.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:54.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:04:54.581 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:54 smithi026 ceph-mon[118879]: pgmap v39: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:54.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.392595506Z level=info msg="Executing migration" id="create entity_events table" 2023-09-23T22:04:54.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.43468991Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-09-23T22:04:54.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.45988818Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T22:04:54.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.460096778Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T22:04:54.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.4849671Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T22:04:54.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.485121524Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T22:04:54.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.51013268Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-09-23T22:04:54.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.577234378Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-09-23T22:04:54.979 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.687530756Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-09-23T22:04:54.979 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.711502805Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T22:04:54.979 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.761693002Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-09-23T22:04:54.979 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.820409337Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T22:04:54.979 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.905588281Z level=info msg="Executing migration" id="Drop public config table" 2023-09-23T22:04:54.979 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:54 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:54.979292577Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-09-23T22:04:55.248 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.021365919Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-09-23T22:04:55.248 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.046847379Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T22:04:55.248 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.063694447Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-09-23T22:04:55.248 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.088897339Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-09-23T22:04:55.248 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.122406888Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-09-23T22:04:55.249 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.139233867Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-09-23T22:04:55.249 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.156084701Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-09-23T22:04:55.249 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.181258797Z level=info msg="Executing migration" id="add share column" 2023-09-23T22:04:55.249 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.206424246Z level=info msg="Executing migration" id="create default alerting folders" 2023-09-23T22:04:55.249 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.248393791Z level=info msg="Executing migration" id="create file table" 2023-09-23T22:04:55.283 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T22:04:55.499 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.273865299Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-09-23T22:04:55.499 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.298988815Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-09-23T22:04:55.499 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.340842399Z level=info msg="Executing migration" id="create file_meta table" 2023-09-23T22:04:55.500 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.366087247Z level=info msg="Executing migration" id="file table idx: path key" 2023-09-23T22:04:55.500 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.441331157Z level=info msg="Executing migration" id="set path collation in file table" 2023-09-23T22:04:55.500 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.500072584Z level=info msg="Executing migration" id="managed permissions migration" 2023-09-23T22:04:55.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T22:04:55.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T22:04:55.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T22:04:55.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:04:55.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T22:04:55.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T22:04:55.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T22:04:55.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: fsmap foo:0 2023-09-23T22:04:55.501 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: Saving service mds.foo spec with placement count:2 2023-09-23T22:04:55.501 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:55 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:55.686 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:04:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T22:04:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T22:04:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T22:04:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:04:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T22:04:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T22:04:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T22:04:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: fsmap foo:0 2023-09-23T22:04:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: Saving service mds.foo spec with placement count:2 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: fsmap foo:0 2023-09-23T22:04:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: Saving service mds.foo spec with placement count:2 2023-09-23T22:04:55.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:55 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:55.775 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.583581886Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-09-23T22:04:55.776 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.609203598Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-09-23T22:04:55.776 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.65930754Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-09-23T22:04:55.776 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.726388797Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-09-23T22:04:56.036 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.776692329Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-09-23T22:04:56.036 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.888616467Z level=info msg="Executing migration" id="update group index for alert rules" 2023-09-23T22:04:56.036 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.919304883Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-09-23T22:04:56.036 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.936068621Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-09-23T22:04:56.036 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.96959415Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-09-23T22:04:56.036 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:55 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:55.98651395Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-09-23T22:04:56.337 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.036776223Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-09-23T22:04:56.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.053647402Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-09-23T22:04:56.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.128863876Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-09-23T22:04:56.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.243187619Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-09-23T22:04:56.338 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.279584926Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-09-23T22:04:56.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:56 smithi070 ceph-mon[105170]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T22:04:56.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:56 smithi070 ceph-mon[105170]: pgmap v43: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:56.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.338531264Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-09-23T22:04:56.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.405901711Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-09-23T22:04:56.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.475088947Z level=info msg="Executing migration" id="create folder table" 2023-09-23T22:04:56.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.551056419Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-09-23T22:04:56.623 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.623285504Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-09-23T22:04:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:56 smithi026 ceph-mon[118879]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T22:04:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:56 smithi026 ceph-mon[118879]: pgmap v43: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:56.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:56 smithi026 ceph-mon[100157]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T22:04:56.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:56 smithi026 ceph-mon[100157]: pgmap v43: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:56.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.665224156Z level=info msg="Executing migration" id="Update folder title length" 2023-09-23T22:04:56.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.707169273Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-09-23T22:04:56.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=migrator t=2023-09-23T22:04:56.757306581Z level=info msg="migrations completed" performed=485 skipped=0 duration=20.091269441s 2023-09-23T22:04:56.873 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=sqlstore t=2023-09-23T22:04:56.757916447Z level=info msg="Created default organization" 2023-09-23T22:04:57.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=plugin.loader t=2023-09-23T22:04:56.885119911Z level=info msg="Plugin registered" pluginID=input 2023-09-23T22:04:57.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=plugin.loader t=2023-09-23T22:04:56.914624039Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T22:04:57.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=plugin.loader t=2023-09-23T22:04:56.914656193Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T22:04:57.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=secrets t=2023-09-23T22:04:56.9146927Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T22:04:57.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=query_data t=2023-09-23T22:04:56.916437598Z level=info msg="Query Service initialization" 2023-09-23T22:04:57.374 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:56 smithi070 conmon[151585]: logger=live.push_http t=2023-09-23T22:04:56.91908749Z level=info msg="Live Push Gateway initialization" 2023-09-23T22:04:58.123 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=infra.usagestats.collector t=2023-09-23T22:04:57.73383426Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T22:04:58.123 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=provisioning.datasources t=2023-09-23T22:04:57.734205184Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T22:04:58.123 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=provisioning.datasources t=2023-09-23T22:04:57.766885424Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T22:04:58.123 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=provisioning.datasources t=2023-09-23T22:04:57.844675313Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-09-23T22:04:58.123 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=provisioning.alerting t=2023-09-23T22:04:57.893027907Z level=info msg="starting to provision alerting" 2023-09-23T22:04:58.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=provisioning.alerting t=2023-09-23T22:04:57.893068904Z level=info msg="finished to provision alerting" 2023-09-23T22:04:58.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=grafanaStorageLogger t=2023-09-23T22:04:57.893267777Z level=info msg="storage starting" 2023-09-23T22:04:58.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=ngalert.state.manager t=2023-09-23T22:04:57.893408022Z level=info msg="Warming state cache for startup" 2023-09-23T22:04:58.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=http.server t=2023-09-23T22:04:57.896354649Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T22:04:58.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=sqlstore.transactions t=2023-09-23T22:04:57.904144444Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T22:04:58.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=ngalert.state.manager t=2023-09-23T22:04:57.960838886Z level=info msg="State cache has been initialized" states=0 duration=67.429995ms 2023-09-23T22:04:58.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=ticker t=2023-09-23T22:04:57.960897798Z level=info msg=starting first_tick=2023-09-23T22:05:00Z 2023-09-23T22:04:58.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:04:57 smithi070 conmon[151585]: logger=ngalert.multiorg.alertmanager t=2023-09-23T22:04:57.960905719Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T22:04:58.259 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:04:58 smithi026 conmon[175842]: ts=2023-09-23T22:04:58.083Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003019606s 2023-09-23T22:04:58.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:04:58.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:04:58.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.rancfk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:04:58.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.rancfk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: Deploying daemon mds.foo.smithi026.rancfk on smithi026 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[118879]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:04:58.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.rancfk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:04:58.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.rancfk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:04:58.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:04:58.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: Deploying daemon mds.foo.smithi026.rancfk on smithi026 2023-09-23T22:04:58.621 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:04:58 smithi026 ceph-mon[100157]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:04:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:04:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:04:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.rancfk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:04:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.rancfk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:04:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:04:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: Deploying daemon mds.foo.smithi026.rancfk on smithi026 2023-09-23T22:04:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:04:58 smithi070 ceph-mon[105170]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:04:58.640 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T22:04:59.547 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[100157]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.ehaqek", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:05:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.ehaqek", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:05:01.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[118879]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.ehaqek", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:05:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.ehaqek", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:05:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:01 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:01.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:01 smithi070 ceph-mon[105170]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:01.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:01 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:01.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:01 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:01.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:01 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:01.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:01 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.ehaqek", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:05:01.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:01 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.ehaqek", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T22:05:01.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:01 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:01.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:05:01 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:05:02.059 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T22:05:02.068 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T22:05:02.068 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T22:05:02.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T22:05:02.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: Deploying daemon mds.foo.smithi070.ehaqek on smithi070 2023-09-23T22:05:02.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: mds.? [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] up:boot 2023-09-23T22:05:02.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: daemon mds.foo.smithi026.rancfk assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T22:05:02.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T22:05:02.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T22:05:02.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: Cluster is now healthy 2023-09-23T22:05:02.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: fsmap foo:0 1 up:standby 2023-09-23T22:05:02.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:05:02.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: fsmap foo:1 {0=foo.smithi026.rancfk=up:creating} 2023-09-23T22:05:02.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:02 smithi070 ceph-mon[105170]: daemon mds.foo.smithi026.rancfk is now active in filesystem foo as rank 0 2023-09-23T22:05:02.432 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: Deploying daemon mds.foo.smithi070.ehaqek on smithi070 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: mds.? [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] up:boot 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: daemon mds.foo.smithi026.rancfk assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: Cluster is now healthy 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: fsmap foo:0 1 up:standby 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: fsmap foo:1 {0=foo.smithi026.rancfk=up:creating} 2023-09-23T22:05:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[100157]: daemon mds.foo.smithi026.rancfk is now active in filesystem foo as rank 0 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: Deploying daemon mds.foo.smithi070.ehaqek on smithi070 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: mds.? [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] up:boot 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: daemon mds.foo.smithi026.rancfk assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: Cluster is now healthy 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: fsmap foo:0 1 up:standby 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: fsmap foo:1 {0=foo.smithi026.rancfk=up:creating} 2023-09-23T22:05:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:02 smithi026 ceph-mon[118879]: daemon mds.foo.smithi026.rancfk is now active in filesystem foo as rank 0 2023-09-23T22:05:03.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:03 smithi026 ceph-mon[100157]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:03.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:03 smithi026 ceph-mon[100157]: mds.? [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] up:active 2023-09-23T22:05:03.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:03 smithi026 ceph-mon[100157]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 2023-09-23T22:05:03.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:03 smithi026 ceph-mon[118879]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:03.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:03 smithi026 ceph-mon[118879]: mds.? [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] up:active 2023-09-23T22:05:03.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:03 smithi026 ceph-mon[118879]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 2023-09-23T22:05:03.496 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:03 smithi070 ceph-mon[105170]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:03.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:03 smithi070 ceph-mon[105170]: mds.? [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] up:active 2023-09-23T22:05:03.497 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:03 smithi070 ceph-mon[105170]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 2023-09-23T22:05:04.143 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:05:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-09-23T22:05:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3372128317' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:05:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: mds.? [v2:172.21.15.70:6832/253331223,v1:172.21.15.70:6833/253331223] up:boot 2023-09-23T22:05:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:05:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:04 smithi070 ceph-mon[105170]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:04.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3372128317' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: mds.? [v2:172.21.15.70:6832/253331223,v1:172.21.15.70:6833/253331223] up:boot 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[118879]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:04.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3372128317' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: mds.? [v2:172.21.15.70:6832/253331223,v1:172.21.15.70:6833/253331223] up:boot 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:04.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:05:04.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:04 smithi026 ceph-mon[100157]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:04.940 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":20,"active_gid":24341,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6800","nonce":3418308803},{"type":"v1","addr":"172.21.15.26:6801","nonce":3418308803}]},"active_addr":"172.21.15.26:6801/3418308803","active_change":"2023-09-23T22:03:52.071373+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24344,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":59,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":3996051568}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":771693589}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":3618866901}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":1692711239}]}]} 2023-09-23T22:05:04.945 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T22:05:04.945 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T22:05:04.945 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:05:05.393 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:06.894 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:06 smithi026 ceph-mon[100157]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2023-09-23T22:05:06.895 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:06 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:06.895 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:06 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:06.895 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:06 smithi026 ceph-mon[118879]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2023-09-23T22:05:06.895 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:06 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:06.895 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:06 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:06 smithi070 ceph-mon[105170]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 4 op/s 2023-09-23T22:05:07.162 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:06 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.162 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:06 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.331 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:05:07.331 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":67,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:04:55.393401+0000","last_up_change":"2023-09-23T22:03:17.723571+0000","last_in_change":"2023-09-23T22:03:00.614896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:04:04.880309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:04:53.313440+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:04:53.801900+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6803","nonce":3939081623}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6805","nonce":3939081623}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6809","nonce":3939081623}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6807","nonce":3939081623}]},"public_addr":"172.21.15.26:6803/3939081623","cluster_addr":"172.21.15.26:6805/3939081623","heartbeat_back_addr":"172.21.15.26:6809/3939081623","heartbeat_front_addr":"172.21.15.26:6807/3939081623","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6811","nonce":2870847047}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6813","nonce":2870847047}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6817","nonce":2870847047}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6815","nonce":2870847047}]},"public_addr":"172.21.15.26:6811/2870847047","cluster_addr":"172.21.15.26:6813/2870847047","heartbeat_back_addr":"172.21.15.26:6817/2870847047","heartbeat_front_addr":"172.21.15.26:6815/2870847047","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6819","nonce":3354962178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6821","nonce":3354962178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6825","nonce":3354962178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6823","nonce":3354962178}]},"public_addr":"172.21.15.26:6819/3354962178","cluster_addr":"172.21.15.26:6821/3354962178","heartbeat_back_addr":"172.21.15.26:6825/3354962178","heartbeat_front_addr":"172.21.15.26:6823/3354962178","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6827","nonce":861409764}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6829","nonce":861409764}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6833","nonce":861409764}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6831","nonce":861409764}]},"public_addr":"172.21.15.26:6827/861409764","cluster_addr":"172.21.15.26:6829/861409764","heartbeat_back_addr":"172.21.15.26:6833/861409764","heartbeat_front_addr":"172.21.15.26:6831/861409764","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6801","nonce":1461751884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6803","nonce":1461751884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6807","nonce":1461751884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6805","nonce":1461751884}]},"public_addr":"172.21.15.70:6801/1461751884","cluster_addr":"172.21.15.70:6803/1461751884","heartbeat_back_addr":"172.21.15.70:6807/1461751884","heartbeat_front_addr":"172.21.15.70:6805/1461751884","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6817","nonce":572290869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6819","nonce":572290869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6823","nonce":572290869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6821","nonce":572290869}]},"public_addr":"172.21.15.70:6817/572290869","cluster_addr":"172.21.15.70:6819/572290869","heartbeat_back_addr":"172.21.15.70:6823/572290869","heartbeat_front_addr":"172.21.15.70:6821/572290869","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6800/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3350176691":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3419561437":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/577898280":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/1887156644":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:05:07.906 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T22:05:07.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:05:07.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: mds.? [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] up:active 2023-09-23T22:05:07.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:07.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:07.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1612197' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: mds.? [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] up:active 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:07.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:07.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:07.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:07.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:07.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:07 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1612197' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:05:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: mds.? [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] up:active 2023-09-23T22:05:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:08.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:08.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:08.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:08.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:08.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:08.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:08.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:07 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1612197' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:05:08.523 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:08.899 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:08 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:08] "GET /metrics HTTP/1.1" 200 238612 "" "Prometheus/2.43.0" 2023-09-23T22:05:09.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:09 smithi026 ceph-mon[100157]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:09.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:09 smithi026 ceph-mon[100157]: mds.? [v2:172.21.15.70:6832/253331223,v1:172.21.15.70:6833/253331223] up:standby 2023-09-23T22:05:09.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:09 smithi026 ceph-mon[100157]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:09.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:09 smithi026 ceph-mon[118879]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:09.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:09 smithi026 ceph-mon[118879]: mds.? [v2:172.21.15.70:6832/253331223,v1:172.21.15.70:6833/253331223] up:standby 2023-09-23T22:05:09.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:09 smithi026 ceph-mon[118879]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:09 smithi070 ceph-mon[105170]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:09 smithi070 ceph-mon[105170]: mds.? [v2:172.21.15.70:6832/253331223,v1:172.21.15.70:6833/253331223] up:standby 2023-09-23T22:05:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:09 smithi070 ceph-mon[105170]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:05:10.302 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:05:10.302 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":67,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:04:55.393401+0000","last_up_change":"2023-09-23T22:03:17.723571+0000","last_in_change":"2023-09-23T22:03:00.614896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:04:04.880309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:04:53.313440+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:04:53.801900+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6803","nonce":3939081623}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6805","nonce":3939081623}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6809","nonce":3939081623}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":3939081623},{"type":"v1","addr":"172.21.15.26:6807","nonce":3939081623}]},"public_addr":"172.21.15.26:6803/3939081623","cluster_addr":"172.21.15.26:6805/3939081623","heartbeat_back_addr":"172.21.15.26:6809/3939081623","heartbeat_front_addr":"172.21.15.26:6807/3939081623","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6811","nonce":2870847047}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6813","nonce":2870847047}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6817","nonce":2870847047}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":2870847047},{"type":"v1","addr":"172.21.15.26:6815","nonce":2870847047}]},"public_addr":"172.21.15.26:6811/2870847047","cluster_addr":"172.21.15.26:6813/2870847047","heartbeat_back_addr":"172.21.15.26:6817/2870847047","heartbeat_front_addr":"172.21.15.26:6815/2870847047","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6819","nonce":3354962178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6821","nonce":3354962178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6825","nonce":3354962178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":3354962178},{"type":"v1","addr":"172.21.15.26:6823","nonce":3354962178}]},"public_addr":"172.21.15.26:6819/3354962178","cluster_addr":"172.21.15.26:6821/3354962178","heartbeat_back_addr":"172.21.15.26:6825/3354962178","heartbeat_front_addr":"172.21.15.26:6823/3354962178","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6827","nonce":861409764}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6829","nonce":861409764}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6833","nonce":861409764}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":861409764},{"type":"v1","addr":"172.21.15.26:6831","nonce":861409764}]},"public_addr":"172.21.15.26:6827/861409764","cluster_addr":"172.21.15.26:6829/861409764","heartbeat_back_addr":"172.21.15.26:6833/861409764","heartbeat_front_addr":"172.21.15.26:6831/861409764","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6801","nonce":1461751884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6803","nonce":1461751884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6807","nonce":1461751884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6805","nonce":1461751884}]},"public_addr":"172.21.15.70:6801/1461751884","cluster_addr":"172.21.15.70:6803/1461751884","heartbeat_back_addr":"172.21.15.70:6807/1461751884","heartbeat_front_addr":"172.21.15.70:6805/1461751884","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6817","nonce":572290869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6819","nonce":572290869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6823","nonce":572290869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6821","nonce":572290869}]},"public_addr":"172.21.15.70:6817/572290869","cluster_addr":"172.21.15.70:6819/572290869","heartbeat_back_addr":"172.21.15.70:6823/572290869","heartbeat_front_addr":"172.21.15.70:6821/572290869","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:6800/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3350176691":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3419561437":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/577898280":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/1887156644":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:05:10.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T22:05:10.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T22:05:10.932 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T22:05:10.932 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T22:05:10.933 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T22:05:10.933 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T22:05:10.934 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T22:05:10.934 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T22:05:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:10 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:10 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:10 smithi026 ceph-mon[118879]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:10 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3201043685' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:05:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:10 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:10 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:10 smithi026 ceph-mon[100157]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:10 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3201043685' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:05:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:10 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:10 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:10 smithi070 ceph-mon[105170]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:10 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3201043685' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:05:11.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:05:11 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:05:12.200 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:12.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:12.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:12.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:12.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:12.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:11 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:12.252 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:12.314 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:12.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:11 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:12.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:11 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:12.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:11 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:12.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:11 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:12.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:11 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:12.578 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:12.580 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:12.581 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:12.581 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:12.582 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:12.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:12 smithi026 ceph-mon[100157]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:13.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:12 smithi026 ceph-mon[118879]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:13.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:12 smithi070 ceph-mon[105170]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:15.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:15 smithi026 ceph-mon[100157]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:15.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:15 smithi026 ceph-mon[118879]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:15.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:15 smithi070 ceph-mon[105170]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T22:05:16.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:16 smithi026 ceph-mon[100157]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:05:16.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:16 smithi026 ceph-mon[118879]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:05:17.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:16 smithi070 ceph-mon[105170]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:05:18.661 INFO:teuthology.orchestra.run.smithi026.stdout:128849018919 2023-09-23T22:05:18.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T22:05:18.896 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:18 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:18] "GET /metrics HTTP/1.1" 200 238612 "" "Prometheus/2.43.0" 2023-09-23T22:05:19.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:19 smithi026 ceph-mon[100157]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:05:19.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:19 smithi026 ceph-mon[118879]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:05:19.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:19 smithi070 ceph-mon[105170]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T22:05:19.702 INFO:teuthology.orchestra.run.smithi026.stdout:77309411377 2023-09-23T22:05:19.702 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T22:05:19.842 INFO:teuthology.orchestra.run.smithi026.stdout:163208757284 2023-09-23T22:05:19.842 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T22:05:21.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:21 smithi026 ceph-mon[118879]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:21.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:21 smithi026 ceph-mon[100157]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:21.396 INFO:teuthology.orchestra.run.smithi026.stdout:55834574902 2023-09-23T22:05:21.396 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T22:05:21.539 INFO:teuthology.orchestra.run.smithi026.stdout:34359738428 2023-09-23T22:05:21.539 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T22:05:21.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:21 smithi070 ceph-mon[105170]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:21.637 INFO:teuthology.orchestra.run.smithi026.stdout:231928234011 2023-09-23T22:05:21.637 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T22:05:21.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:05:21 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:05:22.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:22 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:22.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:22 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:22.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:22 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:22.795 INFO:teuthology.orchestra.run.smithi026.stdout:197568495647 2023-09-23T22:05:22.795 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T22:05:23.058 INFO:teuthology.orchestra.run.smithi026.stdout:107374182444 2023-09-23T22:05:23.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T22:05:23.138 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:23.138 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:23.146 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:23.246 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:23.411 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:23.412 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:23.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:23 smithi026 ceph-mon[118879]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:23.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:23 smithi026 ceph-mon[100157]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:23 smithi070 ceph-mon[105170]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:23.665 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:25.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:25 smithi026 ceph-mon[118879]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:25 smithi026 ceph-mon[100157]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:25 smithi070 ceph-mon[105170]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:26.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:26 smithi026 ceph-mon[100157]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:26.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:26 smithi026 ceph-mon[118879]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:26 smithi070 ceph-mon[105170]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:27.833 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:28.207 INFO:teuthology.orchestra.run.smithi026.stdout:77309411380 2023-09-23T22:05:28.504 INFO:teuthology.orchestra.run.smithi026.stdout:163208757286 2023-09-23T22:05:28.955 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:28 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:28] "GET /metrics HTTP/1.1" 200 238605 "" "Prometheus/2.43.0" 2023-09-23T22:05:29.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:29 smithi026 ceph-mon[100157]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:29.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:29 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3445654295' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:05:29.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:29 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/347068590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:05:29.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:29 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/419060113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:05:29.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:29 smithi026 ceph-mon[118879]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:29.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:29 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3445654295' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:05:29.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:29 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/347068590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:05:29.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:29 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/419060113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:05:29.329 INFO:teuthology.orchestra.run.smithi026.stdout:197568495649 2023-09-23T22:05:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:29 smithi070 ceph-mon[105170]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:29 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3445654295' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:05:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:29 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/347068590' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:05:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:29 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/419060113' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:05:30.095 INFO:teuthology.orchestra.run.smithi026.stdout:128849018922 2023-09-23T22:05:30.117 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757284 got 163208757286 for osd.5 2023-09-23T22:05:30.117 DEBUG:teuthology.parallel:result is None 2023-09-23T22:05:30.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:30 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2596060173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:05:30.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:30 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2596060173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:05:30.186 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411377 got 77309411380 for osd.2 2023-09-23T22:05:30.186 DEBUG:teuthology.parallel:result is None 2023-09-23T22:05:30.216 INFO:teuthology.orchestra.run.smithi026.stdout:231928234013 2023-09-23T22:05:30.309 INFO:teuthology.orchestra.run.smithi026.stdout:55834574904 2023-09-23T22:05:30.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:30 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2596060173' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:05:31.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:31 smithi026 ceph-mon[118879]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:31.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:31 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/168837206' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:05:31.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:31 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2816258436' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:05:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:31 smithi026 ceph-mon[100157]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:31 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/168837206' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:05:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:31 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2816258436' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:05:31.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:31 smithi070 ceph-mon[105170]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:31.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:31 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/168837206' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:05:31.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:31 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2816258436' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:05:31.799 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495647 got 197568495649 for osd.6 2023-09-23T22:05:31.799 DEBUG:teuthology.parallel:result is None 2023-09-23T22:05:31.870 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574902 got 55834574904 for osd.1 2023-09-23T22:05:31.870 DEBUG:teuthology.parallel:result is None 2023-09-23T22:05:31.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:05:31 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:05:32.590 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234011 got 231928234013 for osd.7 2023-09-23T22:05:32.590 DEBUG:teuthology.parallel:result is None 2023-09-23T22:05:32.634 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018919 got 128849018922 for osd.4 2023-09-23T22:05:32.634 DEBUG:teuthology.parallel:result is None 2023-09-23T22:05:32.677 INFO:teuthology.orchestra.run.smithi026.stdout:34359738430 2023-09-23T22:05:33.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:33 smithi026 ceph-mon[100157]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:33.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:33 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2218596613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:05:33.244 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738428 got 34359738430 for osd.0 2023-09-23T22:05:33.244 DEBUG:teuthology.parallel:result is None 2023-09-23T22:05:33.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:33 smithi026 ceph-mon[118879]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:33.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:33 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2218596613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:05:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:33 smithi070 ceph-mon[105170]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:33 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2218596613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:05:33.801 INFO:teuthology.orchestra.run.smithi026.stdout:107374182446 2023-09-23T22:05:34.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:34 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/610720757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:05:34.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:34 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/610720757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:05:34.511 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182444 got 107374182446 for osd.3 2023-09-23T22:05:34.512 DEBUG:teuthology.parallel:result is None 2023-09-23T22:05:34.512 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:05:34.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:05:34.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:34 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/610720757' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:05:34.875 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:35.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:35 smithi026 ceph-mon[100157]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:35.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:35 smithi026 ceph-mon[118879]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:35 smithi070 ceph-mon[105170]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:36.652 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:05:36.653 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:05:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:36 smithi026 ceph-mon[118879]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:36 smithi026 ceph-mon[100157]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:37.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:36 smithi070 ceph-mon[105170]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:37.195 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":63,"stamp":"2023-09-23T22:05:36.106441+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52072,"kb_used_data":7144,"kb_used_omap":0,"kb_used_meta":44864,"kb_avail":749909144,"statfs":{"total":767960285184,"available":767906963456,"internally_reserved":0,"allocated":7315456,"data_stored":4115659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":45940736},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003317"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.405597+0000","last_change":"2023-09-23T22:04:54.404859+0000","last_active":"2023-09-23T22:04:54.405597+0000","last_peered":"2023-09-23T22:04:54.405597+0000","last_clean":"2023-09-23T22:04:54.405597+0000","last_became_active":"2023-09-23T22:04:54.404604+0000","last_became_peered":"2023-09-23T22:04:54.404604+0000","last_unstale":"2023-09-23T22:04:54.405597+0000","last_undegraded":"2023-09-23T22:04:54.405597+0000","last_fullsized":"2023-09-23T22:04:54.405597+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:34.228549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.671975+0000","last_change":"2023-09-23T22:04:54.384572+0000","last_active":"2023-09-23T22:05:01.671975+0000","last_peered":"2023-09-23T22:05:01.671975+0000","last_clean":"2023-09-23T22:05:01.671975+0000","last_became_active":"2023-09-23T22:04:54.384302+0000","last_became_peered":"2023-09-23T22:04:54.384302+0000","last_unstale":"2023-09-23T22:05:01.671975+0000","last_undegraded":"2023-09-23T22:05:01.671975+0000","last_fullsized":"2023-09-23T22:05:01.671975+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:32.575153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.411577+0000","last_change":"2023-09-23T22:04:54.409325+0000","last_active":"2023-09-23T22:04:54.411577+0000","last_peered":"2023-09-23T22:04:54.411577+0000","last_clean":"2023-09-23T22:04:54.411577+0000","last_became_active":"2023-09-23T22:04:54.408996+0000","last_became_peered":"2023-09-23T22:04:54.408996+0000","last_unstale":"2023-09-23T22:04:54.411577+0000","last_undegraded":"2023-09-23T22:04:54.411577+0000","last_fullsized":"2023-09-23T22:04:54.411577+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:24.585073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.399505+0000","last_change":"2023-09-23T22:04:54.375952+0000","last_active":"2023-09-23T22:04:55.399505+0000","last_peered":"2023-09-23T22:04:55.399505+0000","last_clean":"2023-09-23T22:04:55.399505+0000","last_became_active":"2023-09-23T22:04:54.375649+0000","last_became_peered":"2023-09-23T22:04:54.375649+0000","last_unstale":"2023-09-23T22:04:55.399505+0000","last_undegraded":"2023-09-23T22:04:55.399505+0000","last_fullsized":"2023-09-23T22:04:55.399505+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:59.563466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.404288+0000","last_change":"2023-09-23T22:04:54.404097+0000","last_active":"2023-09-23T22:04:54.404288+0000","last_peered":"2023-09-23T22:04:54.404288+0000","last_clean":"2023-09-23T22:04:54.404288+0000","last_became_active":"2023-09-23T22:04:54.403823+0000","last_became_peered":"2023-09-23T22:04:54.403823+0000","last_unstale":"2023-09-23T22:04:54.404288+0000","last_undegraded":"2023-09-23T22:04:54.404288+0000","last_fullsized":"2023-09-23T22:04:54.404288+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:55.802552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.673194+0000","last_change":"2023-09-23T22:04:54.379680+0000","last_active":"2023-09-23T22:05:01.673194+0000","last_peered":"2023-09-23T22:05:01.673194+0000","last_clean":"2023-09-23T22:05:01.673194+0000","last_became_active":"2023-09-23T22:04:54.379383+0000","last_became_peered":"2023-09-23T22:04:54.379383+0000","last_unstale":"2023-09-23T22:05:01.673194+0000","last_undegraded":"2023-09-23T22:05:01.673194+0000","last_fullsized":"2023-09-23T22:05:01.673194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.282795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.405612+0000","last_change":"2023-09-23T22:04:54.404957+0000","last_active":"2023-09-23T22:04:54.405612+0000","last_peered":"2023-09-23T22:04:54.405612+0000","last_clean":"2023-09-23T22:04:54.405612+0000","last_became_active":"2023-09-23T22:04:54.404677+0000","last_became_peered":"2023-09-23T22:04:54.404677+0000","last_unstale":"2023-09-23T22:04:54.405612+0000","last_undegraded":"2023-09-23T22:04:54.405612+0000","last_fullsized":"2023-09-23T22:04:54.405612+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:17:26.547113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.672705+0000","last_change":"2023-09-23T22:04:54.382175+0000","last_active":"2023-09-23T22:05:01.672705+0000","last_peered":"2023-09-23T22:05:01.672705+0000","last_clean":"2023-09-23T22:05:01.672705+0000","last_became_active":"2023-09-23T22:04:54.381883+0000","last_became_peered":"2023-09-23T22:04:54.381883+0000","last_unstale":"2023-09-23T22:05:01.672705+0000","last_undegraded":"2023-09-23T22:05:01.672705+0000","last_fullsized":"2023-09-23T22:05:01.672705+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:48.043436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407153+0000","last_change":"2023-09-23T22:04:54.406726+0000","last_active":"2023-09-23T22:04:54.407153+0000","last_peered":"2023-09-23T22:04:54.407153+0000","last_clean":"2023-09-23T22:04:54.407153+0000","last_became_active":"2023-09-23T22:04:54.406409+0000","last_became_peered":"2023-09-23T22:04:54.406409+0000","last_unstale":"2023-09-23T22:04:54.407153+0000","last_undegraded":"2023-09-23T22:04:54.407153+0000","last_fullsized":"2023-09-23T22:04:54.407153+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:22.664231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.755929+0000","last_change":"2023-09-23T22:04:54.381881+0000","last_active":"2023-09-23T22:04:55.755929+0000","last_peered":"2023-09-23T22:04:55.755929+0000","last_clean":"2023-09-23T22:04:55.755929+0000","last_became_active":"2023-09-23T22:04:54.381585+0000","last_became_peered":"2023-09-23T22:04:54.381585+0000","last_unstale":"2023-09-23T22:04:55.755929+0000","last_undegraded":"2023-09-23T22:04:55.755929+0000","last_fullsized":"2023-09-23T22:04:55.755929+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:58:50.421302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.404155+0000","last_change":"2023-09-23T22:04:54.402894+0000","last_active":"2023-09-23T22:04:54.404155+0000","last_peered":"2023-09-23T22:04:54.404155+0000","last_clean":"2023-09-23T22:04:54.404155+0000","last_became_active":"2023-09-23T22:04:54.402475+0000","last_became_peered":"2023-09-23T22:04:54.402475+0000","last_unstale":"2023-09-23T22:04:54.404155+0000","last_undegraded":"2023-09-23T22:04:54.404155+0000","last_fullsized":"2023-09-23T22:04:54.404155+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:12.503263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.756165+0000","last_change":"2023-09-23T22:04:54.382294+0000","last_active":"2023-09-23T22:04:55.756165+0000","last_peered":"2023-09-23T22:04:55.756165+0000","last_clean":"2023-09-23T22:04:55.756165+0000","last_became_active":"2023-09-23T22:04:54.382013+0000","last_became_peered":"2023-09-23T22:04:54.382013+0000","last_unstale":"2023-09-23T22:04:55.756165+0000","last_undegraded":"2023-09-23T22:04:55.756165+0000","last_fullsized":"2023-09-23T22:04:55.756165+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:25.314566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.401488+0000","last_change":"2023-09-23T22:04:06.152239+0000","last_active":"2023-09-23T22:04:54.401488+0000","last_peered":"2023-09-23T22:04:54.401488+0000","last_clean":"2023-09-23T22:04:54.401488+0000","last_became_active":"2023-09-23T22:04:06.151906+0000","last_became_peered":"2023-09-23T22:04:06.151906+0000","last_unstale":"2023-09-23T22:04:54.401488+0000","last_undegraded":"2023-09-23T22:04:54.401488+0000","last_fullsized":"2023-09-23T22:04:54.401488+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:29.292018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.527739+0000","last_change":"2023-09-23T22:04:54.527443+0000","last_active":"2023-09-23T22:04:54.527739+0000","last_peered":"2023-09-23T22:04:54.527739+0000","last_clean":"2023-09-23T22:04:54.527739+0000","last_became_active":"2023-09-23T22:04:54.527123+0000","last_became_peered":"2023-09-23T22:04:54.527123+0000","last_unstale":"2023-09-23T22:04:54.527739+0000","last_undegraded":"2023-09-23T22:04:54.527739+0000","last_fullsized":"2023-09-23T22:04:54.527739+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:34.117223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407910+0000","last_change":"2023-09-23T22:04:06.211004+0000","last_active":"2023-09-23T22:04:54.407910+0000","last_peered":"2023-09-23T22:04:54.407910+0000","last_clean":"2023-09-23T22:04:54.407910+0000","last_became_active":"2023-09-23T22:04:06.210652+0000","last_became_peered":"2023-09-23T22:04:06.210652+0000","last_unstale":"2023-09-23T22:04:54.407910+0000","last_undegraded":"2023-09-23T22:04:54.407910+0000","last_fullsized":"2023-09-23T22:04:54.407910+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:32.721396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.673217+0000","last_change":"2023-09-23T22:04:54.387001+0000","last_active":"2023-09-23T22:05:01.673217+0000","last_peered":"2023-09-23T22:05:01.673217+0000","last_clean":"2023-09-23T22:05:01.673217+0000","last_became_active":"2023-09-23T22:04:54.386704+0000","last_became_peered":"2023-09-23T22:04:54.386704+0000","last_unstale":"2023-09-23T22:05:01.673217+0000","last_undegraded":"2023-09-23T22:05:01.673217+0000","last_fullsized":"2023-09-23T22:05:01.673217+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:18.520541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526363+0000","last_change":"2023-09-23T22:04:54.525852+0000","last_active":"2023-09-23T22:04:54.526363+0000","last_peered":"2023-09-23T22:04:54.526363+0000","last_clean":"2023-09-23T22:04:54.526363+0000","last_became_active":"2023-09-23T22:04:54.525524+0000","last_became_peered":"2023-09-23T22:04:54.525524+0000","last_unstale":"2023-09-23T22:04:54.526363+0000","last_undegraded":"2023-09-23T22:04:54.526363+0000","last_fullsized":"2023-09-23T22:04:54.526363+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:23.828759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407542+0000","last_change":"2023-09-23T22:04:54.385876+0000","last_active":"2023-09-23T22:04:54.407542+0000","last_peered":"2023-09-23T22:04:54.407542+0000","last_clean":"2023-09-23T22:04:54.407542+0000","last_became_active":"2023-09-23T22:04:54.385588+0000","last_became_peered":"2023-09-23T22:04:54.385588+0000","last_unstale":"2023-09-23T22:04:54.407542+0000","last_undegraded":"2023-09-23T22:04:54.407542+0000","last_fullsized":"2023-09-23T22:04:54.407542+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:47.856644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526683+0000","last_change":"2023-09-23T22:04:06.250190+0000","last_active":"2023-09-23T22:04:54.526683+0000","last_peered":"2023-09-23T22:04:54.526683+0000","last_clean":"2023-09-23T22:04:54.526683+0000","last_became_active":"2023-09-23T22:04:06.249826+0000","last_became_peered":"2023-09-23T22:04:06.249826+0000","last_unstale":"2023-09-23T22:04:54.526683+0000","last_undegraded":"2023-09-23T22:04:54.526683+0000","last_fullsized":"2023-09-23T22:04:54.526683+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:16.888898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.414890+0000","last_change":"2023-09-23T22:04:54.414198+0000","last_active":"2023-09-23T22:04:54.414890+0000","last_peered":"2023-09-23T22:04:54.414890+0000","last_clean":"2023-09-23T22:04:54.414890+0000","last_became_active":"2023-09-23T22:04:54.413874+0000","last_became_peered":"2023-09-23T22:04:54.413874+0000","last_unstale":"2023-09-23T22:04:54.414890+0000","last_undegraded":"2023-09-23T22:04:54.414890+0000","last_fullsized":"2023-09-23T22:04:54.414890+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:20.609941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.400154+0000","last_change":"2023-09-23T22:04:54.383681+0000","last_active":"2023-09-23T22:04:54.400154+0000","last_peered":"2023-09-23T22:04:54.400154+0000","last_clean":"2023-09-23T22:04:54.400154+0000","last_became_active":"2023-09-23T22:04:54.383420+0000","last_became_peered":"2023-09-23T22:04:54.383420+0000","last_unstale":"2023-09-23T22:04:54.400154+0000","last_undegraded":"2023-09-23T22:04:54.400154+0000","last_fullsized":"2023-09-23T22:04:54.400154+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:48.031496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526581+0000","last_change":"2023-09-23T22:04:06.250172+0000","last_active":"2023-09-23T22:04:54.526581+0000","last_peered":"2023-09-23T22:04:54.526581+0000","last_clean":"2023-09-23T22:04:54.526581+0000","last_became_active":"2023-09-23T22:04:06.249764+0000","last_became_peered":"2023-09-23T22:04:06.249764+0000","last_unstale":"2023-09-23T22:04:54.526581+0000","last_undegraded":"2023-09-23T22:04:54.526581+0000","last_fullsized":"2023-09-23T22:04:54.526581+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:28:11.590738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.527561+0000","last_change":"2023-09-23T22:04:54.527318+0000","last_active":"2023-09-23T22:04:54.527561+0000","last_peered":"2023-09-23T22:04:54.527561+0000","last_clean":"2023-09-23T22:04:54.527561+0000","last_became_active":"2023-09-23T22:04:54.527023+0000","last_became_peered":"2023-09-23T22:04:54.527023+0000","last_unstale":"2023-09-23T22:04:54.527561+0000","last_undegraded":"2023-09-23T22:04:54.527561+0000","last_fullsized":"2023-09-23T22:04:54.527561+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:42.048739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398928+0000","last_change":"2023-09-23T22:04:54.378787+0000","last_active":"2023-09-23T22:04:54.398928+0000","last_peered":"2023-09-23T22:04:54.398928+0000","last_clean":"2023-09-23T22:04:54.398928+0000","last_became_active":"2023-09-23T22:04:54.378507+0000","last_became_peered":"2023-09-23T22:04:54.378507+0000","last_unstale":"2023-09-23T22:04:54.398928+0000","last_undegraded":"2023-09-23T22:04:54.398928+0000","last_fullsized":"2023-09-23T22:04:54.398928+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:27.250805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":275,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526579+0000","last_change":"2023-09-23T22:03:21.650107+0000","last_active":"2023-09-23T22:04:54.526579+0000","last_peered":"2023-09-23T22:04:54.526579+0000","last_clean":"2023-09-23T22:04:54.526579+0000","last_became_active":"2023-09-23T22:03:21.649660+0000","last_became_peered":"2023-09-23T22:03:21.649660+0000","last_unstale":"2023-09-23T22:04:54.526579+0000","last_undegraded":"2023-09-23T22:04:54.526579+0000","last_fullsized":"2023-09-23T22:04:54.526579+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:13.053339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.395218+0000","last_change":"2023-09-23T22:04:06.154819+0000","last_active":"2023-09-23T22:04:54.395218+0000","last_peered":"2023-09-23T22:04:54.395218+0000","last_clean":"2023-09-23T22:04:54.395218+0000","last_became_active":"2023-09-23T22:04:06.154506+0000","last_became_peered":"2023-09-23T22:04:06.154506+0000","last_unstale":"2023-09-23T22:04:54.395218+0000","last_undegraded":"2023-09-23T22:04:54.395218+0000","last_fullsized":"2023-09-23T22:04:54.395218+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:49.420305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.413574+0000","last_change":"2023-09-23T22:04:54.411513+0000","last_active":"2023-09-23T22:04:54.413574+0000","last_peered":"2023-09-23T22:04:54.413574+0000","last_clean":"2023-09-23T22:04:54.413574+0000","last_became_active":"2023-09-23T22:04:54.411210+0000","last_became_peered":"2023-09-23T22:04:54.411210+0000","last_unstale":"2023-09-23T22:04:54.413574+0000","last_undegraded":"2023-09-23T22:04:54.413574+0000","last_fullsized":"2023-09-23T22:04:54.413574+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:14.742116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407936+0000","last_change":"2023-09-23T22:04:54.385915+0000","last_active":"2023-09-23T22:04:54.407936+0000","last_peered":"2023-09-23T22:04:54.407936+0000","last_clean":"2023-09-23T22:04:54.407936+0000","last_became_active":"2023-09-23T22:04:54.385615+0000","last_became_peered":"2023-09-23T22:04:54.385615+0000","last_unstale":"2023-09-23T22:04:54.407936+0000","last_undegraded":"2023-09-23T22:04:54.407936+0000","last_fullsized":"2023-09-23T22:04:54.407936+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:39.881906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.399062+0000","last_change":"2023-09-23T22:04:06.250115+0000","last_active":"2023-09-23T22:04:54.399062+0000","last_peered":"2023-09-23T22:04:54.399062+0000","last_clean":"2023-09-23T22:04:54.399062+0000","last_became_active":"2023-09-23T22:04:06.249760+0000","last_became_peered":"2023-09-23T22:04:06.249760+0000","last_unstale":"2023-09-23T22:04:54.399062+0000","last_undegraded":"2023-09-23T22:04:54.399062+0000","last_fullsized":"2023-09-23T22:04:54.399062+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:48.266985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.416900+0000","last_change":"2023-09-23T22:04:54.416646+0000","last_active":"2023-09-23T22:04:54.416900+0000","last_peered":"2023-09-23T22:04:54.416900+0000","last_clean":"2023-09-23T22:04:54.416900+0000","last_became_active":"2023-09-23T22:04:54.416275+0000","last_became_peered":"2023-09-23T22:04:54.416275+0000","last_unstale":"2023-09-23T22:04:54.416900+0000","last_undegraded":"2023-09-23T22:04:54.416900+0000","last_fullsized":"2023-09-23T22:04:54.416900+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:26.326726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398236+0000","last_change":"2023-09-23T22:04:54.387074+0000","last_active":"2023-09-23T22:04:54.398236+0000","last_peered":"2023-09-23T22:04:54.398236+0000","last_clean":"2023-09-23T22:04:54.398236+0000","last_became_active":"2023-09-23T22:04:54.386207+0000","last_became_peered":"2023-09-23T22:04:54.386207+0000","last_unstale":"2023-09-23T22:04:54.398236+0000","last_undegraded":"2023-09-23T22:04:54.398236+0000","last_fullsized":"2023-09-23T22:04:54.398236+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:49.554877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.395062+0000","last_change":"2023-09-23T22:04:06.153745+0000","last_active":"2023-09-23T22:04:54.395062+0000","last_peered":"2023-09-23T22:04:54.395062+0000","last_clean":"2023-09-23T22:04:54.395062+0000","last_became_active":"2023-09-23T22:04:06.153371+0000","last_became_peered":"2023-09-23T22:04:06.153371+0000","last_unstale":"2023-09-23T22:04:54.395062+0000","last_undegraded":"2023-09-23T22:04:54.395062+0000","last_fullsized":"2023-09-23T22:04:54.395062+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:36.366505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.415049+0000","last_change":"2023-09-23T22:04:54.414044+0000","last_active":"2023-09-23T22:04:54.415049+0000","last_peered":"2023-09-23T22:04:54.415049+0000","last_clean":"2023-09-23T22:04:54.415049+0000","last_became_active":"2023-09-23T22:04:54.413453+0000","last_became_peered":"2023-09-23T22:04:54.413453+0000","last_unstale":"2023-09-23T22:04:54.415049+0000","last_undegraded":"2023-09-23T22:04:54.415049+0000","last_fullsized":"2023-09-23T22:04:54.415049+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:31.563303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.397076+0000","last_change":"2023-09-23T22:04:54.379188+0000","last_active":"2023-09-23T22:04:54.397076+0000","last_peered":"2023-09-23T22:04:54.397076+0000","last_clean":"2023-09-23T22:04:54.397076+0000","last_became_active":"2023-09-23T22:04:54.378612+0000","last_became_peered":"2023-09-23T22:04:54.378612+0000","last_unstale":"2023-09-23T22:04:54.397076+0000","last_undegraded":"2023-09-23T22:04:54.397076+0000","last_fullsized":"2023-09-23T22:04:54.397076+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:03.596200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407426+0000","last_change":"2023-09-23T22:04:06.250575+0000","last_active":"2023-09-23T22:04:54.407426+0000","last_peered":"2023-09-23T22:04:54.407426+0000","last_clean":"2023-09-23T22:04:54.407426+0000","last_became_active":"2023-09-23T22:04:06.249857+0000","last_became_peered":"2023-09-23T22:04:06.249857+0000","last_unstale":"2023-09-23T22:04:54.407426+0000","last_undegraded":"2023-09-23T22:04:54.407426+0000","last_fullsized":"2023-09-23T22:04:54.407426+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:52.618997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.414891+0000","last_change":"2023-09-23T22:04:54.413503+0000","last_active":"2023-09-23T22:04:54.414891+0000","last_peered":"2023-09-23T22:04:54.414891+0000","last_clean":"2023-09-23T22:04:54.414891+0000","last_became_active":"2023-09-23T22:04:54.413192+0000","last_became_peered":"2023-09-23T22:04:54.413192+0000","last_unstale":"2023-09-23T22:04:54.414891+0000","last_undegraded":"2023-09-23T22:04:54.414891+0000","last_fullsized":"2023-09-23T22:04:54.414891+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:11.852260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398176+0000","last_change":"2023-09-23T22:04:54.387587+0000","last_active":"2023-09-23T22:04:54.398176+0000","last_peered":"2023-09-23T22:04:54.398176+0000","last_clean":"2023-09-23T22:04:54.398176+0000","last_became_active":"2023-09-23T22:04:54.387313+0000","last_became_peered":"2023-09-23T22:04:54.387313+0000","last_unstale":"2023-09-23T22:04:54.398176+0000","last_undegraded":"2023-09-23T22:04:54.398176+0000","last_fullsized":"2023-09-23T22:04:54.398176+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:53.756703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.409873+0000","last_change":"2023-09-23T22:04:54.409258+0000","last_active":"2023-09-23T22:04:54.409873+0000","last_peered":"2023-09-23T22:04:54.409873+0000","last_clean":"2023-09-23T22:04:54.409873+0000","last_became_active":"2023-09-23T22:04:54.408978+0000","last_became_peered":"2023-09-23T22:04:54.408978+0000","last_unstale":"2023-09-23T22:04:54.409873+0000","last_undegraded":"2023-09-23T22:04:54.409873+0000","last_fullsized":"2023-09-23T22:04:54.409873+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:30.355826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398452+0000","last_change":"2023-09-23T22:04:54.381395+0000","last_active":"2023-09-23T22:04:54.398452+0000","last_peered":"2023-09-23T22:04:54.398452+0000","last_clean":"2023-09-23T22:04:54.398452+0000","last_became_active":"2023-09-23T22:04:54.381159+0000","last_became_peered":"2023-09-23T22:04:54.381159+0000","last_unstale":"2023-09-23T22:04:54.398452+0000","last_undegraded":"2023-09-23T22:04:54.398452+0000","last_fullsized":"2023-09-23T22:04:54.398452+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:35.437640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.405618+0000","last_change":"2023-09-23T22:04:54.405009+0000","last_active":"2023-09-23T22:04:54.405618+0000","last_peered":"2023-09-23T22:04:54.405618+0000","last_clean":"2023-09-23T22:04:54.405618+0000","last_became_active":"2023-09-23T22:04:54.404744+0000","last_became_peered":"2023-09-23T22:04:54.404744+0000","last_unstale":"2023-09-23T22:04:54.405618+0000","last_undegraded":"2023-09-23T22:04:54.405618+0000","last_fullsized":"2023-09-23T22:04:54.405618+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:54.736713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.402301+0000","last_change":"2023-09-23T22:04:54.382501+0000","last_active":"2023-09-23T22:04:54.402301+0000","last_peered":"2023-09-23T22:04:54.402301+0000","last_clean":"2023-09-23T22:04:54.402301+0000","last_became_active":"2023-09-23T22:04:54.382201+0000","last_became_peered":"2023-09-23T22:04:54.382201+0000","last_unstale":"2023-09-23T22:04:54.402301+0000","last_undegraded":"2023-09-23T22:04:54.402301+0000","last_fullsized":"2023-09-23T22:04:54.402301+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:00.811365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.404196+0000","last_change":"2023-09-23T22:04:54.402068+0000","last_active":"2023-09-23T22:04:54.404196+0000","last_peered":"2023-09-23T22:04:54.404196+0000","last_clean":"2023-09-23T22:04:54.404196+0000","last_became_active":"2023-09-23T22:04:54.401785+0000","last_became_peered":"2023-09-23T22:04:54.401785+0000","last_unstale":"2023-09-23T22:04:54.404196+0000","last_undegraded":"2023-09-23T22:04:54.404196+0000","last_fullsized":"2023-09-23T22:04:54.404196+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:25.421013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.672529+0000","last_change":"2023-09-23T22:04:54.386352+0000","last_active":"2023-09-23T22:05:01.672529+0000","last_peered":"2023-09-23T22:05:01.672529+0000","last_clean":"2023-09-23T22:05:01.672529+0000","last_became_active":"2023-09-23T22:04:54.386056+0000","last_became_peered":"2023-09-23T22:04:54.386056+0000","last_unstale":"2023-09-23T22:05:01.672529+0000","last_undegraded":"2023-09-23T22:05:01.672529+0000","last_fullsized":"2023-09-23T22:05:01.672529+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:47.833333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.414802+0000","last_change":"2023-09-23T22:04:54.414057+0000","last_active":"2023-09-23T22:04:54.414802+0000","last_peered":"2023-09-23T22:04:54.414802+0000","last_clean":"2023-09-23T22:04:54.414802+0000","last_became_active":"2023-09-23T22:04:54.413773+0000","last_became_peered":"2023-09-23T22:04:54.413773+0000","last_unstale":"2023-09-23T22:04:54.414802+0000","last_undegraded":"2023-09-23T22:04:54.414802+0000","last_fullsized":"2023-09-23T22:04:54.414802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.557197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.397202+0000","last_change":"2023-09-23T22:04:54.380397+0000","last_active":"2023-09-23T22:04:54.397202+0000","last_peered":"2023-09-23T22:04:54.397202+0000","last_clean":"2023-09-23T22:04:54.397202+0000","last_became_active":"2023-09-23T22:04:54.379637+0000","last_became_peered":"2023-09-23T22:04:54.379637+0000","last_unstale":"2023-09-23T22:04:54.397202+0000","last_undegraded":"2023-09-23T22:04:54.397202+0000","last_fullsized":"2023-09-23T22:04:54.397202+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:47.162589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.525999+0000","last_change":"2023-09-23T22:04:54.525753+0000","last_active":"2023-09-23T22:04:54.525999+0000","last_peered":"2023-09-23T22:04:54.525999+0000","last_clean":"2023-09-23T22:04:54.525999+0000","last_became_active":"2023-09-23T22:04:54.525487+0000","last_became_peered":"2023-09-23T22:04:54.525487+0000","last_unstale":"2023-09-23T22:04:54.525999+0000","last_undegraded":"2023-09-23T22:04:54.525999+0000","last_fullsized":"2023-09-23T22:04:54.525999+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:53.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.756642+0000","last_change":"2023-09-23T22:04:54.384008+0000","last_active":"2023-09-23T22:04:55.756642+0000","last_peered":"2023-09-23T22:04:55.756642+0000","last_clean":"2023-09-23T22:04:55.756642+0000","last_became_active":"2023-09-23T22:04:54.383702+0000","last_became_peered":"2023-09-23T22:04:54.383702+0000","last_unstale":"2023-09-23T22:04:55.756642+0000","last_undegraded":"2023-09-23T22:04:55.756642+0000","last_fullsized":"2023-09-23T22:04:55.756642+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:00.423911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.410032+0000","last_change":"2023-09-23T22:04:54.409736+0000","last_active":"2023-09-23T22:04:54.410032+0000","last_peered":"2023-09-23T22:04:54.410032+0000","last_clean":"2023-09-23T22:04:54.410032+0000","last_became_active":"2023-09-23T22:04:54.409552+0000","last_became_peered":"2023-09-23T22:04:54.409552+0000","last_unstale":"2023-09-23T22:04:54.410032+0000","last_undegraded":"2023-09-23T22:04:54.410032+0000","last_fullsized":"2023-09-23T22:04:54.410032+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:08.099853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.674070+0000","last_change":"2023-09-23T22:04:54.384587+0000","last_active":"2023-09-23T22:05:01.674070+0000","last_peered":"2023-09-23T22:05:01.674070+0000","last_clean":"2023-09-23T22:05:01.674070+0000","last_became_active":"2023-09-23T22:04:54.384308+0000","last_became_peered":"2023-09-23T22:04:54.384308+0000","last_unstale":"2023-09-23T22:05:01.674070+0000","last_undegraded":"2023-09-23T22:05:01.674070+0000","last_fullsized":"2023-09-23T22:05:01.674070+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:12.465686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.528362+0000","last_change":"2023-09-23T22:04:54.528019+0000","last_active":"2023-09-23T22:04:54.528362+0000","last_peered":"2023-09-23T22:04:54.528362+0000","last_clean":"2023-09-23T22:04:54.528362+0000","last_became_active":"2023-09-23T22:04:54.527666+0000","last_became_peered":"2023-09-23T22:04:54.527666+0000","last_unstale":"2023-09-23T22:04:54.528362+0000","last_undegraded":"2023-09-23T22:04:54.528362+0000","last_fullsized":"2023-09-23T22:04:54.528362+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:32.774493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.756450+0000","last_change":"2023-09-23T22:04:54.387398+0000","last_active":"2023-09-23T22:04:55.756450+0000","last_peered":"2023-09-23T22:04:55.756450+0000","last_clean":"2023-09-23T22:04:55.756450+0000","last_became_active":"2023-09-23T22:04:54.385217+0000","last_became_peered":"2023-09-23T22:04:54.385217+0000","last_unstale":"2023-09-23T22:04:55.756450+0000","last_undegraded":"2023-09-23T22:04:55.756450+0000","last_fullsized":"2023-09-23T22:04:55.756450+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:17.101844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.527908+0000","last_change":"2023-09-23T22:04:54.526893+0000","last_active":"2023-09-23T22:04:54.527908+0000","last_peered":"2023-09-23T22:04:54.527908+0000","last_clean":"2023-09-23T22:04:54.527908+0000","last_became_active":"2023-09-23T22:04:54.526575+0000","last_became_peered":"2023-09-23T22:04:54.526575+0000","last_unstale":"2023-09-23T22:04:54.527908+0000","last_undegraded":"2023-09-23T22:04:54.527908+0000","last_fullsized":"2023-09-23T22:04:54.527908+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:41.368329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.676729+0000","last_change":"2023-09-23T22:04:54.377190+0000","last_active":"2023-09-23T22:05:01.676729+0000","last_peered":"2023-09-23T22:05:01.676729+0000","last_clean":"2023-09-23T22:05:01.676729+0000","last_became_active":"2023-09-23T22:04:54.376865+0000","last_became_peered":"2023-09-23T22:04:54.376865+0000","last_unstale":"2023-09-23T22:05:01.676729+0000","last_undegraded":"2023-09-23T22:05:01.676729+0000","last_fullsized":"2023-09-23T22:05:01.676729+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:52.474939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526300+0000","last_change":"2023-09-23T22:04:54.525997+0000","last_active":"2023-09-23T22:04:54.526300+0000","last_peered":"2023-09-23T22:04:54.526300+0000","last_clean":"2023-09-23T22:04:54.526300+0000","last_became_active":"2023-09-23T22:04:54.525678+0000","last_became_peered":"2023-09-23T22:04:54.525678+0000","last_unstale":"2023-09-23T22:04:54.526300+0000","last_undegraded":"2023-09-23T22:04:54.526300+0000","last_fullsized":"2023-09-23T22:04:54.526300+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:55.326941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526576+0000","last_change":"2023-09-23T22:04:54.387688+0000","last_active":"2023-09-23T22:04:54.526576+0000","last_peered":"2023-09-23T22:04:54.526576+0000","last_clean":"2023-09-23T22:04:54.526576+0000","last_became_active":"2023-09-23T22:04:54.387421+0000","last_became_peered":"2023-09-23T22:04:54.387421+0000","last_unstale":"2023-09-23T22:04:54.526576+0000","last_undegraded":"2023-09-23T22:04:54.526576+0000","last_fullsized":"2023-09-23T22:04:54.526576+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:22.411746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.528219+0000","last_change":"2023-09-23T22:04:54.527953+0000","last_active":"2023-09-23T22:04:54.528219+0000","last_peered":"2023-09-23T22:04:54.528219+0000","last_clean":"2023-09-23T22:04:54.528219+0000","last_became_active":"2023-09-23T22:04:54.527667+0000","last_became_peered":"2023-09-23T22:04:54.527667+0000","last_unstale":"2023-09-23T22:04:54.528219+0000","last_undegraded":"2023-09-23T22:04:54.528219+0000","last_fullsized":"2023-09-23T22:04:54.528219+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:14.312715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.401489+0000","last_change":"2023-09-23T22:04:54.383142+0000","last_active":"2023-09-23T22:04:54.401489+0000","last_peered":"2023-09-23T22:04:54.401489+0000","last_clean":"2023-09-23T22:04:54.401489+0000","last_became_active":"2023-09-23T22:04:54.382866+0000","last_became_peered":"2023-09-23T22:04:54.382866+0000","last_unstale":"2023-09-23T22:04:54.401489+0000","last_undegraded":"2023-09-23T22:04:54.401489+0000","last_fullsized":"2023-09-23T22:04:54.401489+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:05.081316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.528372+0000","last_change":"2023-09-23T22:04:54.526801+0000","last_active":"2023-09-23T22:04:54.528372+0000","last_peered":"2023-09-23T22:04:54.528372+0000","last_clean":"2023-09-23T22:04:54.528372+0000","last_became_active":"2023-09-23T22:04:54.526516+0000","last_became_peered":"2023-09-23T22:04:54.526516+0000","last_unstale":"2023-09-23T22:04:54.528372+0000","last_undegraded":"2023-09-23T22:04:54.528372+0000","last_fullsized":"2023-09-23T22:04:54.528372+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:21.836536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.673577+0000","last_change":"2023-09-23T22:04:54.384846+0000","last_active":"2023-09-23T22:05:01.673577+0000","last_peered":"2023-09-23T22:05:01.673577+0000","last_clean":"2023-09-23T22:05:01.673577+0000","last_became_active":"2023-09-23T22:04:54.384616+0000","last_became_peered":"2023-09-23T22:04:54.384616+0000","last_unstale":"2023-09-23T22:05:01.673577+0000","last_undegraded":"2023-09-23T22:05:01.673577+0000","last_fullsized":"2023-09-23T22:05:01.673577+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:33.659945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.405631+0000","last_change":"2023-09-23T22:04:54.404971+0000","last_active":"2023-09-23T22:04:54.405631+0000","last_peered":"2023-09-23T22:04:54.405631+0000","last_clean":"2023-09-23T22:04:54.405631+0000","last_became_active":"2023-09-23T22:04:54.404689+0000","last_became_peered":"2023-09-23T22:04:54.404689+0000","last_unstale":"2023-09-23T22:04:54.405631+0000","last_undegraded":"2023-09-23T22:04:54.405631+0000","last_fullsized":"2023-09-23T22:04:54.405631+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:39.928048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"67'6","reported_seq":17,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.675061+0000","last_change":"2023-09-23T22:04:54.381132+0000","last_active":"2023-09-23T22:05:01.675061+0000","last_peered":"2023-09-23T22:05:01.675061+0000","last_clean":"2023-09-23T22:05:01.675061+0000","last_became_active":"2023-09-23T22:04:54.380838+0000","last_became_peered":"2023-09-23T22:04:54.380838+0000","last_unstale":"2023-09-23T22:05:01.675061+0000","last_undegraded":"2023-09-23T22:05:01.675061+0000","last_fullsized":"2023-09-23T22:05:01.675061+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:41.334974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.415068+0000","last_change":"2023-09-23T22:04:54.414266+0000","last_active":"2023-09-23T22:04:54.415068+0000","last_peered":"2023-09-23T22:04:54.415068+0000","last_clean":"2023-09-23T22:04:54.415068+0000","last_became_active":"2023-09-23T22:04:54.413779+0000","last_became_peered":"2023-09-23T22:04:54.413779+0000","last_unstale":"2023-09-23T22:04:54.415068+0000","last_undegraded":"2023-09-23T22:04:54.415068+0000","last_fullsized":"2023-09-23T22:04:54.415068+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:31.084009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":14,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.674470+0000","last_change":"2023-09-23T22:04:54.384065+0000","last_active":"2023-09-23T22:05:01.674470+0000","last_peered":"2023-09-23T22:05:01.674470+0000","last_clean":"2023-09-23T22:05:01.674470+0000","last_became_active":"2023-09-23T22:04:54.383777+0000","last_became_peered":"2023-09-23T22:04:54.383777+0000","last_unstale":"2023-09-23T22:05:01.674470+0000","last_undegraded":"2023-09-23T22:05:01.674470+0000","last_fullsized":"2023-09-23T22:05:01.674470+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:49.984604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526277+0000","last_change":"2023-09-23T22:04:54.525962+0000","last_active":"2023-09-23T22:04:54.526277+0000","last_peered":"2023-09-23T22:04:54.526277+0000","last_clean":"2023-09-23T22:04:54.526277+0000","last_became_active":"2023-09-23T22:04:54.525741+0000","last_became_peered":"2023-09-23T22:04:54.525741+0000","last_unstale":"2023-09-23T22:04:54.526277+0000","last_undegraded":"2023-09-23T22:04:54.526277+0000","last_fullsized":"2023-09-23T22:04:54.526277+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:53:05.805798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.395321+0000","last_change":"2023-09-23T22:04:54.382361+0000","last_active":"2023-09-23T22:04:54.395321+0000","last_peered":"2023-09-23T22:04:54.395321+0000","last_clean":"2023-09-23T22:04:54.395321+0000","last_became_active":"2023-09-23T22:04:54.382033+0000","last_became_peered":"2023-09-23T22:04:54.382033+0000","last_unstale":"2023-09-23T22:04:54.395321+0000","last_undegraded":"2023-09-23T22:04:54.395321+0000","last_fullsized":"2023-09-23T22:04:54.395321+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:51.412662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.415728+0000","last_change":"2023-09-23T22:04:54.412428+0000","last_active":"2023-09-23T22:04:54.415728+0000","last_peered":"2023-09-23T22:04:54.415728+0000","last_clean":"2023-09-23T22:04:54.415728+0000","last_became_active":"2023-09-23T22:04:54.412224+0000","last_became_peered":"2023-09-23T22:04:54.412224+0000","last_unstale":"2023-09-23T22:04:54.415728+0000","last_undegraded":"2023-09-23T22:04:54.415728+0000","last_fullsized":"2023-09-23T22:04:54.415728+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:10.264695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.399611+0000","last_change":"2023-09-23T22:04:54.389665+0000","last_active":"2023-09-23T22:04:54.399611+0000","last_peered":"2023-09-23T22:04:54.399611+0000","last_clean":"2023-09-23T22:04:54.399611+0000","last_became_active":"2023-09-23T22:04:54.389467+0000","last_became_peered":"2023-09-23T22:04:54.389467+0000","last_unstale":"2023-09-23T22:04:54.399611+0000","last_undegraded":"2023-09-23T22:04:54.399611+0000","last_fullsized":"2023-09-23T22:04:54.399611+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:45.129155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.406317+0000","last_change":"2023-09-23T22:04:54.405930+0000","last_active":"2023-09-23T22:04:54.406317+0000","last_peered":"2023-09-23T22:04:54.406317+0000","last_clean":"2023-09-23T22:04:54.406317+0000","last_became_active":"2023-09-23T22:04:54.405654+0000","last_became_peered":"2023-09-23T22:04:54.405654+0000","last_unstale":"2023-09-23T22:04:54.406317+0000","last_undegraded":"2023-09-23T22:04:54.406317+0000","last_fullsized":"2023-09-23T22:04:54.406317+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:17.173425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398286+0000","last_change":"2023-09-23T22:04:54.388512+0000","last_active":"2023-09-23T22:04:54.398286+0000","last_peered":"2023-09-23T22:04:54.398286+0000","last_clean":"2023-09-23T22:04:54.398286+0000","last_became_active":"2023-09-23T22:04:54.388281+0000","last_became_peered":"2023-09-23T22:04:54.388281+0000","last_unstale":"2023-09-23T22:04:54.398286+0000","last_undegraded":"2023-09-23T22:04:54.398286+0000","last_fullsized":"2023-09-23T22:04:54.398286+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:43.537168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.528058+0000","last_change":"2023-09-23T22:04:54.527728+0000","last_active":"2023-09-23T22:04:54.528058+0000","last_peered":"2023-09-23T22:04:54.528058+0000","last_clean":"2023-09-23T22:04:54.528058+0000","last_became_active":"2023-09-23T22:04:54.527430+0000","last_became_peered":"2023-09-23T22:04:54.527430+0000","last_unstale":"2023-09-23T22:04:54.528058+0000","last_undegraded":"2023-09-23T22:04:54.528058+0000","last_fullsized":"2023-09-23T22:04:54.528058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:30.298965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407485+0000","last_change":"2023-09-23T22:04:54.387554+0000","last_active":"2023-09-23T22:04:54.407485+0000","last_peered":"2023-09-23T22:04:54.407485+0000","last_clean":"2023-09-23T22:04:54.407485+0000","last_became_active":"2023-09-23T22:04:54.387274+0000","last_became_peered":"2023-09-23T22:04:54.387274+0000","last_unstale":"2023-09-23T22:04:54.407485+0000","last_undegraded":"2023-09-23T22:04:54.407485+0000","last_fullsized":"2023-09-23T22:04:54.407485+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:20.494449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.415495+0000","last_change":"2023-09-23T22:04:54.411754+0000","last_active":"2023-09-23T22:04:54.415495+0000","last_peered":"2023-09-23T22:04:54.415495+0000","last_clean":"2023-09-23T22:04:54.415495+0000","last_became_active":"2023-09-23T22:04:54.411472+0000","last_became_peered":"2023-09-23T22:04:54.411472+0000","last_unstale":"2023-09-23T22:04:54.415495+0000","last_undegraded":"2023-09-23T22:04:54.415495+0000","last_fullsized":"2023-09-23T22:04:54.415495+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:40.722110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.756045+0000","last_change":"2023-09-23T22:04:54.380166+0000","last_active":"2023-09-23T22:04:55.756045+0000","last_peered":"2023-09-23T22:04:55.756045+0000","last_clean":"2023-09-23T22:04:55.756045+0000","last_became_active":"2023-09-23T22:04:54.379878+0000","last_became_peered":"2023-09-23T22:04:54.379878+0000","last_unstale":"2023-09-23T22:04:55.756045+0000","last_undegraded":"2023-09-23T22:04:55.756045+0000","last_fullsized":"2023-09-23T22:04:55.756045+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:07.644159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234014,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6524,"kb_used_data":1268,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93738628,"statfs":{"total":95995035648,"available":95988355072,"internally_reserved":0,"allocated":1298432,"data_stored":884042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.33500000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.748,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.58999999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.34100000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69099999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.70099999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40200000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.80000000000000004,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54700000000000004,"15min":0.54400000000000004},"min":{"1min":0.37,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.78800000000000003,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.55800000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57799999999999996,"15min":0.58099999999999996},"min":{"1min":0.42799999999999999,"5min":0.38,"15min":0.38},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.71799999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46700000000000003,"15min":0.46500000000000002},"min":{"1min":0.25600000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.502,"15min":0.504},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.67000000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.751},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.30299999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.76600000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.59299999999999997,"15min":0.60899999999999999},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.75800000000000001,"5min":3.0459999999999998,"15min":3.0459999999999998},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.57399999999999995,"15min":0.57599999999999996},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.79000000000000004}]}]},{"osd":6,"up_from":46,"seq":197568495650,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6580,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738572,"statfs":{"total":95995035648,"available":95988297728,"internally_reserved":0,"allocated":1290240,"data_stored":884219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.47499999999999998,"15min":0.47699999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.65700000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.502,"15min":0.504},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.60199999999999998,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.75600000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.48499999999999999,"15min":0.49199999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.61799999999999999,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.495,"5min":0.499,"15min":0.499},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.59499999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.504,"15min":0.50600000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73899999999999999,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.52000000000000002,"15min":0.52400000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67400000000000004,"5min":0.753,"15min":0.753},"last":0.622}]},{"osd":3,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.76600000000000001,"15min":0.81100000000000005},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.77900000000000003,"5min":7.6429999999999998,"15min":7.6429999999999998},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39300000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64700000000000002,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.69999999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.495,"5min":0.41899999999999998,"15min":0.40699999999999997},"min":{"1min":0.33900000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.56200000000000006}]},{"osd":5,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.44500000000000001,"15min":0.43099999999999999},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.171,"5min":1.171,"15min":1.171},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.435,"15min":0.42899999999999999},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.61299999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:05:20 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.47199999999999998,"15min":0.46600000000000003},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.61399999999999999,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54900000000000004,"15min":0.54600000000000004},"min":{"1min":0.38200000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69399999999999995,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.73199999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574905,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6752,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738400,"statfs":{"total":95995035648,"available":95988121600,"internally_reserved":0,"allocated":679936,"data_stored":292245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.33500000000000002},"min":{"1min":0.252,"5min":0.214,"15min":0.214},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.47399999999999998,"15min":0.39200000000000002},"min":{"1min":0.25,"5min":0.193,"15min":0.193},"max":{"1min":10.122,"5min":10.122,"15min":10.122},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:05:30 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.435,"15min":0.42199999999999999},"min":{"1min":0.26900000000000002,"5min":0.218,"15min":0.218},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.496},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.47299999999999998,"15min":0.42299999999999999},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68799999999999994,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":0.438}]},{"osd":3,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.45500000000000002,"15min":0.46700000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.53400000000000003,"15min":0.54800000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:05:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.60799999999999998,"15min":0.61799999999999999},"min":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.64300000000000002,"15min":0.625},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.0570000000000004,"5min":6.0570000000000004,"15min":6.0570000000000004},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:04:34 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":0.997,"15min":1.022},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":6.1609999999999996,"5min":7.3479999999999999,"15min":7.3479999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.97299999999999998,"15min":1.0309999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.1850000000000001,"5min":6.9539999999999997,"15min":6.9539999999999997},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.65500000000000003,"15min":0.66500000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.89800000000000002,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.70599999999999996,"15min":0.68300000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":6.1929999999999996,"5min":6.1929999999999996,"15min":6.1929999999999996},"last":0.58299999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:05:18 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.91300000000000003,"15min":0.96499999999999997},"min":{"1min":0.43099999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.92300000000000004,"5min":6.117,"15min":6.117},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.67000000000000004,"15min":0.67300000000000004},"min":{"1min":0.46999999999999997,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.627}]}]},{"osd":0,"up_from":8,"seq":34359738431,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7340,"kb_used_data":1252,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93737812,"statfs":{"total":95995035648,"available":95987519488,"internally_reserved":0,"allocated":1282048,"data_stored":883189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:05:12 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.371,"15min":0.36199999999999999},"min":{"1min":0.23799999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.65400000000000003,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.254},{"interface":"front","average":{"1min":0.435,"5min":0.39500000000000002,"15min":0.371},"min":{"1min":0.26800000000000002,"5min":0.193,"15min":0.193},"max":{"1min":1.6359999999999999,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:05:30 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.41599999999999998,"15min":0.40400000000000003},"min":{"1min":0.255,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.96,"5min":1.96,"15min":1.96},"last":0.311},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.442,"15min":0.41999999999999998},"min":{"1min":0.29899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.68899999999999995,"5min":1.593,"15min":1.593},"last":0.33100000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:04:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.497,"15min":0.502},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.49399999999999999,"15min":0.499},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:05:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.41299999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83099999999999996,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.57599999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.80100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:04:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.63700000000000001,"15min":0.64700000000000002},"min":{"1min":0.44900000000000001,"5min":0.439,"15min":0.439},"max":{"1min":0.81799999999999995,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.629,"15min":0.63900000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76700000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.54700000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:04:59 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.63,"15min":0.63600000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.86199999999999999,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61299999999999999,"15min":0.61699999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.73699999999999999,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.63900000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:05:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.67000000000000004,"15min":0.68000000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.877,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.66700000000000004,"15min":0.67300000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.86699999999999999,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.56799999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411381,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6568,"kb_used_data":672,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738584,"statfs":{"total":95995035648,"available":95988310016,"internally_reserved":0,"allocated":688128,"data_stored":292833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:05:31 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.38800000000000001,"15min":0.38600000000000001},"min":{"1min":0.245,"5min":0.222,"15min":0.222},"max":{"1min":0.66800000000000004,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.374},{"interface":"front","average":{"1min":0.371,"5min":0.40200000000000002,"15min":0.39800000000000002},"min":{"1min":0.26000000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.60499999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.34000000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:05:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":0.58699999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.433,"15min":0.42499999999999999},"min":{"1min":0.27600000000000002,"5min":0.193,"15min":0.193},"max":{"1min":0.79100000000000004,"5min":1.121,"15min":1.121},"last":0.434}]},{"osd":3,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.46300000000000002,"15min":0.47199999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.50900000000000001,"15min":0.52000000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.71999999999999997,"5min":0.745,"15min":0.745},"last":0.47899999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56200000000000006,"15min":0.55600000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.74399999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:04:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.069,"5min":1.069,"15min":1.069},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.71999999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:05:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60199999999999998,"15min":0.60499999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:05:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.61699999999999999,"15min":0.625},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.92800000000000005},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.63300000000000001,"15min":0.64000000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.78200000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182447,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6116,"kb_used_data":668,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739036,"statfs":{"total":95995035648,"available":95988772864,"internally_reserved":0,"allocated":684032,"data_stored":292335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.40899999999999997,"15min":0.41699999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.622,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.44900000000000001,"15min":0.46000000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.42999999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.46400000000000002,"15min":0.47999999999999998},"min":{"1min":0.32600000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55700000000000005,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.46700000000000003,"15min":0.47899999999999998},"min":{"1min":0.29599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.621,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.50700000000000001,"15min":0.52500000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.58699999999999997,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.52400000000000002,"15min":0.48299999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.2160000000000002,"5min":7.2160000000000002,"15min":7.2160000000000002},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.59599999999999997,"15min":0.60799999999999998},"min":{"1min":0.45100000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.78600000000000003,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.60799999999999998,"15min":0.625},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.74099999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.66100000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:04:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.63,"15min":0.64200000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80100000000000005,"5min":1.022,"15min":1.022},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.65500000000000003,"15min":0.66700000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85699999999999998,"5min":1.002,"15min":1.002},"last":0.52400000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:04:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.64700000000000002,"15min":0.66000000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.89300000000000002,"5min":1.091,"15min":1.091},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.61699999999999999,"15min":0.626},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.78300000000000003,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.61199999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:05:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.61199999999999999,"15min":0.62},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.84799999999999998,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.66600000000000004,"15min":0.67400000000000004},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.752,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.56000000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018923,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6200,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738952,"statfs":{"total":95995035648,"available":95988686848,"internally_reserved":0,"allocated":704512,"data_stored":293927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:05:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.51300000000000001,"15min":0.52300000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.67500000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.53800000000000003,"15min":0.52700000000000002},"min":{"1min":0.36799999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.61699999999999999,"5min":3.1240000000000001,"15min":3.1240000000000001},"last":0.67200000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:05:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.52200000000000002,"15min":0.53800000000000003},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.69399999999999995,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.56200000000000006,"15min":0.58799999999999997},"min":{"1min":0.318,"5min":0.315,"15min":0.315},"max":{"1min":0.59699999999999998,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":0.59599999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:05:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.56000000000000005,"15min":0.57799999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.65600000000000003,"5min":2.2629999999999999,"15min":2.2629999999999999},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.55200000000000005,"15min":0.54400000000000004},"min":{"1min":0.38,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.60699999999999998,"5min":3.0910000000000002,"15min":3.0910000000000002},"last":0.56699999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:05:15 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.56699999999999995,"15min":0.55500000000000005},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63500000000000001,"5min":3.1499999999999999,"15min":3.1499999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:04:38 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.42099999999999999,"15min":0.41199999999999998},"min":{"1min":0.317,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60199999999999998,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.39500000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.73099999999999998,"5min":3.0470000000000002,"15min":3.0470000000000002},"last":0.47499999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.46700000000000003,"15min":0.45700000000000002},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.434},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.35299999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":2.2440000000000002,"15min":2.2440000000000002},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:05:23 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.68500000000000005,"15min":0.71199999999999997},"min":{"1min":0.375,"5min":0.313,"15min":0.313},"max":{"1min":0.85099999999999998,"5min":3.9609999999999999,"15min":3.9609999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56699999999999995,"15min":0.56999999999999995},"min":{"1min":0.42299999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.72299999999999998,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.5}]}]},{"osd":5,"up_from":38,"seq":163208757287,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5992,"kb_used_data":672,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739160,"statfs":{"total":95995035648,"available":95988899840,"internally_reserved":0,"allocated":688128,"data_stored":292869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.54600000000000004,"15min":0.55700000000000005},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.752,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.51700000000000002,"15min":0.52300000000000002},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.67600000000000005,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.45700000000000002,"5min":0.54900000000000004,"15min":0.56399999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.59299999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.55700000000000005,"15min":0.56699999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.625},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.87,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.57099999999999995,"15min":0.57699999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.69199999999999995,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.56899999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:04:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.41199999999999998,"15min":0.40400000000000003},"min":{"1min":0.30199999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.441,"15min":0.42899999999999999},"min":{"1min":0.34899999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.71299999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:04:58 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.47199999999999998,"15min":0.46600000000000003},"min":{"1min":0.36899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.501,"15min":0.499},"min":{"1min":0.36099999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.79400000000000004,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.66700000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.58299999999999996,"15min":0.59199999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.81200000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.70399999999999996,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.65000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:05:37.200 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:05:37.560 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:37.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:37 smithi026 ceph-mon[118879]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:05:37.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:37 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:37.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:37 smithi026 ceph-mon[100157]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:05:37.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:37 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:38.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:37 smithi070 ceph-mon[105170]: from='client.14778 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:05:38.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:37 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:38.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:38 smithi026 ceph-mon[100157]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:38.863 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:38 smithi026 ceph-mon[118879]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:38.864 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:38 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:38] "GET /metrics HTTP/1.1" 200 238573 "" "Prometheus/2.43.0" 2023-09-23T22:05:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:38 smithi070 ceph-mon[105170]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:05:39.242 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:05:39.786 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":64,"stamp":"2023-09-23T22:05:38.106770+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52072,"kb_used_data":7144,"kb_used_omap":0,"kb_used_meta":44864,"kb_avail":749909144,"statfs":{"total":767960285184,"available":767906963456,"internally_reserved":0,"allocated":7315456,"data_stored":4115659,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":45940736},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002884"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.405597+0000","last_change":"2023-09-23T22:04:54.404859+0000","last_active":"2023-09-23T22:04:54.405597+0000","last_peered":"2023-09-23T22:04:54.405597+0000","last_clean":"2023-09-23T22:04:54.405597+0000","last_became_active":"2023-09-23T22:04:54.404604+0000","last_became_peered":"2023-09-23T22:04:54.404604+0000","last_unstale":"2023-09-23T22:04:54.405597+0000","last_undegraded":"2023-09-23T22:04:54.405597+0000","last_fullsized":"2023-09-23T22:04:54.405597+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:34.228549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.671975+0000","last_change":"2023-09-23T22:04:54.384572+0000","last_active":"2023-09-23T22:05:01.671975+0000","last_peered":"2023-09-23T22:05:01.671975+0000","last_clean":"2023-09-23T22:05:01.671975+0000","last_became_active":"2023-09-23T22:04:54.384302+0000","last_became_peered":"2023-09-23T22:04:54.384302+0000","last_unstale":"2023-09-23T22:05:01.671975+0000","last_undegraded":"2023-09-23T22:05:01.671975+0000","last_fullsized":"2023-09-23T22:05:01.671975+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:32.575153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.411577+0000","last_change":"2023-09-23T22:04:54.409325+0000","last_active":"2023-09-23T22:04:54.411577+0000","last_peered":"2023-09-23T22:04:54.411577+0000","last_clean":"2023-09-23T22:04:54.411577+0000","last_became_active":"2023-09-23T22:04:54.408996+0000","last_became_peered":"2023-09-23T22:04:54.408996+0000","last_unstale":"2023-09-23T22:04:54.411577+0000","last_undegraded":"2023-09-23T22:04:54.411577+0000","last_fullsized":"2023-09-23T22:04:54.411577+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:24.585073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.399505+0000","last_change":"2023-09-23T22:04:54.375952+0000","last_active":"2023-09-23T22:04:55.399505+0000","last_peered":"2023-09-23T22:04:55.399505+0000","last_clean":"2023-09-23T22:04:55.399505+0000","last_became_active":"2023-09-23T22:04:54.375649+0000","last_became_peered":"2023-09-23T22:04:54.375649+0000","last_unstale":"2023-09-23T22:04:55.399505+0000","last_undegraded":"2023-09-23T22:04:55.399505+0000","last_fullsized":"2023-09-23T22:04:55.399505+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:59.563466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.404288+0000","last_change":"2023-09-23T22:04:54.404097+0000","last_active":"2023-09-23T22:04:54.404288+0000","last_peered":"2023-09-23T22:04:54.404288+0000","last_clean":"2023-09-23T22:04:54.404288+0000","last_became_active":"2023-09-23T22:04:54.403823+0000","last_became_peered":"2023-09-23T22:04:54.403823+0000","last_unstale":"2023-09-23T22:04:54.404288+0000","last_undegraded":"2023-09-23T22:04:54.404288+0000","last_fullsized":"2023-09-23T22:04:54.404288+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:55.802552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.673194+0000","last_change":"2023-09-23T22:04:54.379680+0000","last_active":"2023-09-23T22:05:01.673194+0000","last_peered":"2023-09-23T22:05:01.673194+0000","last_clean":"2023-09-23T22:05:01.673194+0000","last_became_active":"2023-09-23T22:04:54.379383+0000","last_became_peered":"2023-09-23T22:04:54.379383+0000","last_unstale":"2023-09-23T22:05:01.673194+0000","last_undegraded":"2023-09-23T22:05:01.673194+0000","last_fullsized":"2023-09-23T22:05:01.673194+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.282795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.405612+0000","last_change":"2023-09-23T22:04:54.404957+0000","last_active":"2023-09-23T22:04:54.405612+0000","last_peered":"2023-09-23T22:04:54.405612+0000","last_clean":"2023-09-23T22:04:54.405612+0000","last_became_active":"2023-09-23T22:04:54.404677+0000","last_became_peered":"2023-09-23T22:04:54.404677+0000","last_unstale":"2023-09-23T22:04:54.405612+0000","last_undegraded":"2023-09-23T22:04:54.405612+0000","last_fullsized":"2023-09-23T22:04:54.405612+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:17:26.547113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.672705+0000","last_change":"2023-09-23T22:04:54.382175+0000","last_active":"2023-09-23T22:05:01.672705+0000","last_peered":"2023-09-23T22:05:01.672705+0000","last_clean":"2023-09-23T22:05:01.672705+0000","last_became_active":"2023-09-23T22:04:54.381883+0000","last_became_peered":"2023-09-23T22:04:54.381883+0000","last_unstale":"2023-09-23T22:05:01.672705+0000","last_undegraded":"2023-09-23T22:05:01.672705+0000","last_fullsized":"2023-09-23T22:05:01.672705+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:48.043436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407153+0000","last_change":"2023-09-23T22:04:54.406726+0000","last_active":"2023-09-23T22:04:54.407153+0000","last_peered":"2023-09-23T22:04:54.407153+0000","last_clean":"2023-09-23T22:04:54.407153+0000","last_became_active":"2023-09-23T22:04:54.406409+0000","last_became_peered":"2023-09-23T22:04:54.406409+0000","last_unstale":"2023-09-23T22:04:54.407153+0000","last_undegraded":"2023-09-23T22:04:54.407153+0000","last_fullsized":"2023-09-23T22:04:54.407153+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:22.664231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.755929+0000","last_change":"2023-09-23T22:04:54.381881+0000","last_active":"2023-09-23T22:04:55.755929+0000","last_peered":"2023-09-23T22:04:55.755929+0000","last_clean":"2023-09-23T22:04:55.755929+0000","last_became_active":"2023-09-23T22:04:54.381585+0000","last_became_peered":"2023-09-23T22:04:54.381585+0000","last_unstale":"2023-09-23T22:04:55.755929+0000","last_undegraded":"2023-09-23T22:04:55.755929+0000","last_fullsized":"2023-09-23T22:04:55.755929+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:58:50.421302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.404155+0000","last_change":"2023-09-23T22:04:54.402894+0000","last_active":"2023-09-23T22:04:54.404155+0000","last_peered":"2023-09-23T22:04:54.404155+0000","last_clean":"2023-09-23T22:04:54.404155+0000","last_became_active":"2023-09-23T22:04:54.402475+0000","last_became_peered":"2023-09-23T22:04:54.402475+0000","last_unstale":"2023-09-23T22:04:54.404155+0000","last_undegraded":"2023-09-23T22:04:54.404155+0000","last_fullsized":"2023-09-23T22:04:54.404155+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:12.503263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.756165+0000","last_change":"2023-09-23T22:04:54.382294+0000","last_active":"2023-09-23T22:04:55.756165+0000","last_peered":"2023-09-23T22:04:55.756165+0000","last_clean":"2023-09-23T22:04:55.756165+0000","last_became_active":"2023-09-23T22:04:54.382013+0000","last_became_peered":"2023-09-23T22:04:54.382013+0000","last_unstale":"2023-09-23T22:04:55.756165+0000","last_undegraded":"2023-09-23T22:04:55.756165+0000","last_fullsized":"2023-09-23T22:04:55.756165+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:25.314566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.401488+0000","last_change":"2023-09-23T22:04:06.152239+0000","last_active":"2023-09-23T22:04:54.401488+0000","last_peered":"2023-09-23T22:04:54.401488+0000","last_clean":"2023-09-23T22:04:54.401488+0000","last_became_active":"2023-09-23T22:04:06.151906+0000","last_became_peered":"2023-09-23T22:04:06.151906+0000","last_unstale":"2023-09-23T22:04:54.401488+0000","last_undegraded":"2023-09-23T22:04:54.401488+0000","last_fullsized":"2023-09-23T22:04:54.401488+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:29.292018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.527739+0000","last_change":"2023-09-23T22:04:54.527443+0000","last_active":"2023-09-23T22:04:54.527739+0000","last_peered":"2023-09-23T22:04:54.527739+0000","last_clean":"2023-09-23T22:04:54.527739+0000","last_became_active":"2023-09-23T22:04:54.527123+0000","last_became_peered":"2023-09-23T22:04:54.527123+0000","last_unstale":"2023-09-23T22:04:54.527739+0000","last_undegraded":"2023-09-23T22:04:54.527739+0000","last_fullsized":"2023-09-23T22:04:54.527739+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:34.117223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407910+0000","last_change":"2023-09-23T22:04:06.211004+0000","last_active":"2023-09-23T22:04:54.407910+0000","last_peered":"2023-09-23T22:04:54.407910+0000","last_clean":"2023-09-23T22:04:54.407910+0000","last_became_active":"2023-09-23T22:04:06.210652+0000","last_became_peered":"2023-09-23T22:04:06.210652+0000","last_unstale":"2023-09-23T22:04:54.407910+0000","last_undegraded":"2023-09-23T22:04:54.407910+0000","last_fullsized":"2023-09-23T22:04:54.407910+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:32.721396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.673217+0000","last_change":"2023-09-23T22:04:54.387001+0000","last_active":"2023-09-23T22:05:01.673217+0000","last_peered":"2023-09-23T22:05:01.673217+0000","last_clean":"2023-09-23T22:05:01.673217+0000","last_became_active":"2023-09-23T22:04:54.386704+0000","last_became_peered":"2023-09-23T22:04:54.386704+0000","last_unstale":"2023-09-23T22:05:01.673217+0000","last_undegraded":"2023-09-23T22:05:01.673217+0000","last_fullsized":"2023-09-23T22:05:01.673217+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:18.520541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526363+0000","last_change":"2023-09-23T22:04:54.525852+0000","last_active":"2023-09-23T22:04:54.526363+0000","last_peered":"2023-09-23T22:04:54.526363+0000","last_clean":"2023-09-23T22:04:54.526363+0000","last_became_active":"2023-09-23T22:04:54.525524+0000","last_became_peered":"2023-09-23T22:04:54.525524+0000","last_unstale":"2023-09-23T22:04:54.526363+0000","last_undegraded":"2023-09-23T22:04:54.526363+0000","last_fullsized":"2023-09-23T22:04:54.526363+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:23.828759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407542+0000","last_change":"2023-09-23T22:04:54.385876+0000","last_active":"2023-09-23T22:04:54.407542+0000","last_peered":"2023-09-23T22:04:54.407542+0000","last_clean":"2023-09-23T22:04:54.407542+0000","last_became_active":"2023-09-23T22:04:54.385588+0000","last_became_peered":"2023-09-23T22:04:54.385588+0000","last_unstale":"2023-09-23T22:04:54.407542+0000","last_undegraded":"2023-09-23T22:04:54.407542+0000","last_fullsized":"2023-09-23T22:04:54.407542+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:15:47.856644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526683+0000","last_change":"2023-09-23T22:04:06.250190+0000","last_active":"2023-09-23T22:04:54.526683+0000","last_peered":"2023-09-23T22:04:54.526683+0000","last_clean":"2023-09-23T22:04:54.526683+0000","last_became_active":"2023-09-23T22:04:06.249826+0000","last_became_peered":"2023-09-23T22:04:06.249826+0000","last_unstale":"2023-09-23T22:04:54.526683+0000","last_undegraded":"2023-09-23T22:04:54.526683+0000","last_fullsized":"2023-09-23T22:04:54.526683+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:16.888898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.414890+0000","last_change":"2023-09-23T22:04:54.414198+0000","last_active":"2023-09-23T22:04:54.414890+0000","last_peered":"2023-09-23T22:04:54.414890+0000","last_clean":"2023-09-23T22:04:54.414890+0000","last_became_active":"2023-09-23T22:04:54.413874+0000","last_became_peered":"2023-09-23T22:04:54.413874+0000","last_unstale":"2023-09-23T22:04:54.414890+0000","last_undegraded":"2023-09-23T22:04:54.414890+0000","last_fullsized":"2023-09-23T22:04:54.414890+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:20.609941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.400154+0000","last_change":"2023-09-23T22:04:54.383681+0000","last_active":"2023-09-23T22:04:54.400154+0000","last_peered":"2023-09-23T22:04:54.400154+0000","last_clean":"2023-09-23T22:04:54.400154+0000","last_became_active":"2023-09-23T22:04:54.383420+0000","last_became_peered":"2023-09-23T22:04:54.383420+0000","last_unstale":"2023-09-23T22:04:54.400154+0000","last_undegraded":"2023-09-23T22:04:54.400154+0000","last_fullsized":"2023-09-23T22:04:54.400154+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:48.031496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526581+0000","last_change":"2023-09-23T22:04:06.250172+0000","last_active":"2023-09-23T22:04:54.526581+0000","last_peered":"2023-09-23T22:04:54.526581+0000","last_clean":"2023-09-23T22:04:54.526581+0000","last_became_active":"2023-09-23T22:04:06.249764+0000","last_became_peered":"2023-09-23T22:04:06.249764+0000","last_unstale":"2023-09-23T22:04:54.526581+0000","last_undegraded":"2023-09-23T22:04:54.526581+0000","last_fullsized":"2023-09-23T22:04:54.526581+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:28:11.590738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.527561+0000","last_change":"2023-09-23T22:04:54.527318+0000","last_active":"2023-09-23T22:04:54.527561+0000","last_peered":"2023-09-23T22:04:54.527561+0000","last_clean":"2023-09-23T22:04:54.527561+0000","last_became_active":"2023-09-23T22:04:54.527023+0000","last_became_peered":"2023-09-23T22:04:54.527023+0000","last_unstale":"2023-09-23T22:04:54.527561+0000","last_undegraded":"2023-09-23T22:04:54.527561+0000","last_fullsized":"2023-09-23T22:04:54.527561+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:42.048739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398928+0000","last_change":"2023-09-23T22:04:54.378787+0000","last_active":"2023-09-23T22:04:54.398928+0000","last_peered":"2023-09-23T22:04:54.398928+0000","last_clean":"2023-09-23T22:04:54.398928+0000","last_became_active":"2023-09-23T22:04:54.378507+0000","last_became_peered":"2023-09-23T22:04:54.378507+0000","last_unstale":"2023-09-23T22:04:54.398928+0000","last_undegraded":"2023-09-23T22:04:54.398928+0000","last_fullsized":"2023-09-23T22:04:54.398928+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:27.250805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"59'128","reported_seq":275,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526579+0000","last_change":"2023-09-23T22:03:21.650107+0000","last_active":"2023-09-23T22:04:54.526579+0000","last_peered":"2023-09-23T22:04:54.526579+0000","last_clean":"2023-09-23T22:04:54.526579+0000","last_became_active":"2023-09-23T22:03:21.649660+0000","last_became_peered":"2023-09-23T22:03:21.649660+0000","last_unstale":"2023-09-23T22:04:54.526579+0000","last_undegraded":"2023-09-23T22:04:54.526579+0000","last_fullsized":"2023-09-23T22:04:54.526579+0000","mapping_epoch":57,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:13.053339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.395218+0000","last_change":"2023-09-23T22:04:06.154819+0000","last_active":"2023-09-23T22:04:54.395218+0000","last_peered":"2023-09-23T22:04:54.395218+0000","last_clean":"2023-09-23T22:04:54.395218+0000","last_became_active":"2023-09-23T22:04:06.154506+0000","last_became_peered":"2023-09-23T22:04:06.154506+0000","last_unstale":"2023-09-23T22:04:54.395218+0000","last_undegraded":"2023-09-23T22:04:54.395218+0000","last_fullsized":"2023-09-23T22:04:54.395218+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:49.420305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.413574+0000","last_change":"2023-09-23T22:04:54.411513+0000","last_active":"2023-09-23T22:04:54.413574+0000","last_peered":"2023-09-23T22:04:54.413574+0000","last_clean":"2023-09-23T22:04:54.413574+0000","last_became_active":"2023-09-23T22:04:54.411210+0000","last_became_peered":"2023-09-23T22:04:54.411210+0000","last_unstale":"2023-09-23T22:04:54.413574+0000","last_undegraded":"2023-09-23T22:04:54.413574+0000","last_fullsized":"2023-09-23T22:04:54.413574+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:14.742116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407936+0000","last_change":"2023-09-23T22:04:54.385915+0000","last_active":"2023-09-23T22:04:54.407936+0000","last_peered":"2023-09-23T22:04:54.407936+0000","last_clean":"2023-09-23T22:04:54.407936+0000","last_became_active":"2023-09-23T22:04:54.385615+0000","last_became_peered":"2023-09-23T22:04:54.385615+0000","last_unstale":"2023-09-23T22:04:54.407936+0000","last_undegraded":"2023-09-23T22:04:54.407936+0000","last_fullsized":"2023-09-23T22:04:54.407936+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:39.881906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.399062+0000","last_change":"2023-09-23T22:04:06.250115+0000","last_active":"2023-09-23T22:04:54.399062+0000","last_peered":"2023-09-23T22:04:54.399062+0000","last_clean":"2023-09-23T22:04:54.399062+0000","last_became_active":"2023-09-23T22:04:06.249760+0000","last_became_peered":"2023-09-23T22:04:06.249760+0000","last_unstale":"2023-09-23T22:04:54.399062+0000","last_undegraded":"2023-09-23T22:04:54.399062+0000","last_fullsized":"2023-09-23T22:04:54.399062+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:48.266985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.416900+0000","last_change":"2023-09-23T22:04:54.416646+0000","last_active":"2023-09-23T22:04:54.416900+0000","last_peered":"2023-09-23T22:04:54.416900+0000","last_clean":"2023-09-23T22:04:54.416900+0000","last_became_active":"2023-09-23T22:04:54.416275+0000","last_became_peered":"2023-09-23T22:04:54.416275+0000","last_unstale":"2023-09-23T22:04:54.416900+0000","last_undegraded":"2023-09-23T22:04:54.416900+0000","last_fullsized":"2023-09-23T22:04:54.416900+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:26.326726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398236+0000","last_change":"2023-09-23T22:04:54.387074+0000","last_active":"2023-09-23T22:04:54.398236+0000","last_peered":"2023-09-23T22:04:54.398236+0000","last_clean":"2023-09-23T22:04:54.398236+0000","last_became_active":"2023-09-23T22:04:54.386207+0000","last_became_peered":"2023-09-23T22:04:54.386207+0000","last_unstale":"2023-09-23T22:04:54.398236+0000","last_undegraded":"2023-09-23T22:04:54.398236+0000","last_fullsized":"2023-09-23T22:04:54.398236+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:49.554877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.395062+0000","last_change":"2023-09-23T22:04:06.153745+0000","last_active":"2023-09-23T22:04:54.395062+0000","last_peered":"2023-09-23T22:04:54.395062+0000","last_clean":"2023-09-23T22:04:54.395062+0000","last_became_active":"2023-09-23T22:04:06.153371+0000","last_became_peered":"2023-09-23T22:04:06.153371+0000","last_unstale":"2023-09-23T22:04:54.395062+0000","last_undegraded":"2023-09-23T22:04:54.395062+0000","last_fullsized":"2023-09-23T22:04:54.395062+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:36.366505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.415049+0000","last_change":"2023-09-23T22:04:54.414044+0000","last_active":"2023-09-23T22:04:54.415049+0000","last_peered":"2023-09-23T22:04:54.415049+0000","last_clean":"2023-09-23T22:04:54.415049+0000","last_became_active":"2023-09-23T22:04:54.413453+0000","last_became_peered":"2023-09-23T22:04:54.413453+0000","last_unstale":"2023-09-23T22:04:54.415049+0000","last_undegraded":"2023-09-23T22:04:54.415049+0000","last_fullsized":"2023-09-23T22:04:54.415049+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:31.563303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.397076+0000","last_change":"2023-09-23T22:04:54.379188+0000","last_active":"2023-09-23T22:04:54.397076+0000","last_peered":"2023-09-23T22:04:54.397076+0000","last_clean":"2023-09-23T22:04:54.397076+0000","last_became_active":"2023-09-23T22:04:54.378612+0000","last_became_peered":"2023-09-23T22:04:54.378612+0000","last_unstale":"2023-09-23T22:04:54.397076+0000","last_undegraded":"2023-09-23T22:04:54.397076+0000","last_fullsized":"2023-09-23T22:04:54.397076+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:03.596200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407426+0000","last_change":"2023-09-23T22:04:06.250575+0000","last_active":"2023-09-23T22:04:54.407426+0000","last_peered":"2023-09-23T22:04:54.407426+0000","last_clean":"2023-09-23T22:04:54.407426+0000","last_became_active":"2023-09-23T22:04:06.249857+0000","last_became_peered":"2023-09-23T22:04:06.249857+0000","last_unstale":"2023-09-23T22:04:54.407426+0000","last_undegraded":"2023-09-23T22:04:54.407426+0000","last_fullsized":"2023-09-23T22:04:54.407426+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:20:52.618997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.414891+0000","last_change":"2023-09-23T22:04:54.413503+0000","last_active":"2023-09-23T22:04:54.414891+0000","last_peered":"2023-09-23T22:04:54.414891+0000","last_clean":"2023-09-23T22:04:54.414891+0000","last_became_active":"2023-09-23T22:04:54.413192+0000","last_became_peered":"2023-09-23T22:04:54.413192+0000","last_unstale":"2023-09-23T22:04:54.414891+0000","last_undegraded":"2023-09-23T22:04:54.414891+0000","last_fullsized":"2023-09-23T22:04:54.414891+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:11.852260+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398176+0000","last_change":"2023-09-23T22:04:54.387587+0000","last_active":"2023-09-23T22:04:54.398176+0000","last_peered":"2023-09-23T22:04:54.398176+0000","last_clean":"2023-09-23T22:04:54.398176+0000","last_became_active":"2023-09-23T22:04:54.387313+0000","last_became_peered":"2023-09-23T22:04:54.387313+0000","last_unstale":"2023-09-23T22:04:54.398176+0000","last_undegraded":"2023-09-23T22:04:54.398176+0000","last_fullsized":"2023-09-23T22:04:54.398176+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:53.756703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.409873+0000","last_change":"2023-09-23T22:04:54.409258+0000","last_active":"2023-09-23T22:04:54.409873+0000","last_peered":"2023-09-23T22:04:54.409873+0000","last_clean":"2023-09-23T22:04:54.409873+0000","last_became_active":"2023-09-23T22:04:54.408978+0000","last_became_peered":"2023-09-23T22:04:54.408978+0000","last_unstale":"2023-09-23T22:04:54.409873+0000","last_undegraded":"2023-09-23T22:04:54.409873+0000","last_fullsized":"2023-09-23T22:04:54.409873+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:30.355826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398452+0000","last_change":"2023-09-23T22:04:54.381395+0000","last_active":"2023-09-23T22:04:54.398452+0000","last_peered":"2023-09-23T22:04:54.398452+0000","last_clean":"2023-09-23T22:04:54.398452+0000","last_became_active":"2023-09-23T22:04:54.381159+0000","last_became_peered":"2023-09-23T22:04:54.381159+0000","last_unstale":"2023-09-23T22:04:54.398452+0000","last_undegraded":"2023-09-23T22:04:54.398452+0000","last_fullsized":"2023-09-23T22:04:54.398452+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:35.437640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.405618+0000","last_change":"2023-09-23T22:04:54.405009+0000","last_active":"2023-09-23T22:04:54.405618+0000","last_peered":"2023-09-23T22:04:54.405618+0000","last_clean":"2023-09-23T22:04:54.405618+0000","last_became_active":"2023-09-23T22:04:54.404744+0000","last_became_peered":"2023-09-23T22:04:54.404744+0000","last_unstale":"2023-09-23T22:04:54.405618+0000","last_undegraded":"2023-09-23T22:04:54.405618+0000","last_fullsized":"2023-09-23T22:04:54.405618+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:54.736713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.402301+0000","last_change":"2023-09-23T22:04:54.382501+0000","last_active":"2023-09-23T22:04:54.402301+0000","last_peered":"2023-09-23T22:04:54.402301+0000","last_clean":"2023-09-23T22:04:54.402301+0000","last_became_active":"2023-09-23T22:04:54.382201+0000","last_became_peered":"2023-09-23T22:04:54.382201+0000","last_unstale":"2023-09-23T22:04:54.402301+0000","last_undegraded":"2023-09-23T22:04:54.402301+0000","last_fullsized":"2023-09-23T22:04:54.402301+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:00.811365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.404196+0000","last_change":"2023-09-23T22:04:54.402068+0000","last_active":"2023-09-23T22:04:54.404196+0000","last_peered":"2023-09-23T22:04:54.404196+0000","last_clean":"2023-09-23T22:04:54.404196+0000","last_became_active":"2023-09-23T22:04:54.401785+0000","last_became_peered":"2023-09-23T22:04:54.401785+0000","last_unstale":"2023-09-23T22:04:54.404196+0000","last_undegraded":"2023-09-23T22:04:54.404196+0000","last_fullsized":"2023-09-23T22:04:54.404196+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:25.421013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.672529+0000","last_change":"2023-09-23T22:04:54.386352+0000","last_active":"2023-09-23T22:05:01.672529+0000","last_peered":"2023-09-23T22:05:01.672529+0000","last_clean":"2023-09-23T22:05:01.672529+0000","last_became_active":"2023-09-23T22:04:54.386056+0000","last_became_peered":"2023-09-23T22:04:54.386056+0000","last_unstale":"2023-09-23T22:05:01.672529+0000","last_undegraded":"2023-09-23T22:05:01.672529+0000","last_fullsized":"2023-09-23T22:05:01.672529+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:47.833333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.414802+0000","last_change":"2023-09-23T22:04:54.414057+0000","last_active":"2023-09-23T22:04:54.414802+0000","last_peered":"2023-09-23T22:04:54.414802+0000","last_clean":"2023-09-23T22:04:54.414802+0000","last_became_active":"2023-09-23T22:04:54.413773+0000","last_became_peered":"2023-09-23T22:04:54.413773+0000","last_unstale":"2023-09-23T22:04:54.414802+0000","last_undegraded":"2023-09-23T22:04:54.414802+0000","last_fullsized":"2023-09-23T22:04:54.414802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:17.557197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.397202+0000","last_change":"2023-09-23T22:04:54.380397+0000","last_active":"2023-09-23T22:04:54.397202+0000","last_peered":"2023-09-23T22:04:54.397202+0000","last_clean":"2023-09-23T22:04:54.397202+0000","last_became_active":"2023-09-23T22:04:54.379637+0000","last_became_peered":"2023-09-23T22:04:54.379637+0000","last_unstale":"2023-09-23T22:04:54.397202+0000","last_undegraded":"2023-09-23T22:04:54.397202+0000","last_fullsized":"2023-09-23T22:04:54.397202+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:47.162589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.525999+0000","last_change":"2023-09-23T22:04:54.525753+0000","last_active":"2023-09-23T22:04:54.525999+0000","last_peered":"2023-09-23T22:04:54.525999+0000","last_clean":"2023-09-23T22:04:54.525999+0000","last_became_active":"2023-09-23T22:04:54.525487+0000","last_became_peered":"2023-09-23T22:04:54.525487+0000","last_unstale":"2023-09-23T22:04:54.525999+0000","last_undegraded":"2023-09-23T22:04:54.525999+0000","last_fullsized":"2023-09-23T22:04:54.525999+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:53.439886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.756642+0000","last_change":"2023-09-23T22:04:54.384008+0000","last_active":"2023-09-23T22:04:55.756642+0000","last_peered":"2023-09-23T22:04:55.756642+0000","last_clean":"2023-09-23T22:04:55.756642+0000","last_became_active":"2023-09-23T22:04:54.383702+0000","last_became_peered":"2023-09-23T22:04:54.383702+0000","last_unstale":"2023-09-23T22:04:55.756642+0000","last_undegraded":"2023-09-23T22:04:55.756642+0000","last_fullsized":"2023-09-23T22:04:55.756642+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:00.423911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.410032+0000","last_change":"2023-09-23T22:04:54.409736+0000","last_active":"2023-09-23T22:04:54.410032+0000","last_peered":"2023-09-23T22:04:54.410032+0000","last_clean":"2023-09-23T22:04:54.410032+0000","last_became_active":"2023-09-23T22:04:54.409552+0000","last_became_peered":"2023-09-23T22:04:54.409552+0000","last_unstale":"2023-09-23T22:04:54.410032+0000","last_undegraded":"2023-09-23T22:04:54.410032+0000","last_fullsized":"2023-09-23T22:04:54.410032+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:08.099853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.674070+0000","last_change":"2023-09-23T22:04:54.384587+0000","last_active":"2023-09-23T22:05:01.674070+0000","last_peered":"2023-09-23T22:05:01.674070+0000","last_clean":"2023-09-23T22:05:01.674070+0000","last_became_active":"2023-09-23T22:04:54.384308+0000","last_became_peered":"2023-09-23T22:04:54.384308+0000","last_unstale":"2023-09-23T22:05:01.674070+0000","last_undegraded":"2023-09-23T22:05:01.674070+0000","last_fullsized":"2023-09-23T22:05:01.674070+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:12.465686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.528362+0000","last_change":"2023-09-23T22:04:54.528019+0000","last_active":"2023-09-23T22:04:54.528362+0000","last_peered":"2023-09-23T22:04:54.528362+0000","last_clean":"2023-09-23T22:04:54.528362+0000","last_became_active":"2023-09-23T22:04:54.527666+0000","last_became_peered":"2023-09-23T22:04:54.527666+0000","last_unstale":"2023-09-23T22:04:54.528362+0000","last_undegraded":"2023-09-23T22:04:54.528362+0000","last_fullsized":"2023-09-23T22:04:54.528362+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:32.774493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.756450+0000","last_change":"2023-09-23T22:04:54.387398+0000","last_active":"2023-09-23T22:04:55.756450+0000","last_peered":"2023-09-23T22:04:55.756450+0000","last_clean":"2023-09-23T22:04:55.756450+0000","last_became_active":"2023-09-23T22:04:54.385217+0000","last_became_peered":"2023-09-23T22:04:54.385217+0000","last_unstale":"2023-09-23T22:04:55.756450+0000","last_undegraded":"2023-09-23T22:04:55.756450+0000","last_fullsized":"2023-09-23T22:04:55.756450+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:17.101844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.527908+0000","last_change":"2023-09-23T22:04:54.526893+0000","last_active":"2023-09-23T22:04:54.527908+0000","last_peered":"2023-09-23T22:04:54.527908+0000","last_clean":"2023-09-23T22:04:54.527908+0000","last_became_active":"2023-09-23T22:04:54.526575+0000","last_became_peered":"2023-09-23T22:04:54.526575+0000","last_unstale":"2023-09-23T22:04:54.527908+0000","last_undegraded":"2023-09-23T22:04:54.527908+0000","last_fullsized":"2023-09-23T22:04:54.527908+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:41.368329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":19,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.676729+0000","last_change":"2023-09-23T22:04:54.377190+0000","last_active":"2023-09-23T22:05:01.676729+0000","last_peered":"2023-09-23T22:05:01.676729+0000","last_clean":"2023-09-23T22:05:01.676729+0000","last_became_active":"2023-09-23T22:04:54.376865+0000","last_became_peered":"2023-09-23T22:04:54.376865+0000","last_unstale":"2023-09-23T22:05:01.676729+0000","last_undegraded":"2023-09-23T22:05:01.676729+0000","last_fullsized":"2023-09-23T22:05:01.676729+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:52.474939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526300+0000","last_change":"2023-09-23T22:04:54.525997+0000","last_active":"2023-09-23T22:04:54.526300+0000","last_peered":"2023-09-23T22:04:54.526300+0000","last_clean":"2023-09-23T22:04:54.526300+0000","last_became_active":"2023-09-23T22:04:54.525678+0000","last_became_peered":"2023-09-23T22:04:54.525678+0000","last_unstale":"2023-09-23T22:04:54.526300+0000","last_undegraded":"2023-09-23T22:04:54.526300+0000","last_fullsized":"2023-09-23T22:04:54.526300+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:55.326941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526576+0000","last_change":"2023-09-23T22:04:54.387688+0000","last_active":"2023-09-23T22:04:54.526576+0000","last_peered":"2023-09-23T22:04:54.526576+0000","last_clean":"2023-09-23T22:04:54.526576+0000","last_became_active":"2023-09-23T22:04:54.387421+0000","last_became_peered":"2023-09-23T22:04:54.387421+0000","last_unstale":"2023-09-23T22:04:54.526576+0000","last_undegraded":"2023-09-23T22:04:54.526576+0000","last_fullsized":"2023-09-23T22:04:54.526576+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:22.411746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.528219+0000","last_change":"2023-09-23T22:04:54.527953+0000","last_active":"2023-09-23T22:04:54.528219+0000","last_peered":"2023-09-23T22:04:54.528219+0000","last_clean":"2023-09-23T22:04:54.528219+0000","last_became_active":"2023-09-23T22:04:54.527667+0000","last_became_peered":"2023-09-23T22:04:54.527667+0000","last_unstale":"2023-09-23T22:04:54.528219+0000","last_undegraded":"2023-09-23T22:04:54.528219+0000","last_fullsized":"2023-09-23T22:04:54.528219+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:14.312715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.401489+0000","last_change":"2023-09-23T22:04:54.383142+0000","last_active":"2023-09-23T22:04:54.401489+0000","last_peered":"2023-09-23T22:04:54.401489+0000","last_clean":"2023-09-23T22:04:54.401489+0000","last_became_active":"2023-09-23T22:04:54.382866+0000","last_became_peered":"2023-09-23T22:04:54.382866+0000","last_unstale":"2023-09-23T22:04:54.401489+0000","last_undegraded":"2023-09-23T22:04:54.401489+0000","last_fullsized":"2023-09-23T22:04:54.401489+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:05.081316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.528372+0000","last_change":"2023-09-23T22:04:54.526801+0000","last_active":"2023-09-23T22:04:54.528372+0000","last_peered":"2023-09-23T22:04:54.528372+0000","last_clean":"2023-09-23T22:04:54.528372+0000","last_became_active":"2023-09-23T22:04:54.526516+0000","last_became_peered":"2023-09-23T22:04:54.526516+0000","last_unstale":"2023-09-23T22:04:54.528372+0000","last_undegraded":"2023-09-23T22:04:54.528372+0000","last_fullsized":"2023-09-23T22:04:54.528372+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:21.836536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":13,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.673577+0000","last_change":"2023-09-23T22:04:54.384846+0000","last_active":"2023-09-23T22:05:01.673577+0000","last_peered":"2023-09-23T22:05:01.673577+0000","last_clean":"2023-09-23T22:05:01.673577+0000","last_became_active":"2023-09-23T22:04:54.384616+0000","last_became_peered":"2023-09-23T22:04:54.384616+0000","last_unstale":"2023-09-23T22:05:01.673577+0000","last_undegraded":"2023-09-23T22:05:01.673577+0000","last_fullsized":"2023-09-23T22:05:01.673577+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:33.659945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.405631+0000","last_change":"2023-09-23T22:04:54.404971+0000","last_active":"2023-09-23T22:04:54.405631+0000","last_peered":"2023-09-23T22:04:54.405631+0000","last_clean":"2023-09-23T22:04:54.405631+0000","last_became_active":"2023-09-23T22:04:54.404689+0000","last_became_peered":"2023-09-23T22:04:54.404689+0000","last_unstale":"2023-09-23T22:04:54.405631+0000","last_undegraded":"2023-09-23T22:04:54.405631+0000","last_fullsized":"2023-09-23T22:04:54.405631+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:39.928048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"67'6","reported_seq":17,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.675061+0000","last_change":"2023-09-23T22:04:54.381132+0000","last_active":"2023-09-23T22:05:01.675061+0000","last_peered":"2023-09-23T22:05:01.675061+0000","last_clean":"2023-09-23T22:05:01.675061+0000","last_became_active":"2023-09-23T22:04:54.380838+0000","last_became_peered":"2023-09-23T22:04:54.380838+0000","last_unstale":"2023-09-23T22:05:01.675061+0000","last_undegraded":"2023-09-23T22:05:01.675061+0000","last_fullsized":"2023-09-23T22:05:01.675061+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:41.334974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.415068+0000","last_change":"2023-09-23T22:04:54.414266+0000","last_active":"2023-09-23T22:04:54.415068+0000","last_peered":"2023-09-23T22:04:54.415068+0000","last_clean":"2023-09-23T22:04:54.415068+0000","last_became_active":"2023-09-23T22:04:54.413779+0000","last_became_peered":"2023-09-23T22:04:54.413779+0000","last_unstale":"2023-09-23T22:04:54.415068+0000","last_undegraded":"2023-09-23T22:04:54.415068+0000","last_fullsized":"2023-09-23T22:04:54.415068+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:31.084009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":14,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:05:01.674470+0000","last_change":"2023-09-23T22:04:54.384065+0000","last_active":"2023-09-23T22:05:01.674470+0000","last_peered":"2023-09-23T22:05:01.674470+0000","last_clean":"2023-09-23T22:05:01.674470+0000","last_became_active":"2023-09-23T22:04:54.383777+0000","last_became_peered":"2023-09-23T22:04:54.383777+0000","last_unstale":"2023-09-23T22:05:01.674470+0000","last_undegraded":"2023-09-23T22:05:01.674470+0000","last_fullsized":"2023-09-23T22:05:01.674470+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:49.984604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.526277+0000","last_change":"2023-09-23T22:04:54.525962+0000","last_active":"2023-09-23T22:04:54.526277+0000","last_peered":"2023-09-23T22:04:54.526277+0000","last_clean":"2023-09-23T22:04:54.526277+0000","last_became_active":"2023-09-23T22:04:54.525741+0000","last_became_peered":"2023-09-23T22:04:54.525741+0000","last_unstale":"2023-09-23T22:04:54.526277+0000","last_undegraded":"2023-09-23T22:04:54.526277+0000","last_fullsized":"2023-09-23T22:04:54.526277+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:53:05.805798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.395321+0000","last_change":"2023-09-23T22:04:54.382361+0000","last_active":"2023-09-23T22:04:54.395321+0000","last_peered":"2023-09-23T22:04:54.395321+0000","last_clean":"2023-09-23T22:04:54.395321+0000","last_became_active":"2023-09-23T22:04:54.382033+0000","last_became_peered":"2023-09-23T22:04:54.382033+0000","last_unstale":"2023-09-23T22:04:54.395321+0000","last_undegraded":"2023-09-23T22:04:54.395321+0000","last_fullsized":"2023-09-23T22:04:54.395321+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:51.412662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.415728+0000","last_change":"2023-09-23T22:04:54.412428+0000","last_active":"2023-09-23T22:04:54.415728+0000","last_peered":"2023-09-23T22:04:54.415728+0000","last_clean":"2023-09-23T22:04:54.415728+0000","last_became_active":"2023-09-23T22:04:54.412224+0000","last_became_peered":"2023-09-23T22:04:54.412224+0000","last_unstale":"2023-09-23T22:04:54.415728+0000","last_undegraded":"2023-09-23T22:04:54.415728+0000","last_fullsized":"2023-09-23T22:04:54.415728+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:10.264695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.399611+0000","last_change":"2023-09-23T22:04:54.389665+0000","last_active":"2023-09-23T22:04:54.399611+0000","last_peered":"2023-09-23T22:04:54.399611+0000","last_clean":"2023-09-23T22:04:54.399611+0000","last_became_active":"2023-09-23T22:04:54.389467+0000","last_became_peered":"2023-09-23T22:04:54.389467+0000","last_unstale":"2023-09-23T22:04:54.399611+0000","last_undegraded":"2023-09-23T22:04:54.399611+0000","last_fullsized":"2023-09-23T22:04:54.399611+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:45.129155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.406317+0000","last_change":"2023-09-23T22:04:54.405930+0000","last_active":"2023-09-23T22:04:54.406317+0000","last_peered":"2023-09-23T22:04:54.406317+0000","last_clean":"2023-09-23T22:04:54.406317+0000","last_became_active":"2023-09-23T22:04:54.405654+0000","last_became_peered":"2023-09-23T22:04:54.405654+0000","last_unstale":"2023-09-23T22:04:54.406317+0000","last_undegraded":"2023-09-23T22:04:54.406317+0000","last_fullsized":"2023-09-23T22:04:54.406317+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:17.173425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.398286+0000","last_change":"2023-09-23T22:04:54.388512+0000","last_active":"2023-09-23T22:04:54.398286+0000","last_peered":"2023-09-23T22:04:54.398286+0000","last_clean":"2023-09-23T22:04:54.398286+0000","last_became_active":"2023-09-23T22:04:54.388281+0000","last_became_peered":"2023-09-23T22:04:54.388281+0000","last_unstale":"2023-09-23T22:04:54.398286+0000","last_undegraded":"2023-09-23T22:04:54.398286+0000","last_fullsized":"2023-09-23T22:04:54.398286+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:43.537168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.528058+0000","last_change":"2023-09-23T22:04:54.527728+0000","last_active":"2023-09-23T22:04:54.528058+0000","last_peered":"2023-09-23T22:04:54.528058+0000","last_clean":"2023-09-23T22:04:54.528058+0000","last_became_active":"2023-09-23T22:04:54.527430+0000","last_became_peered":"2023-09-23T22:04:54.527430+0000","last_unstale":"2023-09-23T22:04:54.528058+0000","last_undegraded":"2023-09-23T22:04:54.528058+0000","last_fullsized":"2023-09-23T22:04:54.528058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:30.298965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.407485+0000","last_change":"2023-09-23T22:04:54.387554+0000","last_active":"2023-09-23T22:04:54.407485+0000","last_peered":"2023-09-23T22:04:54.407485+0000","last_clean":"2023-09-23T22:04:54.407485+0000","last_became_active":"2023-09-23T22:04:54.387274+0000","last_became_peered":"2023-09-23T22:04:54.387274+0000","last_unstale":"2023-09-23T22:04:54.407485+0000","last_undegraded":"2023-09-23T22:04:54.407485+0000","last_fullsized":"2023-09-23T22:04:54.407485+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:20.494449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":10,"reported_epoch":66,"state":"active+clean","last_fresh":"2023-09-23T22:04:54.415495+0000","last_change":"2023-09-23T22:04:54.411754+0000","last_active":"2023-09-23T22:04:54.415495+0000","last_peered":"2023-09-23T22:04:54.415495+0000","last_clean":"2023-09-23T22:04:54.415495+0000","last_became_active":"2023-09-23T22:04:54.411472+0000","last_became_peered":"2023-09-23T22:04:54.411472+0000","last_unstale":"2023-09-23T22:04:54.415495+0000","last_undegraded":"2023-09-23T22:04:54.415495+0000","last_fullsized":"2023-09-23T22:04:54.415495+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:40.722110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":12,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T22:04:55.756045+0000","last_change":"2023-09-23T22:04:54.380166+0000","last_active":"2023-09-23T22:04:55.756045+0000","last_peered":"2023-09-23T22:04:55.756045+0000","last_clean":"2023-09-23T22:04:55.756045+0000","last_became_active":"2023-09-23T22:04:54.379878+0000","last_became_peered":"2023-09-23T22:04:54.379878+0000","last_unstale":"2023-09-23T22:04:55.756045+0000","last_undegraded":"2023-09-23T22:04:55.756045+0000","last_fullsized":"2023-09-23T22:04:55.756045+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:07.644159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234015,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6524,"kb_used_data":1268,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93738628,"statfs":{"total":95995035648,"available":95988355072,"internally_reserved":0,"allocated":1298432,"data_stored":884042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.52700000000000002,"15min":0.52300000000000002},"min":{"1min":0.33500000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.748,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.47699999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.52900000000000003,"15min":0.53300000000000003},"min":{"1min":0.34100000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69099999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.33400000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.44700000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.40200000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.80000000000000004,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.54700000000000004,"15min":0.54400000000000004},"min":{"1min":0.37,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.78800000000000003,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73499999999999999,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57799999999999996,"15min":0.58099999999999996},"min":{"1min":0.42799999999999999,"5min":0.38,"15min":0.38},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.53600000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.46700000000000003,"15min":0.46500000000000002},"min":{"1min":0.25600000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.502,"15min":0.504},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.57999999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.48499999999999999,"15min":0.48699999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.504},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.30299999999999999,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.59299999999999997,"15min":0.60899999999999999},"min":{"1min":0.376,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.75800000000000001,"5min":3.0459999999999998,"15min":3.0459999999999998},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.57399999999999995,"15min":0.57599999999999996},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.59599999999999997}]}]},{"osd":6,"up_from":46,"seq":197568495651,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6580,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738572,"statfs":{"total":95995035648,"available":95988297728,"internally_reserved":0,"allocated":1290240,"data_stored":884219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.47499999999999998,"15min":0.47699999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.65700000000000003,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.502,"15min":0.504},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.60199999999999998,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.76000000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.48499999999999999,"15min":0.49199999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.61799999999999999,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.495,"5min":0.499,"15min":0.499},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.77400000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.504,"15min":0.50600000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73899999999999999,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.84099999999999997},{"interface":"front","average":{"1min":0.499,"5min":0.52000000000000002,"15min":0.52400000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.67400000000000004,"5min":0.753,"15min":0.753},"last":0.70099999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.76600000000000001,"15min":0.81100000000000005},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.77900000000000003,"5min":7.6429999999999998,"15min":7.6429999999999998},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.39300000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64700000000000002,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.73899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.39200000000000002,"15min":0.38400000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.41899999999999998,"15min":0.40699999999999997},"min":{"1min":0.33900000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.109,"5min":1.109,"15min":1.109},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.44500000000000001,"15min":0.43099999999999999},"min":{"1min":0.32000000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.171,"5min":1.171,"15min":1.171},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.435,"15min":0.42899999999999999},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.36699999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:05:20 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.47199999999999998,"15min":0.46600000000000003},"min":{"1min":0.37,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.61399999999999999,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.46000000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54900000000000004,"15min":0.54600000000000004},"min":{"1min":0.38200000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69399999999999995,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.47599999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574906,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6752,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738400,"statfs":{"total":95995035648,"available":95988121600,"internally_reserved":0,"allocated":679936,"data_stored":292245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:05:07 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.33500000000000002},"min":{"1min":0.252,"5min":0.214,"15min":0.214},"max":{"1min":1.1499999999999999,"5min":1.1499999999999999,"15min":1.1499999999999999},"last":0.439},{"interface":"front","average":{"1min":0.81699999999999995,"5min":0.47399999999999998,"15min":0.39200000000000002},"min":{"1min":0.25,"5min":0.193,"15min":0.193},"max":{"1min":10.122,"5min":10.122,"15min":10.122},"last":0.28799999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:05:30 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.435,"15min":0.42199999999999999},"min":{"1min":0.26900000000000002,"5min":0.218,"15min":0.218},"max":{"1min":1.254,"5min":1.254,"15min":1.254},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.47299999999999998,"15min":0.42299999999999999},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68799999999999994,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":0.34999999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.45500000000000002,"15min":0.46700000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.212,"5min":1.212,"15min":1.212},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.53400000000000003,"15min":0.54800000000000004},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:05:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.60799999999999998,"15min":0.61799999999999999},"min":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0029999999999999,"5min":1.0029999999999999,"15min":1.0029999999999999},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.64300000000000002,"15min":0.625},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.0570000000000004,"5min":6.0570000000000004,"15min":6.0570000000000004},"last":0.55500000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:05:35 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.90800000000000003,"15min":0.99199999999999999},"min":{"1min":0.40000000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.90600000000000003,"5min":7.3479999999999999,"15min":7.3479999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.88400000000000001,"15min":1.0009999999999999},"min":{"1min":0.39400000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":1.298,"5min":6.9539999999999997,"15min":6.9539999999999997},"last":0.51100000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.65500000000000003,"15min":0.66500000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":0.89800000000000002,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.84299999999999997,"5min":0.70599999999999996,"15min":0.68300000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":6.1929999999999996,"5min":6.1929999999999996,"15min":6.1929999999999996},"last":0.52300000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:05:18 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.91300000000000003,"15min":0.96499999999999997},"min":{"1min":0.43099999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.92300000000000004,"5min":6.117,"15min":6.117},"last":0.628},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.67000000000000004,"15min":0.67300000000000004},"min":{"1min":0.46999999999999997,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.62}]}]},{"osd":0,"up_from":8,"seq":34359738432,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7340,"kb_used_data":1252,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93737812,"statfs":{"total":95995035648,"available":95987519488,"internally_reserved":0,"allocated":1282048,"data_stored":883189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:05:12 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.371,"15min":0.36199999999999999},"min":{"1min":0.23799999999999999,"5min":0.17499999999999999,"15min":0.17499999999999999},"max":{"1min":0.65400000000000003,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.253},{"interface":"front","average":{"1min":0.435,"5min":0.39500000000000002,"15min":0.371},"min":{"1min":0.26800000000000002,"5min":0.193,"15min":0.193},"max":{"1min":1.6359999999999999,"5min":1.6359999999999999,"15min":1.6359999999999999},"last":0.36599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:05:30 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.41599999999999998,"15min":0.40400000000000003},"min":{"1min":0.255,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.96,"5min":1.96,"15min":1.96},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.442,"15min":0.41999999999999998},"min":{"1min":0.29899999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.68899999999999995,"5min":1.593,"15min":1.593},"last":0.29499999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:04:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.497,"15min":0.502},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.49399999999999999,"15min":0.499},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:05:14 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58699999999999997},"min":{"1min":0.41299999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.83099999999999996,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58199999999999996,"15min":0.57599999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.80100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:04:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.63700000000000001,"15min":0.64700000000000002},"min":{"1min":0.44900000000000001,"5min":0.439,"15min":0.439},"max":{"1min":0.81799999999999995,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.629,"15min":0.63900000000000001},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.76700000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:04:59 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.63,"15min":0.63600000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.86199999999999999,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61299999999999999,"15min":0.61699999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.73699999999999999,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.46400000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:05:20 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.67000000000000004,"15min":0.68000000000000005},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.877,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.66700000000000004,"15min":0.67300000000000004},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.86699999999999999,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.439}]}]},{"osd":2,"up_from":18,"seq":77309411382,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6568,"kb_used_data":672,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738584,"statfs":{"total":95995035648,"available":95988310016,"internally_reserved":0,"allocated":688128,"data_stored":292833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:05:31 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.38800000000000001,"15min":0.38600000000000001},"min":{"1min":0.245,"5min":0.222,"15min":0.222},"max":{"1min":0.66800000000000004,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.371,"5min":0.40200000000000002,"15min":0.39800000000000002},"min":{"1min":0.26000000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.60499999999999998,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:05:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.41299999999999998,"15min":0.40899999999999997},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":0.58699999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.433,"15min":0.42499999999999999},"min":{"1min":0.27600000000000002,"5min":0.193,"15min":0.193},"max":{"1min":0.79100000000000004,"5min":1.121,"15min":1.121},"last":0.376}]},{"osd":3,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.437,"5min":0.46300000000000002,"15min":0.47199999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.50900000000000001,"15min":0.52000000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.71999999999999997,"5min":0.745,"15min":0.745},"last":0.496}]},{"osd":4,"last update":"Sat Sep 23 22:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54800000000000004,"15min":0.54600000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56200000000000006,"15min":0.55600000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.95599999999999996,"5min":0.95599999999999996,"15min":0.95599999999999996},"last":0.629}]},{"osd":5,"last update":"Sat Sep 23 22:05:36 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56799999999999995,"15min":0.56899999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.94599999999999995,"5min":1.069,"15min":1.069},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.46700000000000003,"5min":0.371,"15min":0.371},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.66500000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:05:00 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.60699999999999998,"15min":0.61099999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.96499999999999997,"5min":0.96499999999999997,"15min":0.96499999999999997},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.60199999999999998,"15min":0.60499999999999998},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:05:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.61699999999999999,"15min":0.625},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.63300000000000001,"15min":0.64000000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.71399999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182448,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6116,"kb_used_data":668,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739036,"statfs":{"total":95995035648,"available":95988772864,"internally_reserved":0,"allocated":684032,"data_stored":292335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.40899999999999997,"15min":0.41699999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.622,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.44900000000000001,"15min":0.46000000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":1.075,"5min":1.075,"15min":1.075},"last":0.28999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.46400000000000002,"15min":0.47999999999999998},"min":{"1min":0.32600000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.55700000000000005,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.46700000000000003,"15min":0.47899999999999998},"min":{"1min":0.29599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.621,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.39900000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:04:49 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":0.50700000000000001,"15min":0.52500000000000002},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.58699999999999997,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.439},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.52400000000000002,"15min":0.48299999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.2160000000000002,"5min":7.2160000000000002,"15min":7.2160000000000002},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:05:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.59599999999999997,"15min":0.60799999999999998},"min":{"1min":0.45100000000000001,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.78600000000000003,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.60799999999999998,"15min":0.625},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.74099999999999999,"5min":1.1479999999999999,"15min":1.1479999999999999},"last":0.44900000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:04:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.63,"15min":0.64200000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.80100000000000005,"5min":1.022,"15min":1.022},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.65500000000000003,"15min":0.66700000000000004},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.85699999999999998,"5min":1.002,"15min":1.002},"last":0.46899999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:04:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.64700000000000002,"15min":0.66000000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.89300000000000002,"5min":1.091,"15min":1.091},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.61699999999999999,"15min":0.626},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.78300000000000003,"5min":1.0389999999999999,"15min":1.0389999999999999},"last":0.46100000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:05:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.61199999999999999,"15min":0.62},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.84799999999999998,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.66600000000000004,"15min":0.67400000000000004},"min":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"max":{"1min":0.752,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.621}]}]},{"osd":4,"up_from":30,"seq":128849018924,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6200,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738952,"statfs":{"total":95995035648,"available":95988686848,"internally_reserved":0,"allocated":704512,"data_stored":293927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:05:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.51300000000000001,"15min":0.52300000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.67500000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.53800000000000003,"15min":0.52700000000000002},"min":{"1min":0.36799999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.61699999999999999,"5min":3.1240000000000001,"15min":3.1240000000000001},"last":0.62}]},{"osd":1,"last update":"Sat Sep 23 22:05:15 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":0.52200000000000002,"15min":0.53800000000000003},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.69399999999999995,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.56200000000000006,"15min":0.58799999999999997},"min":{"1min":0.318,"5min":0.315,"15min":0.315},"max":{"1min":0.59699999999999998,"5min":2.2109999999999999,"15min":2.2109999999999999},"last":0.64000000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:05:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.56000000000000005,"15min":0.57799999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.65600000000000003,"5min":2.2629999999999999,"15min":2.2629999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.55200000000000005,"15min":0.54400000000000004},"min":{"1min":0.38,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.60699999999999998,"5min":3.0910000000000002,"15min":3.0910000000000002},"last":0.53100000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:05:15 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.56699999999999995,"15min":0.55500000000000005},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.63500000000000001,"5min":3.1499999999999999,"15min":3.1499999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54600000000000004,"15min":0.54200000000000004},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.69999999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:04:38 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.42099999999999999,"15min":0.41199999999999998},"min":{"1min":0.317,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60199999999999998,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.39500000000000002,"5min":0.318,"15min":0.318},"max":{"1min":0.73099999999999998,"5min":3.0470000000000002,"15min":3.0470000000000002},"last":0.66300000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:04:56 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.46700000000000003,"15min":0.45700000000000002},"min":{"1min":0.35699999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.35299999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.78300000000000003,"5min":2.2440000000000002,"15min":2.2440000000000002},"last":0.48199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:05:23 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.68500000000000005,"15min":0.71199999999999997},"min":{"1min":0.375,"5min":0.313,"15min":0.313},"max":{"1min":0.85099999999999998,"5min":3.9609999999999999,"15min":3.9609999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56699999999999995,"15min":0.56999999999999995},"min":{"1min":0.42299999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.72299999999999998,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.55000000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757288,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5992,"kb_used_data":672,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739160,"statfs":{"total":95995035648,"available":95988899840,"internally_reserved":0,"allocated":688128,"data_stored":292869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:05:38 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.54000000000000004,"15min":0.55500000000000005},"min":{"1min":0.38,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.74399999999999999,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.443},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.53100000000000003,"15min":0.52700000000000002},"min":{"1min":0.34999999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.52500000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:05:38 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":0.56299999999999994,"15min":0.56899999999999995},"min":{"1min":0.40100000000000002,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.6679999999999999,"5min":1.6679999999999999,"15min":1.6679999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.54700000000000004,"15min":0.54900000000000004},"min":{"1min":0.47599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.59199999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:05:38 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.55100000000000005,"15min":0.56499999999999995},"min":{"1min":0.34999999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69999999999999996,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.59699999999999998,"15min":0.58799999999999997},"min":{"1min":0.45700000000000002,"5min":0.374,"15min":0.374},"max":{"1min":1.639,"5min":1.639,"15min":1.639},"last":0.47299999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:05:38 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.57699999999999996,"15min":0.57899999999999996},"min":{"1min":0.42099999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.028,"5min":1.028,"15min":1.028},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.61099999999999999,"15min":0.60699999999999998},"min":{"1min":0.44600000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":0.56999999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:05:38 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.42699999999999999,"15min":0.40899999999999997},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.46400000000000002,"15min":0.437},"min":{"1min":0.33900000000000002,"5min":0.315,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:04:58 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.47199999999999998,"15min":0.46600000000000003},"min":{"1min":0.36899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.501,"15min":0.499},"min":{"1min":0.36099999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.79400000000000004,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.66900000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:05:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.58299999999999996,"15min":0.59199999999999997},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.81200000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.626},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.54100000000000004,"15min":0.54300000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.70399999999999996,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.80300000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:05:39.791 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:05:39.792 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T22:05:39.792 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T22:05:39.792 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T22:05:39.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:39 smithi026 ceph-mon[100157]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:05:39.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:39 smithi026 ceph-mon[118879]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:05:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:39 smithi070 ceph-mon[105170]: from='client.14784 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:05:40.155 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:40.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:40 smithi026 ceph-mon[118879]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:40.681 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:40 smithi026 ceph-mon[100157]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:41.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:40 smithi070 ceph-mon[105170]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:41.722 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:05:41.722 INFO:teuthology.orchestra.run.smithi026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T22:05:41.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:05:41 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:05:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:41 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2818600639' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:05:42.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:41 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2818600639' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:05:42.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:41 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2818600639' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:05:42.328 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T22:05:42.328 INFO:teuthology.run_tasks:Running task print... 2023-09-23T22:05:42.337 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T22:05:42.338 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T22:05:42.383 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T22:05:42.383 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T22:05:42.384 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T22:05:42.387 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T22:05:42.388 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T22:05:42.388 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:05:42.388 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T22:05:42.388 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T22:05:42.389 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T22:05:42.389 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T22:05:42.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:05:42.408 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T22:05:42.409 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:05:42.466 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T22:05:42.466 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T22:05:42.524 INFO:tasks.workunit:timeout=3h 2023-09-23T22:05:42.524 INFO:tasks.workunit:cleanup=True 2023-09-23T22:05:42.524 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T22:05:42.584 INFO:tasks.workunit.client.0.smithi026.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T22:05:42.762 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:43.033 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:42 smithi026 ceph-mon[100157]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:43.033 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:42 smithi026 ceph-mon[118879]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:43.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:42 smithi070 ceph-mon[105170]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:44.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:44 smithi026 conmon[100586]: [23/Sep/2023:22:05:44] ENGINE Bus STOPPING 2023-09-23T22:05:44.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:44 smithi026 conmon[100586]: [23/Sep/2023:22:05:44] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:05:44.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:44 smithi026 conmon[100586]: [23/Sep/2023:22:05:44] ENGINE Bus STOPPED 2023-09-23T22:05:44.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:44 smithi026 conmon[100586]: [23/Sep/2023:22:05:44] ENGINE Bus STARTING 2023-09-23T22:05:44.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:44 smithi026 conmon[100586]: [23/Sep/2023:22:05:44] ENGINE Serving on http://:::9283 2023-09-23T22:05:44.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:44 smithi026 conmon[100586]: [23/Sep/2023:22:05:44] ENGINE Bus STARTED 2023-09-23T22:05:44.909 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T22:05:45.314 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[118879]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3494031367' entity='client.admin' 2023-09-23T22:05:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[100157]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3494031367' entity='client.admin' 2023-09-23T22:05:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:45 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:45 smithi070 ceph-mon[105170]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:45 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3494031367' entity='client.admin' 2023-09-23T22:05:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:45 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:45 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:45 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:45 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:46 smithi026 ceph-mon[100157]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:46 smithi026 ceph-mon[118879]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:47.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:46 smithi070 ceph-mon[105170]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:47.609 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T22:05:47.971 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:48.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:48 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:48] "GET /metrics HTTP/1.1" 200 238573 "" "Prometheus/2.43.0" 2023-09-23T22:05:49.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:49 smithi026 ceph-mon[100157]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:49.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:49 smithi026 ceph-mon[118879]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:49.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:49 smithi070 ceph-mon[105170]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:49.750 INFO:teuthology.orchestra.run.smithi026.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:05:50.666 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:05:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[118879]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:05:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[118879]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:05:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[118879]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:05:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[118879]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[100157]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:05:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[100157]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:05:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[100157]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:05:50.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:50 smithi026 ceph-mon[100157]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:51.062 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:05:51.077 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T22:05:51.105 INFO:tasks.workunit.client.0.smithi026.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T22:05:51.107 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T22:05:51.107 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T22:05:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:50 smithi070 ceph-mon[105170]: from='client.14808 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:05:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:50 smithi070 ceph-mon[105170]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:05:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:50 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:50 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:05:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:50 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:05:51.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:50 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:05:51.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:50 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:05:51.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:50 smithi070 ceph-mon[105170]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T22:05:51.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:50 smithi070 ceph-mon[105170]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:51.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:05:51 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:05:51.923 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T22:05:52.015 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T22:05:52.058 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T22:05:52.059 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T22:05:52.059 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T22:05:52.118 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T22:05:52.122 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T22:05:52.122 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T22:05:52.185 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T22:05:52.186 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T22:05:52.186 DEBUG:teuthology.orchestra.run.smithi026:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T22:05:52.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T22:05:52.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T22:05:52.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T22:05:52.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:52.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T22:05:52.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T22:05:52.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T22:05:52.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:52.247 INFO:tasks.workunit.client.0.smithi026.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T22:05:52.248 INFO:tasks.workunit.client.0.smithi026.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T22:05:52.248 INFO:tasks.workunit.client.0.smithi026.stderr:+ TMPDIR=/tmp/rbd_import_export_192007 2023-09-23T22:05:52.248 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm -rf /tmp/rbd_import_export_192007 2023-09-23T22:05:52.248 INFO:tasks.workunit.client.0.smithi026.stderr:+ mkdir /tmp/rbd_import_export_192007 2023-09-23T22:05:52.249 INFO:tasks.workunit.client.0.smithi026.stderr:+ trap 'rm -rf /tmp/rbd_import_export_192007' INT TERM EXIT 2023-09-23T22:05:52.249 INFO:tasks.workunit.client.0.smithi026.stderr:+ mkdir foo.192007 2023-09-23T22:05:52.250 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import foo.192007 foo.dir 2023-09-23T22:05:52.320 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: cannot import a directory 2023-09-23T22:05:52.320 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 0% complete...failed. 2023-09-23T22:05:52.320 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: import failed: (21) Is a directory 2023-09-23T22:05:52.322 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:05:52.322 INFO:tasks.workunit.client.0.smithi026.stderr:+ rmdir foo.192007 2023-09-23T22:05:52.335 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_192007/img bs=1k count=1 seek=10 2023-09-23T22:05:52.336 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:05:52.336 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:05:52.336 INFO:tasks.workunit.client.0.smithi026.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.4573e-05 s, 23.0 MB/s 2023-09-23T22:05:52.336 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_192007/img bs=1k count=10 seek=100 2023-09-23T22:05:52.337 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records in 2023-09-23T22:05:52.337 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records out 2023-09-23T22:05:52.337 INFO:tasks.workunit.client.0.smithi026.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.7992e-05 s, 177 MB/s 2023-09-23T22:05:52.337 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_192007/img bs=1k count=100 seek=1000 2023-09-23T22:05:52.338 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records in 2023-09-23T22:05:52.338 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records out 2023-09-23T22:05:52.338 INFO:tasks.workunit.client.0.smithi026.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000199362 s, 361 MB/s 2023-09-23T22:05:52.338 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_192007/img bs=1k seek=10000 2023-09-23T22:05:52.339 INFO:tasks.workunit.client.0.smithi026.stderr:139+1 records in 2023-09-23T22:05:52.339 INFO:tasks.workunit.client.0.smithi026.stderr:139+1 records out 2023-09-23T22:05:52.339 INFO:tasks.workunit.client.0.smithi026.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000349609 s, 410 MB/s 2023-09-23T22:05:52.339 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_192007/img bs=1k seek=100000 2023-09-23T22:05:52.367 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records in 2023-09-23T22:05:52.368 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records out 2023-09-23T22:05:52.368 INFO:tasks.workunit.client.0.smithi026.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0278466 s, 2.6 MB/s 2023-09-23T22:05:52.368 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_192007/img bs=1k seek=1000000 2023-09-23T22:05:52.369 INFO:tasks.workunit.client.0.smithi026.stderr:189+1 records in 2023-09-23T22:05:52.369 INFO:tasks.workunit.client.0.smithi026.stderr:189+1 records out 2023-09-23T22:05:52.369 INFO:tasks.workunit.client.0.smithi026.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000473046 s, 410 MB/s 2023-09-23T22:05:52.369 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T22:05:52.405 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 0% complete...failed. 2023-09-23T22:05:52.405 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T22:05:52.408 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:05:52.408 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format /tmp/rbd_import_export_192007/img testimg 2023-09-23T22:05:52.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T22:05:52.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2023-09-23T22:05:52.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T22:05:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:05:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[118879]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T22:05:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-23T22:05:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T22:05:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[118879]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T22:05:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[118879]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T22:05:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[100157]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T22:05:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-23T22:05:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T22:05:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[100157]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T22:05:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:53 smithi026 ceph-mon[100157]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T22:05:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:53 smithi070 ceph-mon[105170]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:53 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T22:05:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:53 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2023-09-23T22:05:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:53 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T22:05:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:53 smithi070 ceph-mon[105170]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T22:05:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:53 smithi070 ceph-mon[105170]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T22:05:54.699 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T22:05:54.701 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:05:54.703 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg /tmp/rbd_import_export_192007/img2 2023-09-23T22:05:55.004 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T22:05:55.007 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg - 2023-09-23T22:05:55.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:55 smithi026 ceph-mon[118879]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:55.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:55 smithi026 ceph-mon[118879]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T22:05:55.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:55 smithi026 ceph-mon[100157]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:55.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:55 smithi026 ceph-mon[100157]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T22:05:55.414 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (67s) 44s ago 109s 16.9M - 0.25.0 c8568f914cd2 7a14521d2de9 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (79s) 45s ago 78s 86.0M - 9.4.7 2c41d148cca3 d197cdcf40c1 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.rancfk smithi026 running (54s) 44s ago 54s 16.3M - 17.2.6-1125-gc9bb1bcf 444539aec5bf a9885df09771 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.ehaqek smithi070 running (52s) 45s ago 51s 14.7M - 17.2.6-1125-gc9bb1bcf 444539aec5bf 812a5d180a78 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443 running (5m) 45s ago 5m 441M - 17.2.6-1125-gc9bb1bcf 444539aec5bf bc3ea1afcb46 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:9283 running (7m) 44s ago 7m 493M - 17.2.6-1125-gc9bb1bcf 444539aec5bf d8b44817b175 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (7m) 44s ago 7m 84.6M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf feb74e5008f7 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (6m) 45s ago 6m 55.2M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf b61f3bc30723 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (5m) 44s ago 5m 56.3M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf fce3aa9fda1a 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (103s) 44s ago 103s 15.8M - 1.5.0 0da6a335fe13 89b44a6ca08e 2023-09-23T22:05:55.415 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (100s) 45s ago 99s 15.1M - 1.5.0 0da6a335fe13 6bac96aa7b4f 2023-09-23T22:05:55.416 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (5m) 44s ago 5m 55.8M 4037M 17.2.6-1125-gc9bb1bcf 444539aec5bf 67a4086eb002 2023-09-23T22:05:55.416 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (5m) 44s ago 5m 58.0M 4037M 17.2.6-1125-gc9bb1bcf 444539aec5bf 02b1d9e99db9 2023-09-23T22:05:55.416 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (4m) 44s ago 4m 49.3M 4037M 17.2.6-1125-gc9bb1bcf 444539aec5bf 50e18b584478 2023-09-23T22:05:55.416 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (4m) 44s ago 4m 48.0M 4037M 17.2.6-1125-gc9bb1bcf 444539aec5bf 57db6dad61fc 2023-09-23T22:05:55.416 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (3m) 45s ago 3m 51.7M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf 92889bd7f5ee 2023-09-23T22:05:55.416 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (3m) 45s ago 3m 48.2M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf 628c1ee2ccfc 2023-09-23T22:05:55.416 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (3m) 45s ago 3m 49.5M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf 5b472aeb7f69 2023-09-23T22:05:55.416 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (2m) 45s ago 2m 51.3M 4036M 17.2.6-1125-gc9bb1bcf 444539aec5bf 2a3425d84687 2023-09-23T22:05:55.416 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (62s) 45s ago 2m 28.2M - 2.43.0 a07b618ecd1d a37f53e6e83a 2023-09-23T22:05:55.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:55 smithi070 ceph-mon[105170]: pgmap v74: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:55.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:55 smithi070 ceph-mon[105170]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T22:05:55.853 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T22:05:55.857 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T22:05:55.949 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T22:05:55.952 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_192007/img /tmp/rbd_import_export_192007/img2 2023-09-23T22:05:56.687 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_192007/img /tmp/rbd_import_export_192007/img3 2023-09-23T22:05:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:56 smithi026 ceph-mon[100157]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:05:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:56 smithi026 ceph-mon[100157]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:56 smithi026 ceph-mon[118879]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:05:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:56 smithi026 ceph-mon[118879]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:56 smithi070 ceph-mon[105170]: from='client.24740 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:05:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:56 smithi070 ceph-mon[105170]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:05:57.264 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/img2 /tmp/rbd_import_export_192007/img3 2023-09-23T22:05:57.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:57 smithi026 ceph-mon[118879]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:05:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:57 smithi026 ceph-mon[100157]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:05:58.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:57 smithi070 ceph-mon[105170]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:05:58.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:05:58 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:05:58] "GET /metrics HTTP/1.1" 200 238564 "" "Prometheus/2.43.0" 2023-09-23T22:05:58.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:58 smithi026 ceph-mon[100157]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s 2023-09-23T22:05:58.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:58 smithi026 ceph-mon[118879]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s 2023-09-23T22:05:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:58 smithi070 ceph-mon[105170]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 198 KiB/s rd, 85 KiB/s wr, 50 op/s 2023-09-23T22:05:59.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:59 smithi026 ceph-mon[100157]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:05:59.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:05:59 smithi026 ceph-mon[100157]: Cluster is now healthy 2023-09-23T22:05:59.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:59 smithi026 ceph-mon[118879]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:05:59.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:05:59 smithi026 ceph-mon[118879]: Cluster is now healthy 2023-09-23T22:06:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:59 smithi070 ceph-mon[105170]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T22:06:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:05:59 smithi070 ceph-mon[105170]: Cluster is now healthy 2023-09-23T22:06:00.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:00 smithi026 ceph-mon[118879]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-23T22:06:00.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:00 smithi026 ceph-mon[118879]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T22:06:00.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:00 smithi026 ceph-mon[100157]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-23T22:06:00.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:00 smithi026 ceph-mon[100157]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T22:06:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:00 smithi070 ceph-mon[105170]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-23T22:06:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:00 smithi070 ceph-mon[105170]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T22:06:01.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:06:01 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:06:02.020 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format - testimg 2023-09-23T22:06:02.442 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:06:02.443 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:06:02.446 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg /tmp/rbd_import_export_192007/img2 2023-09-23T22:06:02.746 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:02.750 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg - 2023-09-23T22:06:03.187 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T22:06:03.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:03 smithi026 ceph-mon[118879]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-23T22:06:03.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:03 smithi026 ceph-mon[100157]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-23T22:06:03.565 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:03.570 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T22:06:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:03 smithi070 ceph-mon[105170]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 64 KiB/s wr, 38 op/s 2023-09-23T22:06:03.683 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T22:06:03.686 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_192007/img /tmp/rbd_import_export_192007/img2 2023-09-23T22:06:03.698 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:06:04.456 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_192007/img /tmp/rbd_import_export_192007/img3 2023-09-23T22:06:04.976 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/img /tmp/rbd_import_export_192007/img2 /tmp/rbd_import_export_192007/img3 2023-09-23T22:06:05.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:05 smithi026 ceph-mon[118879]: pgmap v81: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 224 KiB/s rd, 104 KiB/s wr, 65 op/s 2023-09-23T22:06:05.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:05 smithi026 ceph-mon[100157]: pgmap v81: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 224 KiB/s rd, 104 KiB/s wr, 65 op/s 2023-09-23T22:06:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:05 smithi070 ceph-mon[105170]: pgmap v81: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 224 KiB/s rd, 104 KiB/s wr, 65 op/s 2023-09-23T22:06:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:06 smithi026 ceph-mon[100157]: pgmap v82: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 222 KiB/s rd, 103 KiB/s wr, 64 op/s 2023-09-23T22:06:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:06 smithi026 ceph-mon[118879]: pgmap v82: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 222 KiB/s rd, 103 KiB/s wr, 64 op/s 2023-09-23T22:06:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:06 smithi070 ceph-mon[105170]: pgmap v82: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 222 KiB/s rd, 103 KiB/s wr, 64 op/s 2023-09-23T22:06:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:06:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:06:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:06:08.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:06:08 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:08] "GET /metrics HTTP/1.1" 200 239168 "" "Prometheus/2.43.0" 2023-09-23T22:06:08.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:08 smithi026 ceph-mon[118879]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T22:06:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:08 smithi026 ceph-mon[100157]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T22:06:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:08 smithi070 ceph-mon[105170]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T22:06:10.852 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd help export 2023-09-23T22:06:10.852 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -q export-format 2023-09-23T22:06:10.873 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_192007/img bs=1k count=10 seek=100 2023-09-23T22:06:10.874 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records in 2023-09-23T22:06:10.874 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records out 2023-09-23T22:06:10.874 INFO:tasks.workunit.client.0.smithi026.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.3372e-05 s, 140 MB/s 2023-09-23T22:06:10.874 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format /tmp/rbd_import_export_192007/img testimg 2023-09-23T22:06:10.932 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:06:10.932 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:06:10.934 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap create testimg@snap 2023-09-23T22:06:11.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:11 smithi026 ceph-mon[100157]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T22:06:11.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:11 smithi026 ceph-mon[118879]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T22:06:11.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:11 smithi070 ceph-mon[105170]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T22:06:11.689 INFO:tasks.workunit.client.0.smithi026.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T22:06:11.696 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T22:06:11.750 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd image-meta list testimg 2023-09-23T22:06:11.791 INFO:tasks.workunit.client.0.smithi026.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T22:06:11.792 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T22:06:11.792 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T22:06:11.792 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' 2023-09-23T22:06:11.792 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192007/img_v2 2023-09-23T22:06:11.835 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:11.838 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192007/img_v2 testimg_import 2023-09-23T22:06:11.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:06:11 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:11] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:06:12.696 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:06:12.699 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T22:06:12.736 INFO:tasks.workunit.client.0.smithi026.stdout:rbd image 'testimg_import': 2023-09-23T22:06:12.736 INFO:tasks.workunit.client.0.smithi026.stdout: size 110 KiB in 1 objects 2023-09-23T22:06:12.736 INFO:tasks.workunit.client.0.smithi026.stdout: order 22 (4 MiB objects) 2023-09-23T22:06:12.736 INFO:tasks.workunit.client.0.smithi026.stdout: snapshot_count: 1 2023-09-23T22:06:12.736 INFO:tasks.workunit.client.0.smithi026.stdout: id: 3a3ee0b08e84 2023-09-23T22:06:12.736 INFO:tasks.workunit.client.0.smithi026.stdout: block_name_prefix: rbd_data.3a3ee0b08e84 2023-09-23T22:06:12.736 INFO:tasks.workunit.client.0.smithi026.stdout: format: 2 2023-09-23T22:06:12.736 INFO:tasks.workunit.client.0.smithi026.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T22:06:12.736 INFO:tasks.workunit.client.0.smithi026.stdout: op_features: 2023-09-23T22:06:12.737 INFO:tasks.workunit.client.0.smithi026.stdout: flags: 2023-09-23T22:06:12.737 INFO:tasks.workunit.client.0.smithi026.stdout: create_timestamp: Sat Sep 23 22:06:11 2023 2023-09-23T22:06:12.737 INFO:tasks.workunit.client.0.smithi026.stdout: access_timestamp: Sat Sep 23 22:06:11 2023 2023-09-23T22:06:12.737 INFO:tasks.workunit.client.0.smithi026.stdout: modify_timestamp: Sat Sep 23 22:06:11 2023 2023-09-23T22:06:12.739 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import@snap 2023-09-23T22:06:12.777 INFO:tasks.workunit.client.0.smithi026.stdout:rbd image 'testimg_import': 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: size 110 KiB in 1 objects 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: order 22 (4 MiB objects) 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: snapshot_count: 1 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: id: 3a3ee0b08e84 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: block_name_prefix: rbd_data.3a3ee0b08e84 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: format: 2 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: op_features: 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: flags: 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: create_timestamp: Sat Sep 23 22:06:11 2023 2023-09-23T22:06:12.778 INFO:tasks.workunit.client.0.smithi026.stdout: access_timestamp: Sat Sep 23 22:06:11 2023 2023-09-23T22:06:12.779 INFO:tasks.workunit.client.0.smithi026.stdout: modify_timestamp: Sat Sep 23 22:06:11 2023 2023-09-23T22:06:12.779 INFO:tasks.workunit.client.0.smithi026.stdout: protected: False 2023-09-23T22:06:12.781 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd image-meta list testimg_import 2023-09-23T22:06:12.824 INFO:tasks.workunit.client.0.smithi026.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T22:06:12.824 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T22:06:12.824 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T22:06:12.824 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' 2023-09-23T22:06:12.824 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T22:06:12.824 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T22:06:12.825 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T22:06:12.825 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T22:06:12.825 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T22:06:12.825 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T22:06:12.825 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' ']' 2023-09-23T22:06:12.825 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg_import /tmp/rbd_import_export_192007/img_import 2023-09-23T22:06:12.861 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T22:06:12.865 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192007/img /tmp/rbd_import_export_192007/img_import 2023-09-23T22:06:12.866 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_192007/img /tmp/rbd_import_export_192007/img_import 2023-09-23T22:06:12.866 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/img --format %b 2023-09-23T22:06:12.866 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=24 2023-09-23T22:06:12.868 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/img_import --format %b 2023-09-23T22:06:12.868 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=224 2023-09-23T22:06:12.869 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=200 2023-09-23T22:06:12.869 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=200 2023-09-23T22:06:12.869 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T22:06:12.869 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_192007/img_snap 2023-09-23T22:06:12.907 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T22:06:12.910 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_192007/img_snap_import 2023-09-23T22:06:12.951 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T22:06:12.954 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192007/img_snap /tmp/rbd_import_export_192007/img_snap_import 2023-09-23T22:06:12.954 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_192007/img_snap /tmp/rbd_import_export_192007/img_snap_import 2023-09-23T22:06:12.955 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/img_snap --format %b 2023-09-23T22:06:12.956 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=224 2023-09-23T22:06:12.956 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/img_snap_import --format %b 2023-09-23T22:06:12.957 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=224 2023-09-23T22:06:12.957 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:12.957 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:12.957 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:06:12.958 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/img_v2 2023-09-23T22:06:12.958 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/img_import 2023-09-23T22:06:12.958 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/img_snap 2023-09-23T22:06:12.959 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/img_snap_import 2023-09-23T22:06:12.960 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T22:06:12.984 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:12 smithi026 ceph-mon[100157]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T22:06:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:12 smithi026 ceph-mon[100157]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T22:06:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:12 smithi026 ceph-mon[118879]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T22:06:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:12 smithi026 ceph-mon[118879]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T22:06:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:12 smithi070 ceph-mon[105170]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T22:06:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:12 smithi070 ceph-mon[105170]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T22:06:13.689 INFO:tasks.workunit.client.0.smithi026.stderr: Removing snap: 100% complete...done. 2023-09-23T22:06:13.696 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:06:13.792 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:13.795 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap rm testimg@snap 2023-09-23T22:06:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:13 smithi026 ceph-mon[118879]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T22:06:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:13 smithi026 ceph-mon[100157]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T22:06:14.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:13 smithi070 ceph-mon[105170]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T22:06:14.696 INFO:tasks.workunit.client.0.smithi026.stderr: Removing snap: 100% complete...done. 2023-09-23T22:06:14.706 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T22:06:14.820 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:14.825 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --order 20 /tmp/rbd_import_export_192007/img testimg 2023-09-23T22:06:14.893 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:06:14.894 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:06:14.896 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192007/img_v2 2023-09-23T22:06:14.953 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:14.958 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192007/img_v2 testimg_import 2023-09-23T22:06:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:14 smithi026 ceph-mon[118879]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:06:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:14 smithi026 ceph-mon[118879]: pgmap v89: 73 pgs: 73 active+clean; 799 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 8.0 KiB/s wr, 109 op/s 2023-09-23T22:06:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:14 smithi026 ceph-mon[100157]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:06:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:14 smithi026 ceph-mon[100157]: pgmap v89: 73 pgs: 73 active+clean; 799 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 8.0 KiB/s wr, 109 op/s 2023-09-23T22:06:15.030 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:06:15.034 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T22:06:15.034 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep order 2023-09-23T22:06:15.034 INFO:tasks.workunit.client.0.smithi026.stderr:+ awk '{print $2}' 2023-09-23T22:06:15.035 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 20 2023-09-23T22:06:15.078 INFO:tasks.workunit.client.0.smithi026.stdout:20 2023-09-23T22:06:15.078 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/img_v2 2023-09-23T22:06:15.079 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:06:15.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:14 smithi070 ceph-mon[105170]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T22:06:15.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:14 smithi070 ceph-mon[105170]: pgmap v89: 73 pgs: 73 active+clean; 799 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 8.0 KiB/s wr, 109 op/s 2023-09-23T22:06:15.168 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:15.172 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T22:06:15.261 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:15.264 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_192007/img testimg 2023-09-23T22:06:15.318 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:06:15.322 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd info testimg 2023-09-23T22:06:15.322 INFO:tasks.workunit.client.0.smithi026.stderr:++ grep features 2023-09-23T22:06:15.361 INFO:tasks.workunit.client.0.smithi026.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T22:06:15.361 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' 2023-09-23T22:06:15.361 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192007/img_v2 2023-09-23T22:06:15.401 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:15.406 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192007/img_v2 testimg_import 2023-09-23T22:06:15.460 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:06:15.463 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd info testimg_import 2023-09-23T22:06:15.463 INFO:tasks.workunit.client.0.smithi026.stderr:++ grep features 2023-09-23T22:06:15.502 INFO:tasks.workunit.client.0.smithi026.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T22:06:15.502 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' 2023-09-23T22:06:15.502 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' ' features: layering 2023-09-23T22:06:15.502 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' '!=' ' features: layering 2023-09-23T22:06:15.503 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' ']' 2023-09-23T22:06:15.503 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/img_v2 2023-09-23T22:06:15.503 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:06:15.577 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:15.580 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T22:06:15.655 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:15.658 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_192007/img testimg 2023-09-23T22:06:15.725 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:06:15.728 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192007/img_v2 2023-09-23T22:06:15.780 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:15.783 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192007/img_v2 testimg_import 2023-09-23T22:06:15.856 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:06:15.860 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T22:06:15.860 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'stripe unit' 2023-09-23T22:06:15.860 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T22:06:15.902 INFO:tasks.workunit.client.0.smithi026.stdout: stripe unit: 4 KiB 2023-09-23T22:06:15.903 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T22:06:15.903 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'stripe count' 2023-09-23T22:06:15.903 INFO:tasks.workunit.client.0.smithi026.stderr:+ awk '{print $3}' 2023-09-23T22:06:15.903 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 1000 2023-09-23T22:06:15.941 INFO:tasks.workunit.client.0.smithi026.stdout:1000 2023-09-23T22:06:15.942 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/img_v2 2023-09-23T22:06:15.942 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:06:15.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:15 smithi026 ceph-mon[118879]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:06:15.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:15 smithi026 ceph-mon[100157]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:06:16.031 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T22:06:16.035 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T22:06:16.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:15 smithi070 ceph-mon[105170]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T22:06:16.130 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T22:06:16.133 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_192007/img testimg 2023-09-23T22:06:16.191 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:06:16.194 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap create testimg@snap1 2023-09-23T22:06:16.707 INFO:tasks.workunit.client.0.smithi026.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T22:06:16.714 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap create testimg@snap2 2023-09-23T22:06:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:16 smithi026 ceph-mon[118879]: pgmap v91: 73 pgs: 73 active+clean; 799 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s rd, 11 KiB/s wr, 147 op/s 2023-09-23T22:06:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:16 smithi026 ceph-mon[100157]: pgmap v91: 73 pgs: 73 active+clean; 799 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s rd, 11 KiB/s wr, 147 op/s 2023-09-23T22:06:17.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:16 smithi070 ceph-mon[105170]: pgmap v91: 73 pgs: 73 active+clean; 799 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 119 KiB/s rd, 11 KiB/s wr, 147 op/s 2023-09-23T22:06:17.712 INFO:tasks.workunit.client.0.smithi026.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T22:06:17.720 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap protect testimg@snap2 2023-09-23T22:06:17.768 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_192007/snap_protect 2023-09-23T22:06:17.816 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:17.819 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_192007/snap_protect testimg_import 2023-09-23T22:06:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:17 smithi026 ceph-mon[118879]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T22:06:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:17 smithi026 ceph-mon[100157]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T22:06:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:17 smithi070 ceph-mon[105170]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T22:06:18.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:06:18 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:18] "GET /metrics HTTP/1.1" 200 239168 "" "Prometheus/2.43.0" 2023-09-23T22:06:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:18 smithi026 ceph-mon[100157]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T22:06:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:18 smithi026 ceph-mon[100157]: pgmap v94: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 377 KiB/s rd, 46 KiB/s wr, 507 op/s 2023-09-23T22:06:18.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:18 smithi026 ceph-mon[118879]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T22:06:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:18 smithi026 ceph-mon[118879]: pgmap v94: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 377 KiB/s rd, 46 KiB/s wr, 507 op/s 2023-09-23T22:06:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:18 smithi070 ceph-mon[105170]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T22:06:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:18 smithi070 ceph-mon[105170]: pgmap v94: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 377 KiB/s rd, 46 KiB/s wr, 507 op/s 2023-09-23T22:06:19.742 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:06:19.746 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import@snap1 2023-09-23T22:06:19.746 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'protected: False' 2023-09-23T22:06:19.800 INFO:tasks.workunit.client.0.smithi026.stdout: protected: False 2023-09-23T22:06:19.800 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import@snap2 2023-09-23T22:06:19.801 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'protected: True' 2023-09-23T22:06:19.851 INFO:tasks.workunit.client.0.smithi026.stdout: protected: True 2023-09-23T22:06:19.851 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/snap_protect 2023-09-23T22:06:19.852 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T22:06:19.900 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T22:06:19.946 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap purge testimg 2023-09-23T22:06:19.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:19 smithi026 ceph-mon[118879]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T22:06:19.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:19 smithi026 ceph-mon[100157]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T22:06:20.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:19 smithi070 ceph-mon[105170]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T22:06:20.803 INFO:teuthology.orchestra.run.smithi026.stdout:wait for minority of mons to upgrade 2023-09-23T22:06:20.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:20 smithi026 ceph-mon[118879]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T22:06:20.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:20 smithi026 ceph-mon[118879]: pgmap v97: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 418 KiB/s rd, 51 KiB/s wr, 562 op/s 2023-09-23T22:06:20.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:20 smithi026 ceph-mon[118879]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T22:06:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:20 smithi026 ceph-mon[100157]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T22:06:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:20 smithi026 ceph-mon[100157]: pgmap v97: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 418 KiB/s rd, 51 KiB/s wr, 562 op/s 2023-09-23T22:06:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:20 smithi026 ceph-mon[100157]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T22:06:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:20 smithi070 ceph-mon[105170]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T22:06:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:20 smithi070 ceph-mon[105170]: pgmap v97: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 418 KiB/s rd, 51 KiB/s wr, 562 op/s 2023-09-23T22:06:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:20 smithi070 ceph-mon[105170]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T22:06:21.689 INFO:tasks.workunit.client.0.smithi026.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T22:06:21.699 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap purge testimg_import 2023-09-23T22:06:21.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:06:21 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:21] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:06:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:22 smithi026 ceph-mon[100157]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T22:06:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:22 smithi026 ceph-mon[100157]: pgmap v100: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:22 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:22 smithi026 ceph-mon[118879]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T22:06:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:22 smithi026 ceph-mon[118879]: pgmap v100: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:22 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:22 smithi070 ceph-mon[105170]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T22:06:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:22 smithi070 ceph-mon[105170]: pgmap v100: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:22 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:23.698 INFO:tasks.workunit.client.0.smithi026.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T22:06:23.707 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T22:06:23.808 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:23.811 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:06:23.896 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:23.899 INFO:tasks.workunit.client.0.smithi026.stderr:+ tiered=0 2023-09-23T22:06:23.899 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph osd dump 2023-09-23T22:06:23.899 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep '^pool' 2023-09-23T22:06:23.899 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep ''\''rbd'\''' 2023-09-23T22:06:23.899 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep tier 2023-09-23T22:06:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:23 smithi026 ceph-mon[100157]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T22:06:23.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:23 smithi026 ceph-mon[118879]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T22:06:24.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:23 smithi070 ceph-mon[105170]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T22:06:24.351 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_192007/sparse1 2023-09-23T22:06:24.357 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:06:24.357 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:06:24.357 INFO:tasks.workunit.client.0.smithi026.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00501948 s, 209 MB/s 2023-09-23T22:06:24.358 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_192007/sparse2 2023-09-23T22:06:24.363 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:06:24.364 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:06:24.364 INFO:tasks.workunit.client.0.smithi026.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00498423 s, 210 MB/s 2023-09-23T22:06:24.364 INFO:tasks.workunit.client.0.smithi026.stderr:+ truncate /tmp/rbd_import_export_192007/sparse2 -s 2M 2023-09-23T22:06:24.412 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse1 2023-09-23T22:06:24.450 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 0% complete...failed. 2023-09-23T22:06:24.450 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T22:06:24.453 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:06:24.453 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_192007/sparse1 2023-09-23T22:06:24.515 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T22:06:24.515 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:06:24.516 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:06:24.518 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T22:06:24.518 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse1 2023-09-23T22:06:24.518 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T22:06:24.556 INFO:tasks.workunit.client.0.smithi026.stdout:sparse1 2 MiB 2 2023-09-23T22:06:24.557 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse1 2023-09-23T22:06:24.557 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse1 2023-09-23T22:06:24.557 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse1 2023-09-23T22:06:24.558 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:06:24.558 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:06:24.594 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.3b4c35dab915 2023-09-23T22:06:24.595 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.3b4c35dab915 2023-09-23T22:06:24.595 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse1 2023-09-23T22:06:24.595 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse1 2023-09-23T22:06:24.595 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.3b4c35dab915\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:06:24.595 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:06:24.595 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse1 2023-09-23T22:06:24.595 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:06:24.595 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:06:24.632 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:06:24.632 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:06:24.632 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:06:24.632 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:06:24.632 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:06:24.665 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=1 2023-09-23T22:06:24.665 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 1 2023-09-23T22:06:24.665 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T22:06:24.666 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse1 /tmp/rbd_import_export_192007/sparse1.out 2023-09-23T22:06:24.701 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:24.705 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192007/sparse1 /tmp/rbd_import_export_192007/sparse1.out 2023-09-23T22:06:24.705 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_192007/sparse1 /tmp/rbd_import_export_192007/sparse1.out 2023-09-23T22:06:24.707 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/sparse1 --format %b 2023-09-23T22:06:24.708 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=2048 2023-09-23T22:06:24.708 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/sparse1.out --format %b 2023-09-23T22:06:24.709 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=2048 2023-09-23T22:06:24.709 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:24.709 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:24.709 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:06:24.709 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/sparse1.out 2023-09-23T22:06:24.710 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse1 2023-09-23T22:06:24.795 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T22:06:24.798 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse2 2023-09-23T22:06:24.834 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 0% complete...failed. 2023-09-23T22:06:24.835 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T22:06:24.837 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:06:24.837 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_192007/sparse2 2023-09-23T22:06:24.898 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T22:06:24.898 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:06:24.898 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:06:24.901 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T22:06:24.901 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse2 2023-09-23T22:06:24.901 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T22:06:24.936 INFO:tasks.workunit.client.0.smithi026.stdout:sparse2 2 MiB 2 2023-09-23T22:06:24.937 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse2 2023-09-23T22:06:24.937 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse2 2023-09-23T22:06:24.937 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse2 2023-09-23T22:06:24.937 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:06:24.937 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:06:24.973 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.62216dcedba3 2023-09-23T22:06:24.974 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.62216dcedba3 2023-09-23T22:06:24.974 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.62216dcedba3\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:06:24.975 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse2 2023-09-23T22:06:24.975 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse2 2023-09-23T22:06:24.975 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:06:24.975 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse2 2023-09-23T22:06:24.975 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:06:24.975 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:06:24.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:24 smithi026 ceph-mon[118879]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T22:06:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:24 smithi026 ceph-mon[118879]: pgmap v103: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 7.7 KiB/s wr, 134 op/s 2023-09-23T22:06:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:24 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2390537672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:06:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:24 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2390537672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:06:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:24 smithi026 ceph-mon[100157]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T22:06:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:24 smithi026 ceph-mon[100157]: pgmap v103: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 7.7 KiB/s wr, 134 op/s 2023-09-23T22:06:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:24 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2390537672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:06:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:24 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2390537672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:06:25.011 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:06:25.011 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:06:25.011 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:06:25.011 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:06:25.011 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:06:25.043 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=0 2023-09-23T22:06:25.044 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 0 2023-09-23T22:06:25.044 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T22:06:25.044 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse2 /tmp/rbd_import_export_192007/sparse2.out 2023-09-23T22:06:25.081 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:25.084 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192007/sparse2 /tmp/rbd_import_export_192007/sparse2.out 2023-09-23T22:06:25.085 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_192007/sparse2 /tmp/rbd_import_export_192007/sparse2.out 2023-09-23T22:06:25.087 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/sparse2 --format %b 2023-09-23T22:06:25.087 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=2048 2023-09-23T22:06:25.088 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/sparse2.out --format %b 2023-09-23T22:06:25.089 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=2048 2023-09-23T22:06:25.089 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:25.089 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:25.089 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:06:25.089 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/sparse2.out 2023-09-23T22:06:25.090 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse2 2023-09-23T22:06:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:24 smithi070 ceph-mon[105170]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T22:06:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:24 smithi070 ceph-mon[105170]: pgmap v103: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 7.7 KiB/s wr, 134 op/s 2023-09-23T22:06:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:24 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2390537672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:06:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:24 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2390537672' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:06:25.179 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T22:06:25.179 INFO:tasks.workunit.client.0.smithi026.stderr:+ truncate /tmp/rbd_import_export_192007/sparse1 -s 10M 2023-09-23T22:06:25.180 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T22:06:25.268 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:06:25.269 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:06:25.269 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:06:25.271 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T22:06:25.272 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse1 2023-09-23T22:06:25.272 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T22:06:25.307 INFO:tasks.workunit.client.0.smithi026.stdout:sparse1 10 MiB 2 2023-09-23T22:06:25.308 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse1 2023-09-23T22:06:25.308 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse1 2023-09-23T22:06:25.308 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse1 2023-09-23T22:06:25.309 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:06:25.309 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:06:25.344 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.624edfd894a2 2023-09-23T22:06:25.345 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.624edfd894a2 2023-09-23T22:06:25.345 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.624edfd894a2\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:06:25.345 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse1 2023-09-23T22:06:25.345 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse1 2023-09-23T22:06:25.345 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:06:25.346 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse1 2023-09-23T22:06:25.346 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:06:25.346 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:06:25.381 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:06:25.381 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:06:25.382 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:06:25.382 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:06:25.382 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:06:25.413 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=1 2023-09-23T22:06:25.413 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 1 2023-09-23T22:06:25.413 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T22:06:25.414 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse1 /tmp/rbd_import_export_192007/sparse1.out 2023-09-23T22:06:25.452 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:25.455 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192007/sparse1 /tmp/rbd_import_export_192007/sparse1.out 2023-09-23T22:06:25.455 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_192007/sparse1 /tmp/rbd_import_export_192007/sparse1.out 2023-09-23T22:06:25.463 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/sparse1 --format %b 2023-09-23T22:06:25.464 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=2048 2023-09-23T22:06:25.464 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/sparse1.out --format %b 2023-09-23T22:06:25.465 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=2048 2023-09-23T22:06:25.465 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:25.465 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:25.465 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:06:25.466 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/sparse1.out 2023-09-23T22:06:25.466 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse1 2023-09-23T22:06:25.548 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T22:06:25.551 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_192007/sparse2 oflag=append conv=notrunc 2023-09-23T22:06:25.562 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:06:25.562 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:06:25.562 INFO:tasks.workunit.client.0.smithi026.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00997538 s, 210 MB/s 2023-09-23T22:06:25.563 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T22:06:25.647 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:06:25.647 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:06:25.648 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:06:25.650 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T22:06:25.650 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse2 2023-09-23T22:06:25.650 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T22:06:25.691 INFO:tasks.workunit.client.0.smithi026.stdout:sparse2 4 MiB 2 2023-09-23T22:06:25.692 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse2 2023-09-23T22:06:25.692 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse2 2023-09-23T22:06:25.692 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse2 2023-09-23T22:06:25.692 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:06:25.693 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:06:25.728 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.3bd9cc683f23 2023-09-23T22:06:25.729 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.3bd9cc683f23 2023-09-23T22:06:25.729 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse2 2023-09-23T22:06:25.729 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse2 2023-09-23T22:06:25.729 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.3bd9cc683f23\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:06:25.729 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:06:25.730 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse2 2023-09-23T22:06:25.730 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:06:25.730 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:06:25.766 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:06:25.766 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:06:25.766 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:06:25.766 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:06:25.766 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:06:25.798 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects='0 2023-09-23T22:06:25.798 INFO:tasks.workunit.client.0.smithi026.stderr:2 2023-09-23T22:06:25.799 INFO:tasks.workunit.client.0.smithi026.stderr:3' 2023-09-23T22:06:25.799 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 0 2 3 2023-09-23T22:06:25.799 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T22:06:25.799 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse2 /tmp/rbd_import_export_192007/sparse2.out 2023-09-23T22:06:25.861 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T22:06:25.864 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_192007/sparse2 /tmp/rbd_import_export_192007/sparse2.out 2023-09-23T22:06:25.864 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_192007/sparse2 /tmp/rbd_import_export_192007/sparse2.out 2023-09-23T22:06:25.867 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/sparse2 --format %b 2023-09-23T22:06:25.868 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=6144 2023-09-23T22:06:25.869 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/sparse2.out --format %b 2023-09-23T22:06:25.870 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=6144 2023-09-23T22:06:25.870 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:25.870 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:06:25.870 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:06:25.870 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/sparse2.out 2023-09-23T22:06:25.871 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse2 2023-09-23T22:06:25.952 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T22:06:25.955 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T22:06:25.956 INFO:tasks.workunit.client.0.smithi026.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T22:06:25.956 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_192007/sparse1 sparse 2023-09-23T22:06:26.014 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T22:06:26.014 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:06:26.015 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:06:26.017 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse 2023-09-23T22:06:26.017 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T22:06:26.018 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T22:06:26.018 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:06:26.018 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:06:26.053 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.3c06d83451d3 2023-09-23T22:06:26.054 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.3c06d83451d3 2023-09-23T22:06:26.054 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse 2023-09-23T22:06:26.054 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse 2023-09-23T22:06:26.054 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.3c06d83451d3\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:06:26.054 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:06:26.054 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse 2023-09-23T22:06:26.055 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:06:26.055 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:06:26.090 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:06:26.090 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:06:26.090 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:06:26.090 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:06:26.090 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:06:26.122 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=1 2023-09-23T22:06:26.122 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 1 2023-09-23T22:06:26.122 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T22:06:26.122 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse 2023-09-23T22:06:26.204 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T22:06:26.207 INFO:tasks.workunit.client.0.smithi026.stdout:zeros import through stdin to sparse image 2023-09-23T22:06:26.207 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T22:06:26.207 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T22:06:26.208 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format - sparse 2023-09-23T22:06:26.252 INFO:tasks.workunit.client.0.smithi026.stderr:4+0 records in 2023-09-23T22:06:26.253 INFO:tasks.workunit.client.0.smithi026.stderr:4+0 records out 2023-09-23T22:06:26.253 INFO:tasks.workunit.client.0.smithi026.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.044451 s, 94.4 MB/s 2023-09-23T22:06:26.262 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:06:26.262 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:06:26.265 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse 2023-09-23T22:06:26.265 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T22:06:26.265 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T22:06:26.265 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:06:26.266 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:06:26.301 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.62c9659ab8c2 2023-09-23T22:06:26.301 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.62c9659ab8c2 2023-09-23T22:06:26.301 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse 2023-09-23T22:06:26.301 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse 2023-09-23T22:06:26.301 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.62c9659ab8c2\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:06:26.301 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:06:26.302 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse 2023-09-23T22:06:26.302 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:06:26.302 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:06:26.336 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:06:26.336 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:06:26.337 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:06:26.337 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:06:26.337 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:06:26.368 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects= 2023-09-23T22:06:26.369 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 2023-09-23T22:06:26.369 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T22:06:26.369 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse 2023-09-23T22:06:26.447 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:26.449 INFO:tasks.workunit.client.0.smithi026.stdout:zeros export to sparse file 2023-09-23T22:06:26.450 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo 'zeros export to sparse file' 2023-09-23T22:06:26.450 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T22:06:26.487 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:06:26.489 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd info sparse 2023-09-23T22:06:26.489 INFO:tasks.workunit.client.0.smithi026.stderr:++ grep block_name_prefix 2023-09-23T22:06:26.489 INFO:tasks.workunit.client.0.smithi026.stderr:++ awk '{print $NF;}' 2023-09-23T22:06:26.525 INFO:tasks.workunit.client.0.smithi026.stderr:+ prefix=rbd_data.3c48d82b4d11 2023-09-23T22:06:26.525 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T22:06:26.526 INFO:tasks.workunit.client.0.smithi026.stderr:++ get_image_data_pool sparse 2023-09-23T22:06:26.526 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T22:06:26.526 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T22:06:26.526 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep 'data_pool: ' 2023-09-23T22:06:26.527 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T22:06:26.561 INFO:tasks.workunit.client.0.smithi026.stderr:++ data_pool= 2023-09-23T22:06:26.561 INFO:tasks.workunit.client.0.smithi026.stderr:++ '[' -z ']' 2023-09-23T22:06:26.562 INFO:tasks.workunit.client.0.smithi026.stderr:++ data_pool=rbd 2023-09-23T22:06:26.562 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo rbd 2023-09-23T22:06:26.562 INFO:tasks.workunit.client.0.smithi026.stderr:+ rados -p rbd put rbd_data.3c48d82b4d11.000000000000 - 2023-09-23T22:06:26.591 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:06:26.591 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:06:26.591 INFO:tasks.workunit.client.0.smithi026.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0652221 s, 64.3 MB/s 2023-09-23T22:06:26.619 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse 2023-09-23T22:06:26.619 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T22:06:26.620 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T22:06:26.620 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:06:26.620 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:06:26.655 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.3c48d82b4d11 2023-09-23T22:06:26.656 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.3c48d82b4d11 2023-09-23T22:06:26.656 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse 2023-09-23T22:06:26.657 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse 2023-09-23T22:06:26.657 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.3c48d82b4d11\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:06:26.657 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:06:26.657 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse 2023-09-23T22:06:26.657 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:06:26.657 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:06:26.694 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:06:26.694 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:06:26.694 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:06:26.694 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:06:26.695 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:06:26.727 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=0 2023-09-23T22:06:26.727 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 0 2023-09-23T22:06:26.727 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T22:06:26.727 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/sparse 2023-09-23T22:06:26.728 INFO:tasks.workunit.client.0.smithi026.stderr:rm: cannot remove '/tmp/rbd_import_export_192007/sparse': No such file or directory 2023-09-23T22:06:26.728 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:06:26.728 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse /tmp/rbd_import_export_192007/sparse 2023-09-23T22:06:26.763 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T22:06:26.766 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_192007/sparse --format=%b 2023-09-23T22:06:26.767 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 = 0 ']' 2023-09-23T22:06:26.767 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse 2023-09-23T22:06:26.848 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:06:26.851 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_192007/sparse /tmp/rbd_import_export_192007/sparse1 /tmp/rbd_import_export_192007/sparse2 /tmp/rbd_import_export_192007/sparse3 2023-09-23T22:06:26.853 INFO:tasks.workunit.client.0.smithi026.stderr:rm: cannot remove '/tmp/rbd_import_export_192007/sparse3': No such file or directory 2023-09-23T22:06:26.853 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:06:26.853 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo OK 2023-09-23T22:06:26.853 INFO:tasks.workunit.client.0.smithi026.stdout:OK 2023-09-23T22:06:26.854 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm -rf /tmp/rbd_import_export_192007 2023-09-23T22:06:26.855 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T22:06:26.855 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T22:06:26.926 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T22:06:26.926 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T22:06:26.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:26 smithi026 ceph-mon[118879]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T22:06:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:26 smithi026 ceph-mon[118879]: pgmap v105: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 7.0 KiB/s wr, 121 op/s 2023-09-23T22:06:26.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:26 smithi026 ceph-mon[100157]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T22:06:26.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:26 smithi026 ceph-mon[100157]: pgmap v105: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 7.0 KiB/s wr, 121 op/s 2023-09-23T22:06:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:26 smithi070 ceph-mon[105170]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T22:06:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:26 smithi070 ceph-mon[105170]: pgmap v105: 73 pgs: 73 active+clean; 799 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 7.0 KiB/s wr, 121 op/s 2023-09-23T22:06:27.179 DEBUG:teuthology.parallel:result is None 2023-09-23T22:06:27.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:06:27.207 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:06:27.207 DEBUG:teuthology.orchestra.run.smithi026:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:06:27.263 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:06:27.263 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T22:06:27.264 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T22:06:27.264 DEBUG:teuthology.parallel:result is None 2023-09-23T22:06:28.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:06:28 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:28] "GET /metrics HTTP/1.1" 200 239369 "" "Prometheus/2.43.0" 2023-09-23T22:06:29.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:29 smithi026 ceph-mon[100157]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.9 MiB/s wr, 521 op/s 2023-09-23T22:06:29.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:29 smithi026 ceph-mon[118879]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.9 MiB/s wr, 521 op/s 2023-09-23T22:06:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:29 smithi070 ceph-mon[105170]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.9 MiB/s wr, 521 op/s 2023-09-23T22:06:30.180 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T22:06:31.042 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:06:31.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[100157]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 421 op/s 2023-09-23T22:06:31.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[100157]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T22:06:31.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:06:31.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:06:31.259 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:06:31.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[118879]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 421 op/s 2023-09-23T22:06:31.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[118879]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T22:06:31.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:06:31.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:06:31.260 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:31 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:06:31.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:31 smithi070 ceph-mon[105170]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 421 op/s 2023-09-23T22:06:31.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:31 smithi070 ceph-mon[105170]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T22:06:31.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:31 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:06:31.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:31 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:06:31.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:31 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:06:31.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:06:31 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:31] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:06:32.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:32 smithi026 ceph-mon[118879]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T22:06:32.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:32 smithi026 ceph-mon[118879]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T22:06:32.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:32 smithi026 ceph-mon[118879]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T22:06:32.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:32 smithi026 ceph-mon[100157]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T22:06:32.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:32 smithi026 ceph-mon[100157]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T22:06:32.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:32 smithi026 ceph-mon[100157]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T22:06:32.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:32 smithi070 ceph-mon[105170]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T22:06:32.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:32 smithi070 ceph-mon[105170]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T22:06:32.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:32 smithi070 ceph-mon[105170]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T22:06:33.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:33 smithi026 ceph-mon[118879]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi070 2023-09-23T22:06:33.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:33 smithi026 ceph-mon[118879]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 1010 KiB/s rd, 1.4 MiB/s wr, 324 op/s 2023-09-23T22:06:33.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:33 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/658048796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:33.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:33 smithi026 ceph-mon[100157]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi070 2023-09-23T22:06:33.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:33 smithi026 ceph-mon[100157]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 1010 KiB/s rd, 1.4 MiB/s wr, 324 op/s 2023-09-23T22:06:33.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:33 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/658048796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:33 smithi070 ceph-mon[105170]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi070 2023-09-23T22:06:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:33 smithi070 ceph-mon[105170]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 1010 KiB/s rd, 1.4 MiB/s wr, 324 op/s 2023-09-23T22:06:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:33 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/658048796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:35 smithi026 ceph-mon[118879]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 958 KiB/s rd, 1.3 MiB/s wr, 307 op/s 2023-09-23T22:06:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:35 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/474620023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:35.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:35 smithi026 ceph-mon[100157]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 958 KiB/s rd, 1.3 MiB/s wr, 307 op/s 2023-09-23T22:06:35.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:35 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/474620023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:35 smithi070 ceph-mon[105170]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 958 KiB/s rd, 1.3 MiB/s wr, 307 op/s 2023-09-23T22:06:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:35 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/474620023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:36 smithi026 ceph-mon[118879]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 808 KiB/s rd, 1.1 MiB/s wr, 259 op/s 2023-09-23T22:06:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:36 smithi026 ceph-mon[100157]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 808 KiB/s rd, 1.1 MiB/s wr, 259 op/s 2023-09-23T22:06:37.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:36 smithi070 ceph-mon[105170]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail; 808 KiB/s rd, 1.1 MiB/s wr, 259 op/s 2023-09-23T22:06:37.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:37 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:37.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:37 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/4072428859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:37 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:37 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/4072428859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:38.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:37 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:38.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:37 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/4072428859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:38.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:38 smithi026 ceph-mon[100157]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:38.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:06:38 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:38] "GET /metrics HTTP/1.1" 200 239455 "" "Prometheus/2.43.0" 2023-09-23T22:06:38.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:38 smithi026 ceph-mon[118879]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:38 smithi070 ceph-mon[105170]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:39 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1441917320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:39 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1441917320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:39 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1441917320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:41.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:40 smithi026 ceph-mon[118879]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:41.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:40 smithi026 ceph-mon[100157]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:41.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:40 smithi070 ceph-mon[105170]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:41.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:06:41 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:41] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:06:43.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:43 smithi026 ceph-mon[118879]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:43.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:43 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1895483147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:43.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:43 smithi026 ceph-mon[100157]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:43.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:43 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1895483147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:43 smithi070 ceph-mon[105170]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:43 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1895483147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:45 smithi026 ceph-mon[118879]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:45 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3872960136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:45 smithi026 ceph-mon[100157]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:45 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3872960136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:45 smithi070 ceph-mon[105170]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:45 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3872960136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:46 smithi026 ceph-mon[100157]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:46 smithi026 ceph-mon[118879]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:47.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:46 smithi070 ceph-mon[105170]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:47 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2148093568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:47.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:47 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2148093568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:48.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:47 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2148093568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:48.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:06:48 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:48] "GET /metrics HTTP/1.1" 200 239455 "" "Prometheus/2.43.0" 2023-09-23T22:06:48.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:48 smithi026 ceph-mon[118879]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:48.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:48 smithi026 ceph-mon[100157]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:49.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:48 smithi070 ceph-mon[105170]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:49 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3765656944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:49.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:49 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3765656944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:50.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:49 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3765656944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:50 smithi026 ceph-mon[118879]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:50 smithi026 ceph-mon[100157]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:50 smithi070 ceph-mon[105170]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:51.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:06:51 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:51] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:06:52.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:51 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1594121088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:52.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:51 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1594121088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:52.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:51 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1594121088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:53.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:52 smithi026 ceph-mon[100157]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:53.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:52 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:53.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:52 smithi026 ceph-mon[118879]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:53.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:52 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:53.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:52 smithi070 ceph-mon[105170]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:53.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:52 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:06:55.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:55 smithi026 ceph-mon[100157]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:55.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:55 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1431753266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:55 smithi026 ceph-mon[118879]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:55.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:55 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1431753266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:55.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:55 smithi070 ceph-mon[105170]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:55.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:55 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1431753266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:56 smithi026 ceph-mon[118879]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:56 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2176927347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:56 smithi026 ceph-mon[100157]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:56 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2176927347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:56 smithi070 ceph-mon[105170]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:56 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2176927347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:58.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:06:58 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:06:58] "GET /metrics HTTP/1.1" 200 239424 "" "Prometheus/2.43.0" 2023-09-23T22:06:59.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:59 smithi026 ceph-mon[100157]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:59.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:06:59 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2436295802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:59.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:59 smithi026 ceph-mon[118879]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:59.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:06:59 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2436295802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:06:59.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:59 smithi070 ceph-mon[105170]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:06:59.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:06:59 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2436295802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:01.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:01 smithi026 ceph-mon[118879]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:01 smithi026 ceph-mon[100157]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:01.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:01 smithi070 ceph-mon[105170]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:01.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:01 smithi070 conmon[108367]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:07:01] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:07:02.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:02 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3150069721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:02 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3150069721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:02.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:02 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3150069721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:03.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:03 smithi026 ceph-mon[100157]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:03.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:03 smithi026 ceph-mon[118879]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:03 smithi070 ceph-mon[105170]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:04.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:04 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1083679543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:04.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:04 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1083679543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:04.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:04 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1083679543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:05.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:05 smithi026 ceph-mon[100157]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:05.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:05 smithi026 ceph-mon[118879]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:05 smithi070 ceph-mon[105170]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:06 smithi070 ceph-mon[105170]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:06 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2173577368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:06 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:06.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:06 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:07:06.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:06 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:07:06.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:06 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[100157]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2173577368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:07:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:07:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[118879]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2173577368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:07:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:07:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:06 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:07 smithi026 ceph-mon[100157]: Upgrade: Updating mgr.x 2023-09-23T22:07:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:07 smithi026 ceph-mon[100157]: Deploying daemon mgr.x on smithi070 2023-09-23T22:07:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:07 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:07 smithi026 ceph-mon[118879]: Upgrade: Updating mgr.x 2023-09-23T22:07:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:07 smithi026 ceph-mon[118879]: Deploying daemon mgr.x on smithi070 2023-09-23T22:07:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:07 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:07.986 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:07 smithi070 ceph-mon[105170]: Upgrade: Updating mgr.x 2023-09-23T22:07:07.986 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:07 smithi070 ceph-mon[105170]: Deploying daemon mgr.x on smithi070 2023-09-23T22:07:07.986 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:07 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:08.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:07 smithi070 systemd[1]: Stopping Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:08.964 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:08 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:07:08.964 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:08 smithi070 bash[158304]: bc3ea1afcb46b6b2a5f2767a25b6486ace36227d59db4ca75fe111e1fa87ae0b 2023-09-23T22:07:08.965 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:08 smithi070 ceph-mon[105170]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:08.965 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:08 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2698385847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:08.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:08 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:07:08] "GET /metrics HTTP/1.1" 200 239370 "" "Prometheus/2.43.0" 2023-09-23T22:07:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:08 smithi026 ceph-mon[100157]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:08 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2698385847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:08.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:08 smithi026 ceph-mon[118879]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:08.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:08 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2698385847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:09.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:08 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T22:07:09.374 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:08 smithi070 systemd[1]: Stopped Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:09.374 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:09 smithi070 systemd[1]: Starting Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:10.012 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:09 smithi070 ceph-mgr[158722]: -- 172.21.15.70:0/2726478222 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5562b50a11e0 con 0x5562b5078c00 2023-09-23T22:07:10.013 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:09 smithi070 bash[158657]: 57a112d98af9b4ad5d2b1536676cc09019ee7ae3a713b820a045a3ad84163b34 2023-09-23T22:07:10.013 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:09 smithi070 systemd[1]: Started Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:10.013 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:09 smithi070 conmon[158698]: 2023-09-23T22:07:09.834+0000 7f8758d8a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:07:10.013 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:09 smithi070 conmon[158698]: 2023-09-23T22:07:09.920+0000 7f8758d8a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:07:10.326 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:10 smithi070 conmon[158698]: 2023-09-23T22:07:10.012+0000 7f8758d8a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:07:10.326 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:10 smithi070 conmon[158698]: 2023-09-23T22:07:10.121+0000 7f8758d8a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:07:10.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:10 smithi070 conmon[158698]: 2023-09-23T22:07:10.326+0000 7f8758d8a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:07:11.069 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:10 smithi070 conmon[158698]: 2023-09-23T22:07:10.717+0000 7f8758d8a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:07:11.069 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:10 smithi070 conmon[158698]: 2023-09-23T22:07:10.872+0000 7f8758d8a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:07:11.069 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:10 smithi070 conmon[158698]: 2023-09-23T22:07:10.967+0000 7f8758d8a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:07:11.070 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:10 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:11.070 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:10 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:11.070 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:10 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:11.070 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:10 smithi070 ceph-mon[105170]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:11.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:10 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:11.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:10 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:11.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:10 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:11.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:10 smithi026 ceph-mon[100157]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:11.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:10 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:11.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:10 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:11.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:10 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:11.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:10 smithi026 ceph-mon[118879]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:11.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:11 smithi070 conmon[158698]: 2023-09-23T22:07:11.068+0000 7f8758d8a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:07:11.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:11 smithi070 conmon[158698]: 2023-09-23T22:07:11.541+0000 7f8758d8a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:07:11.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:11 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3581604609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:12.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:11 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3581604609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:12.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:11 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3581604609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:12.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:12 smithi070 conmon[158698]: 2023-09-23T22:07:12.268+0000 7f8758d8a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:07:12.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:12 smithi070 conmon[158698]: 2023-09-23T22:07:12.357+0000 7f8758d8a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:07:13.313 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:13 smithi070 conmon[158698]: 2023-09-23T22:07:13.216+0000 7f8758d8a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:07:13.314 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:13 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:13.314 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:13 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:13.314 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:13 smithi070 ceph-mon[105170]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:13.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:13 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:13.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:13 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:13.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:13 smithi026 ceph-mon[100157]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:13.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:13 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:13.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:13 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:13.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:13 smithi026 ceph-mon[118879]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:13.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:13 smithi070 conmon[158698]: 2023-09-23T22:07:13.312+0000 7f8758d8a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:07:13.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:13 smithi070 conmon[158698]: 2023-09-23T22:07:13.494+0000 7f8758d8a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:07:14.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:13 smithi070 conmon[158698]: 2023-09-23T22:07:13.796+0000 7f8758d8a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:07:14.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:14 smithi070 conmon[158698]: 2023-09-23T22:07:14.037+0000 7f8758d8a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:07:14.500 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:14 smithi070 conmon[158698]: 2023-09-23T22:07:14.195+0000 7f8758d8a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:07:14.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:14 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:14 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.500 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:14 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2175976355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:14.501 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:14 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.501 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:14 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.501 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:14 smithi070 ceph-mon[105170]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2175976355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[118879]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2175976355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:14 smithi026 ceph-mon[100157]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:14.872 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:14 smithi070 conmon[158698]: 2023-09-23T22:07:14.499+0000 7f8758d8a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:07:15.574 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:15 smithi070 conmon[158698]: 2023-09-23T22:07:15.246+0000 7f8758d8a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:07:15.575 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:15 smithi070 conmon[158698]: 2023-09-23T22:07:15.378+0000 7f8758d8a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:07:15.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:15 smithi070 conmon[158698]: 2023-09-23T22:07:15.574+0000 7f8758d8a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:07:15.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:15 smithi070 conmon[158698]: 2023-09-23T22:07:15.699+0000 7f8758d8a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:07:16.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:15 smithi070 conmon[158698]: 2023-09-23T22:07:15.942+0000 7f8758d8a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:07:16.962 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:16 smithi070 conmon[158698]: 2023-09-23T22:07:16.860+0000 7f8758d8a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:07:16.962 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:16 smithi070 conmon[158698]: [23/Sep/2023:22:07:16] ENGINE Bus STARTING 2023-09-23T22:07:16.963 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:16 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1266352899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:16.963 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:16 smithi070 ceph-mon[105170]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:16 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1266352899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:16.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:16 smithi026 ceph-mon[118879]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:16 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1266352899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:16 smithi026 ceph-mon[100157]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:17.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:16 smithi070 conmon[158698]: CherryPy Checker: 2023-09-23T22:07:17.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:16 smithi070 conmon[158698]: The Application mounted at '' has an empty config. 2023-09-23T22:07:17.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:16 smithi070 conmon[158698]: 2023-09-23T22:07:17.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:16 smithi070 conmon[158698]: [23/Sep/2023:22:07:16] ENGINE Serving on http://:::9283 2023-09-23T22:07:17.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:16 smithi070 conmon[158698]: [23/Sep/2023:22:07:16] ENGINE Bus STARTED 2023-09-23T22:07:17.692 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: [23/Sep/2023:22:07:17] ENGINE Bus STOPPING 2023-09-23T22:07:17.692 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: [23/Sep/2023:22:07:17] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:07:17.692 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: [23/Sep/2023:22:07:17] ENGINE Bus STOPPED 2023-09-23T22:07:17.944 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:17 smithi026 conmon[100586]: 2023-09-23T22:07:17.649+0000 7f18ccbee700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T22:07:17.944 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:17 smithi026 conmon[100586]: ignoring --setuser ceph since I am not root 2023-09-23T22:07:17.944 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:17 smithi026 conmon[100586]: ignoring --setgroup ceph since I am not root 2023-09-23T22:07:17.944 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mgr[100610]: -- 172.21.15.26:0/484841242 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5649554b09c0 con 0x564955488800 2023-09-23T22:07:17.945 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:17 smithi026 conmon[100586]: 2023-09-23T22:07:17.846+0000 7f8a584251c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:07:17.945 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:17 smithi026 conmon[100586]: 2023-09-23T22:07:17.942+0000 7f8a584251c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:07:17.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: Standby manager daemon x restarted 2023-09-23T22:07:17.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: Standby manager daemon x started 2023-09-23T22:07:17.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:07:17.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:07:17.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:07:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:07:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:07:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T22:07:17.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: mgrmap e21: x(active, starting, since 0.00718098s) 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:07:17.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:07:17.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:07:17.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:07:17.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:07:17.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:07:17.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: Standby manager daemon x restarted 2023-09-23T22:07:17.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: Standby manager daemon x started 2023-09-23T22:07:17.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:07:17.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T22:07:17.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: mgrmap e21: x(active, starting, since 0.00718098s) 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:07:17.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:07:17.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:07:17.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:07:17.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:07:17.951 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:07:17.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:07:17.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:07:17.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:07:17.951 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:07:17.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:07:17.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[118879]: Manager daemon x is now available 2023-09-23T22:07:17.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:07:17.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:07:17.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:07:17.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:07:17.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:07:17.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:17 smithi026 ceph-mon[100157]: Manager daemon x is now available 2023-09-23T22:07:18.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: [23/Sep/2023:22:07:17] ENGINE Bus STARTING 2023-09-23T22:07:18.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b69fd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b69fd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b69fd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b69fd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b31f6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b31f6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b31f6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b31f6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b31f6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: 2023-09-23T22:07:17.796+0000 7f86b69fd700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:18.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: [23/Sep/2023:22:07:17] ENGINE Serving on http://:::9283 2023-09-23T22:07:18.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:17 smithi070 conmon[158698]: [23/Sep/2023:22:07:17] ENGINE Bus STARTED 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: Standby manager daemon x restarted 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: Standby manager daemon x started 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:07:18.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.24341 172.21.15.26:0/3000051935' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: mgrmap e21: x(active, starting, since 0.00718098s) 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:07:18.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:07:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:17 smithi070 ceph-mon[105170]: Manager daemon x is now available 2023-09-23T22:07:18.234 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:18 smithi026 conmon[100586]: 2023-09-23T22:07:18.053+0000 7f8a584251c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:07:18.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:18 smithi026 conmon[100586]: 2023-09-23T22:07:18.179+0000 7f8a584251c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:07:18.706 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:18 smithi026 conmon[100586]: 2023-09-23T22:07:18.395+0000 7f8a584251c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:07:18.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:18 smithi026 conmon[100586]: 2023-09-23T22:07:18.796+0000 7f8a584251c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:07:18.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:18 smithi026 conmon[100586]: 2023-09-23T22:07:18.903+0000 7f8a584251c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:07:18.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:18.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: Deploying cephadm binary to smithi070 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: Deploying cephadm binary to smithi026 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/4114762731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: mgrmap e22: x(active, since 1.0082s) 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[118879]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:18.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:18.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:18.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:18.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:18.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:18.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: Deploying cephadm binary to smithi070 2023-09-23T22:07:18.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: Deploying cephadm binary to smithi026 2023-09-23T22:07:18.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:18.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:18.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/4114762731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:18.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: mgrmap e22: x(active, since 1.0082s) 2023-09-23T22:07:18.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:18 smithi026 ceph-mon[100157]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: Deploying cephadm binary to smithi070 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: Deploying cephadm binary to smithi026 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/4114762731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:19.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: mgrmap e22: x(active, since 1.0082s) 2023-09-23T22:07:19.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:18 smithi070 ceph-mon[105170]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:19.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:19 smithi026 conmon[100586]: 2023-09-23T22:07:19.053+0000 7f8a584251c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:07:19.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:19 smithi026 conmon[100586]: 2023-09-23T22:07:19.166+0000 7f8a584251c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:07:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:19 smithi026 conmon[100586]: 2023-09-23T22:07:19.677+0000 7f8a584251c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:07:20.655 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:20 smithi026 conmon[100586]: 2023-09-23T22:07:20.398+0000 7f8a584251c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:07:20.656 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:20 smithi026 conmon[100586]: 2023-09-23T22:07:20.612+0000 7f8a584251c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:07:20.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[118879]: [23/Sep/2023:22:07:18] ENGINE Bus STARTING 2023-09-23T22:07:20.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[118879]: [23/Sep/2023:22:07:18] ENGINE Serving on http://172.21.15.70:8765 2023-09-23T22:07:20.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[118879]: [23/Sep/2023:22:07:19] ENGINE Serving on https://172.21.15.70:7150 2023-09-23T22:07:20.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[118879]: [23/Sep/2023:22:07:19] ENGINE Bus STARTED 2023-09-23T22:07:20.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[118879]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:20.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[118879]: mgrmap e23: x(active, since 2s) 2023-09-23T22:07:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[100157]: [23/Sep/2023:22:07:18] ENGINE Bus STARTING 2023-09-23T22:07:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[100157]: [23/Sep/2023:22:07:18] ENGINE Serving on http://172.21.15.70:8765 2023-09-23T22:07:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[100157]: [23/Sep/2023:22:07:19] ENGINE Serving on https://172.21.15.70:7150 2023-09-23T22:07:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[100157]: [23/Sep/2023:22:07:19] ENGINE Bus STARTED 2023-09-23T22:07:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[100157]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:20 smithi026 ceph-mon[100157]: mgrmap e23: x(active, since 2s) 2023-09-23T22:07:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:20 smithi070 ceph-mon[105170]: [23/Sep/2023:22:07:18] ENGINE Bus STARTING 2023-09-23T22:07:21.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:20 smithi070 ceph-mon[105170]: [23/Sep/2023:22:07:18] ENGINE Serving on http://172.21.15.70:8765 2023-09-23T22:07:21.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:20 smithi070 ceph-mon[105170]: [23/Sep/2023:22:07:19] ENGINE Serving on https://172.21.15.70:7150 2023-09-23T22:07:21.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:20 smithi070 ceph-mon[105170]: [23/Sep/2023:22:07:19] ENGINE Bus STARTED 2023-09-23T22:07:21.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:20 smithi070 ceph-mon[105170]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:21.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:20 smithi070 ceph-mon[105170]: mgrmap e23: x(active, since 2s) 2023-09-23T22:07:21.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:21 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2276513580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:21.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:21 smithi070 conmon[158698]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:07:21] "GET /metrics HTTP/1.1" 200 30872 "" "Prometheus/2.43.0" 2023-09-23T22:07:21.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:21 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2276513580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:21.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:21 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2276513580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:21.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:21 smithi026 conmon[100586]: 2023-09-23T22:07:21.661+0000 7f8a584251c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:07:21.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:21 smithi026 conmon[100586]: 2023-09-23T22:07:21.775+0000 7f8a584251c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:07:22.364 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:22 smithi026 conmon[100586]: 2023-09-23T22:07:22.004+0000 7f8a584251c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:07:22.693 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:22 smithi026 conmon[100586]: 2023-09-23T22:07:22.363+0000 7f8a584251c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:07:22.693 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:22 smithi026 conmon[100586]: 2023-09-23T22:07:22.472+0000 7f8a584251c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:07:22.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:22 smithi026 ceph-mon[118879]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:22.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:22 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:22.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:22 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:22.694 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:22 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:22.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:22 smithi026 conmon[100586]: 2023-09-23T22:07:22.820+0000 7f8a584251c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:07:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:22 smithi026 ceph-mon[100157]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:22 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:22 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:22 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:22 smithi070 ceph-mon[105170]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:22 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:22 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:22 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:23 smithi026 conmon[100586]: 2023-09-23T22:07:23.166+0000 7f8a584251c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:07:23.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.948 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi026 to 2501M 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1741482740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.949 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:23 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi026 to 2501M 2023-09-23T22:07:23.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1741482740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:23.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi026 to 2501M 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1741482740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:23.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:23 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:24.412 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:24 smithi026 conmon[100586]: 2023-09-23T22:07:24.005+0000 7f8a584251c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:07:24.413 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:24 smithi026 conmon[100586]: 2023-09-23T22:07:24.165+0000 7f8a584251c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:07:24.694 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:24 smithi026 conmon[100586]: 2023-09-23T22:07:24.412+0000 7f8a584251c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:07:24.694 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:24 smithi026 conmon[100586]: 2023-09-23T22:07:24.552+0000 7f8a584251c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:07:24.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:24 smithi026 conmon[100586]: 2023-09-23T22:07:24.829+0000 7f8a584251c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:07:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:24 smithi026 ceph-mon[100157]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:24 smithi026 ceph-mon[118879]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:25.028 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:24 smithi070 ceph-mon[105170]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:25.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:25 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3790856772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:25.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:25 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3790856772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:25.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:25 smithi026 conmon[100586]: 2023-09-23T22:07:25.891+0000 7f8a584251c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:07:25.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:25 smithi026 conmon[100586]: [23/Sep/2023:22:07:25] ENGINE Bus STARTING 2023-09-23T22:07:26.071 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:25 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3790856772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:26.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:26 smithi026 conmon[100586]: CherryPy Checker: 2023-09-23T22:07:26.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:26 smithi026 conmon[100586]: The Application mounted at '' has an empty config. 2023-09-23T22:07:26.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:26 smithi026 conmon[100586]: 2023-09-23T22:07:26.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:26 smithi026 conmon[100586]: [23/Sep/2023:22:07:26] ENGINE Serving on http://:::9283 2023-09-23T22:07:26.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:26 smithi026 conmon[100586]: [23/Sep/2023:22:07:26] ENGINE Bus STARTED 2023-09-23T22:07:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: Standby manager daemon y started 2023-09-23T22:07:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:07:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:26.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:26.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:26 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:07:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: Standby manager daemon y started 2023-09-23T22:07:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:07:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:07:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:07:26.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:07:26.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:26.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:26.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:26.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:26.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:07:26.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:26.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: Standby manager daemon y started 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.26:0/780909735' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T22:07:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:26 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:07:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Adjusting osd_memory_target on smithi070 to 2500M 2023-09-23T22:07:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:07:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:07:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: mgrmap e24: x(active, since 9s), standbys: y 2023-09-23T22:07:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:07:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:07:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:07:27.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T22:07:27.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[100157]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:27.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Adjusting osd_memory_target on smithi070 to 2500M 2023-09-23T22:07:27.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:07:27.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:07:27.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: mgrmap e24: x(active, since 9s), standbys: y 2023-09-23T22:07:27.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:07:27.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:07:27.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:07:27.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:07:27.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:07:27.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:07:27.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:07:27.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:07:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T22:07:27.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:27 smithi026 ceph-mon[118879]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Adjusting osd_memory_target on smithi070 to 2500M 2023-09-23T22:07:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:07:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:07:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: mgrmap e24: x(active, since 9s), standbys: y 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T22:07:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: Reconfiguring daemon alertmanager.a on smithi026 2023-09-23T22:07:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:27 smithi070 ceph-mon[105170]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:28.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:28 smithi026 conmon[100586]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:07:28] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:07:28.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:28 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2146456425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:28 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2146456425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:29.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:28 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2146456425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:29.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:29 smithi026 ceph-mon[118879]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:29.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:29 smithi026 ceph-mon[100157]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:29 smithi070 ceph-mon[105170]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:30.407 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:30 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:30.714 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:30 smithi026 conmon[175842]: ts=2023-09-23T22:07:30.407Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:07:30.715 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:30 smithi026 bash[199880]: 7a14521d2de95daf65a67e600dbb8afc1448c76c60199f2079d3a55800eba154 2023-09-23T22:07:30.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:30 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/45954479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:30 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/45954479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:31.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:30 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/45954479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:31.485 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T22:07:31.485 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:31.485 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 systemd[1]: Starting Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:31 smithi070 ceph-mon[105170]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:31.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:31 smithi070 conmon[158698]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:07:31] "GET /metrics HTTP/1.1" 200 30872 "" "Prometheus/2.43.0" 2023-09-23T22:07:31.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:31 smithi026 ceph-mon[118879]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:31.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:31 smithi026 ceph-mon[100157]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:31.985 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 conmon[200253]: ts=2023-09-23T22:07:31.749Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T22:07:31.985 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 conmon[200253]: ts=2023-09-23T22:07:31.749Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T22:07:31.985 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 conmon[200253]: ts=2023-09-23T22:07:31.750Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.26 port=9094 2023-09-23T22:07:31.985 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 conmon[200253]: ts=2023-09-23T22:07:31.751Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T22:07:31.986 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 bash[200212]: 1cbbb123afed59eec2f699cf2bf0b346962c29b842a8f3957a7991c084cca582 2023-09-23T22:07:31.986 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 systemd[1]: Started Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:31.986 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 conmon[200253]: ts=2023-09-23T22:07:31.780Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:07:31.986 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 conmon[200253]: ts=2023-09-23T22:07:31.781Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T22:07:31.986 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 conmon[200253]: ts=2023-09-23T22:07:31.783Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T22:07:31.986 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:31 smithi026 conmon[200253]: ts=2023-09-23T22:07:31.784Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T22:07:32.436 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:32 smithi026 systemd[1]: Stopping Ceph node-exporter.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:33.039 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:32 smithi026 bash[200484]: 89b44a6ca08ed7ec41a741f146d68f6edaa46d936211c90f9d39323054583702 2023-09-23T22:07:33.040 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:32 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:07:33.040 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T22:07:33.040 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 systemd[1]: Stopped Ceph node-exporter.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:33.040 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 systemd[1]: Starting Ceph node-exporter.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:33.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[100157]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T22:07:33.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[100157]: Reconfiguring daemon node-exporter.a on smithi026 2023-09-23T22:07:33.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:33.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[118879]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T22:07:33.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[118879]: Reconfiguring daemon node-exporter.a on smithi026 2023-09-23T22:07:33.041 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:32 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:32 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:32 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:32 smithi070 ceph-mon[105170]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T22:07:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:32 smithi070 ceph-mon[105170]: Reconfiguring daemon node-exporter.a on smithi026 2023-09-23T22:07:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:32 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3521628189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[118879]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[118879]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:07:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3521628189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[100157]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:33 smithi026 ceph-mon[100157]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:07:33.989 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200253]: ts=2023-09-23T22:07:33.751Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000253183s 2023-09-23T22:07:33.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.614Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T22:07:33.989 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.614Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.615Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.615Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.615Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.615Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.615Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.615Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T22:07:33.990 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T22:07:33.991 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T22:07:33.992 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T22:07:33.993 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T22:07:33.993 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T22:07:33.993 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T22:07:33.993 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T22:07:33.993 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T22:07:33.993 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T22:07:33.993 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T22:07:33.993 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T22:07:33.993 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T22:07:33.994 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T22:07:33.994 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T22:07:33.994 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T22:07:33.994 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T22:07:33.994 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T22:07:33.994 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T22:07:33.995 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T22:07:33.995 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 conmon[200873]: ts=2023-09-23T22:07:33.616Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T22:07:33.995 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 bash[200832]: 68b72a59fe0873927ff143decba2fbae87a12feddfafd97eaa08a4e4f32644da 2023-09-23T22:07:33.995 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:07:33 smithi026 systemd[1]: Started Ceph node-exporter.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:33 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3521628189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:33 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:33 smithi070 ceph-mon[105170]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:33 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:33 smithi070 ceph-mon[105170]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:07:35.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:35 smithi070 ceph-mon[105170]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T22:07:35.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:35 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/472269128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:35 smithi026 ceph-mon[118879]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T22:07:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:35 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/472269128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:35 smithi026 ceph-mon[100157]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T22:07:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:35 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/472269128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:36.692 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 systemd[1]: Stopping Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:36 smithi026 ceph-mon[118879]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:36 smithi026 ceph-mon[100157]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:37.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:36 smithi070 ceph-mon[105170]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:37.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.696Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:07:37.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.696Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:07:37.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.696Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:07:37.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.696Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:07:37.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.696Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:07:37.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.696Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:07:37.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.696Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T22:07:37.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.696Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T22:07:37.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.696Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T22:07:37.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.697Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:07:37.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.697Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T22:07:37.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:36 smithi070 conmon[153497]: ts=2023-09-23T22:07:36.697Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T22:07:37.623 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:37 smithi070 bash[170590]: a37f53e6e83abe9cbf8a32f5b0f487c9ed1726b74cbae36b5a865780b6901573 2023-09-23T22:07:37.624 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:37 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T22:07:37.624 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:37 smithi070 systemd[1]: Stopped Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:37.624 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:37 smithi070 systemd[1]: Starting Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:38.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.023Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.024Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.024Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi070 (none))" 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.024Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.024Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.025Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.026Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.027Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.027Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.029Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.029Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.549µs 2023-09-23T22:07:38.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.029Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.029Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 bash[170917]: 2739f7c89df0652bea383e98506c66a3224d1e19b99d8d480827579fddf6fc4c 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 systemd[1]: Started Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.044Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.044Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.044Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.252µs wal_replay_duration=14.974037ms wbl_replay_duration=128ns total_replay_duration=15.012802ms 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.048Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.048Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.048Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.068Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.216033ms db_storage=930ns remote_storage=1.578µs web_handler=410ns query_engine=968ns scrape=378.573µs scrape_sd=135.137µs notify=16.377µs notify_sd=20.652µs rules=19.361585ms tracing=7.842µs 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.068Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T22:07:38.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:38 smithi070 conmon[170958]: ts=2023-09-23T22:07:38.068Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T22:07:38.694 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:38 smithi070 systemd[1]: Stopping Ceph node-exporter.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:38 smithi026 ceph-mon[118879]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:38 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2647862096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:38 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:38 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:38.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:38 smithi026 ceph-mon[100157]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:38.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:38 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2647862096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:38.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:38 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:38.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:38 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:38 smithi070 ceph-mon[105170]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:38 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2647862096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:38 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:38 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:39.623 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:07:39.623 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 bash[171190]: 6bac96aa7b4fc11de3ee9c9b18ae239e0aa3efc43233740cb39f8f91fe9e3b02 2023-09-23T22:07:39.623 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T22:07:39.624 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 systemd[1]: Stopped Ceph node-exporter.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:39.624 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 systemd[1]: Starting Ceph node-exporter.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:39.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:39 smithi070 ceph-mon[105170]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:07:39.955 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:39 smithi070 ceph-mon[105170]: Reconfiguring daemon node-exporter.b on smithi070 2023-09-23T22:07:39.955 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.918Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.918Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.918Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.918Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.918Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T22:07:39.956 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T22:07:39.957 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T22:07:39.958 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T22:07:39.959 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T22:07:39.960 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T22:07:39.960 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T22:07:39.960 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T22:07:39.960 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 conmon[171560]: ts=2023-09-23T22:07:39.919Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T22:07:39.960 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 bash[171519]: 523ab3522b38d07e9103b568c8890cca5acb62a24d03575d0d6602626af2a36b 2023-09-23T22:07:39.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:39 smithi026 ceph-mon[100157]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:07:39.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:39 smithi026 ceph-mon[100157]: Reconfiguring daemon node-exporter.b on smithi070 2023-09-23T22:07:39.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:39 smithi026 ceph-mon[118879]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T22:07:39.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:39 smithi026 ceph-mon[118879]: Reconfiguring daemon node-exporter.b on smithi070 2023-09-23T22:07:40.373 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:07:39 smithi070 systemd[1]: Started Ceph node-exporter.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:40.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:40 smithi026 ceph-mon[118879]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:40.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:40 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:40.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:40 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:40.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:40 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1322945852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:40.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:40 smithi026 ceph-mon[100157]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:40.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:40 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:40.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:40 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:40.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:40 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1322945852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:41.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:40 smithi070 ceph-mon[105170]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:41.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:40 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:41.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:40 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:41.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:40 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1322945852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:41 smithi070 ceph-mon[105170]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:07:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:41 smithi070 ceph-mon[105170]: Reconfiguring daemon grafana.a on smithi070 2023-09-23T22:07:41.985 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:41 smithi026 conmon[200253]: ts=2023-09-23T22:07:41.754Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002461518s 2023-09-23T22:07:41.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:41 smithi026 ceph-mon[118879]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:07:41.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:41 smithi026 ceph-mon[118879]: Reconfiguring daemon grafana.a on smithi070 2023-09-23T22:07:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:41 smithi026 ceph-mon[100157]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T22:07:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:41 smithi026 ceph-mon[100157]: Reconfiguring daemon grafana.a on smithi070 2023-09-23T22:07:42.532 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:42 smithi070 systemd[1]: Stopping Ceph grafana.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:42.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:42 smithi070 ceph-mon[105170]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:42.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:42 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2864057870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:42 smithi026 ceph-mon[118879]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:42 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2864057870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:42.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:42 smithi026 ceph-mon[100157]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:42.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:42 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2864057870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:43.291 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:42 smithi070 bash[172028]: d197cdcf40c153ee431dd38f6a3aa551d7173992efec9236b006407c8c4f4169 2023-09-23T22:07:43.292 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:42 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:07:43.292 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T22:07:43.292 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 systemd[1]: Stopped Ceph grafana.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:43.292 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 systemd[1]: Starting Ceph grafana.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:43.552 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[158698]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:07:43] "GET /metrics HTTP/1.1" 200 33762 "" "Prometheus/2.43.0" 2023-09-23T22:07:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:43 smithi026 ceph-mon[100157]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:43.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:43 smithi026 ceph-mon[118879]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:44.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:43 smithi070 ceph-mon[105170]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:44.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[158698]: [23/Sep/2023:22:07:43] ENGINE Bus STOPPING 2023-09-23T22:07:44.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[158698]: [23/Sep/2023:22:07:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:07:44.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[158698]: [23/Sep/2023:22:07:43] ENGINE Bus STOPPED 2023-09-23T22:07:44.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[158698]: [23/Sep/2023:22:07:43] ENGINE Bus STARTING 2023-09-23T22:07:44.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[158698]: [23/Sep/2023:22:07:43] ENGINE Serving on http://:::9283 2023-09-23T22:07:44.124 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[158698]: [23/Sep/2023:22:07:43] ENGINE Bus STARTED 2023-09-23T22:07:44.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 bash[172355]: 42940e65ed5c414390e26bf204820b13b4332b52525f37d419b2dfc73104658f 2023-09-23T22:07:44.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 systemd[1]: Started Ceph grafana.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.819769657Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.8199698Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.819993516Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.82000063Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.82000679Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.820012905Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.82001905Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.820024905Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.820031054Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.820036957Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.820042408Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T22:07:44.125 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.820047884Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.820053635Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=settings t=2023-09-23T22:07:43.820059423Z level=info msg="App mode production" 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=sqlstore t=2023-09-23T22:07:43.820104581Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=sqlstore t=2023-09-23T22:07:43.820122256Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=migrator t=2023-09-23T22:07:43.834614497Z level=info msg="Starting DB migrations" 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=migrator t=2023-09-23T22:07:43.839570222Z level=info msg="migrations completed" performed=0 skipped=481 duration=481.289µs 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=sqlstore t=2023-09-23T22:07:43.839915918Z level=info msg="Created default organization" 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=plugin.loader t=2023-09-23T22:07:43.897003057Z level=info msg="Plugin registered" pluginID=input 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=plugin.loader t=2023-09-23T22:07:43.924374675Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=plugin.loader t=2023-09-23T22:07:43.924395772Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=secrets t=2023-09-23T22:07:43.924428912Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T22:07:44.126 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=query_data t=2023-09-23T22:07:43.926072877Z level=info msg="Query Service initialization" 2023-09-23T22:07:44.127 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:43 smithi070 conmon[172396]: logger=live.push_http t=2023-09-23T22:07:43.928485595Z level=info msg="Live Push Gateway initialization" 2023-09-23T22:07:44.941 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:44.941 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:44.941 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:07:44.941 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:07:44.942 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:44 smithi070 ceph-mon[105170]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:07:44.943 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:44 smithi070 conmon[172396]: logger=infra.usagestats.collector t=2023-09-23T22:07:44.627358131Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T22:07:44.943 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:44 smithi070 conmon[172396]: logger=provisioning.datasources t=2023-09-23T22:07:44.764855543Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-09-23T22:07:44.943 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:44 smithi070 conmon[172396]: logger=provisioning.datasources t=2023-09-23T22:07:44.764928648Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-09-23T22:07:44.943 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:44 smithi070 conmon[172396]: logger=provisioning.datasources t=2023-09-23T22:07:44.775242661Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T22:07:44.943 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:44 smithi070 conmon[172396]: logger=provisioning.alerting t=2023-09-23T22:07:44.941669554Z level=info msg="starting to provision alerting" 2023-09-23T22:07:44.943 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:44 smithi070 conmon[172396]: logger=provisioning.alerting t=2023-09-23T22:07:44.941708162Z level=info msg="finished to provision alerting" 2023-09-23T22:07:44.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:44.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:07:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:07:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:07:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:07:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:07:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:07:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:07:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:07:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:07:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[118879]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:07:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:07:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T22:07:44.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:07:44.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi026.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T22:07:44.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:44.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:07:44.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:07:44.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:07:44.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T22:07:44.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:44.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:07:44.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:44 smithi026 ceph-mon[100157]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T22:07:45.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:44 smithi070 conmon[172396]: logger=ngalert.state.manager t=2023-09-23T22:07:44.941912465Z level=info msg="Warming state cache for startup" 2023-09-23T22:07:45.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:44 smithi070 conmon[172396]: logger=grafanaStorageLogger t=2023-09-23T22:07:44.942028457Z level=info msg="storage starting" 2023-09-23T22:07:45.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:44 smithi070 conmon[172396]: logger=http.server t=2023-09-23T22:07:44.943741479Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T22:07:45.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:45 smithi070 conmon[172396]: logger=ngalert.state.manager t=2023-09-23T22:07:45.011395218Z level=info msg="State cache has been initialized" states=0 duration=69.482479ms 2023-09-23T22:07:45.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:45 smithi070 conmon[172396]: logger=ngalert.multiorg.alertmanager t=2023-09-23T22:07:45.01144479Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T22:07:45.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:07:45 smithi070 conmon[172396]: logger=ticker t=2023-09-23T22:07:45.011470403Z level=info msg=starting first_tick=2023-09-23T22:07:50Z 2023-09-23T22:07:45.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2537911635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:46.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[118879]: Upgrade: Updating mgr.y 2023-09-23T22:07:46.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:46.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:07:46.219 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:07:46.219 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:07:46.219 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:46.219 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[118879]: Deploying daemon mgr.y on smithi026 2023-09-23T22:07:46.219 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[118879]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:46.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2537911635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:46.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[100157]: Upgrade: Updating mgr.y 2023-09-23T22:07:46.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:46.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:07:46.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:07:46.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:07:46.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:46.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[100157]: Deploying daemon mgr.y on smithi026 2023-09-23T22:07:46.220 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:45 smithi026 ceph-mon[100157]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:46.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:45 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2537911635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:46.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:45 smithi070 ceph-mon[105170]: Upgrade: Updating mgr.y 2023-09-23T22:07:46.220 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:45 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:46.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:45 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:07:46.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:45 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:07:46.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:45 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:07:46.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:45 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:46.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:45 smithi070 ceph-mon[105170]: Deploying daemon mgr.y on smithi026 2023-09-23T22:07:46.221 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:45 smithi070 ceph-mon[105170]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:47.126 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:46 smithi026 systemd[1]: Stopping Ceph mgr.y for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:47.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:47 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/777186227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:47.733 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:47 smithi026 bash[201623]: d8b44817b175e7fcfe51739e71b21a1a33127806be22c3c15c0f3c4c8fbec879 2023-09-23T22:07:47.733 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:47 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:07:47.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:47 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/777186227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:47.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:47 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/777186227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:48.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:47 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T22:07:48.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:47 smithi026 systemd[1]: Stopped Ceph mgr.y for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:48.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:47 smithi026 systemd[1]: Starting Ceph mgr.y for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:07:48.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:48 smithi026 ceph-mon[100157]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:48.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:48 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:48 smithi026 ceph-mon[118879]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:48 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:48.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:48 smithi026 bash[202006]: 2dff2050e669cea9943d51d5fcffa670fa1a74f08edbad72f6f38d118b83c15a 2023-09-23T22:07:48.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:48 smithi026 systemd[1]: Started Ceph mgr.y for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:07:48.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:48 smithi026 ceph-mgr[202071]: -- 172.21.15.26:0/2463083033 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561ba2ac91e0 con 0x561ba2aa0800 2023-09-23T22:07:48.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:48 smithi026 conmon[202047]: 2023-09-23T22:07:48.647+0000 7ffbe9705200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:07:48.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:48 smithi070 ceph-mon[105170]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:48.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:48 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:48.873 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:48 smithi070 conmon[170958]: ts=2023-09-23T22:07:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:07:49.228 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:48 smithi026 conmon[202047]: 2023-09-23T22:07:48.746+0000 7ffbe9705200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:07:49.228 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:48 smithi026 conmon[202047]: 2023-09-23T22:07:48.865+0000 7ffbe9705200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:07:49.229 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:48 smithi026 conmon[202047]: 2023-09-23T22:07:48.992+0000 7ffbe9705200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:07:49.229 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:49 smithi026 conmon[202047]: 2023-09-23T22:07:49.227+0000 7ffbe9705200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:07:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:49 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:49 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:49 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:49.918 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:49 smithi026 conmon[202047]: 2023-09-23T22:07:49.758+0000 7ffbe9705200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:07:49.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:49 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:49.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:49 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:49.918 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:49 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:49.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:49 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:49.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:49 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:49.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:49 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:50.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:49 smithi026 conmon[202047]: 2023-09-23T22:07:49.917+0000 7ffbe9705200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:07:50.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:50 smithi026 conmon[202047]: 2023-09-23T22:07:50.026+0000 7ffbe9705200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:07:50.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:50 smithi026 conmon[202047]: 2023-09-23T22:07:50.164+0000 7ffbe9705200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:07:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:50 smithi070 ceph-mon[105170]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:50 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/4062565259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:50 smithi026 ceph-mon[100157]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:50 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/4062565259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:50.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:50 smithi026 conmon[202047]: 2023-09-23T22:07:50.742+0000 7ffbe9705200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:07:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:50 smithi026 ceph-mon[118879]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:50 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/4062565259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:51.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:51 smithi026 conmon[202047]: 2023-09-23T22:07:51.595+0000 7ffbe9705200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:07:51.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:51 smithi026 conmon[202047]: 2023-09-23T22:07:51.706+0000 7ffbe9705200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:07:52.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[118879]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:52.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1582410925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:52.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[100157]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:52.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:52 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1582410925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:52 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:52 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:52 smithi070 ceph-mon[105170]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:52 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:52 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:52 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:52 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:52 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1582410925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:52.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:52 smithi026 conmon[202047]: 2023-09-23T22:07:52.751+0000 7ffbe9705200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:07:52.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:52 smithi026 conmon[202047]: 2023-09-23T22:07:52.872+0000 7ffbe9705200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:07:53.470 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:53 smithi026 conmon[202047]: 2023-09-23T22:07:53.108+0000 7ffbe9705200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:07:53.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:53 smithi070 conmon[158698]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:07:53] "GET /metrics HTTP/1.1" 200 33765 "" "Prometheus/2.43.0" 2023-09-23T22:07:53.728 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:53 smithi026 conmon[202047]: 2023-09-23T22:07:53.470+0000 7ffbe9705200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:07:53.728 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:07:53 smithi026 conmon[200253]: ts=2023-09-23T22:07:53.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi026.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.26:8443: connect: connection refused" 2023-09-23T22:07:53.982 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:53 smithi026 conmon[202047]: 2023-09-23T22:07:53.733+0000 7ffbe9705200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:07:53.983 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:53 smithi026 conmon[202047]: 2023-09-23T22:07:53.932+0000 7ffbe9705200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:07:53.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:53 smithi026 ceph-mon[118879]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:53.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:53 smithi026 ceph-mon[100157]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:54.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:53 smithi070 ceph-mon[105170]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:54.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:54 smithi026 conmon[202047]: 2023-09-23T22:07:54.284+0000 7ffbe9705200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:07:55.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:54 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/965652112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:55.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:54 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/965652112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:55.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:54 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/965652112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:55.405 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:55 smithi026 conmon[202047]: 2023-09-23T22:07:55.239+0000 7ffbe9705200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:07:55.405 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:55 smithi026 conmon[202047]: 2023-09-23T22:07:55.405+0000 7ffbe9705200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:07:55.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:55 smithi026 conmon[202047]: 2023-09-23T22:07:55.662+0000 7ffbe9705200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:07:55.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:55 smithi026 conmon[202047]: 2023-09-23T22:07:55.809+0000 7ffbe9705200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:07:56.312 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:55 smithi070 conmon[158698]: 2023-09-23T22:07:55.980+0000 7f874c438700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T22:07:56.312 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:56 smithi070 conmon[158698]: ignoring --setuser ceph since I am not root 2023-09-23T22:07:56.312 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:56 smithi070 conmon[158698]: ignoring --setgroup ceph since I am not root 2023-09-23T22:07:56.312 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mgr[158722]: -- 172.21.15.70:0/44391444 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56447b71f1e0 con 0x56447b6f6800 2023-09-23T22:07:56.312 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:56 smithi070 conmon[158698]: 2023-09-23T22:07:56.144+0000 7f6b3e29d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:07:56.313 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:56 smithi070 conmon[158698]: 2023-09-23T22:07:56.223+0000 7f6b3e29d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:07:56.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:56 smithi026 conmon[202047]: 2023-09-23T22:07:56.099+0000 7ffbe9705200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:07:56.571 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:56 smithi070 conmon[158698]: 2023-09-23T22:07:56.311+0000 7f6b3e29d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:07:56.572 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:56 smithi070 conmon[158698]: 2023-09-23T22:07:56.403+0000 7f6b3e29d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:07:56.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:56 smithi070 conmon[158698]: 2023-09-23T22:07:56.571+0000 7f6b3e29d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:07:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T22:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: from='mgr.25387 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T22:07:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:56 smithi070 ceph-mon[105170]: mgrmap e25: y(active, starting, since 0.0057183s) 2023-09-23T22:07:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:07:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:56.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:07:56.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:07:56.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T22:07:56.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: from='mgr.25387 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T22:07:56.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[118879]: mgrmap e25: y(active, starting, since 0.0057183s) 2023-09-23T22:07:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 83 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:07:56.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:07:56.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' 2023-09-23T22:07:56.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:56.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: from='mgr.25387 172.21.15.70:0/124492856' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:07:56.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T22:07:56.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T22:07:56.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: from='mgr.25387 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T22:07:56.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:56 smithi026 ceph-mon[100157]: mgrmap e25: y(active, starting, since 0.0057183s) 2023-09-23T22:07:57.246 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:56 smithi070 conmon[158698]: 2023-09-23T22:07:56.933+0000 7f6b3e29d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:07:57.247 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:57 smithi070 conmon[158698]: 2023-09-23T22:07:57.061+0000 7f6b3e29d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:07:57.247 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:57 smithi070 conmon[158698]: 2023-09-23T22:07:57.149+0000 7f6b3e29d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:07:57.466 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.179+0000 7ffbe9705200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:07:57.466 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: [23/Sep/2023:22:07:57] ENGINE Bus STARTING 2023-09-23T22:07:57.466 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: CherryPy Checker: 2023-09-23T22:07:57.466 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: The Application mounted at '' has an empty config. 2023-09-23T22:07:57.467 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.467 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: [23/Sep/2023:22:07:57] ENGINE Serving on http://:::9283 2023-09-23T22:07:57.467 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: [23/Sep/2023:22:07:57] ENGINE Bus STARTED 2023-09-23T22:07:57.467 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: [23/Sep/2023:22:07:57] ENGINE Bus STOPPING 2023-09-23T22:07:57.467 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: [23/Sep/2023:22:07:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:07:57.467 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: [23/Sep/2023:22:07:57] ENGINE Bus STOPPED 2023-09-23T22:07:57.467 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: [23/Sep/2023:22:07:57] ENGINE Bus STARTING 2023-09-23T22:07:57.551 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:57 smithi070 conmon[158698]: 2023-09-23T22:07:57.246+0000 7f6b3e29d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:07:57.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb46b77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb46b77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb46b77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb46b77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb46b77700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb43370700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb43370700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb43370700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb43370700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: 2023-09-23T22:07:57.498+0000 7ffb43370700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:07:57.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: [23/Sep/2023:22:07:57] ENGINE Serving on http://:::9283 2023-09-23T22:07:57.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:07:57 smithi026 conmon[202047]: [23/Sep/2023:22:07:57] ENGINE Bus STARTED 2023-09-23T22:07:57.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/715781749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:57.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: Active manager daemon y restarted 2023-09-23T22:07:57.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: Activating manager daemon y 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: mgrmap e26: y(active, starting, since 0.00591489s) 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:07:57.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:07:57.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: Manager daemon y is now available 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/715781749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: Active manager daemon y restarted 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: Activating manager daemon y 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: mgrmap e26: y(active, starting, since 0.00591489s) 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:07:57.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:07:57.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:07:57.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:07:57.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:07:57.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:07:57.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:07:57.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:07:57.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: Manager daemon y is now available 2023-09-23T22:07:57.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:57.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:57.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:57.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:57.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:57 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:57.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:57 smithi070 conmon[158698]: 2023-09-23T22:07:57.683+0000 7f6b3e29d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:07:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/715781749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:07:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: Active manager daemon y restarted 2023-09-23T22:07:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: Activating manager daemon y 2023-09-23T22:07:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T22:07:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: mgrmap e26: y(active, starting, since 0.00591489s) 2023-09-23T22:07:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:07:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T22:07:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:07:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:07:57.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: Manager daemon y is now available 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:07:57.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:57.877 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:57 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:07:58.659 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:58 smithi070 conmon[158698]: 2023-09-23T22:07:58.401+0000 7f6b3e29d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:07:58.659 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:58 smithi070 conmon[158698]: 2023-09-23T22:07:58.491+0000 7f6b3e29d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:07:59.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:07:58 smithi070 conmon[170958]: ts=2023-09-23T22:07:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:07:59.456 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:59 smithi070 conmon[158698]: 2023-09-23T22:07:59.347+0000 7f6b3e29d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:07:59.457 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:59 smithi070 ceph-mon[105170]: mgrmap e27: y(active, since 1.00873s) 2023-09-23T22:07:59.457 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:07:59 smithi070 ceph-mon[105170]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:59.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:59 smithi026 ceph-mon[118879]: mgrmap e27: y(active, since 1.00873s) 2023-09-23T22:07:59.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:07:59 smithi026 ceph-mon[118879]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:59.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:59 smithi026 ceph-mon[100157]: mgrmap e27: y(active, since 1.00873s) 2023-09-23T22:07:59.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:07:59 smithi026 ceph-mon[100157]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:07:59.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:59 smithi070 conmon[158698]: 2023-09-23T22:07:59.455+0000 7f6b3e29d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:07:59.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:59 smithi070 conmon[158698]: 2023-09-23T22:07:59.650+0000 7f6b3e29d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:08:00.365 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:07:59 smithi070 conmon[158698]: 2023-09-23T22:07:59.957+0000 7f6b3e29d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:08:00.365 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:00 smithi070 conmon[158698]: 2023-09-23T22:08:00.198+0000 7f6b3e29d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:08:00.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:00 smithi070 ceph-mon[105170]: [23/Sep/2023:22:07:58] ENGINE Bus STARTING 2023-09-23T22:08:00.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:00 smithi070 ceph-mon[105170]: [23/Sep/2023:22:07:58] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T22:08:00.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:00 smithi070 ceph-mon[105170]: [23/Sep/2023:22:07:58] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T22:08:00.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:00 smithi070 ceph-mon[105170]: [23/Sep/2023:22:07:58] ENGINE Bus STARTED 2023-09-23T22:08:00.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:00 smithi070 ceph-mon[105170]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:00.366 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:00 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2138299930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:00.485 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:08:00 smithi026 conmon[200253]: ts=2023-09-23T22:08:00.429Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=6 2023-09-23T22:08:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[100157]: [23/Sep/2023:22:07:58] ENGINE Bus STARTING 2023-09-23T22:08:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[100157]: [23/Sep/2023:22:07:58] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T22:08:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[100157]: [23/Sep/2023:22:07:58] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T22:08:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[100157]: [23/Sep/2023:22:07:58] ENGINE Bus STARTED 2023-09-23T22:08:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[100157]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2138299930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[118879]: [23/Sep/2023:22:07:58] ENGINE Bus STARTING 2023-09-23T22:08:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[118879]: [23/Sep/2023:22:07:58] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T22:08:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[118879]: [23/Sep/2023:22:07:58] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T22:08:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[118879]: [23/Sep/2023:22:07:58] ENGINE Bus STARTED 2023-09-23T22:08:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[118879]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:00 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2138299930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:00.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:00 smithi070 conmon[158698]: 2023-09-23T22:08:00.364+0000 7f6b3e29d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:08:01.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:00 smithi070 conmon[158698]: 2023-09-23T22:08:00.664+0000 7f6b3e29d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:08:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[100157]: mgrmap e28: y(active, since 3s) 2023-09-23T22:08:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[118879]: mgrmap e28: y(active, since 3s) 2023-09-23T22:08:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:01 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.567 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:01 smithi070 conmon[158698]: 2023-09-23T22:08:01.423+0000 7f6b3e29d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:08:01.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:01 smithi070 ceph-mon[105170]: mgrmap e28: y(active, since 3s) 2023-09-23T22:08:01.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:01 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:01 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:01 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:01 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:01 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:01 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:01.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:01 smithi070 conmon[158698]: 2023-09-23T22:08:01.567+0000 7f6b3e29d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:08:01.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:01 smithi070 conmon[158698]: 2023-09-23T22:08:01.775+0000 7f6b3e29d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:08:02.211 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:01 smithi070 conmon[158698]: 2023-09-23T22:08:01.889+0000 7f6b3e29d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:08:02.211 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:02 smithi070 conmon[158698]: 2023-09-23T22:08:02.108+0000 7f6b3e29d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:08:02.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:02 smithi026 ceph-mon[118879]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:02.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:02 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:02.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:02 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:02.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:02 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3440563783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:02 smithi026 ceph-mon[100157]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:02 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:02 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:02 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3440563783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:02.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:02 smithi070 ceph-mon[105170]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:02.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:02 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:02.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:02 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:02.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:02 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3440563783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:03.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[118879]: Standby manager daemon x started 2023-09-23T22:08:03.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:08:03.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:08:03.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:08:03.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:08:03.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[100157]: Standby manager daemon x started 2023-09-23T22:08:03.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:08:03.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:08:03.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:08:03.215 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:03 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:08:03.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:03 smithi070 ceph-mon[105170]: Standby manager daemon x started 2023-09-23T22:08:03.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:03 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:08:03.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:03 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:08:03.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:03 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:08:03.292 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:03 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3004635088' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:08:03.292 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:02 smithi070 conmon[158698]: 2023-09-23T22:08:02.979+0000 7f6b3e29d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:08:03.293 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:02 smithi070 conmon[158698]: [23/Sep/2023:22:08:02] ENGINE Bus STARTING 2023-09-23T22:08:03.293 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:03 smithi070 conmon[158698]: CherryPy Checker: 2023-09-23T22:08:03.293 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:03 smithi070 conmon[158698]: The Application mounted at '' has an empty config. 2023-09-23T22:08:03.293 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:03 smithi070 conmon[158698]: 2023-09-23T22:08:03.293 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:03 smithi070 conmon[158698]: [23/Sep/2023:22:08:03] ENGINE Serving on http://:::9283 2023-09-23T22:08:03.293 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:03 smithi070 conmon[158698]: [23/Sep/2023:22:08:03] ENGINE Bus STARTED 2023-09-23T22:08:03.293 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:03 smithi070 conmon[158698]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:08:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T22:08:04.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:04 smithi026 ceph-mon[118879]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:04.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:04 smithi026 ceph-mon[118879]: mgrmap e29: y(active, since 6s), standbys: x 2023-09-23T22:08:04.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:04 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:08:04.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:04 smithi026 ceph-mon[100157]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:04.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:04 smithi026 ceph-mon[100157]: mgrmap e29: y(active, since 6s), standbys: x 2023-09-23T22:08:04.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:04 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:08:04.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:04 smithi070 ceph-mon[105170]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:04.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:04 smithi070 ceph-mon[105170]: mgrmap e29: y(active, since 6s), standbys: x 2023-09-23T22:08:04.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:04 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:08:05.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1324532961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:05.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:05.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:05.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:05.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:05.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1324532961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:05.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:05.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:05.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:05.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:05 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:05 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1324532961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:05 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:05 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:05 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:05.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:05 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:08:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:08:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:06.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:06 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.578 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:08:06.578 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:06.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:06 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:07 smithi026 ceph-mon[118879]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:08:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:07 smithi026 ceph-mon[118879]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T22:08:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:07 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1759899634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:07.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:07 smithi026 ceph-mon[100157]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:08:07.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:07 smithi026 ceph-mon[100157]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T22:08:07.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:07 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1759899634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:07.492 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:07 smithi070 ceph-mon[105170]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T22:08:07.492 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:07 smithi070 ceph-mon[105170]: Reconfiguring daemon prometheus.a on smithi070 2023-09-23T22:08:07.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:07 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1759899634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:08.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:08 smithi026 ceph-mon[118879]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:08.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:08 smithi026 ceph-mon[100157]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:08.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:08 smithi070 ceph-mon[105170]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:08.909 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 systemd[1]: Stopping Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:08:09.229 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.909Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.909Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.909Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.909Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.909Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.909Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.909Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.909Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.909Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.910Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:08:09.230 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.910Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T22:08:09.231 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:08 smithi070 conmon[170958]: ts=2023-09-23T22:08:08.910Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T22:08:09.484 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:09 smithi070 bash[176360]: 2739f7c89df0652bea383e98506c66a3224d1e19b99d8d480827579fddf6fc4c 2023-09-23T22:08:09.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:09 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1067975647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:09.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:09 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1067975647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:09.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:09 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1067975647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:10.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:09 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T22:08:10.119 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:09 smithi070 systemd[1]: Stopped Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:08:10.120 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:09 smithi070 systemd[1]: Starting Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:08:10.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:10 smithi026 ceph-mon[100157]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:10.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:10 smithi026 ceph-mon[118879]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:10.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:10 smithi070 ceph-mon[105170]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:10.624 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.236Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T22:08:10.624 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.236Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T22:08:10.624 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.236Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi070 (none))" 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.236Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.236Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.238Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.238Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.239Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.239Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.240Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.240Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.645µs 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.240Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.242Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T22:08:10.625 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 bash[176687]: 3342388f10a7429fcfceadb71f2b970bbcc69708741cabd2004334914574e914 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 systemd[1]: Started Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.256Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.258Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.259Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.259Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.6µs wal_replay_duration=18.04647ms wbl_replay_duration=129ns total_replay_duration=18.084359ms 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.261Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.262Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.262Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.280Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=18.249731ms db_storage=929ns remote_storage=1.472µs web_handler=433ns query_engine=756ns scrape=181.744µs scrape_sd=83.63µs notify=10.598µs notify_sd=9.058µs rules=17.705536ms tracing=6.09µs 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.280Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T22:08:10.626 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:10 smithi070 conmon[176728]: ts=2023-09-23T22:08:10.280Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T22:08:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:11 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:11 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:11 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:08:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:11 smithi070 ceph-mon[105170]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:08:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:11 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:11.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:11.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:11.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:08:11.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[118879]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:08:11.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:08:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[100157]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T22:08:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:11 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:12.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:12 smithi070 ceph-mon[105170]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:12 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2759829930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:12 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:08:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:12 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:08:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:12 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:08:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:12 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:12.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[100157]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:12.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2759829930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:12.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:08:12.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:08:12.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:08:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[118879]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2759829930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:08:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:08:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:08:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:12 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:13.293 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:13 smithi070 systemd[1]: Stopping Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:08:13.559 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:13 smithi070 ceph-mon[105170]: Upgrade: Updating mgr.x 2023-09-23T22:08:13.559 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:13 smithi070 ceph-mon[105170]: Deploying daemon mgr.x on smithi070 2023-09-23T22:08:13.559 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:13 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:13.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:13 smithi026 ceph-mon[100157]: Upgrade: Updating mgr.x 2023-09-23T22:08:13.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:13 smithi026 ceph-mon[100157]: Deploying daemon mgr.x on smithi070 2023-09-23T22:08:13.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:13 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:13.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:13 smithi026 ceph-mon[118879]: Upgrade: Updating mgr.x 2023-09-23T22:08:13.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:13 smithi026 ceph-mon[118879]: Deploying daemon mgr.x on smithi070 2023-09-23T22:08:13.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:13 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:14.296 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:14 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:08:14.296 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:14 smithi070 bash[177377]: 57a112d98af9b4ad5d2b1536676cc09019ee7ae3a713b820a045a3ad84163b34 2023-09-23T22:08:14.297 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:14 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T22:08:14.297 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:14 smithi070 systemd[1]: Stopped Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:08:14.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:14 smithi070 systemd[1]: Starting Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:08:14.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:14 smithi070 ceph-mon[105170]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:14.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:14 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/2389138070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:14 smithi026 ceph-mon[118879]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:14 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/2389138070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:14.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:14 smithi026 ceph-mon[100157]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:14.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:14 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/2389138070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:15.226 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:14 smithi070 ceph-mgr[177793]: -- 172.21.15.70:0/730548839 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56199a7071e0 con 0x56199a6de800 2023-09-23T22:08:15.227 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:14 smithi070 bash[177728]: 754e649f30a99d4abfdc54f9479f8d06134860c1a1024f9c37c1e403492b5f69 2023-09-23T22:08:15.227 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:14 smithi070 systemd[1]: Started Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:08:15.227 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:15 smithi070 conmon[177769]: 2023-09-23T22:08:15.046+0000 7f85e60cb200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:08:15.227 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:15 smithi070 conmon[177769]: 2023-09-23T22:08:15.130+0000 7f85e60cb200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:08:15.227 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:15 smithi070 conmon[177769]: 2023-09-23T22:08:15.226+0000 7f85e60cb200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:08:15.532 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:15 smithi070 conmon[177769]: 2023-09-23T22:08:15.334+0000 7f85e60cb200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:08:15.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:15 smithi070 conmon[177769]: 2023-09-23T22:08:15.532+0000 7f85e60cb200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:08:16.282 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:15 smithi070 conmon[177769]: 2023-09-23T22:08:15.931+0000 7f85e60cb200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:08:16.282 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:16 smithi070 conmon[177769]: 2023-09-23T22:08:16.078+0000 7f85e60cb200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:08:16.282 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:16 smithi070 conmon[177769]: 2023-09-23T22:08:16.181+0000 7f85e60cb200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:08:16.283 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:16 smithi070 conmon[177769]: 2023-09-23T22:08:16.281+0000 7f85e60cb200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:08:16.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:15 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:16.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:15 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:16.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:15 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:16.283 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:15 smithi070 ceph-mon[105170]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:16.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:15 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:16.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:15 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:16.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:15 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:16.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:15 smithi026 ceph-mon[118879]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:16.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:15 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:16.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:15 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:16.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:15 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:16.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:15 smithi026 ceph-mon[100157]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:17.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:16 smithi070 conmon[177769]: 2023-09-23T22:08:16.774+0000 7f85e60cb200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:08:17.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:17 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/793286666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:17.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:17 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/793286666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:17.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:17 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/793286666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:17.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:17 smithi070 conmon[177769]: 2023-09-23T22:08:17.495+0000 7f85e60cb200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:08:17.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:17 smithi070 conmon[177769]: 2023-09-23T22:08:17.584+0000 7f85e60cb200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:08:18.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:18 smithi070 ceph-mon[105170]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:18.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:18 smithi026 ceph-mon[100157]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:18.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:18 smithi026 ceph-mon[118879]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:18.740 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:18 smithi070 conmon[177769]: 2023-09-23T22:08:18.454+0000 7f85e60cb200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:08:18.740 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:18 smithi070 conmon[177769]: 2023-09-23T22:08:18.551+0000 7f85e60cb200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:08:19.045 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:18 smithi070 conmon[177769]: 2023-09-23T22:08:18.739+0000 7f85e60cb200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:08:19.344 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:19 smithi070 conmon[177769]: 2023-09-23T22:08:19.044+0000 7f85e60cb200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:08:19.344 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:19 smithi070 conmon[177769]: 2023-09-23T22:08:19.290+0000 7f85e60cb200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:08:19.594 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:19 smithi070 conmon[177769]: 2023-09-23T22:08:19.461+0000 7f85e60cb200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:08:19.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:19 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:19 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:19 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3008283322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:19.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:19 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.595 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:19 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3008283322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3008283322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:19 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:19.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:19 smithi070 conmon[177769]: 2023-09-23T22:08:19.746+0000 7f85e60cb200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:08:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:08:19] "GET /metrics HTTP/1.1" 200 33755 "" "Prometheus/2.43.0" 2023-09-23T22:08:20.617 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:20 smithi070 conmon[177769]: 2023-09-23T22:08:20.467+0000 7f85e60cb200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:08:20.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:20 smithi070 ceph-mon[105170]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:20.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:20 smithi026 ceph-mon[118879]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:20.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:20 smithi026 ceph-mon[100157]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:20.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:20 smithi070 conmon[177769]: 2023-09-23T22:08:20.616+0000 7f85e60cb200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:08:20.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:20 smithi070 conmon[177769]: 2023-09-23T22:08:20.829+0000 7f85e60cb200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:08:21.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:20 smithi070 conmon[177769]: 2023-09-23T22:08:20.952+0000 7f85e60cb200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:08:21.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:21 smithi070 conmon[177769]: 2023-09-23T22:08:21.186+0000 7f85e60cb200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:08:21.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:21 smithi026 ceph-mon[118879]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:21.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:21 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/3754865599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:21.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:21 smithi026 ceph-mon[100157]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:21.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:21 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/3754865599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:22.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:21 smithi070 ceph-mon[105170]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:22.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:21 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/3754865599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:22.498 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:22 smithi070 conmon[177769]: [23/Sep/2023:22:08:22] ENGINE Bus STARTING 2023-09-23T22:08:22.498 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:22 smithi070 conmon[177769]: 2023-09-23T22:08:22.108+0000 7f85e60cb200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:08:22.498 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:22 smithi070 conmon[177769]: CherryPy Checker: 2023-09-23T22:08:22.498 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:22 smithi070 conmon[177769]: The Application mounted at '' has an empty config. 2023-09-23T22:08:22.498 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:22 smithi070 conmon[177769]: 2023-09-23T22:08:22.498 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:22 smithi070 conmon[177769]: [23/Sep/2023:22:08:22] ENGINE Serving on http://:::9283 2023-09-23T22:08:22.498 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:22 smithi070 conmon[177769]: [23/Sep/2023:22:08:22] ENGINE Bus STARTED 2023-09-23T22:08:22.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: Standby manager daemon x restarted 2023-09-23T22:08:22.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: Standby manager daemon x started 2023-09-23T22:08:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:08:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:08:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:08:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:08:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:22.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:08:22.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:08:22.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:08:22.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T22:08:22.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:08:22.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:22 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:08:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: Standby manager daemon x restarted 2023-09-23T22:08:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: Standby manager daemon x started 2023-09-23T22:08:22.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:08:22.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:08:22.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:08:22.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:08:22.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:22.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:22.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:22.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:08:22.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:08:22.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:08:22.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T22:08:22.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:08:22.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:08:22.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: Standby manager daemon x restarted 2023-09-23T22:08:22.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: Standby manager daemon x started 2023-09-23T22:08:22.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:08:22.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:08:22.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:08:22.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3333096847' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:08:22.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:22.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:22.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:22.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:08:22.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:22.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:08:22.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T22:08:22.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T22:08:22.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:08:22.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:22 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T22:08:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[100157]: Upgrade: Setting container_image for all mgr 2023-09-23T22:08:24.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T22:08:24.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:08:24.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[100157]: mgrmap e30: y(active, since 25s), standbys: x 2023-09-23T22:08:24.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T22:08:24.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[100157]: Upgrade: It appears safe to stop mon.a 2023-09-23T22:08:24.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[100157]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:24.002 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[100157]: from='client.? 172.21.15.26:0/1125901373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:24.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[118879]: Upgrade: Setting container_image for all mgr 2023-09-23T22:08:24.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T22:08:24.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:08:24.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[118879]: mgrmap e30: y(active, since 25s), standbys: x 2023-09-23T22:08:24.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T22:08:24.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[118879]: Upgrade: It appears safe to stop mon.a 2023-09-23T22:08:24.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[118879]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:24.003 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:23 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/1125901373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:24.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:23 smithi070 ceph-mon[105170]: Upgrade: Setting container_image for all mgr 2023-09-23T22:08:24.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:23 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T22:08:24.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:23 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:08:24.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:23 smithi070 ceph-mon[105170]: mgrmap e30: y(active, since 25s), standbys: x 2023-09-23T22:08:24.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:23 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T22:08:24.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:23 smithi070 ceph-mon[105170]: Upgrade: It appears safe to stop mon.a 2023-09-23T22:08:24.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:23 smithi070 ceph-mon[105170]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:24.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:23 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/1125901373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:25.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[118879]: Upgrade: Updating mon.a 2023-09-23T22:08:25.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:25.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:08:25.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:08:25.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[118879]: Deploying daemon mon.a on smithi026 2023-09-23T22:08:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[100157]: Upgrade: Updating mon.a 2023-09-23T22:08:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[100157]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:08:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:08:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[100157]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:25 smithi026 ceph-mon[100157]: Deploying daemon mon.a on smithi026 2023-09-23T22:08:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:25 smithi070 ceph-mon[105170]: Upgrade: Updating mon.a 2023-09-23T22:08:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:25 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='mgr.y' 2023-09-23T22:08:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:25 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:08:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:25 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:08:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:25 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:25 smithi070 ceph-mon[105170]: Deploying daemon mon.a on smithi026 2023-09-23T22:08:25.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:25 smithi026 systemd[1]: Stopping Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:08:25.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:25 smithi026 conmon[100133]: 2023-09-23T22:08:25.976+0000 7f5089acc700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:08:25.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:25 smithi026 conmon[100133]: 2023-09-23T22:08:25.976+0000 7f5089acc700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T22:08:26.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:26 smithi026 bash[215357]: feb74e5008f79cab8ca4326132018939160eb19ea214a55769fe24434c065172 2023-09-23T22:08:26.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:26 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T22:08:26.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:26 smithi026 systemd[1]: Stopped Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:08:26.969 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:26 smithi026 systemd[1]: Starting Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:08:27.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 bash[215710]: 0028efef866221db5b1dde28b65e86961eb64fd91d4ea3ceaca98040eb313454 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 systemd[1]: Started Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: pidfile_write: ignore empty --pid-file 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: load: jerasure load: lrc 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: RocksDB version: 7.9.2 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Git sha 0 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: DB SUMMARY 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: DB Session ID: QVQBXIBG2JA6BRPTR4M2 2023-09-23T22:08:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1178 Bytes 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 503 ; 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.env: 0x5589f72cd720 2023-09-23T22:08:27.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.info_log: 0x5589fa703420 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.statistics: (nil) 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.use_fsync: 0 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:08:27.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.db_log_dir: 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.wal_dir: 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:08:27.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.write_buffer_manager: 0x5589f99925a0 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:08:27.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.unordered_write: 0 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.row_cache: None 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.wal_filter: None 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.wal_compression: 0 2023-09-23T22:08:27.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:08:27.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:08:27.743 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_open_files: -1 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Compression algorithms supported: 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: kZSTD supported: 0 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:08:27.744 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.merge_operator: 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_filter: None 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:08:27.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5589fa703520) 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_type: 0 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_index_type: 0 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_shortening: 1 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:08:27.746 INFO:journalctl@ceph.mon.a.smithi026.stdout: checksum: 4 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: no_block_cache: 0 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache: 0x5589f9a15350 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_options: 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: capacity : 536870912 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_shard_bits : 4 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: strict_capacity_limit : 0 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_cache_compressed: (nil) 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: persistent_cache: (nil) 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size: 4096 2023-09-23T22:08:27.747 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_size_deviation: 10 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_restart_interval: 16 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: index_block_restart_interval: 1 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_block_size: 4096 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: partition_filters: 0 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: use_delta_encoding: 1 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: filter_policy: bloomfilter 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: whole_key_filtering: 1 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: verify_compression: 0 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: format_version: 5 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_index_compression: 1 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: block_align: 0 2023-09-23T22:08:27.748 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_auto_readahead_size: 262144 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout: prepopulate_block_cache: 0 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout: initial_auto_readahead_size: 8192 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression: NoCompression 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.num_levels: 7 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:08:27.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:08:27.750 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:08:27.750 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:08:27.750 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:08:27.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:08:27.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:08:27.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:08:27.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:08:27.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:08:27.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:08:27.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:08:27.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:08:27.751 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:08:27.752 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:08:27.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:08:27.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:08:27.755 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.ttl: 2592000 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:08:27.756 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12633, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 323a2072-3ea7-4e7c-808e-252eb8138c13 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506907472080, "job": 1, "event": "recovery_started", "wal_files": [31]} 2023-09-23T22:08:27.757 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506907472490, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 1597, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12625, "largest_seqno": 12633, "table_properties": {"data_size": 495, "index_size": 33, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 205, "raw_average_key_size": 22, "raw_value_size": 304, "raw_average_value_size": 33, "num_data_blocks": 1, "num_entries": 9, "num_filter_entries": 9, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695506907, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "323a2072-3ea7-4e7c-808e-252eb8138c13", "db_session_id": "QVQBXIBG2JA6BRPTR4M2", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506907472582, "job": 1, "event": "recovery_finished"} 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5589f9ab0000 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: DB pointer 0x5589f9a9e000 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** DB Stats ** 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:08:27.758 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: L0 1/0 1.56 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-09-23T22:08:27.759 INFO:journalctl@ceph.mon.a.smithi026.stdout: L6 1/0 6.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: Sum 2/0 6.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.8 0.00 0.00 1 0.000 0 0 0.0 0.0 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:08:27.760 INFO:journalctl@ceph.mon.a.smithi026.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: Cumulative compaction: 0.00 GB write, 0.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: Interval compaction: 0.00 GB write, 0.53 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache BinnedLRUCache@0x5589f9a15350#7 capacity: 512.00 MB usage: 0.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.12 KB,2.38419e-05%) Misc(1,0.00 KB,0%) 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:08:27.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: starting mon.a rank 0 at public addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] at bind addrs [v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: mon.a@-1(???) e3 preinit fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: mon.a@-1(???).mds e9 new map 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: mon.a@-1(???).mds e9 print_map 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout: e9 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout: legacy client fscid: 1 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout: Filesystem 'foo' (1) 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout: fs_name foo 2023-09-23T22:08:27.762 INFO:journalctl@ceph.mon.a.smithi026.stdout: epoch 8 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: created 2023-09-23T22:04:54.392171+0000 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: modified 2023-09-23T22:05:06.659218+0000 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: tableserver 0 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: root 0 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: session_timeout 60 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: session_autoclose 300 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_file_size 1099511627776 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: required_client_features {} 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: last_failure 0 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: last_failure_osd_epoch 0 2023-09-23T22:08:27.763 INFO:journalctl@ceph.mon.a.smithi026.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: max_mds 1 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: in 0 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: up {0=14646} 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: failed 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: damaged 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: stopped 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: data_pools [4] 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: metadata_pool 3 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: inline_data disabled 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: balancer 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: bal_rank_mask -1 2023-09-23T22:08:27.764 INFO:journalctl@ceph.mon.a.smithi026.stdout: standby_count_wanted 1 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout: [mds.foo.smithi026.rancfk{0:14646} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout: Standby daemons: 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout: 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout: [mds.foo.smithi070.ehaqek{-1:24520} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.70:6832/253331223,v1:172.21.15.70:6833/253331223] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: mon.a@-1(???).osd e88 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: mon.a@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: mon.a@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: mon.a@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-23T22:08:27.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta expand map: {default=false} 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta from 'false' to 'false' 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta expanded map: {default=false} 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta expand map: {default=info} 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta from 'info' to 'info' 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta expanded map: {default=info} 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta expand map: {default=daemon} 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta expand map: {default=debug} 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T22:08:27.766 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:27 smithi026 ceph-mon[215775]: expand_channel_meta expanded map: {default=debug} 2023-09-23T22:08:29.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:28 smithi070 conmon[176728]: ts=2023-09-23T22:08:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:08:29.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:08:29] "GET /metrics HTTP/1.1" 200 33761 "" "Prometheus/2.43.0" 2023-09-23T22:08:30.843 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:30 smithi026 conmon[202047]: ignoring --setuser ceph since I am not root 2023-09-23T22:08:30.843 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:30 smithi026 conmon[202047]: ignoring --setgroup ceph since I am not root 2023-09-23T22:08:30.843 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mgr[202071]: -- 172.21.15.26:0/1570105924 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a99e9ab1e0 con 0x55a99e982c00 2023-09-23T22:08:30.843 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:30 smithi026 conmon[202047]: 2023-09-23T22:08:30.744+0000 7f47dbd46200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:08:30.844 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:30 smithi026 conmon[202047]: 2023-09-23T22:08:30.841+0000 7f47dbd46200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:08:30.845 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: from='client.15864 172.21.15.26:0/35544085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:30.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:30.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:30.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/173361777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:30.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:30.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: mon.a calling monitor election 2023-09-23T22:08:30.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:08:30.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:08:30.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:08:30.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T22:08:30.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: mgrmap e30: y(active, since 33s), standbys: x 2023-09-23T22:08:30.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: mgrmap e31: y(active, since 33s), standbys: x 2023-09-23T22:08:30.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: overall HEALTH_OK 2023-09-23T22:08:30.850 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[118879]: from='mgr.25616 ' entity='' 2023-09-23T22:08:30.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: from='client.15864 172.21.15.26:0/35544085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:30.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:30.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/173361777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: mon.a calling monitor election 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: mgrmap e30: y(active, since 33s), standbys: x 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: mgrmap e31: y(active, since 33s), standbys: x 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: overall HEALTH_OK 2023-09-23T22:08:30.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:30 smithi026 ceph-mon[215775]: from='mgr.25616 ' entity='' 2023-09-23T22:08:30.860 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:30 smithi070 conmon[177769]: ignoring --setuser ceph since I am not root 2023-09-23T22:08:30.860 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:30 smithi070 conmon[177769]: ignoring --setgroup ceph since I am not root 2023-09-23T22:08:30.860 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:30 smithi070 conmon[177769]: 2023-09-23T22:08:30.691+0000 7f1fa52a0200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:30 smithi070 conmon[177769]: 2023-09-23T22:08:30.770+0000 7f1fa52a0200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: from='client.15864 172.21.15.26:0/35544085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: from='mgr.25616 172.21.15.26:0/449796305' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/173361777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: mon.a calling monitor election 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:08:30.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:08:30.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T22:08:30.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: mgrmap e30: y(active, since 33s), standbys: x 2023-09-23T22:08:30.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: mgrmap e31: y(active, since 33s), standbys: x 2023-09-23T22:08:30.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: overall HEALTH_OK 2023-09-23T22:08:30.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:30 smithi070 ceph-mon[105170]: from='mgr.25616 ' entity='' 2023-09-23T22:08:30.997 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T22:08:31.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:30 smithi070 conmon[177769]: 2023-09-23T22:08:30.859+0000 7f1fa52a0200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:08:31.120 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:30 smithi070 conmon[177769]: 2023-09-23T22:08:30.951+0000 7f1fa52a0200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:08:31.134 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:30 smithi026 conmon[202047]: 2023-09-23T22:08:30.950+0000 7f47dbd46200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T22:08:31.135 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:31 smithi026 conmon[202047]: 2023-09-23T22:08:31.076+0000 7f47dbd46200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T22:08:31.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:31 smithi070 conmon[177769]: 2023-09-23T22:08:31.119+0000 7f1fa52a0200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:08:31.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:31 smithi026 conmon[202047]: 2023-09-23T22:08:31.288+0000 7f47dbd46200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T22:08:31.691 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:08:31.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:31 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/461474228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:31.790 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:31 smithi070 conmon[177769]: 2023-09-23T22:08:31.478+0000 7f1fa52a0200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:08:31.791 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:31 smithi070 conmon[177769]: 2023-09-23T22:08:31.608+0000 7f1fa52a0200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:08:31.791 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:31 smithi070 conmon[177769]: 2023-09-23T22:08:31.696+0000 7f1fa52a0200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:08:31.931 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/461474228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:31.932 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:31 smithi026 conmon[202047]: 2023-09-23T22:08:31.772+0000 7f47dbd46200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T22:08:31.932 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:31 smithi026 conmon[202047]: 2023-09-23T22:08:31.931+0000 7f47dbd46200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T22:08:31.932 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:31 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/461474228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:08:32.069 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:08:32.185 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:32 smithi026 conmon[202047]: 2023-09-23T22:08:32.058+0000 7f47dbd46200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T22:08:32.185 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:32 smithi026 conmon[202047]: 2023-09-23T22:08:32.185+0000 7f47dbd46200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:08:32.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:31 smithi070 conmon[177769]: 2023-09-23T22:08:31.789+0000 7f1fa52a0200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T22:08:32.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:32 smithi070 conmon[177769]: 2023-09-23T22:08:32.209+0000 7f1fa52a0200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:08:32.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:32 smithi026 conmon[202047]: 2023-09-23T22:08:32.720+0000 7f47dbd46200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T22:08:33.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:32 smithi070 conmon[177769]: 2023-09-23T22:08:32.859+0000 7f1fa52a0200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:08:33.123 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:32 smithi070 conmon[177769]: 2023-09-23T22:08:32.941+0000 7f1fa52a0200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:08:33.979 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:33 smithi070 conmon[177769]: 2023-09-23T22:08:33.711+0000 7f1fa52a0200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:08:33.979 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:33 smithi070 conmon[177769]: 2023-09-23T22:08:33.805+0000 7f1fa52a0200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:08:33.984 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:33 smithi026 conmon[202047]: 2023-09-23T22:08:33.527+0000 7f47dbd46200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T22:08:33.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:33 smithi026 conmon[202047]: 2023-09-23T22:08:33.638+0000 7f47dbd46200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T22:08:34.259 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:33 smithi070 conmon[177769]: 2023-09-23T22:08:33.978+0000 7f1fa52a0200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:08:34.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:34 smithi070 conmon[177769]: 2023-09-23T22:08:34.258+0000 7f1fa52a0200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:08:34.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:34 smithi070 conmon[177769]: 2023-09-23T22:08:34.485+0000 7f1fa52a0200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:08:34.902 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:34 smithi070 conmon[177769]: 2023-09-23T22:08:34.634+0000 7f1fa52a0200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:08:34.976 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:34 smithi026 conmon[202047]: 2023-09-23T22:08:34.638+0000 7f47dbd46200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:08:34.976 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:34 smithi026 conmon[202047]: 2023-09-23T22:08:34.751+0000 7f47dbd46200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T22:08:35.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:34 smithi026 conmon[202047]: 2023-09-23T22:08:34.976+0000 7f47dbd46200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T22:08:35.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:34 smithi070 conmon[177769]: 2023-09-23T22:08:34.901+0000 7f1fa52a0200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:08:35.592 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:35 smithi026 conmon[202047]: 2023-09-23T22:08:35.333+0000 7f47dbd46200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T22:08:35.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:35 smithi070 conmon[177769]: 2023-09-23T22:08:35.604+0000 7f1fa52a0200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:08:35.873 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:35 smithi070 conmon[177769]: 2023-09-23T22:08:35.736+0000 7f1fa52a0200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:08:35.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:35 smithi026 conmon[202047]: 2023-09-23T22:08:35.592+0000 7f47dbd46200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T22:08:35.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:35 smithi026 conmon[202047]: 2023-09-23T22:08:35.778+0000 7f47dbd46200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T22:08:36.269 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:35 smithi070 conmon[177769]: 2023-09-23T22:08:35.932+0000 7f1fa52a0200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:08:36.270 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:36 smithi070 conmon[177769]: 2023-09-23T22:08:36.047+0000 7f1fa52a0200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:08:36.485 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:36 smithi026 conmon[202047]: 2023-09-23T22:08:36.120+0000 7f47dbd46200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T22:08:36.623 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:36 smithi070 conmon[177769]: 2023-09-23T22:08:36.269+0000 7f1fa52a0200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:08:37.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:36 smithi026 conmon[202047]: 2023-09-23T22:08:36.965+0000 7f47dbd46200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T22:08:37.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:37 smithi026 conmon[202047]: 2023-09-23T22:08:37.128+0000 7f47dbd46200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T22:08:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[215775]: Standby manager daemon x restarted 2023-09-23T22:08:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[215775]: Standby manager daemon x started 2023-09-23T22:08:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[215775]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:08:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[215775]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:08:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[215775]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:08:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[215775]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:08:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[118879]: Standby manager daemon x restarted 2023-09-23T22:08:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[118879]: Standby manager daemon x started 2023-09-23T22:08:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:08:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:08:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:08:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:37 smithi026 ceph-mon[118879]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:08:37.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:37 smithi070 ceph-mon[105170]: Standby manager daemon x restarted 2023-09-23T22:08:37.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:37 smithi070 ceph-mon[105170]: Standby manager daemon x started 2023-09-23T22:08:37.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:37 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T22:08:37.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:37 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T22:08:37.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:37 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T22:08:37.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:37 smithi070 ceph-mon[105170]: from='mgr.? 172.21.15.70:0/3242561094' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T22:08:37.624 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:37 smithi070 conmon[177769]: 2023-09-23T22:08:37.129+0000 7f1fa52a0200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:08:37.624 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:37 smithi070 conmon[177769]: [23/Sep/2023:22:08:37] ENGINE Bus STARTING 2023-09-23T22:08:37.625 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:37 smithi070 conmon[177769]: CherryPy Checker: 2023-09-23T22:08:37.625 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:37 smithi070 conmon[177769]: The Application mounted at '' has an empty config. 2023-09-23T22:08:37.625 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:37 smithi070 conmon[177769]: 2023-09-23T22:08:37.625 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:37 smithi070 conmon[177769]: [23/Sep/2023:22:08:37] ENGINE Serving on http://:::9283 2023-09-23T22:08:37.625 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:08:37 smithi070 conmon[177769]: [23/Sep/2023:22:08:37] ENGINE Bus STARTED 2023-09-23T22:08:37.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:37 smithi026 conmon[202047]: 2023-09-23T22:08:37.385+0000 7f47dbd46200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T22:08:37.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:37 smithi026 conmon[202047]: 2023-09-23T22:08:37.527+0000 7f47dbd46200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T22:08:38.201 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:37 smithi026 conmon[202047]: 2023-09-23T22:08:37.812+0000 7f47dbd46200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T22:08:38.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:38 smithi026 ceph-mon[215775]: mgrmap e32: y(active, since 40s), standbys: x 2023-09-23T22:08:38.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:38 smithi026 ceph-mon[118879]: mgrmap e32: y(active, since 40s), standbys: x 2023-09-23T22:08:38.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:38 smithi070 ceph-mon[105170]: mgrmap e32: y(active, since 40s), standbys: x 2023-09-23T22:08:39.057 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:38 smithi070 conmon[105126]: 2023-09-23T22:08:38.901+0000 7f50ecc8f700 -1 mon.b@1(peon).osd e89 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:08:39.058 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:38 smithi070 conmon[176728]: ts=2023-09-23T22:08:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:08:39.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:38 smithi026 conmon[118855]: 2023-09-23T22:08:38.901+0000 7f212b68a700 -1 mon.c@2(peon).osd e89 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T22:08:39.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:38 smithi026 conmon[202047]: 2023-09-23T22:08:38.884+0000 7f47dbd46200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T22:08:39.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:38 smithi026 conmon[202047]: [23/Sep/2023:22:08:38] ENGINE Bus STARTING 2023-09-23T22:08:39.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4739879700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4739879700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4739879700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4739879700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4739879700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4735871700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.204 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4735871700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.205 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4735871700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.205 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4735871700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.205 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.057+0000 7f4735871700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T22:08:39.205 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: CherryPy Checker: 2023-09-23T22:08:39.205 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: The Application mounted at '' has an empty config. 2023-09-23T22:08:39.205 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.205 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: [23/Sep/2023:22:08:39] ENGINE Serving on http://:::9283 2023-09-23T22:08:39.205 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: [23/Sep/2023:22:08:39] ENGINE Bus STARTED 2023-09-23T22:08:39.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: Active manager daemon y restarted 2023-09-23T22:08:39.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: Activating manager daemon y 2023-09-23T22:08:39.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:08:39.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: mgrmap e33: y(active, starting, since 0.00566s), standbys: x 2023-09-23T22:08:39.206 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: failed to encode map e89 with expected crc 2023-09-23T22:08:39.207 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:08:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: Active manager daemon y restarted 2023-09-23T22:08:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: Activating manager daemon y 2023-09-23T22:08:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:08:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: mgrmap e33: y(active, starting, since 0.00566s), standbys: x 2023-09-23T22:08:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: failed to encode map e89 with expected crc 2023-09-23T22:08:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:08:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:08:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:08:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:08:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:08:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:08:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:08:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:08:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:08:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:08:39.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:08:39.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:08:39.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:08:39.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:08:39.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:08:39.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:08:39.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:08:39.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:08:39.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: Manager daemon y is now available 2023-09-23T22:08:39.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:39.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:39.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:39.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:08:39.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:08:39.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:08:39.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:08:39.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:08:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: Manager daemon y is now available 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:08:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:08:39.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: Active manager daemon y restarted 2023-09-23T22:08:39.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: Activating manager daemon y 2023-09-23T22:08:39.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:08:39.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: mgrmap e33: y(active, starting, since 0.00566s), standbys: x 2023-09-23T22:08:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: failed to encode map e89 with expected crc 2023-09-23T22:08:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:08:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:08:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:08:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi026.rancfk"}]: dispatch 2023-09-23T22:08:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi070.ehaqek"}]: dispatch 2023-09-23T22:08:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T22:08:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T22:08:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T22:08:39.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T22:08:39.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: Manager daemon y is now available 2023-09-23T22:08:39.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:39.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:39.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:39.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T22:08:39.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T22:08:39.913 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:08:39] "GET /metrics HTTP/1.1" 200 31077 "" "Prometheus/2.43.0" 2023-09-23T22:08:39.913 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:39 smithi026 conmon[202047]: 2023-09-23T22:08:39.909+0000 7f47954a6700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T22:08:39.928 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:08:39.929 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (68s) - 4m 17.4M - 0.25.0 c8568f914cd2 1cbbb123afed 2023-09-23T22:08:39.929 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (56s) - 4m 82.2M - 9.4.7 2c41d148cca3 42940e65ed5c 2023-09-23T22:08:39.929 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.rancfk smithi026 running (3m) - 3m 17.8M - 17.2.6-1125-gc9bb1bcf 444539aec5bf a9885df09771 2023-09-23T22:08:39.929 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.ehaqek smithi070 running (3m) - 3m 16.1M - 17.2.6-1125-gc9bb1bcf 444539aec5bf 812a5d180a78 2023-09-23T22:08:39.929 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (25s) - 8m 44.7M - 18.2.0-460-gbb0daeff e32c2a64c126 754e649f30a9 2023-09-23T22:08:39.929 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (51s) - 10m 475M - 18.2.0-460-gbb0daeff e32c2a64c126 2dff2050e669 2023-09-23T22:08:39.929 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (10m) - 10m 103M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf feb74e5008f7 2023-09-23T22:08:39.930 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (8m) - 8m 72.1M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf b61f3bc30723 2023-09-23T22:08:39.930 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (8m) - 8m 69.6M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf fce3aa9fda1a 2023-09-23T22:08:39.930 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (66s) - 4m 8610k - 1.5.0 0da6a335fe13 68b72a59fe08 2023-09-23T22:08:39.930 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (60s) - 4m 15.8M - 1.5.0 0da6a335fe13 523ab3522b38 2023-09-23T22:08:39.930 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (8m) - 8m 75.5M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 67a4086eb002 2023-09-23T22:08:39.931 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (7m) - 7m 100M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 02b1d9e99db9 2023-09-23T22:08:39.931 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (7m) - 7m 70.7M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 50e18b584478 2023-09-23T22:08:39.931 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (6m) - 6m 63.3M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 57db6dad61fc 2023-09-23T22:08:39.931 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (6m) - 6m 68.6M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 92889bd7f5ee 2023-09-23T22:08:39.931 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (6m) - 6m 85.3M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 628c1ee2ccfc 2023-09-23T22:08:39.931 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (5m) - 5m 70.0M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 5b472aeb7f69 2023-09-23T22:08:39.931 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (5m) - 5m 89.9M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 2a3425d84687 2023-09-23T22:08:39.932 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (30s) - 4m 39.6M - 2.43.0 a07b618ecd1d 3342388f10a7 2023-09-23T22:08:40.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[215775]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[215775]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[215775]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[215775]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[215775]: [23/Sep/2023:22:08:39] ENGINE Bus STARTING 2023-09-23T22:08:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[215775]: mgrmap e34: y(active, since 1.01031s), standbys: x 2023-09-23T22:08:40.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[118879]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[118879]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[118879]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[118879]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[118879]: [23/Sep/2023:22:08:39] ENGINE Bus STARTING 2023-09-23T22:08:40.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:40 smithi026 ceph-mon[118879]: mgrmap e34: y(active, since 1.01031s), standbys: x 2023-09-23T22:08:40.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:40 smithi070 ceph-mon[105170]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:40 smithi070 ceph-mon[105170]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:40 smithi070 ceph-mon[105170]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:40 smithi070 ceph-mon[105170]: failed to encode map e89 with expected crc 2023-09-23T22:08:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:40 smithi070 ceph-mon[105170]: [23/Sep/2023:22:08:39] ENGINE Bus STARTING 2023-09-23T22:08:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:40 smithi070 ceph-mon[105170]: mgrmap e34: y(active, since 1.01031s), standbys: x 2023-09-23T22:08:40.685 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T22:08:41.341 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:08:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[118879]: [23/Sep/2023:22:08:39] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T22:08:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[118879]: [23/Sep/2023:22:08:39] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T22:08:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[118879]: [23/Sep/2023:22:08:39] ENGINE Bus STARTED 2023-09-23T22:08:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[118879]: from='client.25784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:08:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[118879]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:41.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[215775]: [23/Sep/2023:22:08:39] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T22:08:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[215775]: [23/Sep/2023:22:08:39] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T22:08:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[215775]: [23/Sep/2023:22:08:39] ENGINE Bus STARTED 2023-09-23T22:08:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[215775]: from='client.25784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:08:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:41 smithi026 ceph-mon[215775]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:41.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:41 smithi070 ceph-mon[105170]: [23/Sep/2023:22:08:39] ENGINE Serving on https://172.21.15.26:7150 2023-09-23T22:08:41.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:41 smithi070 ceph-mon[105170]: [23/Sep/2023:22:08:39] ENGINE Serving on http://172.21.15.26:8765 2023-09-23T22:08:41.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:41 smithi070 ceph-mon[105170]: [23/Sep/2023:22:08:39] ENGINE Bus STARTED 2023-09-23T22:08:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:41 smithi070 ceph-mon[105170]: from='client.25784 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:08:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:41 smithi070 ceph-mon[105170]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:42.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:42 smithi026 ceph-mon[215775]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:42.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:42 smithi026 ceph-mon[215775]: mgrmap e35: y(active, since 2s), standbys: x 2023-09-23T22:08:42.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:42 smithi026 ceph-mon[118879]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:42.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:42 smithi026 ceph-mon[118879]: mgrmap e35: y(active, since 2s), standbys: x 2023-09-23T22:08:42.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:42 smithi070 ceph-mon[105170]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:42.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:42 smithi070 ceph-mon[105170]: mgrmap e35: y(active, since 2s), standbys: x 2023-09-23T22:08:43.295 INFO:teuthology.orchestra.run.smithi026.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:08:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:43 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:43 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:43 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:43 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:43 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:43 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:43 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:43.984 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T22:08:44.358 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:08:44.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[215775]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:44.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[215775]: from='client.25820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:08:44.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[215775]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:08:44.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:44.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:44.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[118879]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:44.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[118879]: from='client.25820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:08:44.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[118879]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:08:44.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:44.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:44 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:44.843 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:44 smithi070 ceph-mon[105170]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:44.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:44 smithi070 ceph-mon[105170]: from='client.25820 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:08:44.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:44 smithi070 ceph-mon[105170]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:08:44.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:44 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:44.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:44 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:46 smithi070 ceph-mon[105170]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:46 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:46 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:46 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:46 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[215775]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[118879]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:46.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:46.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:46.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:46 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:47.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:47 smithi070 ceph-mon[105170]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:08:47.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:47 smithi070 ceph-mon[105170]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:08:47.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:47 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:47 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:47 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:47 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:47 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[118879]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:08:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:08:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[215775]: Updating smithi026:/etc/ceph/ceph.conf 2023-09-23T22:08:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[215775]: Updating smithi070:/etc/ceph/ceph.conf 2023-09-23T22:08:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:47.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:47 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:48.873 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:48 smithi070 conmon[176728]: ts=2023-09-23T22:08:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:08:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:48.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:48.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:48.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:48.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:48.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:48.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:48.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:48.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:48.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:48.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:48.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:48.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:48.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:48.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T22:08:48.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:48.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: Updating smithi026:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:48.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: Updating smithi070:/etc/ceph/ceph.client.admin.keyring 2023-09-23T22:08:48.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: Updating smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:48.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: Updating smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T22:08:48.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:08:48.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:08:48.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:08:48.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:08:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:08:49] "GET /metrics HTTP/1.1" 200 31077 "" "Prometheus/2.43.0" 2023-09-23T22:08:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:50 smithi070 ceph-mon[105170]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:50 smithi026 ceph-mon[118879]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:50 smithi026 ceph-mon[215775]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:52 smithi070 ceph-mon[105170]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:52 smithi026 ceph-mon[118879]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:52 smithi026 ceph-mon[215775]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:54 smithi070 ceph-mon[105170]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:54 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:54 smithi026 ceph-mon[118879]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:54 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:54 smithi026 ceph-mon[215775]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:08:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:56 smithi070 ceph-mon[105170]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:56 smithi026 ceph-mon[118879]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:56 smithi026 ceph-mon[215775]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:58.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:08:58 smithi070 ceph-mon[105170]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:58.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:08:58 smithi070 conmon[176728]: ts=2023-09-23T22:08:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:08:58.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:08:58 smithi026 ceph-mon[118879]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:58.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:08:58 smithi026 ceph-mon[215775]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:08:59.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:08:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:08:59] "GET /metrics HTTP/1.1" 200 33756 "" "Prometheus/2.43.0" 2023-09-23T22:09:00.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:00 smithi070 ceph-mon[105170]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:00.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:00 smithi026 ceph-mon[118879]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:00.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:00 smithi026 ceph-mon[215775]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:02.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:02 smithi070 ceph-mon[105170]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:02.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:02 smithi026 ceph-mon[118879]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:02.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:02 smithi026 ceph-mon[215775]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:04 smithi070 ceph-mon[105170]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:04.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:04 smithi026 ceph-mon[118879]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:04.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:04 smithi026 ceph-mon[215775]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:06 smithi070 ceph-mon[105170]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:06 smithi026 ceph-mon[118879]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:06 smithi026 ceph-mon[215775]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:08.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:08 smithi070 ceph-mon[105170]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:08.873 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:09:08 smithi070 conmon[176728]: ts=2023-09-23T22:09:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:09:08.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:08 smithi026 ceph-mon[215775]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:09.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:08 smithi026 ceph-mon[118879]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:09 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:09.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:09:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:09:09] "GET /metrics HTTP/1.1" 200 33751 "" "Prometheus/2.43.0" 2023-09-23T22:09:09.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:09 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:09.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:10.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:10 smithi070 ceph-mon[105170]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:10 smithi026 ceph-mon[215775]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:10 smithi026 ceph-mon[118879]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:12.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:12 smithi070 ceph-mon[105170]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:12 smithi026 ceph-mon[215775]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:12 smithi026 ceph-mon[118879]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:14.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:14 smithi070 ceph-mon[105170]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:14 smithi026 ceph-mon[118879]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:14 smithi026 ceph-mon[215775]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:16 smithi070 ceph-mon[105170]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:16 smithi026 ceph-mon[118879]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:16 smithi026 ceph-mon[215775]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:18 smithi070 ceph-mon[105170]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:18.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:09:18 smithi070 conmon[176728]: ts=2023-09-23T22:09:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:09:18.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:18 smithi026 ceph-mon[118879]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:18 smithi026 ceph-mon[215775]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:09:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:09:19] "GET /metrics HTTP/1.1" 200 33751 "" "Prometheus/2.43.0" 2023-09-23T22:09:20.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:20 smithi070 ceph-mon[105170]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:20.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:20 smithi026 ceph-mon[118879]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:20 smithi026 ceph-mon[215775]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:22.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:22 smithi070 ceph-mon[105170]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:22 smithi026 ceph-mon[215775]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:22 smithi026 ceph-mon[118879]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:24.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:24 smithi070 ceph-mon[105170]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:24.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:24 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:24 smithi026 ceph-mon[215775]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:24 smithi026 ceph-mon[118879]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:24 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:26 smithi070 ceph-mon[105170]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:26.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:26 smithi026 ceph-mon[215775]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:26 smithi026 ceph-mon[118879]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:28.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:28 smithi070 ceph-mon[105170]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:28.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:09:28 smithi070 conmon[176728]: ts=2023-09-23T22:09:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:09:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:28 smithi026 ceph-mon[118879]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:28.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:28 smithi026 ceph-mon[215775]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:29.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:09:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:09:29] "GET /metrics HTTP/1.1" 200 33755 "" "Prometheus/2.43.0" 2023-09-23T22:09:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:30 smithi070 ceph-mon[105170]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:30 smithi026 ceph-mon[118879]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:30.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:30 smithi026 ceph-mon[215775]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:32 smithi070 ceph-mon[105170]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:32.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:32 smithi026 ceph-mon[118879]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:32.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:32 smithi026 ceph-mon[215775]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:34.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:34 smithi070 ceph-mon[105170]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:34.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:34 smithi026 ceph-mon[118879]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:34.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:34 smithi026 ceph-mon[215775]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:36 smithi070 ceph-mon[105170]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:36 smithi026 ceph-mon[118879]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:36 smithi026 ceph-mon[215775]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:38 smithi070 ceph-mon[105170]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:38.873 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:09:38 smithi070 conmon[176728]: ts=2023-09-23T22:09:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:09:38.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:38 smithi026 ceph-mon[215775]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:38 smithi026 ceph-mon[118879]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:39 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:39.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:09:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:09:39] "GET /metrics HTTP/1.1" 200 33754 "" "Prometheus/2.43.0" 2023-09-23T22:09:39.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:39 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:40.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:40 smithi070 ceph-mon[105170]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:40.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:40 smithi026 ceph-mon[215775]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:40.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:40 smithi026 ceph-mon[118879]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:42.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:42 smithi070 ceph-mon[105170]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:42.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:42 smithi026 ceph-mon[215775]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:42 smithi026 ceph-mon[118879]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:44 smithi070 ceph-mon[105170]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:44.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:44 smithi026 ceph-mon[215775]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:44.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:44 smithi026 ceph-mon[118879]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:46.207 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T22:09:46.574 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:09:46.829 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:46 smithi026 ceph-mon[215775]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:46.829 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:46 smithi026 ceph-mon[118879]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:46.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:46 smithi070 ceph-mon[105170]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:48.414 INFO:teuthology.orchestra.run.smithi026.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:09:48.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[215775]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:48.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[118879]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:48 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.873 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:09:48 smithi070 conmon[176728]: ts=2023-09-23T22:09:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:09:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:48 smithi070 ceph-mon[105170]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:09:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:48 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:49.229 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T22:09:49.744 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:09:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:49 smithi070 ceph-mon[105170]: from='client.25826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:09:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:49 smithi070 ceph-mon[105170]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:09:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:09:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:09:49] "GET /metrics HTTP/1.1" 200 33754 "" "Prometheus/2.43.0" 2023-09-23T22:09:49.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:49 smithi026 ceph-mon[118879]: from='client.25826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:09:49.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:49 smithi026 ceph-mon[118879]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:09:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:49 smithi026 ceph-mon[215775]: from='client.25826 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:09:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:49 smithi026 ceph-mon[215775]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:09:50.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:50 smithi026 ceph-mon[118879]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:50 smithi026 ceph-mon[215775]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:50 smithi070 ceph-mon[105170]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:51.364 INFO:teuthology.orchestra.run.smithi026.stdout:wait for majority of mons to upgrade 2023-09-23T22:09:52.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T22:09:52.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:52 smithi026 ceph-mon[118879]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:52.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:52 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:52.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:52 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:52.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:52 smithi026 ceph-mon[215775]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:52.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:52.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:52.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:52 smithi070 ceph-mon[105170]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:52.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:52 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:52.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:52 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:52.744 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:09:54.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:09:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:09:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:09:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:09:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:09:54.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T22:09:54.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:53 smithi070 ceph-mon[105170]: Upgrade: It appears safe to stop mon.c 2023-09-23T22:09:54.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:54.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:54.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:54.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:09:54.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:09:54.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:54.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:09:54.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:09:54.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:09:54.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T22:09:54.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[118879]: Upgrade: It appears safe to stop mon.c 2023-09-23T22:09:54.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:54.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:54.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:54.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:09:54.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:09:54.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:54.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:09:54.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:09:54.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:09:54.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T22:09:54.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:53 smithi026 ceph-mon[215775]: Upgrade: It appears safe to stop mon.c 2023-09-23T22:09:54.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:54 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:54.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:54 smithi026 ceph-mon[118879]: from='client.? 172.21.15.26:0/4148832133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:09:54.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:54.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4148832133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:09:55.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:54 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:09:55.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:54 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/4148832133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:09:56.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[215775]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:56.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[215775]: Upgrade: Updating mon.c 2023-09-23T22:09:56.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:56.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:09:56.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:09:56.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:09:56.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[215775]: Deploying daemon mon.c on smithi026 2023-09-23T22:09:56.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[118879]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:56.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[118879]: Upgrade: Updating mon.c 2023-09-23T22:09:56.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:56.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:09:56.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:09:56.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[118879]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:09:56.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 ceph-mon[118879]: Deploying daemon mon.c on smithi026 2023-09-23T22:09:56.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:56 smithi070 ceph-mon[105170]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:09:56.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:56 smithi070 ceph-mon[105170]: Upgrade: Updating mon.c 2023-09-23T22:09:56.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:56 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:09:56.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:56 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:09:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:56 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:09:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:56 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:09:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:09:56 smithi070 ceph-mon[105170]: Deploying daemon mon.c on smithi026 2023-09-23T22:09:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 systemd[1]: Stopping Ceph mon.c for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:09:57.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 conmon[118855]: 2023-09-23T22:09:56.903+0000 7f2130e95700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:09:57.155 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:56 smithi026 conmon[118855]: 2023-09-23T22:09:56.903+0000 7f2130e95700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T22:09:57.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:57 smithi026 bash[222576]: fce3aa9fda1a2e863b0168b7d431c2cd270a6396cd435059ac9d5aed00391b2d 2023-09-23T22:09:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:57 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T22:09:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:57 smithi026 systemd[1]: Stopped Ceph mon.c for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:09:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:57 smithi026 systemd[1]: Starting Ceph mon.c for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:09:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 bash[222940]: 25f6f9fc0ac743ddec0a6c3048cd4506081ff9dd39b8829d695df36de35336a2 2023-09-23T22:09:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:09:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T22:09:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: pidfile_write: ignore empty --pid-file 2023-09-23T22:09:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 systemd[1]: Started Ceph mon.c for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:09:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: load: jerasure load: lrc 2023-09-23T22:09:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: RocksDB version: 7.9.2 2023-09-23T22:09:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Git sha 0 2023-09-23T22:09:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T22:09:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: DB SUMMARY 2023-09-23T22:09:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: DB Session ID: I3SPBWBDRC3O1JIBA87X 2023-09-23T22:09:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:09:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:09:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1103 Bytes 2023-09-23T22:09:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T22:09:58.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 3446270 ; 2023-09-23T22:09:58.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:09:58.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:09:58.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:09:58.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T22:09:58.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:09:58.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T22:09:58.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.env: 0x556ea4d23720 2023-09-23T22:09:58.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T22:09:58.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.info_log: 0x556ea6fc9420 2023-09-23T22:09:58.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:09:58.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.statistics: (nil) 2023-09-23T22:09:58.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.use_fsync: 0 2023-09-23T22:09:58.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:09:58.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:09:58.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:09:58.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.db_log_dir: 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.wal_dir: 2023-09-23T22:09:58.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:09:58.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:09:58.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:09:58.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:09:58.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:09:58.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:09:58.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:09:58.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:09:58.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.write_buffer_manager: 0x556ea62585a0 2023-09-23T22:09:58.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.unordered_write: 0 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:09:58.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.row_cache: None 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.wal_filter: None 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.wal_compression: 0 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:09:58.743 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:09:58.744 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_open_files: -1 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:09:58.745 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Compression algorithms supported: 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: kZSTD supported: 0 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:09:58.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.merge_operator: 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_filter: None 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556ea6fc9520) 2023-09-23T22:09:58.747 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_type: 0 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_index_type: 0 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_shortening: 1 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: checksum: 4 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: no_block_cache: 0 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache: 0x556ea62db1f0 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:09:58.748 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_options: 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: capacity : 536870912 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_shard_bits : 4 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: strict_capacity_limit : 0 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_cache_compressed: (nil) 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: persistent_cache: (nil) 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size: 4096 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_size_deviation: 10 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_restart_interval: 16 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: index_block_restart_interval: 1 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_block_size: 4096 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: partition_filters: 0 2023-09-23T22:09:58.749 INFO:journalctl@ceph.mon.c.smithi026.stdout: use_delta_encoding: 1 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: filter_policy: bloomfilter 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: whole_key_filtering: 1 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: verify_compression: 0 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: format_version: 5 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_index_compression: 1 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: block_align: 0 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_auto_readahead_size: 262144 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: prepopulate_block_cache: 0 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: initial_auto_readahead_size: 8192 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T22:09:58.750 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression: NoCompression 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.num_levels: 7 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:09:58.751 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:09:58.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:09:58.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T22:09:58.754 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:09:58.755 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:09:58.755 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:09:58.756 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:09:58.757 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.ttl: 2592000 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T22:09:58.758 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13200, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 785aec0b-a8a3-4980-87ae-fdfb3b57e814 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506998328110, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506998341526, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2168178, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13195, "largest_seqno": 14535, "table_properties": {"data_size": 2162603, "index_size": 3340, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10583, "raw_average_key_size": 22, "raw_value_size": 2152190, "raw_average_value_size": 4511, "num_data_blocks": 154, "num_entries": 477, "num_filter_entries": 477, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695506998, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "785aec0b-a8a3-4980-87ae-fdfb3b57e814", "db_session_id": "I3SPBWBDRC3O1JIBA87X", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695506998341663, "job": 1, "event": "recovery_finished"} 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T22:09:58.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556ea6376000 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: DB pointer 0x556ea6364000 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** DB Stats ** 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:09:58.760 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: L0 1/0 2.07 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 398.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: L6 1/0 6.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: Sum 2/0 8.59 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 398.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 398.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T22:09:58.761 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** Compaction Stats [default] ** 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 398.2 0.01 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:09:58.762 INFO:journalctl@ceph.mon.c.smithi026.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout: Cumulative compaction: 0.00 GB write, 120.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout: Interval compaction: 0.00 GB write, 120.98 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache BinnedLRUCache@0x556ea62db1f0#7 capacity: 512.00 MB usage: 5.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: starting mon.c rank 2 at public addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] at bind addrs [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: mon.c@-1(???) e3 preinit fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: mon.c@-1(???).mds e9 new map 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: mon.c@-1(???).mds e9 print_map 2023-09-23T22:09:58.763 INFO:journalctl@ceph.mon.c.smithi026.stdout: e9 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: legacy client fscid: 1 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: Filesystem 'foo' (1) 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: fs_name foo 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: epoch 8 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: created 2023-09-23T22:04:54.392171+0000 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: modified 2023-09-23T22:05:06.659218+0000 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: tableserver 0 2023-09-23T22:09:58.764 INFO:journalctl@ceph.mon.c.smithi026.stdout: root 0 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: session_timeout 60 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: session_autoclose 300 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_file_size 1099511627776 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: required_client_features {} 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: last_failure 0 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: last_failure_osd_epoch 0 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: max_mds 1 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: in 0 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: up {0=14646} 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: failed 2023-09-23T22:09:58.765 INFO:journalctl@ceph.mon.c.smithi026.stdout: damaged 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: stopped 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: data_pools [4] 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: metadata_pool 3 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: inline_data disabled 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: balancer 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: bal_rank_mask -1 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: standby_count_wanted 1 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: [mds.foo.smithi026.rancfk{0:14646} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: Standby daemons: 2023-09-23T22:09:58.766 INFO:journalctl@ceph.mon.c.smithi026.stdout: 2023-09-23T22:09:58.767 INFO:journalctl@ceph.mon.c.smithi026.stdout: [mds.foo.smithi070.ehaqek{-1:24520} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.70:6832/253331223,v1:172.21.15.70:6833/253331223] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:09:58.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: mon.c@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T22:09:58.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: mon.c@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:09:58.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: mon.c@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:09:58.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: mon.c@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:09:58.767 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:09:58 smithi026 ceph-mon[223014]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-09-23T22:09:59.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:09:58 smithi070 conmon[176728]: ts=2023-09-23T22:09:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:09:59.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:09:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:09:59] "GET /metrics HTTP/1.1" 200 33752 "" "Prometheus/2.43.0" 2023-09-23T22:10:04.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: mon.b calling monitor election 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: mon.a calling monitor election 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: mgrmap e35: y(active, since 84s), standbys: x 2023-09-23T22:10:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: mon.c (rank 2) addr [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] is down (out of quorum) 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/824102850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: mon.b calling monitor election 2023-09-23T22:10:04.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: mon.a calling monitor election 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: mgrmap e35: y(active, since 84s), standbys: x 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:10:04.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:10:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:10:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: mon.c (rank 2) addr [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] is down (out of quorum) 2023-09-23T22:10:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/824102850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:10:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: mon.b calling monitor election 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: mon.a calling monitor election 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:10:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: mgrmap e35: y(active, since 84s), standbys: x 2023-09-23T22:10:04.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T22:10:04.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T22:10:04.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T22:10:04.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: mon.c (rank 2) addr [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0] is down (out of quorum) 2023-09-23T22:10:04.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:04.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:04.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:04.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:04 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/824102850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:10:05.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: mon.c calling monitor election 2023-09-23T22:10:05.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: mon.a calling monitor election 2023-09-23T22:10:05.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: mon.b calling monitor election 2023-09-23T22:10:05.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:10:05.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:10:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:10:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:10:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: mgrmap e35: y(active, since 85s), standbys: x 2023-09-23T22:10:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:10:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:10:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[223014]: overall HEALTH_OK 2023-09-23T22:10:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: mon.c calling monitor election 2023-09-23T22:10:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: mon.a calling monitor election 2023-09-23T22:10:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: mon.b calling monitor election 2023-09-23T22:10:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:10:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:10:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:10:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:10:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: mgrmap e35: y(active, since 85s), standbys: x 2023-09-23T22:10:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:10:05.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:10:05.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:05 smithi026 ceph-mon[215775]: overall HEALTH_OK 2023-09-23T22:10:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: mon.c calling monitor election 2023-09-23T22:10:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: mon.a calling monitor election 2023-09-23T22:10:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: mon.b calling monitor election 2023-09-23T22:10:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:10:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: monmap e3: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:10:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:10:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:10:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: mgrmap e35: y(active, since 85s), standbys: x 2023-09-23T22:10:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T22:10:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: Cluster is now healthy 2023-09-23T22:10:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:05 smithi070 ceph-mon[105170]: overall HEALTH_OK 2023-09-23T22:10:06.453 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T22:10:06.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:06 smithi026 ceph-mon[223014]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:06.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:06 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:06.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:06 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:06.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/974801534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:10:06.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:06 smithi026 ceph-mon[215775]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:06.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:06 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:06.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:06 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:06.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/974801534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:10:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:06 smithi070 ceph-mon[105170]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:06.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:06 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:06.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:06 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:06.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:06 smithi070 ceph-mon[105170]: from='client.? 172.21.15.26:0/974801534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:10:07.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:10:07.862 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:10:08.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:08 smithi026 ceph-mon[223014]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:08.355 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:08 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:08.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:08 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:08.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:08 smithi026 ceph-mon[215775]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:08.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:08 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:08.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:08 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:08.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:08 smithi070 ceph-mon[105170]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:08.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:08 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:08.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:08 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:08.610 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:10:08 smithi026 conmon[202047]: 2023-09-23T22:10:08.352+0000 7f47954a6700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T22:10:09.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:10:08 smithi070 conmon[176728]: ts=2023-09-23T22:10:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:10:09.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:09.612 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:10:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:10:09] "GET /metrics HTTP/1.1" 200 33749 "" "Prometheus/2.43.0" 2023-09-23T22:10:09.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:09 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:10.250 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:10:10.250 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (2m) - 6m 30.5M - 0.25.0 c8568f914cd2 1cbbb123afed 2023-09-23T22:10:10.250 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (2m) - 5m 86.4M - 9.4.7 2c41d148cca3 42940e65ed5c 2023-09-23T22:10:10.250 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.rancfk smithi026 running (5m) - 5m 19.4M - 17.2.6-1125-gc9bb1bcf 444539aec5bf a9885df09771 2023-09-23T22:10:10.251 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.ehaqek smithi070 running (5m) - 5m 16.7M - 17.2.6-1125-gc9bb1bcf 444539aec5bf 812a5d180a78 2023-09-23T22:10:10.251 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (115s) - 9m 447M - 18.2.0-460-gbb0daeff e32c2a64c126 754e649f30a9 2023-09-23T22:10:10.251 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (2m) - 11m 500M - 18.2.0-460-gbb0daeff e32c2a64c126 2dff2050e669 2023-09-23T22:10:10.251 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (103s) - 11m 48.4M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0028efef8662 2023-09-23T22:10:10.251 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (10m) - 10m 79.6M 2048M 17.2.6-1125-gc9bb1bcf 444539aec5bf b61f3bc30723 2023-09-23T22:10:10.251 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (12s) - 10m 23.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 25f6f9fc0ac7 2023-09-23T22:10:10.252 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (2m) - 5m 18.7M - 1.5.0 0da6a335fe13 68b72a59fe08 2023-09-23T22:10:10.252 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (2m) - 5m 18.6M - 1.5.0 0da6a335fe13 523ab3522b38 2023-09-23T22:10:10.252 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (9m) - 9m 81.2M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 67a4086eb002 2023-09-23T22:10:10.252 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (9m) - 9m 106M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 02b1d9e99db9 2023-09-23T22:10:10.252 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (8m) - 8m 74.7M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 50e18b584478 2023-09-23T22:10:10.252 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (8m) - 8m 69.5M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 57db6dad61fc 2023-09-23T22:10:10.252 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (8m) - 8m 73.8M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 92889bd7f5ee 2023-09-23T22:10:10.252 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (7m) - 7m 88.8M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 628c1ee2ccfc 2023-09-23T22:10:10.253 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (7m) - 7m 73.8M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 5b472aeb7f69 2023-09-23T22:10:10.253 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (6m) - 6m 95.0M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 2a3425d84687 2023-09-23T22:10:10.253 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (2m) - 6m 53.5M - 2.43.0 a07b618ecd1d 3342388f10a7 2023-09-23T22:10:10.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:10 smithi026 ceph-mon[223014]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:10.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:10 smithi026 ceph-mon[223014]: mgrmap e36: y(active, since 90s), standbys: x 2023-09-23T22:10:10.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:10 smithi026 ceph-mon[215775]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:10.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:10 smithi026 ceph-mon[215775]: mgrmap e36: y(active, since 90s), standbys: x 2023-09-23T22:10:10.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:10 smithi070 ceph-mon[105170]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:10.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:10 smithi070 ceph-mon[105170]: mgrmap e36: y(active, since 90s), standbys: x 2023-09-23T22:10:11.051 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T22:10:11.429 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:10:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:10:11.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:11.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:11.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:11.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:10:11.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:11.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:11.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:10:11.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:10:11.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:11 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T22:10:11.634 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:10:11.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:11.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:11.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:11.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:10:11.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:11.635 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:11.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:10:11.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:10:11.636 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T22:10:11.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:10:11.636 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:11.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:11.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:11.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:10:11.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:11.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:11.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:10:11.637 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T22:10:11.638 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T22:10:12.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:12 smithi070 ceph-mon[105170]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:12.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:12 smithi070 ceph-mon[105170]: Upgrade: It appears safe to stop mon.b 2023-09-23T22:10:12.638 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:12 smithi026 ceph-mon[215775]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:12.638 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:12 smithi026 ceph-mon[215775]: Upgrade: It appears safe to stop mon.b 2023-09-23T22:10:12.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:12 smithi026 ceph-mon[223014]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:12.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:12 smithi026 ceph-mon[223014]: Upgrade: It appears safe to stop mon.b 2023-09-23T22:10:13.072 INFO:teuthology.orchestra.run.smithi026.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:10:13.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:13 smithi070 ceph-mon[105170]: Upgrade: Updating mon.b 2023-09-23T22:10:13.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:13 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:13.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:13 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:13 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:13 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:13 smithi070 ceph-mon[105170]: Deploying daemon mon.b on smithi070 2023-09-23T22:10:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:13 smithi070 ceph-mon[105170]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:13.715 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T22:10:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[215775]: Upgrade: Updating mon.b 2023-09-23T22:10:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[215775]: Deploying daemon mon.b on smithi070 2023-09-23T22:10:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[223014]: Upgrade: Updating mon.b 2023-09-23T22:10:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[223014]: Deploying daemon mon.b on smithi070 2023-09-23T22:10:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:13 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:14.082 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:10:14.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:13 smithi070 systemd[1]: Stopping Ceph mon.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:10:14.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:14 smithi070 conmon[105126]: 2023-09-23T22:10:14.246+0000 7f50f249a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:10:14.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:14 smithi070 conmon[105126]: 2023-09-23T22:10:14.246+0000 7f50f249a700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T22:10:15.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 bash[192230]: b61f3bc30723a214f18fa899a50721599d5e4b650e73b261a08e0ed7688eec2f 2023-09-23T22:10:15.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T22:10:15.327 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 systemd[1]: Stopped Ceph mon.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:10:15.327 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 systemd[1]: Starting Ceph mon.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:10:16.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:15 smithi026 ceph-mon[215775]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:16.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:15 smithi026 ceph-mon[215775]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:10:16.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:15 smithi026 ceph-mon[215775]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:10:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:15 smithi026 ceph-mon[223014]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:15 smithi026 ceph-mon[223014]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:10:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:15 smithi026 ceph-mon[223014]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:10:16.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: pidfile_write: ignore empty --pid-file 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: load: jerasure load: lrc 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: RocksDB version: 7.9.2 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Git sha 0 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: DB SUMMARY 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: DB Session ID: JPS41V9YPJOC3STLXFJ1 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: CURRENT file: CURRENT 2023-09-23T22:10:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1103 Bytes 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 4240866 ; 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.error_if_exists: 0 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.create_if_missing: 0 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.paranoid_checks: 1 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.env: 0x560ef5b88720 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T22:10:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.info_log: 0x560ef7b51420 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.statistics: (nil) 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.use_fsync: 0 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_log_file_size: 0 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.allow_fallocate: 1 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T22:10:16.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T22:10:16.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.use_direct_reads: 0 2023-09-23T22:10:16.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T22:10:16.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T22:10:16.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.db_log_dir: 2023-09-23T22:10:16.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.wal_dir: 2023-09-23T22:10:16.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T22:10:16.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T22:10:16.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.write_buffer_manager: 0x560ef6de05a0 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T22:10:16.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T22:10:16.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T22:10:16.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T22:10:16.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T22:10:16.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.unordered_write: 0 2023-09-23T22:10:16.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T22:10:16.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T22:10:16.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T22:10:16.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T22:10:16.380 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.row_cache: None 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.wal_filter: None 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.two_write_queues: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.wal_compression: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.atomic_flush: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.log_readahead_size: 0 2023-09-23T22:10:16.381 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_background_jobs: 2 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_background_compactions: -1 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_subcompactions: 1 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T22:10:16.382 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_open_files: -1 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T22:10:16.383 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_background_flushes: -1 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Compression algorithms supported: 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: kZSTD supported: 0 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: kXpressCompression supported: 0 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: kZlibCompression supported: 1 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: kSnappyCompression supported: 1 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: kLZ4Compression supported: 1 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: kBZip2Compression supported: 0 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T22:10:16.384 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.merge_operator: 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_filter: None 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x560ef7b51520) 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout: cache_index_and_filter_blocks: 1 2023-09-23T22:10:16.385 INFO:journalctl@ceph.mon.b.smithi070.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: pin_top_level_index_and_filter: 1 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: index_type: 0 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: data_block_index_type: 0 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: index_shortening: 1 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: checksum: 4 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: no_block_cache: 0 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache: 0x560ef6e63350 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache_name: BinnedLRUCache 2023-09-23T22:10:16.386 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache_options: 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: capacity : 536870912 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: num_shard_bits : 4 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: strict_capacity_limit : 0 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: high_pri_pool_ratio: 0.000 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_cache_compressed: (nil) 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: persistent_cache: (nil) 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_size: 4096 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_size_deviation: 10 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_restart_interval: 16 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: index_block_restart_interval: 1 2023-09-23T22:10:16.387 INFO:journalctl@ceph.mon.b.smithi070.stdout: metadata_block_size: 4096 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: partition_filters: 0 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: use_delta_encoding: 1 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: filter_policy: bloomfilter 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: whole_key_filtering: 1 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: verify_compression: 0 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: read_amp_bytes_per_bit: 0 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: format_version: 5 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: enable_index_compression: 1 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: block_align: 0 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: max_auto_readahead_size: 262144 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: prepopulate_block_cache: 0 2023-09-23T22:10:16.388 INFO:journalctl@ceph.mon.b.smithi070.stdout: initial_auto_readahead_size: 8192 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression: NoCompression 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.num_levels: 7 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T22:10:16.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 bash[192586]: f07a7787aa42b65c49e854f8360068f3eb3ae28e117e263eceda0e22818d80ce 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T22:10:16.390 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T22:10:16.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T22:10:16.392 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T22:10:16.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T22:10:16.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T22:10:16.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T22:10:16.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T22:10:16.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T22:10:16.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T22:10:16.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T22:10:16.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T22:10:16.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.table_properties_collectors: 2023-09-23T22:10:16.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.inplace_update_support: 0 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.bloom_locality: 0 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.max_successive_merges: 0 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.ttl: 2592000 2023-09-23T22:10:16.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T22:10:16.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T22:10:16.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T22:10:16.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.enable_blob_files: false 2023-09-23T22:10:16.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.min_blob_size: 0 2023-09-23T22:10:16.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T22:10:16.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T22:10:16.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T22:10:16.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13306, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3d5e5655-2442-4ad9-a7ff-f53889cc50da 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695507015893056, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T22:10:16.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 systemd[1]: Started Ceph mon.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:10:16.398 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695507015904479, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2634555, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13301, "largest_seqno": 14985, "table_properties": {"data_size": 2627825, "index_size": 4175, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1541, "raw_key_size": 13435, "raw_average_key_size": 22, "raw_value_size": 2614873, "raw_average_value_size": 4358, "num_data_blocks": 191, "num_entries": 600, "num_filter_entries": 600, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695507015, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3d5e5655-2442-4ad9-a7ff-f53889cc50da", "db_session_id": "JPS41V9YPJOC3STLXFJ1", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T22:10:16.398 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695507015904577, "job": 1, "event": "recovery_finished"} 2023-09-23T22:10:16.398 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T22:10:16.398 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T22:10:16.398 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T22:10:16.398 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x560ef6efe000 2023-09-23T22:10:16.398 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: DB pointer 0x560ef6eec000 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** DB Stats ** 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T22:10:16.399 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** Compaction Stats [default] ** 2023-09-23T22:10:16.400 INFO:journalctl@ceph.mon.b.smithi070.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:10:16.400 INFO:journalctl@ceph.mon.b.smithi070.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T22:10:16.400 INFO:journalctl@ceph.mon.b.smithi070.stdout: L0 1/0 2.51 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 520.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T22:10:16.400 INFO:journalctl@ceph.mon.b.smithi070.stdout: L6 1/0 6.52 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T22:10:16.400 INFO:journalctl@ceph.mon.b.smithi070.stdout: Sum 2/0 9.03 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 520.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T22:10:16.400 INFO:journalctl@ceph.mon.b.smithi070.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 520.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T22:10:16.400 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** Compaction Stats [default] ** 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 520.9 0.00 0.00 1 0.005 0 0 0.0 0.0 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T22:10:16.401 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T22:10:16.402 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T22:10:16.402 INFO:journalctl@ceph.mon.b.smithi070.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T22:10:16.402 INFO:journalctl@ceph.mon.b.smithi070.stdout: Cumulative compaction: 0.00 GB write, 181.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:10:16.402 INFO:journalctl@ceph.mon.b.smithi070.stdout: Interval compaction: 0.00 GB write, 181.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T22:10:16.402 INFO:journalctl@ceph.mon.b.smithi070.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T22:10:16.402 INFO:journalctl@ceph.mon.b.smithi070.stdout: Block cache BinnedLRUCache@0x560ef6e63350#7 capacity: 512.00 MB usage: 6.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T22:10:16.402 INFO:journalctl@ceph.mon.b.smithi070.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.78 KB,0.000339746%) IndexBlock(1,4.58 KB,0.000873208%) Misc(1,0.00 KB,0%) 2023-09-23T22:10:16.402 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T22:10:16.402 INFO:journalctl@ceph.mon.b.smithi070.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T22:10:16.403 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: starting mon.b rank 1 at public addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] at bind addrs [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T22:10:16.403 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: mon.b@-1(???) e3 preinit fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T22:10:16.403 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: mon.b@-1(???).mds e9 new map 2023-09-23T22:10:16.403 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: mon.b@-1(???).mds e9 print_map 2023-09-23T22:10:16.403 INFO:journalctl@ceph.mon.b.smithi070.stdout: e9 2023-09-23T22:10:16.403 INFO:journalctl@ceph.mon.b.smithi070.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T22:10:16.403 INFO:journalctl@ceph.mon.b.smithi070.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:10:16.403 INFO:journalctl@ceph.mon.b.smithi070.stdout: legacy client fscid: 1 2023-09-23T22:10:16.403 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T22:10:16.404 INFO:journalctl@ceph.mon.b.smithi070.stdout: Filesystem 'foo' (1) 2023-09-23T22:10:16.404 INFO:journalctl@ceph.mon.b.smithi070.stdout: fs_name foo 2023-09-23T22:10:16.404 INFO:journalctl@ceph.mon.b.smithi070.stdout: epoch 8 2023-09-23T22:10:16.404 INFO:journalctl@ceph.mon.b.smithi070.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T22:10:16.404 INFO:journalctl@ceph.mon.b.smithi070.stdout: created 2023-09-23T22:04:54.392171+0000 2023-09-23T22:10:16.404 INFO:journalctl@ceph.mon.b.smithi070.stdout: modified 2023-09-23T22:05:06.659218+0000 2023-09-23T22:10:16.404 INFO:journalctl@ceph.mon.b.smithi070.stdout: tableserver 0 2023-09-23T22:10:16.404 INFO:journalctl@ceph.mon.b.smithi070.stdout: root 0 2023-09-23T22:10:16.405 INFO:journalctl@ceph.mon.b.smithi070.stdout: session_timeout 60 2023-09-23T22:10:16.405 INFO:journalctl@ceph.mon.b.smithi070.stdout: session_autoclose 300 2023-09-23T22:10:16.405 INFO:journalctl@ceph.mon.b.smithi070.stdout: max_file_size 1099511627776 2023-09-23T22:10:16.405 INFO:journalctl@ceph.mon.b.smithi070.stdout: required_client_features {} 2023-09-23T22:10:16.405 INFO:journalctl@ceph.mon.b.smithi070.stdout: last_failure 0 2023-09-23T22:10:16.405 INFO:journalctl@ceph.mon.b.smithi070.stdout: last_failure_osd_epoch 0 2023-09-23T22:10:16.405 INFO:journalctl@ceph.mon.b.smithi070.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T22:10:16.405 INFO:journalctl@ceph.mon.b.smithi070.stdout: max_mds 1 2023-09-23T22:10:16.405 INFO:journalctl@ceph.mon.b.smithi070.stdout: in 0 2023-09-23T22:10:16.406 INFO:journalctl@ceph.mon.b.smithi070.stdout: up {0=14646} 2023-09-23T22:10:16.406 INFO:journalctl@ceph.mon.b.smithi070.stdout: failed 2023-09-23T22:10:16.406 INFO:journalctl@ceph.mon.b.smithi070.stdout: damaged 2023-09-23T22:10:16.406 INFO:journalctl@ceph.mon.b.smithi070.stdout: stopped 2023-09-23T22:10:16.406 INFO:journalctl@ceph.mon.b.smithi070.stdout: data_pools [4] 2023-09-23T22:10:16.406 INFO:journalctl@ceph.mon.b.smithi070.stdout: metadata_pool 3 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: inline_data disabled 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: balancer 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: bal_rank_mask -1 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: standby_count_wanted 1 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: [mds.foo.smithi026.rancfk{0:14646} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.26:6834/3702519723,v1:172.21.15.26:6835/3702519723] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: Standby daemons: 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout: [mds.foo.smithi070.ehaqek{-1:24520} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.70:6832/253331223,v1:172.21.15.70:6833/253331223] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T22:10:16.407 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: mon.b@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: mon.b@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: mon.b@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: mon.b@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta expand map: {default=false} 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta from 'false' to 'false' 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta expanded map: {default=false} 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta expand map: {default=info} 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta from 'info' to 'info' 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta expanded map: {default=info} 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta expand map: {default=daemon} 2023-09-23T22:10:16.408 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T22:10:16.409 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T22:10:16.409 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta expand map: {default=debug} 2023-09-23T22:10:16.409 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T22:10:16.409 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: expand_channel_meta expanded map: {default=debug} 2023-09-23T22:10:16.409 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:16.409 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:10:16.409 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:15 smithi070 ceph-mon[192652]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:10:17.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: overall HEALTH_OK 2023-09-23T22:10:17.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:10:17.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: mon.a calling monitor election 2023-09-23T22:10:17.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: mon.c calling monitor election 2023-09-23T22:10:17.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: mon.b calling monitor election 2023-09-23T22:10:17.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:10:17.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:10:17.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:10:17.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: monmap e4: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:10:17.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:10:17.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:10:17.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: mgrmap e36: y(active, since 97s), standbys: x 2023-09-23T22:10:17.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: overall HEALTH_OK 2023-09-23T22:10:17.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:17.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:17.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:17.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: overall HEALTH_OK 2023-09-23T22:10:17.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:10:17.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: mon.a calling monitor election 2023-09-23T22:10:17.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: mon.c calling monitor election 2023-09-23T22:10:17.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: mon.b calling monitor election 2023-09-23T22:10:17.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:10:17.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:10:17.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:10:17.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: monmap e4: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:10:17.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:10:17.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:10:17.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: mgrmap e36: y(active, since 97s), standbys: x 2023-09-23T22:10:17.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: overall HEALTH_OK 2023-09-23T22:10:17.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:17.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:17.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:17.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: overall HEALTH_OK 2023-09-23T22:10:17.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T22:10:17.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: mon.a calling monitor election 2023-09-23T22:10:17.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: mon.c calling monitor election 2023-09-23T22:10:17.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: mon.b calling monitor election 2023-09-23T22:10:17.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T22:10:17.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T22:10:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T22:10:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: monmap e4: 3 mons at {a=[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0],b=[v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0],c=[v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]} removed_ranks: {} 2023-09-23T22:10:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: fsmap foo:1 {0=foo.smithi026.rancfk=up:active} 1 up:standby 2023-09-23T22:10:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T22:10:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: mgrmap e36: y(active, since 97s), standbys: x 2023-09-23T22:10:17.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: overall HEALTH_OK 2023-09-23T22:10:17.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:17.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:17.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:18 smithi070 ceph-mon[192652]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:18.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:10:18 smithi070 conmon[176728]: ts=2023-09-23T22:10:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:10:18.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:18 smithi026 ceph-mon[223014]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:18 smithi026 ceph-mon[215775]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:10:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:10:19] "GET /metrics HTTP/1.1" 200 33749 "" "Prometheus/2.43.0" 2023-09-23T22:10:20.428 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:20 smithi070 ceph-mon[192652]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:20.428 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.428 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.428 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.428 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[223014]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[215775]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:21.235 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:10:20 smithi026 conmon[202047]: 2023-09-23T22:10:20.912+0000 7f47954a6700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T22:10:22.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:21 smithi070 ceph-mon[192652]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:22.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:21 smithi026 ceph-mon[215775]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:22.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:21 smithi026 ceph-mon[223014]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T22:10:24.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:24.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: Reconfiguring mon.a (monmap changed)... 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: Reconfiguring daemon mon.a on smithi026 2023-09-23T22:10:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:24.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:24.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:24.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:24.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:24.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: Reconfiguring mon.a (monmap changed)... 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: Reconfiguring daemon mon.a on smithi026 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:24.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:24.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:24.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:10:24.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:24.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: Reconfiguring mon.a (monmap changed)... 2023-09-23T22:10:24.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:24.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:24.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:24.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: Reconfiguring daemon mon.a on smithi026 2023-09-23T22:10:24.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:25.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:25.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:25.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[215775]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T22:10:25.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:10:25.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:10:25.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:25.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[215775]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T22:10:25.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[215775]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:25.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:25.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:25.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[223014]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T22:10:25.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:10:25.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:10:25.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:25.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[223014]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T22:10:25.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:25 smithi026 ceph-mon[223014]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:25 smithi070 ceph-mon[192652]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T22:10:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:10:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:10:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:25 smithi070 ceph-mon[192652]: Reconfiguring daemon mgr.y on smithi026 2023-09-23T22:10:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:25 smithi070 ceph-mon[192652]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[223014]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:10:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:27.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[223014]: Reconfiguring daemon mon.c on smithi026 2023-09-23T22:10:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[215775]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:10:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:27 smithi026 ceph-mon[215775]: Reconfiguring daemon mon.c on smithi026 2023-09-23T22:10:27.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:27 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:27.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:27 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:27.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:27 smithi070 ceph-mon[192652]: Reconfiguring mon.c (monmap changed)... 2023-09-23T22:10:27.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:27 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:27 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:27 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:27 smithi070 ceph-mon[192652]: Reconfiguring daemon mon.c on smithi026 2023-09-23T22:10:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[215775]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[215775]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T22:10:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:10:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:28.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[215775]: Reconfiguring daemon osd.0 on smithi026 2023-09-23T22:10:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[223014]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[223014]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T22:10:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:10:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:28 smithi026 ceph-mon[223014]: Reconfiguring daemon osd.0 on smithi026 2023-09-23T22:10:28.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:10:28 smithi070 conmon[176728]: ts=2023-09-23T22:10:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:10:28.925 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:28 smithi070 ceph-mon[192652]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:28.925 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:28 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:28.925 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:28 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:28.925 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:28 smithi070 ceph-mon[192652]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T22:10:28.925 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:28 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:10:28.926 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:28 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:28.926 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:28 smithi070 ceph-mon[192652]: Reconfiguring daemon osd.0 on smithi026 2023-09-23T22:10:29.864 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:10:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:10:29] "GET /metrics HTTP/1.1" 200 33742 "" "Prometheus/2.43.0" 2023-09-23T22:10:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:29 smithi070 ceph-mon[192652]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T22:10:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:10:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:29 smithi070 ceph-mon[192652]: Reconfiguring daemon osd.1 on smithi026 2023-09-23T22:10:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:29 smithi070 ceph-mon[192652]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:30.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:30.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:30.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[215775]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T22:10:30.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:10:30.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:30.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[215775]: Reconfiguring daemon osd.1 on smithi026 2023-09-23T22:10:30.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[215775]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:30.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:30.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:30.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[223014]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T22:10:30.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:10:30.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:30.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[223014]: Reconfiguring daemon osd.1 on smithi026 2023-09-23T22:10:30.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:29 smithi026 ceph-mon[223014]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:31.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:31.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:31.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[223014]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T22:10:31.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:10:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:31.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[223014]: Reconfiguring daemon osd.2 on smithi026 2023-09-23T22:10:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[215775]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T22:10:31.666 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:10:31.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:31.667 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:31 smithi026 ceph-mon[215775]: Reconfiguring daemon osd.2 on smithi026 2023-09-23T22:10:31.667 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:31 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:31.667 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:31 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:31.668 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:31 smithi070 ceph-mon[192652]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T22:10:31.668 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:31 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:10:31.668 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:31 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:31.668 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:31 smithi070 ceph-mon[192652]: Reconfiguring daemon osd.2 on smithi026 2023-09-23T22:10:32.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[223014]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:32.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[223014]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T22:10:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:10:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[223014]: Reconfiguring daemon osd.3 on smithi026 2023-09-23T22:10:32.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[215775]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:32.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:32.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:32.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[215775]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T22:10:32.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:10:32.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:32.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:32 smithi026 ceph-mon[215775]: Reconfiguring daemon osd.3 on smithi026 2023-09-23T22:10:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:32 smithi070 ceph-mon[192652]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:32 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:32 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:32 smithi070 ceph-mon[192652]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T22:10:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:32 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:10:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:32 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:32.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:32 smithi070 ceph-mon[192652]: Reconfiguring daemon osd.3 on smithi026 2023-09-23T22:10:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[215775]: Reconfiguring mds.foo.smithi026.rancfk (monmap changed)... 2023-09-23T22:10:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.rancfk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:10:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[215775]: Reconfiguring daemon mds.foo.smithi026.rancfk on smithi026 2023-09-23T22:10:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[215775]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[223014]: Reconfiguring mds.foo.smithi026.rancfk (monmap changed)... 2023-09-23T22:10:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.rancfk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:10:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[223014]: Reconfiguring daemon mds.foo.smithi026.rancfk on smithi026 2023-09-23T22:10:33.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:33 smithi026 ceph-mon[223014]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:33 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:33 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:33 smithi070 ceph-mon[192652]: Reconfiguring mds.foo.smithi026.rancfk (monmap changed)... 2023-09-23T22:10:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:33 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi026.rancfk", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:10:34.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:33 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:34.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:33 smithi070 ceph-mon[192652]: Reconfiguring daemon mds.foo.smithi026.rancfk on smithi026 2023-09-23T22:10:34.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:33 smithi070 ceph-mon[192652]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:35 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:35 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:35 smithi070 ceph-mon[192652]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:10:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:35 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:35 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:35 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:35 smithi070 ceph-mon[192652]: Reconfiguring daemon mon.b on smithi070 2023-09-23T22:10:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[223014]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:10:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:35.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[223014]: Reconfiguring daemon mon.b on smithi070 2023-09-23T22:10:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[215775]: Reconfiguring mon.b (monmap changed)... 2023-09-23T22:10:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T22:10:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T22:10:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:35 smithi026 ceph-mon[215775]: Reconfiguring daemon mon.b on smithi070 2023-09-23T22:10:36.840 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:36 smithi070 ceph-mon[192652]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:36.840 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:36.840 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:36.840 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:36 smithi070 ceph-mon[192652]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T22:10:36.840 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:10:36.840 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:10:36.840 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:36.841 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:36 smithi070 ceph-mon[192652]: Reconfiguring daemon mgr.x on smithi070 2023-09-23T22:10:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[215775]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[215775]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T22:10:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:10:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:10:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[215775]: Reconfiguring daemon mgr.x on smithi070 2023-09-23T22:10:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[223014]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[223014]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T22:10:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T22:10:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T22:10:36.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:36.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:36 smithi026 ceph-mon[223014]: Reconfiguring daemon mgr.x on smithi070 2023-09-23T22:10:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:37 smithi070 ceph-mon[192652]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T22:10:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:10:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:37 smithi070 ceph-mon[192652]: Reconfiguring daemon osd.4 on smithi070 2023-09-23T22:10:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:37 smithi070 ceph-mon[192652]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:38.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:38.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:38.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[223014]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T22:10:38.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:10:38.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:38.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[223014]: Reconfiguring daemon osd.4 on smithi070 2023-09-23T22:10:38.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[223014]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[215775]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T22:10:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T22:10:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[215775]: Reconfiguring daemon osd.4 on smithi070 2023-09-23T22:10:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:37 smithi026 ceph-mon[215775]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:39.066 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:10:38 smithi070 conmon[176728]: ts=2023-09-23T22:10:38.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:10:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:39 smithi070 ceph-mon[192652]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T22:10:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:10:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:39 smithi070 ceph-mon[192652]: Reconfiguring daemon osd.5 on smithi070 2023-09-23T22:10:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:39.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:39.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:39.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[223014]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T22:10:39.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:10:39.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:39.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[223014]: Reconfiguring daemon osd.5 on smithi070 2023-09-23T22:10:39.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[215775]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T22:10:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T22:10:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[215775]: Reconfiguring daemon osd.5 on smithi070 2023-09-23T22:10:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:39.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:10:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:10:39] "GET /metrics HTTP/1.1" 200 33740 "" "Prometheus/2.43.0" 2023-09-23T22:10:40.825 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:40 smithi070 ceph-mon[192652]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:40.930 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:40.930 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:40.931 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:10:40.931 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:40.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[223014]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:40.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:40.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:40.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:10:40.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:40.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[215775]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:40.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:40.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:40.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T22:10:40.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:41.798 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:41 smithi070 ceph-mon[192652]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T22:10:41.798 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:41 smithi070 ceph-mon[192652]: Reconfiguring daemon osd.6 on smithi070 2023-09-23T22:10:41.799 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:41.799 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:41.799 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:10:41.799 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:41.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[223014]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T22:10:41.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[223014]: Reconfiguring daemon osd.6 on smithi070 2023-09-23T22:10:41.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:41.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:41.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:10:41.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[215775]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T22:10:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[215775]: Reconfiguring daemon osd.6 on smithi070 2023-09-23T22:10:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T22:10:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:42.846 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:42 smithi070 ceph-mon[192652]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:42.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:42 smithi070 ceph-mon[192652]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T22:10:42.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:42 smithi070 ceph-mon[192652]: Reconfiguring daemon osd.7 on smithi070 2023-09-23T22:10:42.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:42 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:42.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:42 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:42.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:42 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.ehaqek", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:10:42.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:42 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[223014]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[223014]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T22:10:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[223014]: Reconfiguring daemon osd.7 on smithi070 2023-09-23T22:10:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.ehaqek", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:10:42.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[215775]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[215775]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T22:10:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[215775]: Reconfiguring daemon osd.7 on smithi070 2023-09-23T22:10:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi070.ehaqek", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T22:10:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:42 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:43 smithi070 ceph-mon[192652]: Reconfiguring mds.foo.smithi070.ehaqek (monmap changed)... 2023-09-23T22:10:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:43 smithi070 ceph-mon[192652]: Reconfiguring daemon mds.foo.smithi070.ehaqek on smithi070 2023-09-23T22:10:43.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:43 smithi026 ceph-mon[223014]: Reconfiguring mds.foo.smithi070.ehaqek (monmap changed)... 2023-09-23T22:10:43.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:43 smithi026 ceph-mon[223014]: Reconfiguring daemon mds.foo.smithi070.ehaqek on smithi070 2023-09-23T22:10:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:43 smithi026 ceph-mon[215775]: Reconfiguring mds.foo.smithi070.ehaqek (monmap changed)... 2023-09-23T22:10:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:43 smithi026 ceph-mon[215775]: Reconfiguring daemon mds.foo.smithi070.ehaqek on smithi070 2023-09-23T22:10:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:44 smithi070 ceph-mon[192652]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:10:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:44.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[223014]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:44.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:10:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[215775]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:10:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:10:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:10:44.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:10:46.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:46 smithi070 ceph-mon[192652]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:46 smithi026 ceph-mon[223014]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:46 smithi026 ceph-mon[215775]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:48.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:48 smithi070 ceph-mon[192652]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:48.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:10:48 smithi070 conmon[176728]: ts=2023-09-23T22:10:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:10:48.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:48 smithi026 ceph-mon[223014]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:48.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:48 smithi026 ceph-mon[215775]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:10:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:10:49] "GET /metrics HTTP/1.1" 200 33740 "" "Prometheus/2.43.0" 2023-09-23T22:10:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:50 smithi070 ceph-mon[192652]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:50 smithi026 ceph-mon[223014]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:50 smithi026 ceph-mon[215775]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:52 smithi070 ceph-mon[192652]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:52 smithi026 ceph-mon[223014]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:52 smithi026 ceph-mon[215775]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:54 smithi070 ceph-mon[192652]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:54 smithi026 ceph-mon[223014]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:54 smithi026 ceph-mon[215775]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:10:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:56 smithi070 ceph-mon[192652]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:56 smithi026 ceph-mon[223014]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:56 smithi026 ceph-mon[215775]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:58.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:10:58 smithi070 ceph-mon[192652]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:58.873 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:10:58 smithi070 conmon[176728]: ts=2023-09-23T22:10:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:10:58.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:10:58 smithi026 ceph-mon[223014]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:58.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:10:58 smithi026 ceph-mon[215775]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:10:59.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:10:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:10:59] "GET /metrics HTTP/1.1" 200 33754 "" "Prometheus/2.43.0" 2023-09-23T22:11:00.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:00 smithi070 ceph-mon[192652]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:00.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:00 smithi026 ceph-mon[223014]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:00.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:00 smithi026 ceph-mon[215775]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:02.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:02 smithi070 ceph-mon[192652]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:02.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:02 smithi026 ceph-mon[223014]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:02.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:02 smithi026 ceph-mon[215775]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:04 smithi070 ceph-mon[192652]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:04.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:04 smithi026 ceph-mon[215775]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:04.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:04 smithi026 ceph-mon[223014]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:05.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:05 smithi026 ceph-mon[223014]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:05.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:05 smithi026 ceph-mon[215775]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:06.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:05 smithi070 ceph-mon[192652]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:08.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:08 smithi070 ceph-mon[192652]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:08.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:11:08 smithi070 conmon[176728]: ts=2023-09-23T22:11:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:11:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:08 smithi026 ceph-mon[215775]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:08.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:08 smithi026 ceph-mon[223014]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:09.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:11:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:11:09] "GET /metrics HTTP/1.1" 200 33753 "" "Prometheus/2.43.0" 2023-09-23T22:11:09.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:09.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:10.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:10 smithi070 ceph-mon[192652]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:10 smithi026 ceph-mon[223014]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:10 smithi026 ceph-mon[215775]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:12.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:12 smithi070 ceph-mon[192652]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:12 smithi026 ceph-mon[223014]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:12 smithi026 ceph-mon[215775]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:14.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:14 smithi070 ceph-mon[192652]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:14 smithi026 ceph-mon[223014]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:14 smithi026 ceph-mon[215775]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:16.088 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T22:11:16.460 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:11:16.840 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:16 smithi026 ceph-mon[223014]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:16.840 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:16 smithi026 ceph-mon[215775]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:16 smithi070 ceph-mon[192652]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:18.137 INFO:teuthology.orchestra.run.smithi026.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:11:18.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:18 smithi026 ceph-mon[215775]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:18.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:18 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:18.838 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:18 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:18.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:18 smithi026 ceph-mon[223014]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:18.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:18 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:18.839 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:18 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:18.855 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T22:11:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:18 smithi070 ceph-mon[192652]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:18 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:18 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:18.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:11:18 smithi070 conmon[176728]: ts=2023-09-23T22:11:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:11:19.374 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:11:19.825 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:19 smithi026 ceph-mon[215775]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:11:19.826 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:19 smithi026 ceph-mon[215775]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:11:19.826 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:11:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:11:19] "GET /metrics HTTP/1.1" 200 33753 "" "Prometheus/2.43.0" 2023-09-23T22:11:19.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:19 smithi026 ceph-mon[223014]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:11:19.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:19 smithi026 ceph-mon[223014]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:11:19.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:19 smithi070 ceph-mon[192652]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:11:19.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:19 smithi070 ceph-mon[192652]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:11:20.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:20 smithi070 ceph-mon[192652]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:20.952 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:20 smithi026 ceph-mon[215775]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:20.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:20 smithi026 ceph-mon[223014]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:21.075 INFO:teuthology.orchestra.run.smithi026.stdout:wait for all mons to upgrade 2023-09-23T22:11:21.685 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T22:11:22.398 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:11:22.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:22 smithi070 ceph-mon[192652]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:22.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:22 smithi026 ceph-mon[215775]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:22.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:22 smithi026 ceph-mon[223014]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:24.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:24.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T22:11:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T22:11:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T22:11:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T22:11:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T22:11:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:11:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:11:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:24.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:24.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T22:11:24.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T22:11:24.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T22:11:24.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T22:11:24.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:11:24.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:11:24.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:24.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:24.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T22:11:24.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T22:11:24.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T22:11:24.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T22:11:24.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T22:11:24.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T22:11:24.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:11:24.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:24.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:11:24.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:24.563 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T22:11:25.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:25 smithi070 ceph-mon[192652]: Upgrade: Setting container_image for all mon 2023-09-23T22:11:25.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:25 smithi070 ceph-mon[192652]: Upgrade: Setting container_image for all crash 2023-09-23T22:11:25.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:25 smithi070 ceph-mon[192652]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:11:25.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:25 smithi070 ceph-mon[192652]: Upgrade: osd.0 is safe to restart 2023-09-23T22:11:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3628407559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:11:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[215775]: Upgrade: Setting container_image for all mon 2023-09-23T22:11:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[215775]: Upgrade: Setting container_image for all crash 2023-09-23T22:11:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[215775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:11:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[215775]: Upgrade: osd.0 is safe to restart 2023-09-23T22:11:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3628407559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:11:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[223014]: Upgrade: Setting container_image for all mon 2023-09-23T22:11:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[223014]: Upgrade: Setting container_image for all crash 2023-09-23T22:11:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[223014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T22:11:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[223014]: Upgrade: osd.0 is safe to restart 2023-09-23T22:11:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3628407559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T22:11:25.539 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:11:25.924 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:11:26.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[223014]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:26.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[223014]: Upgrade: Updating osd.0 2023-09-23T22:11:26.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:26.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:11:26.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:26.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[223014]: Deploying daemon osd.0 on smithi026 2023-09-23T22:11:26.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[215775]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:26.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[215775]: Upgrade: Updating osd.0 2023-09-23T22:11:26.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:26.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:11:26.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:26.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:26 smithi026 ceph-mon[215775]: Deploying daemon osd.0 on smithi026 2023-09-23T22:11:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:26 smithi070 ceph-mon[192652]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:26 smithi070 ceph-mon[192652]: Upgrade: Updating osd.0 2023-09-23T22:11:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:26 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:26 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T22:11:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:26 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:26 smithi070 ceph-mon[192652]: Deploying daemon osd.0 on smithi026 2023-09-23T22:11:28.170 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (3m) - 7m 30.5M - 0.25.0 c8568f914cd2 1cbbb123afed 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (3m) - 6m 91.3M - 9.4.7 2c41d148cca3 42940e65ed5c 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.rancfk smithi026 running (6m) - 6m 19.5M - 17.2.6-1125-gc9bb1bcf 444539aec5bf a9885df09771 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.ehaqek smithi070 running (6m) - 6m 17.1M - 17.2.6-1125-gc9bb1bcf 444539aec5bf 812a5d180a78 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (3m) - 11m 447M - 18.2.0-460-gbb0daeff e32c2a64c126 754e649f30a9 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (3m) - 13m 504M - 18.2.0-460-gbb0daeff e32c2a64c126 2dff2050e669 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (3m) - 13m 58.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0028efef8662 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (72s) - 11m 28.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 f07a7787aa42 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (90s) - 11m 44.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 25f6f9fc0ac7 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (3m) - 7m 18.8M - 1.5.0 0da6a335fe13 68b72a59fe08 2023-09-23T22:11:28.171 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (3m) - 7m 19.0M - 1.5.0 0da6a335fe13 523ab3522b38 2023-09-23T22:11:28.172 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (10m) - 10m 84.7M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 67a4086eb002 2023-09-23T22:11:28.172 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (10m) - 10m 109M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 02b1d9e99db9 2023-09-23T22:11:28.172 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (10m) - 10m 77.0M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 50e18b584478 2023-09-23T22:11:28.172 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (9m) - 9m 73.1M 2501M 17.2.6-1125-gc9bb1bcf 444539aec5bf 57db6dad61fc 2023-09-23T22:11:28.172 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (9m) - 9m 75.3M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 92889bd7f5ee 2023-09-23T22:11:28.172 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (9m) - 9m 89.9M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 628c1ee2ccfc 2023-09-23T22:11:28.172 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (8m) - 8m 75.0M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 5b472aeb7f69 2023-09-23T22:11:28.172 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (8m) - 8m 96.3M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 2a3425d84687 2023-09-23T22:11:28.172 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (3m) - 7m 53.7M - 2.43.0 a07b618ecd1d 3342388f10a7 2023-09-23T22:11:28.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:28 smithi026 ceph-mon[223014]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:28 smithi026 ceph-mon[215775]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:28.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:28 smithi070 ceph-mon[192652]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:28.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:11:28 smithi070 conmon[176728]: ts=2023-09-23T22:11:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:11:29.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T22:11:29.735 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:11:29.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:29 smithi070 ceph-mon[192652]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:11:29.903 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:11:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:11:29] "GET /metrics HTTP/1.1" 200 33755 "" "Prometheus/2.43.0" 2023-09-23T22:11:29.903 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:29 smithi026 systemd[1]: Stopping Ceph osd.0 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:11:29.904 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:29 smithi026 ceph-mon[215775]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:11:29.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:29 smithi026 ceph-mon[223014]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:11:30.179 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:29 smithi026 conmon[125164]: 2023-09-23T22:11:29.901+0000 7f64ce97a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:11:30.180 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:29 smithi026 conmon[125164]: 2023-09-23T22:11:29.901+0000 7f64ce97a700 -1 osd.0 89 *** Got signal Terminated *** 2023-09-23T22:11:30.180 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:29 smithi026 conmon[125164]: 2023-09-23T22:11:29.901+0000 7f64ce97a700 -1 osd.0 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:11:30.868 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:30 smithi026 ceph-mon[215775]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:30.869 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:30 smithi026 ceph-mon[215775]: osd.0 marked itself down and dead 2023-09-23T22:11:30.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:30 smithi026 ceph-mon[223014]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:30.869 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:30 smithi026 ceph-mon[223014]: osd.0 marked itself down and dead 2023-09-23T22:11:30.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:30 smithi070 ceph-mon[192652]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:30.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:30 smithi070 ceph-mon[192652]: osd.0 marked itself down and dead 2023-09-23T22:11:31.482 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:31 smithi026 bash[235725]: 67a4086eb00277f905e24df03d2efbe849bf3270e899044a086300bcbdb931df 2023-09-23T22:11:31.558 INFO:teuthology.orchestra.run.smithi026.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:11:31.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[215775]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:11:31.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[215775]: failed to encode map e90 with expected crc 2023-09-23T22:11:31.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[215775]: failed to encode map e90 with expected crc 2023-09-23T22:11:31.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[215775]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T22:11:31.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:31.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[223014]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:11:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[223014]: failed to encode map e90 with expected crc 2023-09-23T22:11:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[223014]: failed to encode map e90 with expected crc 2023-09-23T22:11:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[223014]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T22:11:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:31 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:31 smithi070 ceph-mon[192652]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:11:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:31 smithi070 ceph-mon[192652]: failed to encode map e90 with expected crc 2023-09-23T22:11:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:31 smithi070 ceph-mon[192652]: failed to encode map e90 with expected crc 2023-09-23T22:11:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:31 smithi070 ceph-mon[192652]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T22:11:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:31 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:32.517 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T22:11:32.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:32.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:11:32.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[223014]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:32.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: failed to encode map e91 with expected crc 2023-09-23T22:11:32.850 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:32 smithi026 ceph-mon[215775]: failed to encode map e91 with expected crc 2023-09-23T22:11:33.059 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:11:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: pgmap v90: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: from='client.44132 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: failed to encode map e91 with expected crc 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: failed to encode map e91 with expected crc 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: failed to encode map e91 with expected crc 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: failed to encode map e91 with expected crc 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: failed to encode map e91 with expected crc 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: failed to encode map e91 with expected crc 2023-09-23T22:11:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: failed to encode map e91 with expected crc 2023-09-23T22:11:33.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:32 smithi070 ceph-mon[192652]: failed to encode map e91 with expected crc 2023-09-23T22:11:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:33 smithi026 ceph-mon[215775]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:33 smithi026 ceph-mon[223014]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:33 smithi070 ceph-mon[192652]: pgmap v92: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:34.456 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:33 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T22:11:34.457 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:33 smithi026 systemd[1]: Stopped Ceph osd.0 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:11:34.457 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:34 smithi026 systemd[1]: Starting Ceph osd.0 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:11:35.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:35 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:11:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:35 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:11:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:35 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:11:35.735 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:35 smithi026 bash[236904]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:11:35.736 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:35 smithi026 bash[236904]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:11:35.736 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:35 smithi026 bash[236904]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T22:11:35.736 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:35 smithi026 bash[236904]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:11:35.736 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:35 smithi026 bash[236904]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T22:11:35.736 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:35 smithi026 bash[236904]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T22:11:35.736 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:35 smithi026 bash[236904]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T22:11:35.736 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:35 smithi026 bash[236904]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T22:11:36.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:36 smithi070 ceph-mon[192652]: pgmap v93: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:36.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:36 smithi026 ceph-mon[223014]: pgmap v93: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:36.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:36 smithi026 ceph-mon[215775]: pgmap v93: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:36.985 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:36 smithi026 ceph-osd[237405]: -- 172.21.15.26:0/3639236032 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b6f212e4e0 con 0x55b6f2101400 2023-09-23T22:11:36.985 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:36 smithi026 bash[237339]: dd9b16818fa43f894b879ea8eb90e8fca940b3f8846252894af29d0147a17fb6 2023-09-23T22:11:36.985 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:36 smithi026 systemd[1]: Started Ceph osd.0 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:11:37.485 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:37 smithi026 conmon[237380]: 2023-09-23T22:11:37.123+0000 7fd2b2732640 -1 Falling back to public interface 2023-09-23T22:11:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:37.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:37.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:37 smithi026 ceph-mon[215775]: pgmap v94: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:37 smithi026 ceph-mon[223014]: pgmap v94: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:37 smithi070 ceph-mon[192652]: pgmap v94: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:39.123 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:11:38 smithi070 conmon[176728]: ts=2023-09-23T22:11:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:11:39.915 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:11:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:11:39] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.43.0" 2023-09-23T22:11:40.235 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:39 smithi026 conmon[237380]: 2023-09-23T22:11:39.914+0000 7fd2b2732640 -1 osd.0 89 log_to_monitors true 2023-09-23T22:11:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[215775]: pgmap v95: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[215775]: from='osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:11:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[223014]: pgmap v95: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:39 smithi026 ceph-mon[223014]: from='osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:11:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:39 smithi070 ceph-mon[192652]: pgmap v95: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:39 smithi070 ceph-mon[192652]: from='osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T22:11:40.985 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:11:40 smithi026 conmon[237380]: 2023-09-23T22:11:40.699+0000 7fd29fb97700 -1 osd.0 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:11:41.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:41.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:41 smithi070 ceph-mon[192652]: failed to encode map e92 with expected crc 2023-09-23T22:11:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:41 smithi070 ceph-mon[192652]: from='osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:11:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:41 smithi070 ceph-mon[192652]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T22:11:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:41 smithi070 ceph-mon[192652]: from='osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:11:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[223014]: failed to encode map e92 with expected crc 2023-09-23T22:11:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[223014]: from='osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:11:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[223014]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T22:11:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[223014]: from='osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:11:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[215775]: failed to encode map e92 with expected crc 2023-09-23T22:11:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[215775]: from='osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T22:11:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[215775]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T22:11:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:41 smithi026 ceph-mon[215775]: from='osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:11:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:42 smithi070 ceph-mon[192652]: failed to encode map e92 with expected crc 2023-09-23T22:11:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:42 smithi070 ceph-mon[192652]: pgmap v97: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:42 smithi070 ceph-mon[192652]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:11:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:42 smithi070 ceph-mon[192652]: failed to encode map e93 with expected crc 2023-09-23T22:11:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:42 smithi070 ceph-mon[192652]: osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654] boot 2023-09-23T22:11:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:42 smithi070 ceph-mon[192652]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T22:11:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:42 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:11:42.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[215775]: failed to encode map e92 with expected crc 2023-09-23T22:11:42.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[215775]: pgmap v97: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[215775]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:11:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[215775]: failed to encode map e93 with expected crc 2023-09-23T22:11:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[215775]: osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654] boot 2023-09-23T22:11:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[215775]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T22:11:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:11:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[223014]: failed to encode map e92 with expected crc 2023-09-23T22:11:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[223014]: pgmap v97: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:42.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[223014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:11:42.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[223014]: failed to encode map e93 with expected crc 2023-09-23T22:11:42.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[223014]: osd.0 [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654] boot 2023-09-23T22:11:42.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[223014]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T22:11:42.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:42 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T22:11:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:43 smithi070 ceph-mon[192652]: failed to encode map e93 with expected crc 2023-09-23T22:11:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:43 smithi070 ceph-mon[192652]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:43 smithi070 ceph-mon[192652]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:43 smithi070 ceph-mon[192652]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T22:11:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:43 smithi070 ceph-mon[192652]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:43 smithi070 ceph-mon[192652]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[215775]: failed to encode map e93 with expected crc 2023-09-23T22:11:43.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[215775]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[215775]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[215775]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T22:11:43.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[215775]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[215775]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[223014]: failed to encode map e93 with expected crc 2023-09-23T22:11:43.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[223014]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[223014]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[223014]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T22:11:43.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[223014]: failed to encode map e94 with expected crc 2023-09-23T22:11:43.385 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:43 smithi026 ceph-mon[223014]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: pgmap v100: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:44.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:11:44.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: pgmap v100: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:44.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:11:44.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: failed to encode map e94 with expected crc 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: pgmap v100: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:11:44.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:45 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:11:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:45 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:11:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:45 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:11:46.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:46 smithi070 ceph-mon[192652]: pgmap v101: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T22:11:46.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:46 smithi026 ceph-mon[223014]: pgmap v101: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T22:11:46.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:46 smithi026 ceph-mon[215775]: pgmap v101: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T22:11:47.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:47 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T22:11:47.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:47 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:11:47.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:47 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T22:11:47.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:47 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:11:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:47 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T22:11:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:47 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:11:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:48 smithi070 ceph-mon[192652]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:48.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:11:48 smithi070 conmon[176728]: ts=2023-09-23T22:11:48.660Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-09-23T22:11:48.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:11:48 smithi070 conmon[176728]: ts=2023-09-23T22:11:48.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:11:48.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:48 smithi026 ceph-mon[223014]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:48.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:48 smithi026 ceph-mon[215775]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:11:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:11:49] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.43.0" 2023-09-23T22:11:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:50 smithi070 ceph-mon[192652]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:50 smithi026 ceph-mon[223014]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:50 smithi026 ceph-mon[215775]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:52 smithi070 ceph-mon[192652]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:52 smithi026 ceph-mon[223014]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:52.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:52 smithi026 ceph-mon[215775]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:54 smithi070 ceph-mon[192652]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:54 smithi026 ceph-mon[215775]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:54 smithi026 ceph-mon[223014]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:11:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:11:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:56 smithi070 ceph-mon[192652]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:56 smithi026 ceph-mon[215775]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:56 smithi026 ceph-mon[223014]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:58.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:11:58 smithi070 ceph-mon[192652]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:58.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:11:58 smithi070 conmon[176728]: ts=2023-09-23T22:11:58.660Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephOSDDown: template: __alert_CephOSDDown:1:358: executing \"__alert_CephOSDDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2023-09-23T22:11:58.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:11:58 smithi070 conmon[176728]: ts=2023-09-23T22:11:58.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:11:58.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:11:58 smithi026 ceph-mon[215775]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:58.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:11:58 smithi026 ceph-mon[223014]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:11:59.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:11:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:11:59] "GET /metrics HTTP/1.1" 200 33886 "" "Prometheus/2.43.0" 2023-09-23T22:12:00.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:00 smithi070 ceph-mon[192652]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:00.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:00 smithi026 ceph-mon[215775]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:00.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:00 smithi026 ceph-mon[223014]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:02.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:02 smithi070 ceph-mon[192652]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:02.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:02 smithi026 ceph-mon[223014]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:02.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:02 smithi026 ceph-mon[215775]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:04 smithi070 ceph-mon[192652]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:04.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:04 smithi026 ceph-mon[223014]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:04.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:04 smithi026 ceph-mon[215775]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:06 smithi070 ceph-mon[192652]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:06 smithi026 ceph-mon[223014]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:06 smithi026 ceph-mon[215775]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:08.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:08 smithi070 ceph-mon[192652]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:08.873 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:12:08 smithi070 conmon[176728]: ts=2023-09-23T22:12:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:12:08.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:08 smithi026 ceph-mon[223014]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:08 smithi026 ceph-mon[215775]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:09.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:12:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:12:09] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.43.0" 2023-09-23T22:12:09.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:10.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:10 smithi070 ceph-mon[192652]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:10 smithi026 ceph-mon[223014]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:10 smithi026 ceph-mon[215775]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:12.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:12 smithi070 ceph-mon[192652]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:12 smithi026 ceph-mon[223014]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:12.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:12 smithi026 ceph-mon[215775]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:14.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:14 smithi070 ceph-mon[192652]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:14.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:14 smithi026 ceph-mon[223014]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:14.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:14 smithi026 ceph-mon[215775]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:16 smithi070 ceph-mon[192652]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:16 smithi026 ceph-mon[223014]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:16 smithi026 ceph-mon[215775]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:18 smithi070 ceph-mon[192652]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:18.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:12:18 smithi070 conmon[176728]: ts=2023-09-23T22:12:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:12:18.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:18 smithi026 ceph-mon[223014]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:18 smithi026 ceph-mon[215775]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:12:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:12:19] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.43.0" 2023-09-23T22:12:20.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:20 smithi070 ceph-mon[192652]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:20 smithi026 ceph-mon[215775]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:20.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:20 smithi026 ceph-mon[223014]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:22.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:22 smithi070 ceph-mon[192652]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:22 smithi026 ceph-mon[215775]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:22 smithi026 ceph-mon[223014]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:24.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:24 smithi070 ceph-mon[192652]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:24.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:24 smithi026 ceph-mon[215775]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:24 smithi026 ceph-mon[223014]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:26 smithi070 ceph-mon[192652]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:26.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:26 smithi026 ceph-mon[215775]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:26 smithi026 ceph-mon[223014]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:28.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:28 smithi070 ceph-mon[192652]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:28.874 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:12:28 smithi070 conmon[176728]: ts=2023-09-23T22:12:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi026:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)\", cluster_addr=\"172.21.15.26\", device_class=\"ssd\", hostname=\"smithi026\", instance=\"smithi070.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.26\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T22:12:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:28 smithi026 ceph-mon[223014]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:28.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:28 smithi026 ceph-mon[215775]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:29.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:12:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:12:29] "GET /metrics HTTP/1.1" 200 33883 "" "Prometheus/2.43.0" 2023-09-23T22:12:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:30 smithi070 ceph-mon[192652]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:30.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:30 smithi026 ceph-mon[223014]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:30.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:30 smithi026 ceph-mon[215775]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:32 smithi070 ceph-mon[192652]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:32.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:32 smithi026 ceph-mon[215775]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:32.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:32 smithi026 ceph-mon[223014]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:34.849 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:34 smithi026 ceph-mon[223014]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:34.849 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:34 smithi026 ceph-mon[215775]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:34.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:34 smithi070 ceph-mon[192652]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:35.321 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T22:12:35.684 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:12:36.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:36 smithi026 ceph-mon[223014]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:36.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:36 smithi026 ceph-mon[215775]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:36 smithi070 ceph-mon[192652]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:37.456 INFO:teuthology.orchestra.run.smithi026.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:12:38.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T22:12:38.684 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:12:38.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:38.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:12:38.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:12:38.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:38.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:38.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:12:38.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:12:38.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:38.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:38.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:38.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:38.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:38.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:12:38.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:38.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:12:38.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:12:38.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:38.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:38.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:12:38.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:12:38.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:38.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:38.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:38.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:38.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:38.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:12:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:12:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:12:39.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:12:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:12:39] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2023-09-23T22:12:39.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:39 smithi026 ceph-mon[223014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:12:39.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:39 smithi026 ceph-mon[223014]: Upgrade: osd.1 is safe to restart 2023-09-23T22:12:39.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:39.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:39 smithi026 ceph-mon[215775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:12:39.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:39 smithi026 ceph-mon[215775]: Upgrade: osd.1 is safe to restart 2023-09-23T22:12:39.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:39 smithi070 ceph-mon[192652]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T22:12:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:39 smithi070 ceph-mon[192652]: Upgrade: osd.1 is safe to restart 2023-09-23T22:12:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:40.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[215775]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:40.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[215775]: Upgrade: Updating osd.1 2023-09-23T22:12:40.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:40.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:12:40.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:12:40.542 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[215775]: Deploying daemon osd.1 on smithi026 2023-09-23T22:12:40.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[223014]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:40.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[223014]: Upgrade: Updating osd.1 2023-09-23T22:12:40.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:40.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:12:40.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:12:40.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:40 smithi026 ceph-mon[223014]: Deploying daemon osd.1 on smithi026 2023-09-23T22:12:40.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:40 smithi070 ceph-mon[192652]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:40.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:40 smithi070 ceph-mon[192652]: Upgrade: Updating osd.1 2023-09-23T22:12:40.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T22:12:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:12:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:40 smithi070 ceph-mon[192652]: Deploying daemon osd.1 on smithi026 2023-09-23T22:12:41.277 INFO:teuthology.orchestra.run.smithi026.stdout:wait for half of osds to upgrade 2023-09-23T22:12:41.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T22:12:42.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:42 smithi026 ceph-mon[215775]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:42.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:42 smithi026 ceph-mon[223014]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:42.855 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:12:42.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:42 smithi070 ceph-mon[192652]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:43.526 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:43 smithi026 systemd[1]: Stopping Ceph osd.1 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:12:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:43 smithi070 ceph-mon[192652]: osd.1 marked itself down and dead 2023-09-23T22:12:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:43 smithi026 ceph-mon[215775]: osd.1 marked itself down and dead 2023-09-23T22:12:43.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:43 smithi026 ceph-mon[223014]: osd.1 marked itself down and dead 2023-09-23T22:12:43.986 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:43 smithi026 conmon[134913]: 2023-09-23T22:12:43.524+0000 7fdeac856700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:12:43.986 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:43 smithi026 conmon[134913]: 2023-09-23T22:12:43.524+0000 7fdeac856700 -1 osd.1 94 *** Got signal Terminated *** 2023-09-23T22:12:43.986 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:43 smithi026 conmon[134913]: 2023-09-23T22:12:43.524+0000 7fdeac856700 -1 osd.1 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:12:44.485 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:44 smithi026 bash[244690]: 02b1d9e99db9fee3b390f14c86676dcfd5b63a48123df92c73725365dc198707 2023-09-23T22:12:44.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[223014]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:44.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[223014]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:12:44.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[223014]: failed to encode map e95 with expected crc 2023-09-23T22:12:44.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[223014]: failed to encode map e95 with expected crc 2023-09-23T22:12:44.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[223014]: osdmap e95: 8 total, 7 up, 8 in 2023-09-23T22:12:44.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[215775]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:44.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[215775]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:12:44.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[215775]: failed to encode map e95 with expected crc 2023-09-23T22:12:44.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[215775]: failed to encode map e95 with expected crc 2023-09-23T22:12:44.813 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:44 smithi026 ceph-mon[215775]: osdmap e95: 8 total, 7 up, 8 in 2023-09-23T22:12:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:44 smithi070 ceph-mon[192652]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:12:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:44 smithi070 ceph-mon[192652]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:12:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:44 smithi070 ceph-mon[192652]: failed to encode map e95 with expected crc 2023-09-23T22:12:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:44 smithi070 ceph-mon[192652]: failed to encode map e95 with expected crc 2023-09-23T22:12:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:44 smithi070 ceph-mon[192652]: osdmap e95: 8 total, 7 up, 8 in 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.703 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2398528940' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:45.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T22:12:45.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2398528940' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:45.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: failed to encode map e96 with expected crc 2023-09-23T22:12:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2398528940' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:46.735 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:46 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T22:12:46.736 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:46 smithi026 systemd[1]: Stopped Ceph osd.1 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:12:46.736 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:46 smithi026 systemd[1]: Starting Ceph osd.1 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:12:46.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:46 smithi026 ceph-mon[215775]: pgmap v133: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 25 active+undersized, 44 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:12:46.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:46 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:12:46.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:46 smithi026 ceph-mon[223014]: pgmap v133: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 25 active+undersized, 44 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:12:46.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:46 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:12:46.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:46 smithi070 ceph-mon[192652]: pgmap v133: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 25 active+undersized, 44 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:12:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:46 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:12:47.735 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:47 smithi026 bash[245484]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:12:47.736 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:47 smithi026 bash[245484]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T22:12:47.736 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:47 smithi026 bash[245484]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T22:12:47.736 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:47 smithi026 bash[245484]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T22:12:47.736 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:47 smithi026 bash[245484]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T22:12:47.736 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:47 smithi026 bash[245484]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T22:12:47.736 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:47 smithi026 bash[245484]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T22:12:47.736 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:47 smithi026 bash[245484]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T22:12:48.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:48 smithi070 ceph-mon[192652]: pgmap v134: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:48.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1497113012' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:48.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:48 smithi026 ceph-mon[223014]: pgmap v134: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:48.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1497113012' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:48.985 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:48 smithi026 ceph-osd[245975]: -- 172.21.15.26:0/4195618521 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5627c64284e0 con 0x5627c63fac00 2023-09-23T22:12:48.986 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:48 smithi026 bash[245910]: 1fd817f92d4e14c63086e3ebd379fce7ef32e4473c674d7ef0da5d69059283cd 2023-09-23T22:12:48.986 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:48 smithi026 systemd[1]: Started Ceph osd.1 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:12:48.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:48 smithi026 ceph-mon[215775]: pgmap v134: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:48.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1497113012' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:49.611 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:49 smithi026 conmon[245951]: 2023-09-23T22:12:49.235+0000 7f8bddd92640 -1 Falling back to public interface 2023-09-23T22:12:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:12:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:12:49] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2023-09-23T22:12:49.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:49 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:49.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:49 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:49.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:49 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:49.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:49 smithi026 ceph-mon[223014]: pgmap v135: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:49 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:49 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:49 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:49.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:49 smithi026 ceph-mon[215775]: pgmap v135: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:50.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:49 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:50.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:49 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:50.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:49 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:50.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:49 smithi070 ceph-mon[192652]: pgmap v135: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3334910196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:51.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3334910196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:51.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3334910196' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:52.485 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:51 smithi026 conmon[245951]: 2023-09-23T22:12:51.984+0000 7f8bddd92640 -1 osd.1 94 log_to_monitors true 2023-09-23T22:12:52.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: pgmap v136: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:52.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: from='osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:12:52.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2065434970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: pgmap v136: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: from='osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:12:52.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2065434970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:52.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:52 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:12:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: pgmap v136: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: from='osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:12:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2065434970' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:52 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:12:53.183 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:12:52 smithi026 conmon[245951]: 2023-09-23T22:12:52.889+0000 7f8bcd1fb700 -1 osd.1 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:12:53.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[215775]: failed to encode map e97 with expected crc 2023-09-23T22:12:53.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[215775]: failed to encode map e97 with expected crc 2023-09-23T22:12:53.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[215775]: from='osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:12:53.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[215775]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T22:12:53.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[215775]: from='osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:12:53.971 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[215775]: pgmap v138: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 117 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:53.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[223014]: failed to encode map e97 with expected crc 2023-09-23T22:12:53.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[223014]: failed to encode map e97 with expected crc 2023-09-23T22:12:53.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[223014]: from='osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:12:53.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[223014]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T22:12:53.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[223014]: from='osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:12:53.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:53 smithi026 ceph-mon[223014]: pgmap v138: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 117 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:54.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:53 smithi070 ceph-mon[192652]: failed to encode map e97 with expected crc 2023-09-23T22:12:54.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:53 smithi070 ceph-mon[192652]: failed to encode map e97 with expected crc 2023-09-23T22:12:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:53 smithi070 ceph-mon[192652]: from='osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:12:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:53 smithi070 ceph-mon[192652]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T22:12:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:53 smithi070 ceph-mon[192652]: from='osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:12:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:53 smithi070 ceph-mon[192652]: pgmap v138: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 117 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T22:12:55.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:12:55.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: failed to encode map e98 with expected crc 2023-09-23T22:12:55.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792] boot 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: failed to encode map e97 with expected crc 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1540702080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[223014]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: failed to encode map e98 with expected crc 2023-09-23T22:12:55.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792] boot 2023-09-23T22:12:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T22:12:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:12:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: failed to encode map e97 with expected crc 2023-09-23T22:12:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1540702080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:54 smithi026 ceph-mon[215775]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T22:12:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:12:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: failed to encode map e98 with expected crc 2023-09-23T22:12:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: osd.1 [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792] boot 2023-09-23T22:12:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T22:12:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:12:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: failed to encode map e97 with expected crc 2023-09-23T22:12:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:12:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1540702080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:54 smithi070 ceph-mon[192652]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T22:12:55.881 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:12:55 smithi026 conmon[202047]: 2023-09-23T22:12:55.824+0000 7f47954a6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-09-23T22:12:56.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: pgmap v141: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: pgmap v141: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:56.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:12:56.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: failed to encode map e99 with expected crc 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: pgmap v141: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T22:12:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:56.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:56.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:12:56.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:12:56.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:12:56.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:12:56.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:56.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:56.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:12:56.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:12:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:56 smithi026 ceph-mon[215775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:12:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:56 smithi026 ceph-mon[215775]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-09-23T22:12:57.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:56 smithi026 ceph-mon[223014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:12:57.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:56 smithi026 ceph-mon[223014]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-09-23T22:12:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:56 smithi070 ceph-mon[192652]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:12:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:56 smithi070 ceph-mon[192652]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-09-23T22:12:58.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4037799364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:58.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:57 smithi026 ceph-mon[223014]: pgmap v142: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:12:58.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:57 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:12:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4037799364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:57 smithi026 ceph-mon[215775]: pgmap v142: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:12:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:57 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:12:58.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4037799364' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:12:58.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:57 smithi070 ceph-mon[192652]: pgmap v142: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:12:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:57 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:12:59.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:59 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T22:12:59.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:12:59 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:12:59.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:59 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T22:12:59.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:12:59 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:12:59.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:59 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T22:12:59.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:12:59 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:12:59.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:12:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:12:59] "GET /metrics HTTP/1.1" 200 33740 "" "Prometheus/2.43.0" 2023-09-23T22:13:00.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:00 smithi070 ceph-mon[192652]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:00.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3948700195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:00.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:00 smithi026 ceph-mon[223014]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:00.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3948700195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:00 smithi026 ceph-mon[215775]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:00.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3948700195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:02.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:02 smithi070 ceph-mon[192652]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:02.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3953141750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:02.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:02 smithi026 ceph-mon[223014]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:02.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3953141750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:02 smithi026 ceph-mon[215775]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3953141750' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:04 smithi070 ceph-mon[192652]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3705881906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:04 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:04.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:04 smithi026 ceph-mon[223014]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:04.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3705881906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:04.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:04 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:04.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:04 smithi026 ceph-mon[215775]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:04.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3705881906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:04.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:04 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:06 smithi070 ceph-mon[192652]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3818117976' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:06 smithi026 ceph-mon[215775]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3818117976' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:06 smithi026 ceph-mon[223014]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3818117976' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:08.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:08 smithi026 ceph-mon[223014]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:08 smithi026 ceph-mon[215775]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:08 smithi070 ceph-mon[192652]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:09 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1429510749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:09.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:13:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:13:09] "GET /metrics HTTP/1.1" 200 33883 "" "Prometheus/2.43.0" 2023-09-23T22:13:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:09 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1429510749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:09.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:09 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1429510749' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:10.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:10 smithi070 ceph-mon[192652]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:10.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:10 smithi026 ceph-mon[215775]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:10 smithi026 ceph-mon[223014]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:11.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:11 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:13:11.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1544013983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:11.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:11 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:13:11.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1544013983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:11.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:11 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:13:11.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1544013983' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:12.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[223014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:13:12.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[223014]: Upgrade: osd.2 is safe to restart 2023-09-23T22:13:12.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[223014]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:12.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:12.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:13:12.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:13:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[215775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:13:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[215775]: Upgrade: osd.2 is safe to restart 2023-09-23T22:13:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[215775]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:13:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:12 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:13:12.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:12 smithi070 ceph-mon[192652]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T22:13:12.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:12 smithi070 ceph-mon[192652]: Upgrade: osd.2 is safe to restart 2023-09-23T22:13:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:12 smithi070 ceph-mon[192652]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:12 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:12 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T22:13:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:12 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:13:13.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:13 smithi070 ceph-mon[192652]: Upgrade: Updating osd.2 2023-09-23T22:13:13.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:13 smithi070 ceph-mon[192652]: Deploying daemon osd.2 on smithi026 2023-09-23T22:13:13.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2792467400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:13 smithi026 ceph-mon[223014]: Upgrade: Updating osd.2 2023-09-23T22:13:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:13 smithi026 ceph-mon[223014]: Deploying daemon osd.2 on smithi026 2023-09-23T22:13:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2792467400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:13 smithi026 ceph-mon[215775]: Upgrade: Updating osd.2 2023-09-23T22:13:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:13 smithi026 ceph-mon[215775]: Deploying daemon osd.2 on smithi026 2023-09-23T22:13:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2792467400' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:14.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:14 smithi026 ceph-mon[215775]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:14.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:14 smithi026 ceph-mon[223014]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:14.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:14 smithi070 ceph-mon[192652]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:15.623 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:15 smithi026 systemd[1]: Stopping Ceph osd.2 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:13:15.985 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:15 smithi026 conmon[144599]: 2023-09-23T22:13:15.622+0000 7f4550997700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:13:15.985 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:15 smithi026 conmon[144599]: 2023-09-23T22:13:15.622+0000 7f4550997700 -1 osd.2 99 *** Got signal Terminated *** 2023-09-23T22:13:15.985 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:15 smithi026 conmon[144599]: 2023-09-23T22:13:15.622+0000 7f4550997700 -1 osd.2 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:13:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:16 smithi070 ceph-mon[192652]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:16 smithi070 ceph-mon[192652]: osd.2 marked itself down and dead 2023-09-23T22:13:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/699120259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:16 smithi026 ceph-mon[215775]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:16 smithi026 ceph-mon[215775]: osd.2 marked itself down and dead 2023-09-23T22:13:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/699120259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:16 smithi026 ceph-mon[223014]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:16 smithi026 ceph-mon[223014]: osd.2 marked itself down and dead 2023-09-23T22:13:16.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/699120259' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:17.485 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:17 smithi026 bash[252418]: 50e18b5844786b01af15323b6f9f7ebb7c74139b189461a3a87aaede6b21e534 2023-09-23T22:13:17.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:17 smithi070 ceph-mon[192652]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:13:17.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:17 smithi070 ceph-mon[192652]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:17 smithi070 ceph-mon[192652]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:17 smithi070 ceph-mon[192652]: osdmap e100: 8 total, 7 up, 8 in 2023-09-23T22:13:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:17 smithi070 ceph-mon[192652]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:17 smithi070 ceph-mon[192652]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:17 smithi070 ceph-mon[192652]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:17 smithi070 ceph-mon[192652]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[215775]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:13:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[215775]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[215775]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[215775]: osdmap e100: 8 total, 7 up, 8 in 2023-09-23T22:13:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[215775]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[215775]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[215775]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[215775]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[223014]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:13:17.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[223014]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[223014]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[223014]: osdmap e100: 8 total, 7 up, 8 in 2023-09-23T22:13:17.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[223014]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[223014]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[223014]: failed to encode map e100 with expected crc 2023-09-23T22:13:17.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:17 smithi026 ceph-mon[223014]: failed to encode map e100 with expected crc 2023-09-23T22:13:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: pgmap v153: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T22:13:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2102082794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: pgmap v153: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T22:13:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2102082794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: pgmap v153: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T22:13:18.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: failed to encode map e101 with expected crc 2023-09-23T22:13:18.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2102082794' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:19.612 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:19 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T22:13:19.612 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:19 smithi026 systemd[1]: Stopped Ceph osd.2 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:13:19.612 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:19 smithi026 systemd[1]: Starting Ceph osd.2 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:13:19.871 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:13:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:13:19] "GET /metrics HTTP/1.1" 200 33883 "" "Prometheus/2.43.0" 2023-09-23T22:13:20.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:20 smithi026 ceph-mon[223014]: pgmap v155: 73 pgs: 2 stale+active+clean, 4 active+undersized, 67 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:20.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2384824122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:20.583 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:20 smithi026 bash[253087]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:13:20.583 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:20 smithi026 bash[253087]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:13:20.583 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:20 smithi026 bash[253087]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T22:13:20.583 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:20 smithi026 bash[253087]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:13:20.583 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:20 smithi026 bash[253087]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:13:20.583 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:20 smithi026 bash[253087]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T22:13:20.583 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:20 smithi026 bash[253087]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:13:20.583 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:20 smithi026 bash[253087]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T22:13:20.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:20 smithi070 ceph-mon[192652]: pgmap v155: 73 pgs: 2 stale+active+clean, 4 active+undersized, 67 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:20.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2384824122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:20 smithi026 ceph-mon[215775]: pgmap v155: 73 pgs: 2 stale+active+clean, 4 active+undersized, 67 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:20.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2384824122' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:21.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:21 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:13:21.985 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:21 smithi026 ceph-osd[253599]: -- 172.21.15.26:0/665309679 <== mon.0 v2:172.21.15.26:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ef25a744e0 con 0x55ef25a47400 2023-09-23T22:13:21.986 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:21 smithi026 bash[253534]: 82c8c1d0efd8845863a26477a1c21350419a3ace307213c38c9a86a282c23d73 2023-09-23T22:13:21.986 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:21 smithi026 systemd[1]: Started Ceph osd.2 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:13:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:21 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:13:22.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:21 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:13:22.485 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:22 smithi026 conmon[253575]: 2023-09-23T22:13:22.135+0000 7f855c1ff640 -1 Falling back to public interface 2023-09-23T22:13:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:22 smithi026 ceph-mon[223014]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:22 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:22 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:22 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:13:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:22 smithi026 ceph-mon[215775]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:22 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:22.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:22 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:22.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:22 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:13:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:22 smithi070 ceph-mon[192652]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:22 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:22 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:22 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:13:23.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3797189085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3797189085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:24.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3797189085' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:24.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[223014]: pgmap v157: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:24.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:24.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:24.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:24.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:24.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[215775]: pgmap v157: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:24.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:24.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:24.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:24.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:24 smithi070 ceph-mon[192652]: pgmap v157: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:25.235 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:24 smithi026 conmon[253575]: 2023-09-23T22:13:24.894+0000 7f855c1ff640 -1 osd.2 99 log_to_monitors true 2023-09-23T22:13:25.967 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:13:25 smithi026 conmon[253575]: 2023-09-23T22:13:25.667+0000 7f854d66c700 -1 osd.2 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:13:25.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:25 smithi026 ceph-mon[223014]: from='osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:13:25.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/490502558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:25.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:25.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:25.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:25 smithi026 ceph-mon[215775]: from='osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:13:25.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/490502558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:25.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:25.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:25 smithi070 ceph-mon[192652]: from='osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:13:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/490502558' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:26.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[215775]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:26.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[215775]: failed to encode map e102 with expected crc 2023-09-23T22:13:26.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[215775]: failed to encode map e102 with expected crc 2023-09-23T22:13:26.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[215775]: from='osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:13:26.938 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[215775]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T22:13:26.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[215775]: from='osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:13:26.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[223014]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:26.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[223014]: failed to encode map e102 with expected crc 2023-09-23T22:13:26.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[223014]: failed to encode map e102 with expected crc 2023-09-23T22:13:26.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[223014]: from='osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:13:26.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[223014]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T22:13:26.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:26 smithi026 ceph-mon[223014]: from='osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:13:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:26 smithi070 ceph-mon[192652]: pgmap v158: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 118 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:26 smithi070 ceph-mon[192652]: failed to encode map e102 with expected crc 2023-09-23T22:13:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:26 smithi070 ceph-mon[192652]: failed to encode map e102 with expected crc 2023-09-23T22:13:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:26 smithi070 ceph-mon[192652]: from='osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:13:27.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:26 smithi070 ceph-mon[192652]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T22:13:27.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:26 smithi070 ceph-mon[192652]: from='osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:13:27.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[223014]: failed to encode map e102 with expected crc 2023-09-23T22:13:27.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[223014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:13:27.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[223014]: failed to encode map e103 with expected crc 2023-09-23T22:13:27.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[223014]: failed to encode map e103 with expected crc 2023-09-23T22:13:27.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[223014]: osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869] boot 2023-09-23T22:13:27.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[223014]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T22:13:27.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:13:27.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3943791199' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[215775]: failed to encode map e102 with expected crc 2023-09-23T22:13:27.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[215775]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:13:27.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[215775]: failed to encode map e103 with expected crc 2023-09-23T22:13:27.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[215775]: failed to encode map e103 with expected crc 2023-09-23T22:13:27.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[215775]: osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869] boot 2023-09-23T22:13:27.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[215775]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T22:13:27.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:13:27.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3943791199' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:27 smithi070 ceph-mon[192652]: failed to encode map e102 with expected crc 2023-09-23T22:13:28.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:27 smithi070 ceph-mon[192652]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:13:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:27 smithi070 ceph-mon[192652]: failed to encode map e103 with expected crc 2023-09-23T22:13:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:27 smithi070 ceph-mon[192652]: failed to encode map e103 with expected crc 2023-09-23T22:13:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:27 smithi070 ceph-mon[192652]: osd.2 [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869] boot 2023-09-23T22:13:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:27 smithi070 ceph-mon[192652]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T22:13:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:27 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:13:28.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3943791199' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:28.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:13:28 smithi026 conmon[202047]: 2023-09-23T22:13:28.863+0000 7f47954a6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T22:13:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: pgmap v161: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[223014]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: pgmap v161: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: failed to encode map e104 with expected crc 2023-09-23T22:13:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:28 smithi026 ceph-mon[215775]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: pgmap v161: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:29.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T22:13:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:28 smithi070 ceph-mon[192652]: failed to encode map e104 with expected crc 2023-09-23T22:13:29.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:13:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:13:29] "GET /metrics HTTP/1.1" 200 33883 "" "Prometheus/2.43.0" 2023-09-23T22:13:29.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:29.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:29.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: pgmap v163: 73 pgs: 3 active+undersized+degraded, 13 active+undersized, 57 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:29.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1050072673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:13:29.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:13:29.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:29.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:29.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T22:13:29.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: pgmap v163: 73 pgs: 3 active+undersized+degraded, 13 active+undersized, 57 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:29.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:29.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:29 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1050072673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:13:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: pgmap v163: 73 pgs: 3 active+undersized+degraded, 13 active+undersized, 57 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:29 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1050072673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:31.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:31 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded) 2023-09-23T22:13:31.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:31 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:13:31.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:31 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded) 2023-09-23T22:13:31.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:31 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:13:31.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:31 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded) 2023-09-23T22:13:31.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:31 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:13:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:32 smithi070 ceph-mon[192652]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:32 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2605825041' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:32.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:32 smithi026 ceph-mon[215775]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:32.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:32 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2605825041' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:32.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:32 smithi026 ceph-mon[223014]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:32.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:32 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2605825041' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:34.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:34 smithi070 ceph-mon[192652]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:34.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:34 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1719392673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:34.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:34 smithi026 ceph-mon[215775]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:34.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:34 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1719392673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:34.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:34 smithi026 ceph-mon[223014]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:34.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:34 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1719392673' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:36 smithi070 ceph-mon[192652]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:36.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:36 smithi026 ceph-mon[215775]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:36 smithi026 ceph-mon[223014]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:37.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:37 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/179889298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:37.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:37 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/179889298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:37 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/179889298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:38 smithi070 ceph-mon[192652]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:38.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:38 smithi026 ceph-mon[215775]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:38 smithi026 ceph-mon[223014]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:39.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:13:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:13:39] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.43.0" 2023-09-23T22:13:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:40 smithi070 ceph-mon[192652]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3487916429' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:40.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[223014]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:40.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:40.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:40.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3487916429' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:40.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[215775]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3487916429' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:42 smithi070 ceph-mon[192652]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1614852700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:42.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:42 smithi026 ceph-mon[223014]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:42.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1614852700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:42.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:42 smithi026 ceph-mon[215775]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:42.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1614852700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:44.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:44 smithi026 ceph-mon[215775]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:44.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:44 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3635918062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:44.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:44.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:44 smithi026 ceph-mon[223014]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:44.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:44 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3635918062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:44.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:44 smithi070 ceph-mon[192652]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:44 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3635918062' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:45.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[215775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:45.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[215775]: Upgrade: osd.3 is safe to restart 2023-09-23T22:13:45.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:45.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:13:45.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:13:45.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[223014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:45.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[223014]: Upgrade: osd.3 is safe to restart 2023-09-23T22:13:45.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:45.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:13:45.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:45 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:13:45.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:45 smithi070 ceph-mon[192652]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T22:13:45.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:45 smithi070 ceph-mon[192652]: Upgrade: osd.3 is safe to restart 2023-09-23T22:13:45.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:45 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:45 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T22:13:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:45 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:13:46.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:46 smithi070 ceph-mon[192652]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:46 smithi070 ceph-mon[192652]: Upgrade: Updating osd.3 2023-09-23T22:13:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:46 smithi070 ceph-mon[192652]: Deploying daemon osd.3 on smithi026 2023-09-23T22:13:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2933785004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:46 smithi026 ceph-mon[215775]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:46 smithi026 ceph-mon[215775]: Upgrade: Updating osd.3 2023-09-23T22:13:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:46 smithi026 ceph-mon[215775]: Deploying daemon osd.3 on smithi026 2023-09-23T22:13:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2933785004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:46 smithi026 ceph-mon[223014]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:46 smithi026 ceph-mon[223014]: Upgrade: Updating osd.3 2023-09-23T22:13:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:46 smithi026 ceph-mon[223014]: Deploying daemon osd.3 on smithi026 2023-09-23T22:13:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2933785004' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:48 smithi026 ceph-mon[223014]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/373902063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:48 smithi026 ceph-mon[215775]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/373902063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:48.736 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:48 smithi026 systemd[1]: Stopping Ceph osd.3 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:13:48.736 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:48 smithi026 conmon[153746]: 2023-09-23T22:13:48.580+0000 7f738a529700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:13:48.736 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:48 smithi026 conmon[153746]: 2023-09-23T22:13:48.580+0000 7f738a529700 -1 osd.3 104 *** Got signal Terminated *** 2023-09-23T22:13:48.736 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:48 smithi026 conmon[153746]: 2023-09-23T22:13:48.580+0000 7f738a529700 -1 osd.3 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:13:48.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:48 smithi070 ceph-mon[192652]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:48.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/373902063' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:49 smithi070 ceph-mon[192652]: osd.3 marked itself down and dead 2023-09-23T22:13:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:49 smithi026 ceph-mon[215775]: osd.3 marked itself down and dead 2023-09-23T22:13:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:13:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:13:49] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.43.0" 2023-09-23T22:13:49.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:49 smithi026 ceph-mon[223014]: osd.3 marked itself down and dead 2023-09-23T22:13:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:13:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: osdmap e105: 8 total, 7 up, 8 in 2023-09-23T22:13:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:50 smithi070 ceph-mon[192652]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:13:50.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: osdmap e105: 8 total, 7 up, 8 in 2023-09-23T22:13:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[215775]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:13:50.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:13:50.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: osdmap e105: 8 total, 7 up, 8 in 2023-09-23T22:13:50.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:50 smithi026 ceph-mon[223014]: failed to encode map e105 with expected crc 2023-09-23T22:13:50.988 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:50 smithi026 bash[260782]: 57db6dad61fcd8567d0d2a518ffbfff35a79c066aea14f501dae8175cc513147 2023-09-23T22:13:51.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[215775]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[215775]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[215775]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T22:13:51.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[215775]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[215775]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[215775]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[215775]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3337086325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:51 smithi070 ceph-mon[192652]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:51 smithi070 ceph-mon[192652]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:51 smithi070 ceph-mon[192652]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T22:13:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:51 smithi070 ceph-mon[192652]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:51 smithi070 ceph-mon[192652]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:51 smithi070 ceph-mon[192652]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:51 smithi070 ceph-mon[192652]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:51 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3337086325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:51.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[223014]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[223014]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.949 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[223014]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T22:13:51.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[223014]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[223014]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[223014]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[223014]: failed to encode map e106 with expected crc 2023-09-23T22:13:51.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:51 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3337086325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:52 smithi026 ceph-mon[223014]: pgmap v176: 73 pgs: 1 active+undersized+degraded, 15 peering, 3 stale+active+clean, 10 active+undersized, 44 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:13:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:52 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T22:13:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:52 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:13:52.736 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:52 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T22:13:52.736 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:52 smithi026 systemd[1]: Stopped Ceph osd.3 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:13:52.736 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:52 smithi026 systemd[1]: Starting Ceph osd.3 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:13:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:52 smithi026 ceph-mon[215775]: pgmap v176: 73 pgs: 1 active+undersized+degraded, 15 peering, 3 stale+active+clean, 10 active+undersized, 44 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:13:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:52 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T22:13:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:52 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:13:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:52 smithi070 ceph-mon[192652]: pgmap v176: 73 pgs: 1 active+undersized+degraded, 15 peering, 3 stale+active+clean, 10 active+undersized, 44 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:13:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:52 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T22:13:52.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:52 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:13:53.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:53 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/877074480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:53.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:53 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/877074480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:53.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:53 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/877074480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:53.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:53 smithi026 bash[261442]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:13:53.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:53 smithi026 bash[261442]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T22:13:53.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:53 smithi026 bash[261442]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T22:13:53.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:53 smithi026 bash[261442]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T22:13:53.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:53 smithi026 bash[261442]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T22:13:53.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:53 smithi026 bash[261442]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T22:13:53.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:53 smithi026 bash[261442]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T22:13:53.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:53 smithi026 bash[261442]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T22:13:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:54 smithi070 ceph-mon[192652]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 15 peering, 15 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:13:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:54 smithi026 ceph-mon[215775]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 15 peering, 15 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:13:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:54.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:54 smithi026 ceph-osd[261957]: -- 172.21.15.26:0/1810452877 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f0c23304e0 con 0x55f0c2303400 2023-09-23T22:13:54.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:54 smithi026 bash[261892]: 6e18c70e050b1d77ca511a46eed9a69087e83b68146296fb0808d1f570a606cf 2023-09-23T22:13:54.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:54 smithi026 systemd[1]: Started Ceph osd.3 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:13:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:54 smithi026 ceph-mon[223014]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 15 peering, 15 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:13:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:13:55.485 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:55 smithi026 conmon[261933]: 2023-09-23T22:13:55.138+0000 7fd68cbf5640 -1 Falling back to public interface 2023-09-23T22:13:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:13:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[215775]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:13:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1783810776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:55.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:55.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:13:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[223014]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:13:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1783810776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:13:56.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:55 smithi070 ceph-mon[192652]: pgmap v178: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:13:56.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1783810776' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:56 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T22:13:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:56 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T22:13:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:56 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T22:13:57.985 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:57 smithi026 conmon[261933]: 2023-09-23T22:13:57.914+0000 7fd68cbf5640 -1 osd.3 104 log_to_monitors true 2023-09-23T22:13:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:57 smithi026 ceph-mon[215775]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:13:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:57 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:13:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:57 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:57 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:57 smithi026 ceph-mon[223014]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:13:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:57 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:13:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:57 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:57 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:57 smithi070 ceph-mon[192652]: pgmap v179: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:13:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:57 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:13:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:57 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:57 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.982 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:13:58 smithi026 conmon[261933]: 2023-09-23T22:13:58.792+0000 7fd67b85d700 -1 osd.3 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:13:58.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/540251012' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:58.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[215775]: from='osd.3 [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[215775]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/540251012' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[223014]: from='osd.3 [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[223014]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:58.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/540251012' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:13:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:58 smithi070 ceph-mon[192652]: from='osd.3 [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:13:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:58 smithi070 ceph-mon[192652]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:13:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:13:59.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[215775]: failed to encode map e107 with expected crc 2023-09-23T22:13:59.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[215775]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:13:59.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[215775]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T22:13:59.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[215775]: from='osd.3 [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:13:59.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[215775]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:13:59.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[215775]: failed to encode map e107 with expected crc 2023-09-23T22:13:59.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[215775]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:13:59.880 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:13:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:13:59] "GET /metrics HTTP/1.1" 200 33947 "" "Prometheus/2.43.0" 2023-09-23T22:13:59.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[223014]: failed to encode map e107 with expected crc 2023-09-23T22:13:59.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[223014]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:13:59.880 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[223014]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T22:13:59.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[223014]: from='osd.3 [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:13:59.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[223014]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:13:59.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[223014]: failed to encode map e107 with expected crc 2023-09-23T22:13:59.881 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:13:59 smithi026 ceph-mon[223014]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:14:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:59 smithi070 ceph-mon[192652]: failed to encode map e107 with expected crc 2023-09-23T22:14:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:59 smithi070 ceph-mon[192652]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:14:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:59 smithi070 ceph-mon[192652]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T22:14:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:59 smithi070 ceph-mon[192652]: from='osd.3 [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:14:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:59 smithi070 ceph-mon[192652]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:14:00.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:59 smithi070 ceph-mon[192652]: failed to encode map e107 with expected crc 2023-09-23T22:14:00.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:13:59 smithi070 ceph-mon[192652]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T22:14:00.140 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T22:14:00.982 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T22:14:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:00 smithi070 ceph-mon[192652]: failed to encode map e107 with expected crc 2023-09-23T22:14:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:00 smithi070 ceph-mon[192652]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:14:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:00 smithi070 ceph-mon[192652]: failed to encode map e108 with expected crc 2023-09-23T22:14:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:00 smithi070 ceph-mon[192652]: failed to encode map e108 with expected crc 2023-09-23T22:14:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:00 smithi070 ceph-mon[192652]: osd.3 [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469] boot 2023-09-23T22:14:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:00 smithi070 ceph-mon[192652]: osdmap e108: 8 total, 8 up, 8 in 2023-09-23T22:14:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:00 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:14:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:00 smithi070 ceph-mon[192652]: failed to encode map e108 with expected crc 2023-09-23T22:14:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4134373893' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:14:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[215775]: failed to encode map e107 with expected crc 2023-09-23T22:14:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[215775]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:14:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[215775]: failed to encode map e108 with expected crc 2023-09-23T22:14:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[215775]: failed to encode map e108 with expected crc 2023-09-23T22:14:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[215775]: osd.3 [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469] boot 2023-09-23T22:14:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[215775]: osdmap e108: 8 total, 8 up, 8 in 2023-09-23T22:14:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:14:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[215775]: failed to encode map e108 with expected crc 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4134373893' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[223014]: failed to encode map e107 with expected crc 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[223014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[223014]: failed to encode map e108 with expected crc 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[223014]: failed to encode map e108 with expected crc 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[223014]: osd.3 [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469] boot 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[223014]: osdmap e108: 8 total, 8 up, 8 in 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[223014]: failed to encode map e108 with expected crc 2023-09-23T22:14:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4134373893' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T22:14:01.366 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T22:14:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[223014]: pgmap v184: 73 pgs: 16 peering, 2 active+undersized+degraded, 13 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T22:14:01.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T22:14:01.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: failed to encode map e109 with expected crc 2023-09-23T22:14:01.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:01 smithi026 ceph-mon[215775]: pgmap v184: 73 pgs: 16 peering, 2 active+undersized+degraded, 13 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T22:14:02.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: failed to encode map e109 with expected crc 2023-09-23T22:14:02.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: failed to encode map e109 with expected crc 2023-09-23T22:14:02.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T22:14:02.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: failed to encode map e109 with expected crc 2023-09-23T22:14:02.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: failed to encode map e109 with expected crc 2023-09-23T22:14:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: failed to encode map e109 with expected crc 2023-09-23T22:14:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: failed to encode map e109 with expected crc 2023-09-23T22:14:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: failed to encode map e109 with expected crc 2023-09-23T22:14:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: failed to encode map e109 with expected crc 2023-09-23T22:14:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: failed to encode map e109 with expected crc 2023-09-23T22:14:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:01 smithi070 ceph-mon[192652]: pgmap v184: 73 pgs: 16 peering, 2 active+undersized+degraded, 13 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T22:14:02.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:14:02 smithi026 conmon[202047]: 2023-09-23T22:14:02.454+0000 7f47954a6700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:14:03.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:14:03.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T22:14:03.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:14:03.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:03.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:03.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:14:03.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:14:03.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:03.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:14:03.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:03.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:03.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:03.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:14:03.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:14:03.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T22:14:03.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:14:03.598 INFO:teuthology.orchestra.run.smithi026.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:14:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:14:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:14:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:14:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:14:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T22:14:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T22:14:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:14:04.203 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:14:04.560 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:04 smithi070 ceph-mon[192652]: pgmap v185: 73 pgs: 16 peering, 2 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T22:14:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:04 smithi070 ceph-mon[192652]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:14:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:04 smithi070 ceph-mon[192652]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:14:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:04 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:04.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:04 smithi026 ceph-mon[215775]: pgmap v185: 73 pgs: 16 peering, 2 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T22:14:04.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:04 smithi026 ceph-mon[215775]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:14:04.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:04 smithi026 ceph-mon[215775]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:14:04.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:04 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:04.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:04 smithi026 ceph-mon[223014]: pgmap v185: 73 pgs: 16 peering, 2 active+undersized+degraded, 11 active+undersized, 44 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T22:14:04.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:04 smithi026 ceph-mon[223014]: from='client.34452 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:14:04.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:04 smithi026 ceph-mon[223014]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T22:14:04.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:04 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:05.933 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:05 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T22:14:05.933 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:05 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:14:05.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:05 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T22:14:05.934 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:05 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:14:06.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:05 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T22:14:06.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:05 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:14:06.495 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:14:06.495 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (6m) - 10m 30.7M - 0.25.0 c8568f914cd2 1cbbb123afed 2023-09-23T22:14:06.495 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (6m) - 9m 91.1M - 9.4.7 2c41d148cca3 42940e65ed5c 2023-09-23T22:14:06.495 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.rancfk smithi026 running (9m) - 9m 20.2M - 17.2.6-1125-gc9bb1bcf 444539aec5bf a9885df09771 2023-09-23T22:14:06.495 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.ehaqek smithi070 running (9m) - 9m 17.8M - 17.2.6-1125-gc9bb1bcf 444539aec5bf 812a5d180a78 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (5m) - 13m 449M - 18.2.0-460-gbb0daeff e32c2a64c126 754e649f30a9 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (6m) - 15m 514M - 18.2.0-460-gbb0daeff e32c2a64c126 2dff2050e669 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (5m) - 15m 80.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0028efef8662 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (3m) - 14m 55.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 f07a7787aa42 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) - 14m 58.1M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 25f6f9fc0ac7 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) - 9m 19.7M - 1.5.0 0da6a335fe13 68b72a59fe08 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (6m) - 9m 19.0M - 1.5.0 0da6a335fe13 523ab3522b38 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (2m) - 13m 60.1M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 dd9b16818fa4 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (78s) - 13m 60.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 1fd817f92d4e 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (45s) - 12m 49.1M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 82c8c1d0efd8 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (12s) - 12m 11.5M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 6e18c70e050b 2023-09-23T22:14:06.496 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (12m) - 12m 98.5M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 92889bd7f5ee 2023-09-23T22:14:06.497 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (11m) - 11m 107M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 628c1ee2ccfc 2023-09-23T22:14:06.497 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (11m) - 11m 90.0M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 5b472aeb7f69 2023-09-23T22:14:06.497 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (10m) - 10m 114M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 2a3425d84687 2023-09-23T22:14:06.497 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (5m) - 10m 66.3M - 2.43.0 a07b618ecd1d 3342388f10a7 2023-09-23T22:14:06.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:06 smithi026 ceph-mon[223014]: pgmap v186: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:06 smithi026 ceph-mon[215775]: pgmap v186: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:06 smithi070 ceph-mon[192652]: pgmap v186: 73 pgs: 7 peering, 66 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:07.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T22:14:07.515 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:07 smithi026 ceph-mon[223014]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:14:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:07 smithi026 ceph-mon[223014]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:07 smithi026 ceph-mon[215775]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:14:07.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:07 smithi026 ceph-mon[215775]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:07 smithi070 ceph-mon[192652]: from='client.34458 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:14:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:07 smithi070 ceph-mon[192652]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:09.438 INFO:teuthology.orchestra.run.smithi026.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T22:14:09.438 INFO:teuthology.orchestra.run.smithi026.stdout:alertmanager.a smithi026 *:9093,9094 running (6m) - 10m 30.7M - 0.25.0 c8568f914cd2 1cbbb123afed 2023-09-23T22:14:09.438 INFO:teuthology.orchestra.run.smithi026.stdout:grafana.a smithi070 *:3000 running (6m) - 9m 91.1M - 9.4.7 2c41d148cca3 42940e65ed5c 2023-09-23T22:14:09.438 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi026.rancfk smithi026 running (9m) - 9m 20.2M - 17.2.6-1125-gc9bb1bcf 444539aec5bf a9885df09771 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:mds.foo.smithi070.ehaqek smithi070 running (9m) - 9m 17.8M - 17.2.6-1125-gc9bb1bcf 444539aec5bf 812a5d180a78 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.x smithi070 *:8443,9283,8765 running (5m) - 13m 449M - 18.2.0-460-gbb0daeff e32c2a64c126 754e649f30a9 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:mgr.y smithi026 *:8443,9283,8765 running (6m) - 15m 514M - 18.2.0-460-gbb0daeff e32c2a64c126 2dff2050e669 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a smithi026 running (5m) - 15m 80.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0028efef8662 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b smithi070 running (3m) - 14m 55.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 f07a7787aa42 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c smithi026 running (4m) - 14m 58.1M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 25f6f9fc0ac7 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.a smithi026 *:9100 running (6m) - 9m 19.7M - 1.5.0 0da6a335fe13 68b72a59fe08 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:node-exporter.b smithi070 *:9100 running (6m) - 9m 19.0M - 1.5.0 0da6a335fe13 523ab3522b38 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 smithi026 running (2m) - 13m 60.1M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 dd9b16818fa4 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 smithi026 running (80s) - 13m 60.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 1fd817f92d4e 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 smithi026 running (48s) - 12m 49.1M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 82c8c1d0efd8 2023-09-23T22:14:09.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 smithi026 running (15s) - 12m 11.5M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 6e18c70e050b 2023-09-23T22:14:09.440 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 smithi070 running (12m) - 12m 98.5M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 92889bd7f5ee 2023-09-23T22:14:09.440 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 smithi070 running (11m) - 11m 107M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 628c1ee2ccfc 2023-09-23T22:14:09.440 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 smithi070 running (11m) - 11m 90.0M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 5b472aeb7f69 2023-09-23T22:14:09.440 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 smithi070 running (10m) - 10m 114M 2500M 17.2.6-1125-gc9bb1bcf 444539aec5bf 2a3425d84687 2023-09-23T22:14:09.440 INFO:teuthology.orchestra.run.smithi026.stdout:prometheus.a smithi070 *:9095 running (5m) - 10m 66.3M - 2.43.0 a07b618ecd1d 3342388f10a7 2023-09-23T22:14:09.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:14:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:14:09] "GET /metrics HTTP/1.1" 200 33942 "" "Prometheus/2.43.0" 2023-09-23T22:14:10.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T22:14:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:10 smithi070 ceph-mon[192652]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:10 smithi070 ceph-mon[192652]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:14:10.407 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:10.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:10 smithi026 ceph-mon[223014]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:10.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:10.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:10.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:10 smithi026 ceph-mon[223014]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:14:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:10 smithi026 ceph-mon[215775]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:10 smithi026 ceph-mon[215775]: from='client.34464 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:14:12.200 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "mon": { 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr": { 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "osd": { 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)": 4, 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T22:14:12.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:14:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "mds": { 2023-09-23T22:14:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)": 2 2023-09-23T22:14:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:14:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "overall": { 2023-09-23T22:14:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 17.2.6-1125-gc9bb1bcf (c9bb1bcff1cbfd495af3bc0024f437eb1ef23f7f) quincy (stable)": 6, 2023-09-23T22:14:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T22:14:12.202 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:14:12.202 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:14:12.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:12 smithi070 ceph-mon[192652]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:12.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:12 smithi026 ceph-mon[215775]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:12.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:12 smithi026 ceph-mon[223014]: pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:12.763 DEBUG:teuthology.parallel:result is None 2023-09-23T22:14:12.763 INFO:teuthology.run_tasks:Running task print... 2023-09-23T22:14:12.772 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T22:14:12.772 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T22:14:12.831 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T22:14:12.831 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T22:14:12.832 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T22:14:13.005 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3574591288' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:13.006 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3574591288' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:13.196 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:13.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3574591288' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T22:14:14.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:14 smithi070 ceph-mon[192652]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:14.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:14 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:14 smithi026 ceph-mon[215775]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:14 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:14 smithi026 ceph-mon[223014]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:14.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:14 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:15.002 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:14:15.628 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":36,"active_gid":25772,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6800","nonce":450481418},{"type":"v1","addr":"172.21.15.26:6801","nonce":450481418}]},"active_addr":"172.21.15.26:6801/450481418","active_change":"2023-09-23T22:08:38.898677+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25666,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":89,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":3624341455}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":759933880}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":1330146230}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.26:0","nonce":3677843650}]}]} 2023-09-23T22:14:15.631 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T22:14:15.631 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T22:14:15.632 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:14:15.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:15 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1069105135' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:14:15.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:15 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1069105135' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:14:15.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:15 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1069105135' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T22:14:15.990 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:16 smithi070 ceph-mon[192652]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:16 smithi026 ceph-mon[223014]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:16.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:16 smithi026 ceph-mon[215775]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:17.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:17 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:14:17.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:17 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:14:17.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:17 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:14:17.570 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:17 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:17.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:17 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:14:17.571 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:17 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:14:17.571 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:17 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:14:17.571 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:17 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:17.683 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:14:17.683 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":109,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:14:00.774068+0000","last_up_change":"2023-09-23T22:13:59.769180+0000","last_in_change":"2023-09-23T22:03:00.614896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:04:04.880309+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":83,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:04:53.313440+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:04:53.801900+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":108,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6803","nonce":507489654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6805","nonce":507489654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6809","nonce":507489654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6807","nonce":507489654}]},"public_addr":"172.21.15.26:6803/507489654","cluster_addr":"172.21.15.26:6805/507489654","heartbeat_back_addr":"172.21.15.26:6809/507489654","heartbeat_front_addr":"172.21.15.26:6807/507489654","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":94,"up_from":98,"up_thru":108,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":693231792},{"type":"v1","addr":"172.21.15.26:6811","nonce":693231792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":693231792},{"type":"v1","addr":"172.21.15.26:6813","nonce":693231792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":693231792},{"type":"v1","addr":"172.21.15.26:6817","nonce":693231792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":693231792},{"type":"v1","addr":"172.21.15.26:6815","nonce":693231792}]},"public_addr":"172.21.15.26:6811/693231792","cluster_addr":"172.21.15.26:6813/693231792","heartbeat_back_addr":"172.21.15.26:6817/693231792","heartbeat_front_addr":"172.21.15.26:6815/693231792","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":99,"up_from":103,"up_thru":108,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1775189869},{"type":"v1","addr":"172.21.15.26:6819","nonce":1775189869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1775189869},{"type":"v1","addr":"172.21.15.26:6821","nonce":1775189869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1775189869},{"type":"v1","addr":"172.21.15.26:6825","nonce":1775189869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1775189869},{"type":"v1","addr":"172.21.15.26:6823","nonce":1775189869}]},"public_addr":"172.21.15.26:6819/1775189869","cluster_addr":"172.21.15.26:6821/1775189869","heartbeat_back_addr":"172.21.15.26:6825/1775189869","heartbeat_front_addr":"172.21.15.26:6823/1775189869","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":104,"up_from":108,"up_thru":108,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":1956095469},{"type":"v1","addr":"172.21.15.26:6827","nonce":1956095469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":1956095469},{"type":"v1","addr":"172.21.15.26:6829","nonce":1956095469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":1956095469},{"type":"v1","addr":"172.21.15.26:6833","nonce":1956095469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":1956095469},{"type":"v1","addr":"172.21.15.26:6831","nonce":1956095469}]},"public_addr":"172.21.15.26:6827/1956095469","cluster_addr":"172.21.15.26:6829/1956095469","heartbeat_back_addr":"172.21.15.26:6833/1956095469","heartbeat_front_addr":"172.21.15.26:6831/1956095469","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6801","nonce":1461751884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6803","nonce":1461751884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6807","nonce":1461751884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6805","nonce":1461751884}]},"public_addr":"172.21.15.70:6801/1461751884","cluster_addr":"172.21.15.70:6803/1461751884","heartbeat_back_addr":"172.21.15.70:6807/1461751884","heartbeat_front_addr":"172.21.15.70:6805/1461751884","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":105,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6817","nonce":572290869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6819","nonce":572290869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6823","nonce":572290869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6821","nonce":572290869}]},"public_addr":"172.21.15.70:6817/572290869","cluster_addr":"172.21.15.70:6819/572290869","heartbeat_back_addr":"172.21.15.70:6823/572290869","heartbeat_front_addr":"172.21.15.70:6821/572290869","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":94},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":99},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":104},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:0/1453808871":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/575035623":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/771693589":"2023-09-24T22:07:17.644537+0000","172.21.15.70:6835/1647959066":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/2583870620":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/1692711239":"2023-09-24T22:07:17.644537+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3618866901":"2023-09-24T22:07:17.644537+0000","172.21.15.70:0/3676980533":"2023-09-24T22:07:55.974675+0000","172.21.15.70:0/1888936544":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/574101533":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/4237902526":"2023-09-24T22:08:38.898443+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.70:6834/1647959066":"2023-09-24T22:07:55.974675+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3419561437":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.70:0/202130526":"2023-09-24T22:07:55.974675+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.70:0/562032249":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/1211771677":"2023-09-24T22:08:38.898443+0000","172.21.15.26:6801/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3996051568":"2023-09-24T22:07:17.644537+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/1887156644":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/2951021001":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/577898280":"2023-09-24T22:03:52.071180+0000","172.21.15.70:0/3435183798":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/3350176691":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6800/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3418308803":"2023-09-24T22:07:17.644537+0000","172.21.15.26:6801/3418308803":"2023-09-24T22:07:17.644537+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/2951021001":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/472809172":"2023-09-24T22:08:38.898443+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:14:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:17 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:14:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:17 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:14:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:17 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:14:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:17 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:18.305 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T22:14:18.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:14:18.666 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:18.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[215775]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:18.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:14:18.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:14:18.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:14:18.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:18.846 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3955311321' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:14:18.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[223014]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:18.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:14:18.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:14:18.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:14:18.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:18.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3955311321' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:14:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:18 smithi070 ceph-mon[192652]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:18 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:14:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:18 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:14:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:18 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:14:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:18 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:14:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3955311321' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:14:19.848 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:14:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:14:19] "GET /metrics HTTP/1.1" 200 33942 "" "Prometheus/2.43.0" 2023-09-23T22:14:20.243 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:14:20.243 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":109,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:14:00.774068+0000","last_up_change":"2023-09-23T22:13:59.769180+0000","last_in_change":"2023-09-23T22:03:00.614896+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:04:04.880309+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"83","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":83,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:04:53.313440+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:04:53.801900+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":108,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6803","nonce":507489654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6805","nonce":507489654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6809","nonce":507489654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6807","nonce":507489654}]},"public_addr":"172.21.15.26:6803/507489654","cluster_addr":"172.21.15.26:6805/507489654","heartbeat_back_addr":"172.21.15.26:6809/507489654","heartbeat_front_addr":"172.21.15.26:6807/507489654","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":94,"up_from":98,"up_thru":108,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":693231792},{"type":"v1","addr":"172.21.15.26:6811","nonce":693231792}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":693231792},{"type":"v1","addr":"172.21.15.26:6813","nonce":693231792}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":693231792},{"type":"v1","addr":"172.21.15.26:6817","nonce":693231792}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":693231792},{"type":"v1","addr":"172.21.15.26:6815","nonce":693231792}]},"public_addr":"172.21.15.26:6811/693231792","cluster_addr":"172.21.15.26:6813/693231792","heartbeat_back_addr":"172.21.15.26:6817/693231792","heartbeat_front_addr":"172.21.15.26:6815/693231792","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":99,"up_from":103,"up_thru":108,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":1775189869},{"type":"v1","addr":"172.21.15.26:6819","nonce":1775189869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":1775189869},{"type":"v1","addr":"172.21.15.26:6821","nonce":1775189869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":1775189869},{"type":"v1","addr":"172.21.15.26:6825","nonce":1775189869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":1775189869},{"type":"v1","addr":"172.21.15.26:6823","nonce":1775189869}]},"public_addr":"172.21.15.26:6819/1775189869","cluster_addr":"172.21.15.26:6821/1775189869","heartbeat_back_addr":"172.21.15.26:6825/1775189869","heartbeat_front_addr":"172.21.15.26:6823/1775189869","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":104,"up_from":108,"up_thru":108,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":1956095469},{"type":"v1","addr":"172.21.15.26:6827","nonce":1956095469}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":1956095469},{"type":"v1","addr":"172.21.15.26:6829","nonce":1956095469}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":1956095469},{"type":"v1","addr":"172.21.15.26:6833","nonce":1956095469}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":1956095469},{"type":"v1","addr":"172.21.15.26:6831","nonce":1956095469}]},"public_addr":"172.21.15.26:6827/1956095469","cluster_addr":"172.21.15.26:6829/1956095469","heartbeat_back_addr":"172.21.15.26:6833/1956095469","heartbeat_front_addr":"172.21.15.26:6831/1956095469","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6801","nonce":1461751884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6803","nonce":1461751884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6807","nonce":1461751884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":1461751884},{"type":"v1","addr":"172.21.15.70:6805","nonce":1461751884}]},"public_addr":"172.21.15.70:6801/1461751884","cluster_addr":"172.21.15.70:6803/1461751884","heartbeat_back_addr":"172.21.15.70:6807/1461751884","heartbeat_front_addr":"172.21.15.70:6805/1461751884","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":46,"up_thru":105,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6817","nonce":572290869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6819","nonce":572290869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6823","nonce":572290869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":572290869},{"type":"v1","addr":"172.21.15.70:6821","nonce":572290869}]},"public_addr":"172.21.15.70:6817/572290869","cluster_addr":"172.21.15.70:6819/572290869","heartbeat_back_addr":"172.21.15.70:6823/572290869","heartbeat_front_addr":"172.21.15.70:6821/572290869","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":94},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":99},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":104},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/575035623":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/472809172":"2023-09-24T22:08:38.898443+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3419561437":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/771693589":"2023-09-24T22:07:17.644537+0000","172.21.15.70:6835/1647959066":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/2583870620":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/1692711239":"2023-09-24T22:07:17.644537+0000","172.21.15.70:0/202130526":"2023-09-24T22:07:55.974675+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3618866901":"2023-09-24T22:07:17.644537+0000","172.21.15.70:0/3676980533":"2023-09-24T22:07:55.974675+0000","172.21.15.70:0/1888936544":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/574101533":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/4237902526":"2023-09-24T22:08:38.898443+0000","172.21.15.70:6834/1647959066":"2023-09-24T22:07:55.974675+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.70:0/562032249":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/1211771677":"2023-09-24T22:08:38.898443+0000","172.21.15.26:6801/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3996051568":"2023-09-24T22:07:17.644537+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/1887156644":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/577898280":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6800/2951021001":"2023-09-24T22:08:38.898443+0000","172.21.15.70:0/3435183798":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/3350176691":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6800/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3418308803":"2023-09-24T22:07:17.644537+0000","172.21.15.26:6801/3418308803":"2023-09-24T22:07:17.644537+0000","172.21.15.70:0/1453808871":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/2951021001":"2023-09-24T22:08:38.898443+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:14:20.785 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T22:14:20.785 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T22:14:20.786 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T22:14:20.786 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T22:14:20.786 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T22:14:20.787 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T22:14:20.787 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T22:14:20.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T22:14:20.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:20 smithi026 ceph-mon[215775]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:20.830 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3660069110' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:14:20.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:20 smithi026 ceph-mon[223014]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:20.831 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3660069110' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:14:20.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:20 smithi070 ceph-mon[192652]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:20.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3660069110' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:14:22.050 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:22.100 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:22.162 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:22.298 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:22.354 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:22.355 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:22.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:22 smithi070 ceph-mon[192652]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:22.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:22 smithi026 ceph-mon[215775]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:22.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:22 smithi026 ceph-mon[223014]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:23.524 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:23.524 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:24.811 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:24 smithi026 ceph-mon[215775]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:24.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:24.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:24 smithi026 ceph-mon[223014]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:24.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:24.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:24 smithi070 ceph-mon[192652]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:24.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:26.580 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:26 smithi026 ceph-mon[215775]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:26 smithi070 ceph-mon[192652]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:26 smithi026 ceph-mon[223014]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:28.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:28 smithi070 ceph-mon[192652]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:28.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:28 smithi026 ceph-mon[223014]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:28.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:28 smithi026 ceph-mon[215775]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:29.078 INFO:teuthology.orchestra.run.smithi026.stdout:197568495756 2023-09-23T22:14:29.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T22:14:29.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:14:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:14:29] "GET /metrics HTTP/1.1" 200 33945 "" "Prometheus/2.43.0" 2023-09-23T22:14:30.472 INFO:teuthology.orchestra.run.smithi026.stdout:128849019029 2023-09-23T22:14:30.473 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T22:14:30.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:30 smithi026 ceph-mon[215775]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:30.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:30 smithi026 ceph-mon[223014]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:30.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:30 smithi070 ceph-mon[192652]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:31.773 INFO:teuthology.orchestra.run.smithi026.stdout:399431958563 2023-09-23T22:14:31.773 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T22:14:32.290 INFO:teuthology.orchestra.run.smithi026.stdout:463856467976 2023-09-23T22:14:32.290 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T22:14:32.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:32 smithi026 ceph-mon[223014]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:32.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:32 smithi026 ceph-mon[215775]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:32 smithi070 ceph-mon[192652]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:33.224 INFO:teuthology.orchestra.run.smithi026.stdout:442381631503 2023-09-23T22:14:33.224 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T22:14:33.455 INFO:teuthology.orchestra.run.smithi026.stdout:231928234121 2023-09-23T22:14:33.455 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T22:14:33.736 INFO:teuthology.orchestra.run.smithi026.stdout:163208757394 2023-09-23T22:14:33.737 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T22:14:34.135 INFO:teuthology.orchestra.run.smithi026.stdout:420906795030 2023-09-23T22:14:34.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T22:14:34.316 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:34.571 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:34.621 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:34.822 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:34.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:34 smithi070 ceph-mon[192652]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:34.901 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:34 smithi026 ceph-mon[223014]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:34.901 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:34 smithi026 ceph-mon[215775]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:34.925 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:34.986 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:35.829 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:36.748 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:36 smithi026 ceph-mon[215775]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:36.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:36 smithi026 ceph-mon[223014]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:36 smithi070 ceph-mon[192652]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:37.445 INFO:teuthology.orchestra.run.smithi026.stdout:463856467977 2023-09-23T22:14:37.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:37 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1098099106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:14:37.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:37 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1098099106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:14:37.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:37 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1098099106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:14:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:38 smithi070 ceph-mon[192652]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:38.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:38 smithi026 ceph-mon[215775]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:38 smithi026 ceph-mon[223014]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:39.016 INFO:teuthology.orchestra.run.smithi026.stdout:197568495758 2023-09-23T22:14:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:39 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/65327508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:14:39.891 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:39.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:14:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:14:39] "GET /metrics HTTP/1.1" 200 33941 "" "Prometheus/2.43.0" 2023-09-23T22:14:39.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:39.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:39 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/65327508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:14:39.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:39 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/65327508' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:14:40.142 INFO:tasks.cephadm.ceph_manager.ceph:need seq 463856467976 got 463856467977 for osd.3 2023-09-23T22:14:40.143 DEBUG:teuthology.parallel:result is None 2023-09-23T22:14:40.645 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:40 smithi026 ceph-mon[215775]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:40.645 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:40 smithi026 ceph-mon[223014]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:40.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:40 smithi070 ceph-mon[192652]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:41.255 INFO:tasks.cephadm.ceph_manager.ceph:need seq 197568495756 got 197568495758 for osd.6 2023-09-23T22:14:41.255 DEBUG:teuthology.parallel:result is None 2023-09-23T22:14:41.398 INFO:teuthology.orchestra.run.smithi026.stdout:442381631504 2023-09-23T22:14:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/151499027' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:14:41.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/151499027' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:14:41.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/151499027' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:14:42.539 INFO:teuthology.orchestra.run.smithi026.stdout:128849019032 2023-09-23T22:14:42.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:42 smithi026 ceph-mon[215775]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:42.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1668750362' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:14:42.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:42 smithi026 ceph-mon[223014]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:42.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1668750362' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:14:42.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:42 smithi070 ceph-mon[192652]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:42.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1668750362' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:14:43.314 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631503 got 442381631504 for osd.2 2023-09-23T22:14:43.314 DEBUG:teuthology.parallel:result is None 2023-09-23T22:14:43.523 INFO:teuthology.orchestra.run.smithi026.stdout:399431958566 2023-09-23T22:14:43.760 INFO:teuthology.orchestra.run.smithi026.stdout:231928234123 2023-09-23T22:14:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2936281792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:14:43.895 INFO:teuthology.orchestra.run.smithi026.stdout:163208757396 2023-09-23T22:14:43.928 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2936281792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:14:43.928 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2936281792' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:14:44.389 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019029 got 128849019032 for osd.4 2023-09-23T22:14:44.390 DEBUG:teuthology.parallel:result is None 2023-09-23T22:14:44.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:44 smithi026 ceph-mon[215775]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:44.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:44 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/362025257' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:14:44.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:44 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1499080104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:14:44.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:44 smithi026 ceph-mon[223014]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:44.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:44 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/362025257' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:14:44.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:44 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1499080104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:14:44.663 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958563 got 399431958566 for osd.0 2023-09-23T22:14:44.663 DEBUG:teuthology.parallel:result is None 2023-09-23T22:14:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:44 smithi070 ceph-mon[192652]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:44 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/362025257' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:14:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:44 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1499080104' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:14:45.422 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757394 got 163208757396 for osd.5 2023-09-23T22:14:45.422 DEBUG:teuthology.parallel:result is None 2023-09-23T22:14:45.739 INFO:teuthology.orchestra.run.smithi026.stdout:420906795032 2023-09-23T22:14:46.283 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234121 got 231928234123 for osd.7 2023-09-23T22:14:46.283 DEBUG:teuthology.parallel:result is None 2023-09-23T22:14:46.482 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795030 got 420906795032 for osd.1 2023-09-23T22:14:46.482 DEBUG:teuthology.parallel:result is None 2023-09-23T22:14:46.482 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:14:46.482 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:14:46.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:46 smithi026 ceph-mon[215775]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:46.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2660431288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:14:46.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:46 smithi026 ceph-mon[223014]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:46.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2660431288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:14:46.844 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:46 smithi070 ceph-mon[192652]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:46.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2660431288' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:14:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:14:48.606 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:14:48.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:48 smithi070 ceph-mon[192652]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:48.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:48 smithi026 ceph-mon[223014]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:48.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:48 smithi026 ceph-mon[215775]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:49.223 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":207,"stamp":"2023-09-23T22:14:47.009527+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6483,"num_read_kb":14132,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164156,"kb_used_data":22972,"kb_used_omap":31,"kb_used_meta":141088,"kb_avail":749797060,"statfs":{"total":767960285184,"available":767792189440,"internally_reserved":0,"allocated":23523328,"data_stored":19273120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":144475063},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002749"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787529+0000","last_change":"2023-09-23T22:12:54.868920+0000","last_active":"2023-09-23T22:13:59.787529+0000","last_peered":"2023-09-23T22:13:59.787529+0000","last_clean":"2023-09-23T22:13:59.787529+0000","last_became_active":"2023-09-23T22:12:54.868576+0000","last_became_peered":"2023-09-23T22:12:54.868576+0000","last_unstale":"2023-09-23T22:13:59.787529+0000","last_undegraded":"2023-09-23T22:13:59.787529+0000","last_fullsized":"2023-09-23T22:13:59.787529+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:03.825524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787688+0000","last_change":"2023-09-23T22:13:27.664835+0000","last_active":"2023-09-23T22:13:59.787688+0000","last_peered":"2023-09-23T22:13:59.787688+0000","last_clean":"2023-09-23T22:13:59.787688+0000","last_became_active":"2023-09-23T22:13:27.664333+0000","last_became_peered":"2023-09-23T22:13:27.664333+0000","last_unstale":"2023-09-23T22:13:59.787688+0000","last_undegraded":"2023-09-23T22:13:59.787688+0000","last_fullsized":"2023-09-23T22:13:59.787688+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:11.434650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.784662+0000","last_change":"2023-09-23T22:14:00.784662+0000","last_active":"2023-09-23T22:14:00.784662+0000","last_peered":"2023-09-23T22:14:00.784662+0000","last_clean":"2023-09-23T22:14:00.784662+0000","last_became_active":"2023-09-23T22:14:00.784294+0000","last_became_peered":"2023-09-23T22:14:00.784294+0000","last_unstale":"2023-09-23T22:14:00.784662+0000","last_undegraded":"2023-09-23T22:14:00.784662+0000","last_fullsized":"2023-09-23T22:14:00.784662+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:09.123997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.785463+0000","last_change":"2023-09-23T22:14:00.785463+0000","last_active":"2023-09-23T22:14:00.785463+0000","last_peered":"2023-09-23T22:14:00.785463+0000","last_clean":"2023-09-23T22:14:00.785463+0000","last_became_active":"2023-09-23T22:14:00.785128+0000","last_became_peered":"2023-09-23T22:14:00.785128+0000","last_unstale":"2023-09-23T22:14:00.785463+0000","last_undegraded":"2023-09-23T22:14:00.785463+0000","last_fullsized":"2023-09-23T22:14:00.785463+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:46.343842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792878+0000","last_change":"2023-09-23T22:14:00.792878+0000","last_active":"2023-09-23T22:14:00.792878+0000","last_peered":"2023-09-23T22:14:00.792878+0000","last_clean":"2023-09-23T22:14:00.792878+0000","last_became_active":"2023-09-23T22:14:00.792609+0000","last_became_peered":"2023-09-23T22:14:00.792609+0000","last_unstale":"2023-09-23T22:14:00.792878+0000","last_undegraded":"2023-09-23T22:14:00.792878+0000","last_fullsized":"2023-09-23T22:14:00.792878+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:47.586353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781836+0000","last_change":"2023-09-23T22:04:54.379680+0000","last_active":"2023-09-23T22:13:59.781836+0000","last_peered":"2023-09-23T22:13:59.781836+0000","last_clean":"2023-09-23T22:13:59.781836+0000","last_became_active":"2023-09-23T22:04:54.379383+0000","last_became_peered":"2023-09-23T22:04:54.379383+0000","last_unstale":"2023-09-23T22:13:59.781836+0000","last_undegraded":"2023-09-23T22:13:59.781836+0000","last_fullsized":"2023-09-23T22:13:59.781836+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.282795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794701+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:14:00.794701+0000","last_peered":"2023-09-23T22:14:00.794701+0000","last_clean":"2023-09-23T22:14:00.794701+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:14:00.794701+0000","last_undegraded":"2023-09-23T22:14:00.794701+0000","last_fullsized":"2023-09-23T22:14:00.794701+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.782175+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:13:59.782175+0000","last_peered":"2023-09-23T22:13:59.782175+0000","last_clean":"2023-09-23T22:13:59.782175+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:13:59.782175+0000","last_undegraded":"2023-09-23T22:13:59.782175+0000","last_fullsized":"2023-09-23T22:13:59.782175+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.786689+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:14:00.786689+0000","last_peered":"2023-09-23T22:14:00.786689+0000","last_clean":"2023-09-23T22:14:00.786689+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:14:00.786689+0000","last_undegraded":"2023-09-23T22:14:00.786689+0000","last_fullsized":"2023-09-23T22:14:00.786689+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":40,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.780635+0000","last_change":"2023-09-23T22:11:42.696059+0000","last_active":"2023-09-23T22:14:00.780635+0000","last_peered":"2023-09-23T22:14:00.780635+0000","last_clean":"2023-09-23T22:14:00.780635+0000","last_became_active":"2023-09-23T22:11:42.695756+0000","last_became_peered":"2023-09-23T22:11:42.695756+0000","last_unstale":"2023-09-23T22:14:00.780635+0000","last_undegraded":"2023-09-23T22:14:00.780635+0000","last_fullsized":"2023-09-23T22:14:00.780635+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:34.633172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":50,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788995+0000","last_change":"2023-09-23T22:04:54.402894+0000","last_active":"2023-09-23T22:13:59.788995+0000","last_peered":"2023-09-23T22:13:59.788995+0000","last_clean":"2023-09-23T22:13:59.788995+0000","last_became_active":"2023-09-23T22:04:54.402475+0000","last_became_peered":"2023-09-23T22:04:54.402475+0000","last_unstale":"2023-09-23T22:13:59.788995+0000","last_undegraded":"2023-09-23T22:13:59.788995+0000","last_fullsized":"2023-09-23T22:13:59.788995+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:12.503263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789394+0000","last_change":"2023-09-23T22:12:54.870816+0000","last_active":"2023-09-23T22:13:59.789394+0000","last_peered":"2023-09-23T22:13:59.789394+0000","last_clean":"2023-09-23T22:13:59.789394+0000","last_became_active":"2023-09-23T22:12:54.870450+0000","last_became_peered":"2023-09-23T22:12:54.870450+0000","last_unstale":"2023-09-23T22:13:59.789394+0000","last_undegraded":"2023-09-23T22:13:59.789394+0000","last_fullsized":"2023-09-23T22:13:59.789394+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.764353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":234,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786388+0000","last_change":"2023-09-23T22:13:27.664537+0000","last_active":"2023-09-23T22:13:59.786388+0000","last_peered":"2023-09-23T22:13:59.786388+0000","last_clean":"2023-09-23T22:13:59.786388+0000","last_became_active":"2023-09-23T22:13:27.663897+0000","last_became_peered":"2023-09-23T22:13:27.663897+0000","last_unstale":"2023-09-23T22:13:59.786388+0000","last_undegraded":"2023-09-23T22:13:59.786388+0000","last_fullsized":"2023-09-23T22:13:59.786388+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:37.321580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":222,"num_read_kb":1213,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795827+0000","last_change":"2023-09-23T22:14:00.795827+0000","last_active":"2023-09-23T22:14:00.795827+0000","last_peered":"2023-09-23T22:14:00.795827+0000","last_clean":"2023-09-23T22:14:00.795827+0000","last_became_active":"2023-09-23T22:14:00.795225+0000","last_became_peered":"2023-09-23T22:14:00.795225+0000","last_unstale":"2023-09-23T22:14:00.795827+0000","last_undegraded":"2023-09-23T22:14:00.795827+0000","last_fullsized":"2023-09-23T22:14:00.795827+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:58:56.251393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":705,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781094+0000","last_change":"2023-09-23T22:12:54.874377+0000","last_active":"2023-09-23T22:13:59.781094+0000","last_peered":"2023-09-23T22:13:59.781094+0000","last_clean":"2023-09-23T22:13:59.781094+0000","last_became_active":"2023-09-23T22:12:54.873780+0000","last_became_peered":"2023-09-23T22:12:54.873780+0000","last_unstale":"2023-09-23T22:13:59.781094+0000","last_undegraded":"2023-09-23T22:13:59.781094+0000","last_fullsized":"2023-09-23T22:13:59.781094+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:08.681877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796111+0000","last_change":"2023-09-23T22:14:00.796111+0000","last_active":"2023-09-23T22:14:00.796111+0000","last_peered":"2023-09-23T22:14:00.796111+0000","last_clean":"2023-09-23T22:14:00.796111+0000","last_became_active":"2023-09-23T22:14:00.794856+0000","last_became_peered":"2023-09-23T22:14:00.794856+0000","last_unstale":"2023-09-23T22:14:00.796111+0000","last_undegraded":"2023-09-23T22:14:00.796111+0000","last_fullsized":"2023-09-23T22:14:00.796111+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:16.319602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786418+0000","last_change":"2023-09-23T22:11:42.696594+0000","last_active":"2023-09-23T22:13:59.786418+0000","last_peered":"2023-09-23T22:13:59.786418+0000","last_clean":"2023-09-23T22:13:59.786418+0000","last_became_active":"2023-09-23T22:11:42.696334+0000","last_became_peered":"2023-09-23T22:11:42.696334+0000","last_unstale":"2023-09-23T22:13:59.786418+0000","last_undegraded":"2023-09-23T22:13:59.786418+0000","last_fullsized":"2023-09-23T22:13:59.786418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:36.810008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780034+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:13:59.780034+0000","last_peered":"2023-09-23T22:13:59.780034+0000","last_clean":"2023-09-23T22:13:59.780034+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:13:59.780034+0000","last_undegraded":"2023-09-23T22:13:59.780034+0000","last_fullsized":"2023-09-23T22:13:59.780034+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":544,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786909+0000","last_change":"2023-09-23T22:11:42.693882+0000","last_active":"2023-09-23T22:13:59.786909+0000","last_peered":"2023-09-23T22:13:59.786909+0000","last_clean":"2023-09-23T22:13:59.786909+0000","last_became_active":"2023-09-23T22:11:42.693381+0000","last_became_peered":"2023-09-23T22:11:42.693381+0000","last_unstale":"2023-09-23T22:13:59.786909+0000","last_undegraded":"2023-09-23T22:13:59.786909+0000","last_fullsized":"2023-09-23T22:13:59.786909+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:06.246085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789486+0000","last_change":"2023-09-23T22:13:27.666017+0000","last_active":"2023-09-23T22:13:59.789486+0000","last_peered":"2023-09-23T22:13:59.789486+0000","last_clean":"2023-09-23T22:13:59.789486+0000","last_became_active":"2023-09-23T22:13:27.665706+0000","last_became_peered":"2023-09-23T22:13:27.665706+0000","last_unstale":"2023-09-23T22:13:59.789486+0000","last_undegraded":"2023-09-23T22:13:59.789486+0000","last_fullsized":"2023-09-23T22:13:59.789486+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:16.575957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790941+0000","last_change":"2023-09-23T22:14:00.790941+0000","last_active":"2023-09-23T22:14:00.790941+0000","last_peered":"2023-09-23T22:14:00.790941+0000","last_clean":"2023-09-23T22:14:00.790941+0000","last_became_active":"2023-09-23T22:14:00.790616+0000","last_became_peered":"2023-09-23T22:14:00.790616+0000","last_unstale":"2023-09-23T22:14:00.790941+0000","last_undegraded":"2023-09-23T22:14:00.790941+0000","last_fullsized":"2023-09-23T22:14:00.790941+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:53.552619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":458,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783191+0000","last_change":"2023-09-23T22:12:54.861801+0000","last_active":"2023-09-23T22:13:59.783191+0000","last_peered":"2023-09-23T22:13:59.783191+0000","last_clean":"2023-09-23T22:13:59.783191+0000","last_became_active":"2023-09-23T22:12:54.861358+0000","last_became_peered":"2023-09-23T22:12:54.861358+0000","last_unstale":"2023-09-23T22:13:59.783191+0000","last_undegraded":"2023-09-23T22:13:59.783191+0000","last_fullsized":"2023-09-23T22:13:59.783191+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:24.061357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785893+0000","last_change":"2023-09-23T22:04:54.527318+0000","last_active":"2023-09-23T22:13:59.785893+0000","last_peered":"2023-09-23T22:13:59.785893+0000","last_clean":"2023-09-23T22:13:59.785893+0000","last_became_active":"2023-09-23T22:04:54.527023+0000","last_became_peered":"2023-09-23T22:04:54.527023+0000","last_unstale":"2023-09-23T22:13:59.785893+0000","last_undegraded":"2023-09-23T22:13:59.785893+0000","last_fullsized":"2023-09-23T22:13:59.785893+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:42.048739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.789944+0000","last_change":"2023-09-23T22:14:00.789944+0000","last_active":"2023-09-23T22:14:00.789944+0000","last_peered":"2023-09-23T22:14:00.789944+0000","last_clean":"2023-09-23T22:14:00.789944+0000","last_became_active":"2023-09-23T22:14:00.789559+0000","last_became_peered":"2023-09-23T22:14:00.789559+0000","last_unstale":"2023-09-23T22:14:00.789944+0000","last_undegraded":"2023-09-23T22:14:00.789944+0000","last_fullsized":"2023-09-23T22:14:00.789944+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:30.957719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":451,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783108+0000","last_change":"2023-09-23T22:11:42.695484+0000","last_active":"2023-09-23T22:13:59.783108+0000","last_peered":"2023-09-23T22:13:59.783108+0000","last_clean":"2023-09-23T22:13:59.783108+0000","last_became_active":"2023-09-23T22:11:42.694972+0000","last_became_peered":"2023-09-23T22:11:42.694972+0000","last_unstale":"2023-09-23T22:13:59.783108+0000","last_undegraded":"2023-09-23T22:13:59.783108+0000","last_fullsized":"2023-09-23T22:13:59.783108+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:36.787404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":694,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780660+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:13:59.780660+0000","last_peered":"2023-09-23T22:13:59.780660+0000","last_clean":"2023-09-23T22:13:59.780660+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:13:59.780660+0000","last_undegraded":"2023-09-23T22:13:59.780660+0000","last_fullsized":"2023-09-23T22:13:59.780660+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781830+0000","last_change":"2023-09-23T22:12:54.874146+0000","last_active":"2023-09-23T22:13:59.781830+0000","last_peered":"2023-09-23T22:13:59.781830+0000","last_clean":"2023-09-23T22:13:59.781830+0000","last_became_active":"2023-09-23T22:12:54.873782+0000","last_became_peered":"2023-09-23T22:12:54.873782+0000","last_unstale":"2023-09-23T22:13:59.781830+0000","last_undegraded":"2023-09-23T22:13:59.781830+0000","last_fullsized":"2023-09-23T22:13:59.781830+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:25.933932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780900+0000","last_change":"2023-09-23T22:13:27.659751+0000","last_active":"2023-09-23T22:13:59.780900+0000","last_peered":"2023-09-23T22:13:59.780900+0000","last_clean":"2023-09-23T22:13:59.780900+0000","last_became_active":"2023-09-23T22:13:27.659422+0000","last_became_peered":"2023-09-23T22:13:27.659422+0000","last_unstale":"2023-09-23T22:13:59.780900+0000","last_undegraded":"2023-09-23T22:13:59.780900+0000","last_fullsized":"2023-09-23T22:13:59.780900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:25.705909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":378,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.783947+0000","last_change":"2023-09-23T22:14:00.783947+0000","last_active":"2023-09-23T22:14:00.783947+0000","last_peered":"2023-09-23T22:14:00.783947+0000","last_clean":"2023-09-23T22:14:00.783947+0000","last_became_active":"2023-09-23T22:14:00.783319+0000","last_became_peered":"2023-09-23T22:14:00.783319+0000","last_unstale":"2023-09-23T22:14:00.783947+0000","last_undegraded":"2023-09-23T22:14:00.783947+0000","last_fullsized":"2023-09-23T22:14:00.783947+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:02:25.419096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796811+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:14:00.796811+0000","last_peered":"2023-09-23T22:14:00.796811+0000","last_clean":"2023-09-23T22:14:00.796811+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:14:00.796811+0000","last_undegraded":"2023-09-23T22:14:00.796811+0000","last_fullsized":"2023-09-23T22:14:00.796811+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788001+0000","last_change":"2023-09-23T22:11:42.697009+0000","last_active":"2023-09-23T22:13:59.788001+0000","last_peered":"2023-09-23T22:13:59.788001+0000","last_clean":"2023-09-23T22:13:59.788001+0000","last_became_active":"2023-09-23T22:11:42.696588+0000","last_became_peered":"2023-09-23T22:11:42.696588+0000","last_unstale":"2023-09-23T22:13:59.788001+0000","last_undegraded":"2023-09-23T22:13:59.788001+0000","last_fullsized":"2023-09-23T22:13:59.788001+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:57.260787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":750,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780601+0000","last_change":"2023-09-23T22:12:54.877082+0000","last_active":"2023-09-23T22:13:59.780601+0000","last_peered":"2023-09-23T22:13:59.780601+0000","last_clean":"2023-09-23T22:13:59.780601+0000","last_became_active":"2023-09-23T22:12:54.876524+0000","last_became_peered":"2023-09-23T22:12:54.876524+0000","last_unstale":"2023-09-23T22:13:59.780601+0000","last_undegraded":"2023-09-23T22:13:59.780601+0000","last_fullsized":"2023-09-23T22:13:59.780601+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:46.185025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.778046+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:13:59.778046+0000","last_peered":"2023-09-23T22:13:59.778046+0000","last_clean":"2023-09-23T22:13:59.778046+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:13:59.778046+0000","last_undegraded":"2023-09-23T22:13:59.778046+0000","last_fullsized":"2023-09-23T22:13:59.778046+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792797+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:14:00.792797+0000","last_peered":"2023-09-23T22:14:00.792797+0000","last_clean":"2023-09-23T22:14:00.792797+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:14:00.792797+0000","last_undegraded":"2023-09-23T22:14:00.792797+0000","last_fullsized":"2023-09-23T22:14:00.792797+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":555,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.782571+0000","last_change":"2023-09-23T22:12:54.875239+0000","last_active":"2023-09-23T22:13:59.782571+0000","last_peered":"2023-09-23T22:13:59.782571+0000","last_clean":"2023-09-23T22:13:59.782571+0000","last_became_active":"2023-09-23T22:12:54.874675+0000","last_became_peered":"2023-09-23T22:12:54.874675+0000","last_unstale":"2023-09-23T22:13:59.782571+0000","last_undegraded":"2023-09-23T22:13:59.782571+0000","last_fullsized":"2023-09-23T22:13:59.782571+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:21.865417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.791734+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:14:00.791734+0000","last_peered":"2023-09-23T22:14:00.791734+0000","last_clean":"2023-09-23T22:14:00.791734+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:14:00.791734+0000","last_undegraded":"2023-09-23T22:14:00.791734+0000","last_fullsized":"2023-09-23T22:14:00.791734+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795928+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:14:00.795928+0000","last_peered":"2023-09-23T22:14:00.795928+0000","last_clean":"2023-09-23T22:14:00.795928+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:14:00.795928+0000","last_undegraded":"2023-09-23T22:14:00.795928+0000","last_fullsized":"2023-09-23T22:14:00.795928+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.790050+0000","last_change":"2023-09-23T22:12:54.868682+0000","last_active":"2023-09-23T22:13:59.790050+0000","last_peered":"2023-09-23T22:13:59.790050+0000","last_clean":"2023-09-23T22:13:59.790050+0000","last_became_active":"2023-09-23T22:12:54.868339+0000","last_became_peered":"2023-09-23T22:12:54.868339+0000","last_unstale":"2023-09-23T22:13:59.790050+0000","last_undegraded":"2023-09-23T22:13:59.790050+0000","last_fullsized":"2023-09-23T22:13:59.790050+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:53.246678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789109+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:13:59.789109+0000","last_peered":"2023-09-23T22:13:59.789109+0000","last_clean":"2023-09-23T22:13:59.789109+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:13:59.789109+0000","last_undegraded":"2023-09-23T22:13:59.789109+0000","last_fullsized":"2023-09-23T22:13:59.789109+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.791120+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:13:59.791120+0000","last_peered":"2023-09-23T22:13:59.791120+0000","last_clean":"2023-09-23T22:13:59.791120+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:13:59.791120+0000","last_undegraded":"2023-09-23T22:13:59.791120+0000","last_fullsized":"2023-09-23T22:13:59.791120+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785609+0000","last_change":"2023-09-23T22:12:54.880778+0000","last_active":"2023-09-23T22:13:59.785609+0000","last_peered":"2023-09-23T22:13:59.785609+0000","last_clean":"2023-09-23T22:13:59.785609+0000","last_became_active":"2023-09-23T22:12:54.880272+0000","last_became_peered":"2023-09-23T22:12:54.880272+0000","last_unstale":"2023-09-23T22:13:59.785609+0000","last_undegraded":"2023-09-23T22:13:59.785609+0000","last_fullsized":"2023-09-23T22:13:59.785609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:01.030591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792760+0000","last_change":"2023-09-23T22:14:00.792760+0000","last_active":"2023-09-23T22:14:00.792760+0000","last_peered":"2023-09-23T22:14:00.792760+0000","last_clean":"2023-09-23T22:14:00.792760+0000","last_became_active":"2023-09-23T22:14:00.792478+0000","last_became_peered":"2023-09-23T22:14:00.792478+0000","last_unstale":"2023-09-23T22:14:00.792760+0000","last_undegraded":"2023-09-23T22:14:00.792760+0000","last_fullsized":"2023-09-23T22:14:00.792760+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.935338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790875+0000","last_change":"2023-09-23T22:14:00.790875+0000","last_active":"2023-09-23T22:14:00.790875+0000","last_peered":"2023-09-23T22:14:00.790875+0000","last_clean":"2023-09-23T22:14:00.790875+0000","last_became_active":"2023-09-23T22:14:00.790332+0000","last_became_peered":"2023-09-23T22:14:00.790332+0000","last_unstale":"2023-09-23T22:14:00.790875+0000","last_undegraded":"2023-09-23T22:14:00.790875+0000","last_fullsized":"2023-09-23T22:14:00.790875+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:05.904013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786633+0000","last_change":"2023-09-23T22:12:54.871021+0000","last_active":"2023-09-23T22:13:59.786633+0000","last_peered":"2023-09-23T22:13:59.786633+0000","last_clean":"2023-09-23T22:13:59.786633+0000","last_became_active":"2023-09-23T22:12:54.870736+0000","last_became_peered":"2023-09-23T22:12:54.870736+0000","last_unstale":"2023-09-23T22:13:59.786633+0000","last_undegraded":"2023-09-23T22:13:59.786633+0000","last_fullsized":"2023-09-23T22:13:59.786633+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:51.846457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795682+0000","last_change":"2023-09-23T22:14:00.795682+0000","last_active":"2023-09-23T22:14:00.795682+0000","last_peered":"2023-09-23T22:14:00.795682+0000","last_clean":"2023-09-23T22:14:00.795682+0000","last_became_active":"2023-09-23T22:14:00.795377+0000","last_became_peered":"2023-09-23T22:14:00.795377+0000","last_unstale":"2023-09-23T22:14:00.795682+0000","last_undegraded":"2023-09-23T22:14:00.795682+0000","last_fullsized":"2023-09-23T22:14:00.795682+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:33.253920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785440+0000","last_change":"2023-09-23T22:12:54.865320+0000","last_active":"2023-09-23T22:13:59.785440+0000","last_peered":"2023-09-23T22:13:59.785440+0000","last_clean":"2023-09-23T22:13:59.785440+0000","last_became_active":"2023-09-23T22:12:54.865007+0000","last_became_peered":"2023-09-23T22:12:54.865007+0000","last_unstale":"2023-09-23T22:13:59.785440+0000","last_undegraded":"2023-09-23T22:13:59.785440+0000","last_fullsized":"2023-09-23T22:13:59.785440+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:40.963311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787011+0000","last_change":"2023-09-23T22:04:54.384008+0000","last_active":"2023-09-23T22:13:59.787011+0000","last_peered":"2023-09-23T22:13:59.787011+0000","last_clean":"2023-09-23T22:13:59.787011+0000","last_became_active":"2023-09-23T22:04:54.383702+0000","last_became_peered":"2023-09-23T22:04:54.383702+0000","last_unstale":"2023-09-23T22:13:59.787011+0000","last_undegraded":"2023-09-23T22:13:59.787011+0000","last_fullsized":"2023-09-23T22:13:59.787011+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:00.423911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795430+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:14:00.795430+0000","last_peered":"2023-09-23T22:14:00.795430+0000","last_clean":"2023-09-23T22:14:00.795430+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:14:00.795430+0000","last_undegraded":"2023-09-23T22:14:00.795430+0000","last_fullsized":"2023-09-23T22:14:00.795430+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786126+0000","last_change":"2023-09-23T22:12:54.861787+0000","last_active":"2023-09-23T22:13:59.786126+0000","last_peered":"2023-09-23T22:13:59.786126+0000","last_clean":"2023-09-23T22:13:59.786126+0000","last_became_active":"2023-09-23T22:12:54.861377+0000","last_became_peered":"2023-09-23T22:12:54.861377+0000","last_unstale":"2023-09-23T22:13:59.786126+0000","last_undegraded":"2023-09-23T22:13:59.786126+0000","last_fullsized":"2023-09-23T22:13:59.786126+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:02.832226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780468+0000","last_change":"2023-09-23T22:04:54.528019+0000","last_active":"2023-09-23T22:13:59.780468+0000","last_peered":"2023-09-23T22:13:59.780468+0000","last_clean":"2023-09-23T22:13:59.780468+0000","last_became_active":"2023-09-23T22:04:54.527666+0000","last_became_peered":"2023-09-23T22:04:54.527666+0000","last_unstale":"2023-09-23T22:13:59.780468+0000","last_undegraded":"2023-09-23T22:13:59.780468+0000","last_fullsized":"2023-09-23T22:13:59.780468+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:32.774493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786894+0000","last_change":"2023-09-23T22:11:42.694008+0000","last_active":"2023-09-23T22:13:59.786894+0000","last_peered":"2023-09-23T22:13:59.786894+0000","last_clean":"2023-09-23T22:13:59.786894+0000","last_became_active":"2023-09-23T22:11:42.693609+0000","last_became_peered":"2023-09-23T22:11:42.693609+0000","last_unstale":"2023-09-23T22:13:59.786894+0000","last_undegraded":"2023-09-23T22:13:59.786894+0000","last_fullsized":"2023-09-23T22:13:59.786894+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:59.614114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794463+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:14:00.794463+0000","last_peered":"2023-09-23T22:14:00.794463+0000","last_clean":"2023-09-23T22:14:00.794463+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:14:00.794463+0000","last_undegraded":"2023-09-23T22:14:00.794463+0000","last_fullsized":"2023-09-23T22:14:00.794463+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":77,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.784703+0000","last_change":"2023-09-23T22:11:42.701916+0000","last_active":"2023-09-23T22:13:59.784703+0000","last_peered":"2023-09-23T22:13:59.784703+0000","last_clean":"2023-09-23T22:13:59.784703+0000","last_became_active":"2023-09-23T22:11:42.701591+0000","last_became_peered":"2023-09-23T22:11:42.701591+0000","last_unstale":"2023-09-23T22:13:59.784703+0000","last_undegraded":"2023-09-23T22:13:59.784703+0000","last_fullsized":"2023-09-23T22:13:59.784703+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:23.941128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.793040+0000","last_change":"2023-09-23T22:14:00.792449+0000","last_active":"2023-09-23T22:14:00.793040+0000","last_peered":"2023-09-23T22:14:00.793040+0000","last_clean":"2023-09-23T22:14:00.793040+0000","last_became_active":"2023-09-23T22:14:00.792239+0000","last_became_peered":"2023-09-23T22:14:00.792239+0000","last_unstale":"2023-09-23T22:14:00.793040+0000","last_undegraded":"2023-09-23T22:14:00.793040+0000","last_fullsized":"2023-09-23T22:14:00.793040+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:33.488563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783327+0000","last_change":"2023-09-23T22:04:54.387688+0000","last_active":"2023-09-23T22:13:59.783327+0000","last_peered":"2023-09-23T22:13:59.783327+0000","last_clean":"2023-09-23T22:13:59.783327+0000","last_became_active":"2023-09-23T22:04:54.387421+0000","last_became_peered":"2023-09-23T22:04:54.387421+0000","last_unstale":"2023-09-23T22:13:59.783327+0000","last_undegraded":"2023-09-23T22:13:59.783327+0000","last_fullsized":"2023-09-23T22:13:59.783327+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:22.411746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794669+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:14:00.794669+0000","last_peered":"2023-09-23T22:14:00.794669+0000","last_clean":"2023-09-23T22:14:00.794669+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:14:00.794669+0000","last_undegraded":"2023-09-23T22:14:00.794669+0000","last_fullsized":"2023-09-23T22:14:00.794669+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792829+0000","last_change":"2023-09-23T22:14:00.792829+0000","last_active":"2023-09-23T22:14:00.792829+0000","last_peered":"2023-09-23T22:14:00.792829+0000","last_clean":"2023-09-23T22:14:00.792829+0000","last_became_active":"2023-09-23T22:14:00.792462+0000","last_became_peered":"2023-09-23T22:14:00.792462+0000","last_unstale":"2023-09-23T22:14:00.792829+0000","last_undegraded":"2023-09-23T22:14:00.792829+0000","last_fullsized":"2023-09-23T22:14:00.792829+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:55.154145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794351+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:14:00.794351+0000","last_peered":"2023-09-23T22:14:00.794351+0000","last_clean":"2023-09-23T22:14:00.794351+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:14:00.794351+0000","last_undegraded":"2023-09-23T22:14:00.794351+0000","last_fullsized":"2023-09-23T22:14:00.794351+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785117+0000","last_change":"2023-09-23T22:13:27.663039+0000","last_active":"2023-09-23T22:13:59.785117+0000","last_peered":"2023-09-23T22:13:59.785117+0000","last_clean":"2023-09-23T22:13:59.785117+0000","last_became_active":"2023-09-23T22:13:27.662687+0000","last_became_peered":"2023-09-23T22:13:27.662687+0000","last_unstale":"2023-09-23T22:13:59.785117+0000","last_undegraded":"2023-09-23T22:13:59.785117+0000","last_fullsized":"2023-09-23T22:13:59.785117+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:13.432843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788095+0000","last_change":"2023-09-23T22:12:54.866763+0000","last_active":"2023-09-23T22:13:59.788095+0000","last_peered":"2023-09-23T22:13:59.788095+0000","last_clean":"2023-09-23T22:13:59.788095+0000","last_became_active":"2023-09-23T22:12:54.866422+0000","last_became_peered":"2023-09-23T22:12:54.866422+0000","last_unstale":"2023-09-23T22:13:59.788095+0000","last_undegraded":"2023-09-23T22:13:59.788095+0000","last_fullsized":"2023-09-23T22:13:59.788095+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:43.904727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.790220+0000","last_change":"2023-09-23T22:04:54.381132+0000","last_active":"2023-09-23T22:13:59.790220+0000","last_peered":"2023-09-23T22:13:59.790220+0000","last_clean":"2023-09-23T22:13:59.790220+0000","last_became_active":"2023-09-23T22:04:54.380838+0000","last_became_peered":"2023-09-23T22:04:54.380838+0000","last_unstale":"2023-09-23T22:13:59.790220+0000","last_undegraded":"2023-09-23T22:13:59.790220+0000","last_fullsized":"2023-09-23T22:13:59.790220+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:41.334974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780225+0000","last_change":"2023-09-23T22:13:27.661799+0000","last_active":"2023-09-23T22:13:59.780225+0000","last_peered":"2023-09-23T22:13:59.780225+0000","last_clean":"2023-09-23T22:13:59.780225+0000","last_became_active":"2023-09-23T22:13:27.661473+0000","last_became_peered":"2023-09-23T22:13:27.661473+0000","last_unstale":"2023-09-23T22:13:59.780225+0000","last_undegraded":"2023-09-23T22:13:59.780225+0000","last_fullsized":"2023-09-23T22:13:59.780225+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:34.827694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796705+0000","last_change":"2023-09-23T22:14:00.796705+0000","last_active":"2023-09-23T22:14:00.796705+0000","last_peered":"2023-09-23T22:14:00.796705+0000","last_clean":"2023-09-23T22:14:00.796705+0000","last_became_active":"2023-09-23T22:14:00.796252+0000","last_became_peered":"2023-09-23T22:14:00.796252+0000","last_unstale":"2023-09-23T22:14:00.796705+0000","last_undegraded":"2023-09-23T22:14:00.796705+0000","last_fullsized":"2023-09-23T22:14:00.796705+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:58.956676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794963+0000","last_change":"2023-09-23T22:14:00.794963+0000","last_active":"2023-09-23T22:14:00.794963+0000","last_peered":"2023-09-23T22:14:00.794963+0000","last_clean":"2023-09-23T22:14:00.794963+0000","last_became_active":"2023-09-23T22:14:00.794394+0000","last_became_peered":"2023-09-23T22:14:00.794394+0000","last_unstale":"2023-09-23T22:14:00.794963+0000","last_undegraded":"2023-09-23T22:14:00.794963+0000","last_fullsized":"2023-09-23T22:14:00.794963+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:25.119383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780772+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:13:59.780772+0000","last_peered":"2023-09-23T22:13:59.780772+0000","last_clean":"2023-09-23T22:13:59.780772+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:13:59.780772+0000","last_undegraded":"2023-09-23T22:13:59.780772+0000","last_fullsized":"2023-09-23T22:13:59.780772+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790411+0000","last_change":"2023-09-23T22:14:00.790411+0000","last_active":"2023-09-23T22:14:00.790411+0000","last_peered":"2023-09-23T22:14:00.790411+0000","last_clean":"2023-09-23T22:14:00.790411+0000","last_became_active":"2023-09-23T22:14:00.789906+0000","last_became_peered":"2023-09-23T22:14:00.789906+0000","last_unstale":"2023-09-23T22:14:00.790411+0000","last_undegraded":"2023-09-23T22:14:00.790411+0000","last_fullsized":"2023-09-23T22:14:00.790411+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:28:43.440626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.788697+0000","last_change":"2023-09-23T22:14:00.787881+0000","last_active":"2023-09-23T22:14:00.788697+0000","last_peered":"2023-09-23T22:14:00.788697+0000","last_clean":"2023-09-23T22:14:00.788697+0000","last_became_active":"2023-09-23T22:14:00.787498+0000","last_became_peered":"2023-09-23T22:14:00.787498+0000","last_unstale":"2023-09-23T22:14:00.788697+0000","last_undegraded":"2023-09-23T22:14:00.788697+0000","last_fullsized":"2023-09-23T22:14:00.788697+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:19.424721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786273+0000","last_change":"2023-09-23T22:12:54.864958+0000","last_active":"2023-09-23T22:13:59.786273+0000","last_peered":"2023-09-23T22:13:59.786273+0000","last_clean":"2023-09-23T22:13:59.786273+0000","last_became_active":"2023-09-23T22:12:54.864451+0000","last_became_peered":"2023-09-23T22:12:54.864451+0000","last_unstale":"2023-09-23T22:13:59.786273+0000","last_undegraded":"2023-09-23T22:13:59.786273+0000","last_fullsized":"2023-09-23T22:13:59.786273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:32.668061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790097+0000","last_change":"2023-09-23T22:14:00.790097+0000","last_active":"2023-09-23T22:14:00.790097+0000","last_peered":"2023-09-23T22:14:00.790097+0000","last_clean":"2023-09-23T22:14:00.790097+0000","last_became_active":"2023-09-23T22:14:00.789389+0000","last_became_peered":"2023-09-23T22:14:00.789389+0000","last_unstale":"2023-09-23T22:14:00.790097+0000","last_undegraded":"2023-09-23T22:14:00.790097+0000","last_fullsized":"2023-09-23T22:14:00.790097+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:05.673490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792919+0000","last_change":"2023-09-23T22:14:00.792919+0000","last_active":"2023-09-23T22:14:00.792919+0000","last_peered":"2023-09-23T22:14:00.792919+0000","last_clean":"2023-09-23T22:14:00.792919+0000","last_became_active":"2023-09-23T22:14:00.792490+0000","last_became_peered":"2023-09-23T22:14:00.792490+0000","last_unstale":"2023-09-23T22:14:00.792919+0000","last_undegraded":"2023-09-23T22:14:00.792919+0000","last_fullsized":"2023-09-23T22:14:00.792919+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:53.433863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.791655+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:14:00.791655+0000","last_peered":"2023-09-23T22:14:00.791655+0000","last_clean":"2023-09-23T22:14:00.791655+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:14:00.791655+0000","last_undegraded":"2023-09-23T22:14:00.791655+0000","last_fullsized":"2023-09-23T22:14:00.791655+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790313+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:14:00.790313+0000","last_peered":"2023-09-23T22:14:00.790313+0000","last_clean":"2023-09-23T22:14:00.790313+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:14:00.790313+0000","last_undegraded":"2023-09-23T22:14:00.790313+0000","last_fullsized":"2023-09-23T22:14:00.790313+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":86,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787689+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:13:59.787689+0000","last_peered":"2023-09-23T22:13:59.787689+0000","last_clean":"2023-09-23T22:13:59.787689+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:13:59.787689+0000","last_undegraded":"2023-09-23T22:13:59.787689+0000","last_fullsized":"2023-09-23T22:13:59.787689+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13208,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234124,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12896,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":11200,"kb_avail":93732256,"statfs":{"total":95995035648,"available":95981830144,"internally_reserved":0,"allocated":1728512,"data_stored":1203715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11468800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:13:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.55500000000000005,"15min":0.55300000000000005},"min":{"1min":0.377,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.504},{"interface":"front","average":{"1min":0.623,"5min":0.59099999999999997,"15min":0.58599999999999997},"min":{"1min":0.42999999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.42199999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.59399999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.008,"15min":1.008},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":8.5220000000000002},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:13:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.61399999999999999,"15min":0.59599999999999997},"min":{"1min":0.38700000000000001,"5min":0.38600000000000001,"15min":0.35699999999999998},"max":{"1min":0.81000000000000005,"5min":1.3109999999999999,"15min":7.8970000000000002},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.624,"5min":0.61299999999999999,"15min":0.59099999999999997},"min":{"1min":0.42799999999999999,"5min":0.41899999999999998,"15min":0.35599999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":2.9039999999999999},"last":0.63500000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.503,"15min":0.47799999999999998},"min":{"1min":0.32000000000000001,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":0.79000000000000004,"5min":0.81399999999999995,"15min":0.82199999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.50900000000000001,"15min":0.502},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.80800000000000005},"last":0.56000000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.502,"15min":0.48499999999999999},"min":{"1min":0.379,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":0.70499999999999996,"5min":0.71999999999999997,"15min":0.78700000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.57999999999999996,"15min":0.55400000000000005},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29199999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.873},"last":0.68500000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.55300000000000005,"15min":0.55800000000000005},"min":{"1min":0.41799999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.77600000000000002,"5min":0.82999999999999996,"15min":3.0459999999999998},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.625,"15min":0.58999999999999997},"min":{"1min":0.40799999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.81399999999999995,"5min":0.874,"15min":0.874},"last":0.66400000000000003}]}]},{"osd":6,"up_from":46,"seq":197568495760,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12444,"kb_used_data":5780,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93732708,"statfs":{"total":95995035648,"available":95982292992,"internally_reserved":0,"allocated":5918720,"data_stored":5398215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:13:44 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.56499999999999995,"15min":0.57199999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67900000000000005,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.60099999999999998,"15min":0.61099999999999999},"min":{"1min":0.36599999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.75,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.62}]},{"osd":1,"last update":"Sat Sep 23 22:13:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.70999999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.66400000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:13:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.58199999999999996,"15min":0.67600000000000005},"min":{"1min":0.434,"5min":0.379,"15min":0.28699999999999998},"max":{"1min":0.83999999999999997,"5min":1.4299999999999999,"15min":7.6429999999999998},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58699999999999997,"15min":0.57299999999999995},"min":{"1min":0.47799999999999998,"5min":0.36099999999999999,"15min":0.35799999999999998},"max":{"1min":0.752,"5min":0.84099999999999997,"15min":4.5670000000000002},"last":0.68000000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:14:05 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.46800000000000003,"15min":0.432},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.252},"max":{"1min":0.68899999999999995,"5min":0.77700000000000002,"15min":1.718},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.50900000000000001,"15min":0.46600000000000003},"min":{"1min":0.32400000000000001,"5min":0.315,"15min":0.26500000000000001},"max":{"1min":0.76800000000000002,"5min":0.86099999999999999,"15min":1.78},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:14:05 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51800000000000002,"15min":0.48299999999999998},"min":{"1min":0.375,"5min":0.315,"15min":0.29499999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":1.5629999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.51800000000000002,"15min":0.47999999999999998},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29399999999999998},"max":{"1min":0.68500000000000005,"5min":0.81399999999999995,"15min":1.74},"last":0.65100000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:14:42 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.56000000000000005,"15min":0.52100000000000002},"min":{"1min":0.378,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":0.77600000000000002,"5min":0.77900000000000003,"15min":1.7709999999999999},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.60599999999999998,"15min":0.57799999999999996},"min":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.35599999999999998},"max":{"1min":0.78500000000000003,"5min":0.90900000000000003,"15min":1.603},"last":0.73699999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795032,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32828,"kb_used_data":5236,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93712324,"statfs":{"total":95995035648,"available":95961419776,"internally_reserved":0,"allocated":5361664,"data_stored":4813549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":28238196},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.19400000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.20499999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:14:28 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.30499999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27900000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.56100000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.59399999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.50800000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958567,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29324,"kb_used_data":1732,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93715828,"statfs":{"total":95995035648,"available":95965007872,"internally_reserved":0,"allocated":1773568,"data_stored":1210692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":28237674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:13:58 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.496,"5min":0.496,"15min":0.496},"last":0.41299999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.4020000000000001,"5min":3.4020000000000001,"15min":3.4020000000000001},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.48299999999999998,"15min":0.49199999999999999},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.316},"max":{"1min":0.621,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.504},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.57899999999999996,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.495}]},{"osd":4,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.64600000000000002,"15min":0.65000000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.79500000000000004,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.65600000000000003,"15min":0.65900000000000003},"min":{"1min":0.52400000000000002,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.82999999999999996,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.67900000000000005,"15min":0.68500000000000005},"min":{"1min":0.47499999999999998,"5min":0.442,"15min":0.442},"max":{"1min":0.86199999999999999,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.67400000000000004,"15min":0.68100000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.79300000000000004,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.63}]},{"osd":6,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.67100000000000004,"15min":0.67500000000000004},"min":{"1min":0.46300000000000002,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":0.88900000000000001,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.627,"5min":0.70099999999999996,"15min":0.71299999999999997},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.85399999999999998,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.67600000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.71899999999999997,"15min":0.72699999999999998},"min":{"1min":0.5,"5min":0.5,"15min":0.5},"max":{"1min":0.92100000000000004,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.73999999999999999,"15min":0.75},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.90800000000000003,"5min":0.98499999999999999,"15min":0.98499999999999999},"last":0.753}]}]},{"osd":2,"up_from":103,"seq":442381631506,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28656,"kb_used_data":1128,"kb_used_omap":10,"kb_used_meta":27509,"kb_avail":93716496,"statfs":{"total":95995035648,"available":95965691904,"internally_reserved":0,"allocated":1155072,"data_stored":616429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":28169388},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.25800000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.31},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.33700000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":2.948,"5min":2.948,"15min":2.948},"last":0.61699999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":2.931,"5min":2.931,"15min":2.931},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.627}]}]},{"osd":3,"up_from":108,"seq":463856467979,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28572,"kb_used_data":1108,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716580,"statfs":{"total":95995035648,"available":95965777920,"internally_reserved":0,"allocated":1134592,"data_stored":613253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":28110381},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34399999999999997}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.28499999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51200000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57699999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019033,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7772,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93737380,"statfs":{"total":95995035648,"available":95987077120,"internally_reserved":0,"allocated":1134592,"data_stored":613600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.628,"5min":0.628,"15min":0.628},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.8180000000000001,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.63900000000000001,"15min":0.64800000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.75900000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.73199999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.62},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.67200000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:13:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.57099999999999995,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.37,"15min":0.313},"max":{"1min":0.80000000000000004,"5min":0.86199999999999999,"15min":3.1499999999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59599999999999997,"15min":0.56699999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76600000000000001,"5min":0.85099999999999998,"15min":1.0089999999999999},"last":0.60899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48899999999999999,"15min":0.45500000000000002},"min":{"1min":0.29199999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66300000000000003,"5min":0.749,"15min":0.81899999999999995},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.39500000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":0.622,"5min":0.78500000000000003,"15min":3.0470000000000002},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:14:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52100000000000002,"15min":0.49399999999999999},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.29599999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.83099999999999996},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56100000000000005,"15min":0.55000000000000004},"min":{"1min":0.41399999999999998,"5min":0.32100000000000001,"15min":0.28899999999999998},"max":{"1min":0.73799999999999999,"5min":0.83299999999999996,"15min":2.2440000000000002},"last":0.57999999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.58799999999999997,"15min":0.629},"min":{"1min":0.42699999999999999,"5min":0.33600000000000002,"15min":0.313},"max":{"1min":0.79500000000000004,"5min":0.83799999999999997,"15min":3.9609999999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.60299999999999998,"15min":0.58199999999999996},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.81000000000000005,"5min":0.82399999999999995,"15min":0.94999999999999996},"last":0.56599999999999995}]}]},{"osd":5,"up_from":38,"seq":163208757397,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11664,"kb_used_data":5192,"kb_used_omap":0,"kb_used_meta":6464,"kb_avail":93733488,"statfs":{"total":95995035648,"available":95983091712,"internally_reserved":0,"allocated":5316608,"data_stored":4803667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6619136},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.56100000000000005,"15min":0.56499999999999995},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.70899999999999996,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.312},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.627,"15min":0.64000000000000001},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.71399999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.372}]},{"osd":1,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.39400000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.49399999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.59499999999999997,"15min":0.57899999999999996},"min":{"1min":0.46300000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77900000000000003,"5min":0.93999999999999995,"15min":1.028},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.61299999999999999,"15min":0.60299999999999998},"min":{"1min":0.45400000000000001,"5min":0.36599999999999999,"15min":0.33400000000000002},"max":{"1min":0.79200000000000004,"5min":1.6160000000000001,"15min":2.04},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 22:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47399999999999998,"15min":0.44500000000000001},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69199999999999995,"5min":0.70899999999999996,"15min":0.752},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53800000000000003,"15min":0.48999999999999999},"min":{"1min":0.42199999999999999,"5min":0.35599999999999998,"15min":0.28999999999999998},"max":{"1min":0.747,"5min":0.751,"15min":0.81100000000000005},"last":0.51300000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:14:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.55000000000000004,"15min":0.50900000000000001},"min":{"1min":0.44600000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.56699999999999995,"15min":0.53000000000000003},"min":{"1min":0.46400000000000002,"5min":0.34300000000000003,"15min":0.29699999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.48099999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:40 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.59599999999999997,"15min":0.57799999999999996},"min":{"1min":0.437,"5min":0.376,"15min":0.32600000000000001},"max":{"1min":0.73199999999999998,"5min":0.81799999999999995,"15min":0.93700000000000006},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.58599999999999997,"15min":0.55900000000000005},"min":{"1min":0.41799999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.84999999999999998},"last":0.53600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:14:49.228 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:14:49.589 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:49.863 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:14:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:14:49] "GET /metrics HTTP/1.1" 200 33941 "" "Prometheus/2.43.0" 2023-09-23T22:14:49.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:49 smithi026 ceph-mon[215775]: from='client.44501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:14:49.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:49 smithi026 ceph-mon[223014]: from='client.44501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:14:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:49 smithi070 ceph-mon[192652]: from='client.44501 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:14:50.877 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:50 smithi026 ceph-mon[215775]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:50.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:50 smithi026 ceph-mon[223014]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:50 smithi070 ceph-mon[192652]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:14:51.269 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:14:51.907 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":209,"stamp":"2023-09-23T22:14:51.010358+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6483,"num_read_kb":14132,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164156,"kb_used_data":22972,"kb_used_omap":31,"kb_used_meta":141088,"kb_avail":749797060,"statfs":{"total":767960285184,"available":767792189440,"internally_reserved":0,"allocated":23523328,"data_stored":19273120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":144475063},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002553"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787529+0000","last_change":"2023-09-23T22:12:54.868920+0000","last_active":"2023-09-23T22:13:59.787529+0000","last_peered":"2023-09-23T22:13:59.787529+0000","last_clean":"2023-09-23T22:13:59.787529+0000","last_became_active":"2023-09-23T22:12:54.868576+0000","last_became_peered":"2023-09-23T22:12:54.868576+0000","last_unstale":"2023-09-23T22:13:59.787529+0000","last_undegraded":"2023-09-23T22:13:59.787529+0000","last_fullsized":"2023-09-23T22:13:59.787529+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:03.825524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787688+0000","last_change":"2023-09-23T22:13:27.664835+0000","last_active":"2023-09-23T22:13:59.787688+0000","last_peered":"2023-09-23T22:13:59.787688+0000","last_clean":"2023-09-23T22:13:59.787688+0000","last_became_active":"2023-09-23T22:13:27.664333+0000","last_became_peered":"2023-09-23T22:13:27.664333+0000","last_unstale":"2023-09-23T22:13:59.787688+0000","last_undegraded":"2023-09-23T22:13:59.787688+0000","last_fullsized":"2023-09-23T22:13:59.787688+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:11.434650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.784662+0000","last_change":"2023-09-23T22:14:00.784662+0000","last_active":"2023-09-23T22:14:00.784662+0000","last_peered":"2023-09-23T22:14:00.784662+0000","last_clean":"2023-09-23T22:14:00.784662+0000","last_became_active":"2023-09-23T22:14:00.784294+0000","last_became_peered":"2023-09-23T22:14:00.784294+0000","last_unstale":"2023-09-23T22:14:00.784662+0000","last_undegraded":"2023-09-23T22:14:00.784662+0000","last_fullsized":"2023-09-23T22:14:00.784662+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:09.123997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.785463+0000","last_change":"2023-09-23T22:14:00.785463+0000","last_active":"2023-09-23T22:14:00.785463+0000","last_peered":"2023-09-23T22:14:00.785463+0000","last_clean":"2023-09-23T22:14:00.785463+0000","last_became_active":"2023-09-23T22:14:00.785128+0000","last_became_peered":"2023-09-23T22:14:00.785128+0000","last_unstale":"2023-09-23T22:14:00.785463+0000","last_undegraded":"2023-09-23T22:14:00.785463+0000","last_fullsized":"2023-09-23T22:14:00.785463+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:46.343842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792878+0000","last_change":"2023-09-23T22:14:00.792878+0000","last_active":"2023-09-23T22:14:00.792878+0000","last_peered":"2023-09-23T22:14:00.792878+0000","last_clean":"2023-09-23T22:14:00.792878+0000","last_became_active":"2023-09-23T22:14:00.792609+0000","last_became_peered":"2023-09-23T22:14:00.792609+0000","last_unstale":"2023-09-23T22:14:00.792878+0000","last_undegraded":"2023-09-23T22:14:00.792878+0000","last_fullsized":"2023-09-23T22:14:00.792878+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:47.586353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781836+0000","last_change":"2023-09-23T22:04:54.379680+0000","last_active":"2023-09-23T22:13:59.781836+0000","last_peered":"2023-09-23T22:13:59.781836+0000","last_clean":"2023-09-23T22:13:59.781836+0000","last_became_active":"2023-09-23T22:04:54.379383+0000","last_became_peered":"2023-09-23T22:04:54.379383+0000","last_unstale":"2023-09-23T22:13:59.781836+0000","last_undegraded":"2023-09-23T22:13:59.781836+0000","last_fullsized":"2023-09-23T22:13:59.781836+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.282795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794701+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:14:00.794701+0000","last_peered":"2023-09-23T22:14:00.794701+0000","last_clean":"2023-09-23T22:14:00.794701+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:14:00.794701+0000","last_undegraded":"2023-09-23T22:14:00.794701+0000","last_fullsized":"2023-09-23T22:14:00.794701+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.782175+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:13:59.782175+0000","last_peered":"2023-09-23T22:13:59.782175+0000","last_clean":"2023-09-23T22:13:59.782175+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:13:59.782175+0000","last_undegraded":"2023-09-23T22:13:59.782175+0000","last_fullsized":"2023-09-23T22:13:59.782175+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.786689+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:14:00.786689+0000","last_peered":"2023-09-23T22:14:00.786689+0000","last_clean":"2023-09-23T22:14:00.786689+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:14:00.786689+0000","last_undegraded":"2023-09-23T22:14:00.786689+0000","last_fullsized":"2023-09-23T22:14:00.786689+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":40,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.780635+0000","last_change":"2023-09-23T22:11:42.696059+0000","last_active":"2023-09-23T22:14:00.780635+0000","last_peered":"2023-09-23T22:14:00.780635+0000","last_clean":"2023-09-23T22:14:00.780635+0000","last_became_active":"2023-09-23T22:11:42.695756+0000","last_became_peered":"2023-09-23T22:11:42.695756+0000","last_unstale":"2023-09-23T22:14:00.780635+0000","last_undegraded":"2023-09-23T22:14:00.780635+0000","last_fullsized":"2023-09-23T22:14:00.780635+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:34.633172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":50,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788995+0000","last_change":"2023-09-23T22:04:54.402894+0000","last_active":"2023-09-23T22:13:59.788995+0000","last_peered":"2023-09-23T22:13:59.788995+0000","last_clean":"2023-09-23T22:13:59.788995+0000","last_became_active":"2023-09-23T22:04:54.402475+0000","last_became_peered":"2023-09-23T22:04:54.402475+0000","last_unstale":"2023-09-23T22:13:59.788995+0000","last_undegraded":"2023-09-23T22:13:59.788995+0000","last_fullsized":"2023-09-23T22:13:59.788995+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:12.503263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789394+0000","last_change":"2023-09-23T22:12:54.870816+0000","last_active":"2023-09-23T22:13:59.789394+0000","last_peered":"2023-09-23T22:13:59.789394+0000","last_clean":"2023-09-23T22:13:59.789394+0000","last_became_active":"2023-09-23T22:12:54.870450+0000","last_became_peered":"2023-09-23T22:12:54.870450+0000","last_unstale":"2023-09-23T22:13:59.789394+0000","last_undegraded":"2023-09-23T22:13:59.789394+0000","last_fullsized":"2023-09-23T22:13:59.789394+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.764353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":234,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786388+0000","last_change":"2023-09-23T22:13:27.664537+0000","last_active":"2023-09-23T22:13:59.786388+0000","last_peered":"2023-09-23T22:13:59.786388+0000","last_clean":"2023-09-23T22:13:59.786388+0000","last_became_active":"2023-09-23T22:13:27.663897+0000","last_became_peered":"2023-09-23T22:13:27.663897+0000","last_unstale":"2023-09-23T22:13:59.786388+0000","last_undegraded":"2023-09-23T22:13:59.786388+0000","last_fullsized":"2023-09-23T22:13:59.786388+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:37.321580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":222,"num_read_kb":1213,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795827+0000","last_change":"2023-09-23T22:14:00.795827+0000","last_active":"2023-09-23T22:14:00.795827+0000","last_peered":"2023-09-23T22:14:00.795827+0000","last_clean":"2023-09-23T22:14:00.795827+0000","last_became_active":"2023-09-23T22:14:00.795225+0000","last_became_peered":"2023-09-23T22:14:00.795225+0000","last_unstale":"2023-09-23T22:14:00.795827+0000","last_undegraded":"2023-09-23T22:14:00.795827+0000","last_fullsized":"2023-09-23T22:14:00.795827+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:58:56.251393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":705,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781094+0000","last_change":"2023-09-23T22:12:54.874377+0000","last_active":"2023-09-23T22:13:59.781094+0000","last_peered":"2023-09-23T22:13:59.781094+0000","last_clean":"2023-09-23T22:13:59.781094+0000","last_became_active":"2023-09-23T22:12:54.873780+0000","last_became_peered":"2023-09-23T22:12:54.873780+0000","last_unstale":"2023-09-23T22:13:59.781094+0000","last_undegraded":"2023-09-23T22:13:59.781094+0000","last_fullsized":"2023-09-23T22:13:59.781094+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:08.681877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796111+0000","last_change":"2023-09-23T22:14:00.796111+0000","last_active":"2023-09-23T22:14:00.796111+0000","last_peered":"2023-09-23T22:14:00.796111+0000","last_clean":"2023-09-23T22:14:00.796111+0000","last_became_active":"2023-09-23T22:14:00.794856+0000","last_became_peered":"2023-09-23T22:14:00.794856+0000","last_unstale":"2023-09-23T22:14:00.796111+0000","last_undegraded":"2023-09-23T22:14:00.796111+0000","last_fullsized":"2023-09-23T22:14:00.796111+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:16.319602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786418+0000","last_change":"2023-09-23T22:11:42.696594+0000","last_active":"2023-09-23T22:13:59.786418+0000","last_peered":"2023-09-23T22:13:59.786418+0000","last_clean":"2023-09-23T22:13:59.786418+0000","last_became_active":"2023-09-23T22:11:42.696334+0000","last_became_peered":"2023-09-23T22:11:42.696334+0000","last_unstale":"2023-09-23T22:13:59.786418+0000","last_undegraded":"2023-09-23T22:13:59.786418+0000","last_fullsized":"2023-09-23T22:13:59.786418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:36.810008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780034+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:13:59.780034+0000","last_peered":"2023-09-23T22:13:59.780034+0000","last_clean":"2023-09-23T22:13:59.780034+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:13:59.780034+0000","last_undegraded":"2023-09-23T22:13:59.780034+0000","last_fullsized":"2023-09-23T22:13:59.780034+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":544,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786909+0000","last_change":"2023-09-23T22:11:42.693882+0000","last_active":"2023-09-23T22:13:59.786909+0000","last_peered":"2023-09-23T22:13:59.786909+0000","last_clean":"2023-09-23T22:13:59.786909+0000","last_became_active":"2023-09-23T22:11:42.693381+0000","last_became_peered":"2023-09-23T22:11:42.693381+0000","last_unstale":"2023-09-23T22:13:59.786909+0000","last_undegraded":"2023-09-23T22:13:59.786909+0000","last_fullsized":"2023-09-23T22:13:59.786909+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:06.246085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789486+0000","last_change":"2023-09-23T22:13:27.666017+0000","last_active":"2023-09-23T22:13:59.789486+0000","last_peered":"2023-09-23T22:13:59.789486+0000","last_clean":"2023-09-23T22:13:59.789486+0000","last_became_active":"2023-09-23T22:13:27.665706+0000","last_became_peered":"2023-09-23T22:13:27.665706+0000","last_unstale":"2023-09-23T22:13:59.789486+0000","last_undegraded":"2023-09-23T22:13:59.789486+0000","last_fullsized":"2023-09-23T22:13:59.789486+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:16.575957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790941+0000","last_change":"2023-09-23T22:14:00.790941+0000","last_active":"2023-09-23T22:14:00.790941+0000","last_peered":"2023-09-23T22:14:00.790941+0000","last_clean":"2023-09-23T22:14:00.790941+0000","last_became_active":"2023-09-23T22:14:00.790616+0000","last_became_peered":"2023-09-23T22:14:00.790616+0000","last_unstale":"2023-09-23T22:14:00.790941+0000","last_undegraded":"2023-09-23T22:14:00.790941+0000","last_fullsized":"2023-09-23T22:14:00.790941+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:53.552619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":458,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783191+0000","last_change":"2023-09-23T22:12:54.861801+0000","last_active":"2023-09-23T22:13:59.783191+0000","last_peered":"2023-09-23T22:13:59.783191+0000","last_clean":"2023-09-23T22:13:59.783191+0000","last_became_active":"2023-09-23T22:12:54.861358+0000","last_became_peered":"2023-09-23T22:12:54.861358+0000","last_unstale":"2023-09-23T22:13:59.783191+0000","last_undegraded":"2023-09-23T22:13:59.783191+0000","last_fullsized":"2023-09-23T22:13:59.783191+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:24.061357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785893+0000","last_change":"2023-09-23T22:04:54.527318+0000","last_active":"2023-09-23T22:13:59.785893+0000","last_peered":"2023-09-23T22:13:59.785893+0000","last_clean":"2023-09-23T22:13:59.785893+0000","last_became_active":"2023-09-23T22:04:54.527023+0000","last_became_peered":"2023-09-23T22:04:54.527023+0000","last_unstale":"2023-09-23T22:13:59.785893+0000","last_undegraded":"2023-09-23T22:13:59.785893+0000","last_fullsized":"2023-09-23T22:13:59.785893+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:42.048739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.789944+0000","last_change":"2023-09-23T22:14:00.789944+0000","last_active":"2023-09-23T22:14:00.789944+0000","last_peered":"2023-09-23T22:14:00.789944+0000","last_clean":"2023-09-23T22:14:00.789944+0000","last_became_active":"2023-09-23T22:14:00.789559+0000","last_became_peered":"2023-09-23T22:14:00.789559+0000","last_unstale":"2023-09-23T22:14:00.789944+0000","last_undegraded":"2023-09-23T22:14:00.789944+0000","last_fullsized":"2023-09-23T22:14:00.789944+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:30.957719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":451,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783108+0000","last_change":"2023-09-23T22:11:42.695484+0000","last_active":"2023-09-23T22:13:59.783108+0000","last_peered":"2023-09-23T22:13:59.783108+0000","last_clean":"2023-09-23T22:13:59.783108+0000","last_became_active":"2023-09-23T22:11:42.694972+0000","last_became_peered":"2023-09-23T22:11:42.694972+0000","last_unstale":"2023-09-23T22:13:59.783108+0000","last_undegraded":"2023-09-23T22:13:59.783108+0000","last_fullsized":"2023-09-23T22:13:59.783108+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:36.787404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":694,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780660+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:13:59.780660+0000","last_peered":"2023-09-23T22:13:59.780660+0000","last_clean":"2023-09-23T22:13:59.780660+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:13:59.780660+0000","last_undegraded":"2023-09-23T22:13:59.780660+0000","last_fullsized":"2023-09-23T22:13:59.780660+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781830+0000","last_change":"2023-09-23T22:12:54.874146+0000","last_active":"2023-09-23T22:13:59.781830+0000","last_peered":"2023-09-23T22:13:59.781830+0000","last_clean":"2023-09-23T22:13:59.781830+0000","last_became_active":"2023-09-23T22:12:54.873782+0000","last_became_peered":"2023-09-23T22:12:54.873782+0000","last_unstale":"2023-09-23T22:13:59.781830+0000","last_undegraded":"2023-09-23T22:13:59.781830+0000","last_fullsized":"2023-09-23T22:13:59.781830+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:25.933932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780900+0000","last_change":"2023-09-23T22:13:27.659751+0000","last_active":"2023-09-23T22:13:59.780900+0000","last_peered":"2023-09-23T22:13:59.780900+0000","last_clean":"2023-09-23T22:13:59.780900+0000","last_became_active":"2023-09-23T22:13:27.659422+0000","last_became_peered":"2023-09-23T22:13:27.659422+0000","last_unstale":"2023-09-23T22:13:59.780900+0000","last_undegraded":"2023-09-23T22:13:59.780900+0000","last_fullsized":"2023-09-23T22:13:59.780900+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:25.705909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":378,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.783947+0000","last_change":"2023-09-23T22:14:00.783947+0000","last_active":"2023-09-23T22:14:00.783947+0000","last_peered":"2023-09-23T22:14:00.783947+0000","last_clean":"2023-09-23T22:14:00.783947+0000","last_became_active":"2023-09-23T22:14:00.783319+0000","last_became_peered":"2023-09-23T22:14:00.783319+0000","last_unstale":"2023-09-23T22:14:00.783947+0000","last_undegraded":"2023-09-23T22:14:00.783947+0000","last_fullsized":"2023-09-23T22:14:00.783947+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:02:25.419096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796811+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:14:00.796811+0000","last_peered":"2023-09-23T22:14:00.796811+0000","last_clean":"2023-09-23T22:14:00.796811+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:14:00.796811+0000","last_undegraded":"2023-09-23T22:14:00.796811+0000","last_fullsized":"2023-09-23T22:14:00.796811+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788001+0000","last_change":"2023-09-23T22:11:42.697009+0000","last_active":"2023-09-23T22:13:59.788001+0000","last_peered":"2023-09-23T22:13:59.788001+0000","last_clean":"2023-09-23T22:13:59.788001+0000","last_became_active":"2023-09-23T22:11:42.696588+0000","last_became_peered":"2023-09-23T22:11:42.696588+0000","last_unstale":"2023-09-23T22:13:59.788001+0000","last_undegraded":"2023-09-23T22:13:59.788001+0000","last_fullsized":"2023-09-23T22:13:59.788001+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:57.260787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":750,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780601+0000","last_change":"2023-09-23T22:12:54.877082+0000","last_active":"2023-09-23T22:13:59.780601+0000","last_peered":"2023-09-23T22:13:59.780601+0000","last_clean":"2023-09-23T22:13:59.780601+0000","last_became_active":"2023-09-23T22:12:54.876524+0000","last_became_peered":"2023-09-23T22:12:54.876524+0000","last_unstale":"2023-09-23T22:13:59.780601+0000","last_undegraded":"2023-09-23T22:13:59.780601+0000","last_fullsized":"2023-09-23T22:13:59.780601+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:46.185025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.778046+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:13:59.778046+0000","last_peered":"2023-09-23T22:13:59.778046+0000","last_clean":"2023-09-23T22:13:59.778046+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:13:59.778046+0000","last_undegraded":"2023-09-23T22:13:59.778046+0000","last_fullsized":"2023-09-23T22:13:59.778046+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792797+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:14:00.792797+0000","last_peered":"2023-09-23T22:14:00.792797+0000","last_clean":"2023-09-23T22:14:00.792797+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:14:00.792797+0000","last_undegraded":"2023-09-23T22:14:00.792797+0000","last_fullsized":"2023-09-23T22:14:00.792797+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":555,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.782571+0000","last_change":"2023-09-23T22:12:54.875239+0000","last_active":"2023-09-23T22:13:59.782571+0000","last_peered":"2023-09-23T22:13:59.782571+0000","last_clean":"2023-09-23T22:13:59.782571+0000","last_became_active":"2023-09-23T22:12:54.874675+0000","last_became_peered":"2023-09-23T22:12:54.874675+0000","last_unstale":"2023-09-23T22:13:59.782571+0000","last_undegraded":"2023-09-23T22:13:59.782571+0000","last_fullsized":"2023-09-23T22:13:59.782571+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:21.865417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.791734+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:14:00.791734+0000","last_peered":"2023-09-23T22:14:00.791734+0000","last_clean":"2023-09-23T22:14:00.791734+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:14:00.791734+0000","last_undegraded":"2023-09-23T22:14:00.791734+0000","last_fullsized":"2023-09-23T22:14:00.791734+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795928+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:14:00.795928+0000","last_peered":"2023-09-23T22:14:00.795928+0000","last_clean":"2023-09-23T22:14:00.795928+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:14:00.795928+0000","last_undegraded":"2023-09-23T22:14:00.795928+0000","last_fullsized":"2023-09-23T22:14:00.795928+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.790050+0000","last_change":"2023-09-23T22:12:54.868682+0000","last_active":"2023-09-23T22:13:59.790050+0000","last_peered":"2023-09-23T22:13:59.790050+0000","last_clean":"2023-09-23T22:13:59.790050+0000","last_became_active":"2023-09-23T22:12:54.868339+0000","last_became_peered":"2023-09-23T22:12:54.868339+0000","last_unstale":"2023-09-23T22:13:59.790050+0000","last_undegraded":"2023-09-23T22:13:59.790050+0000","last_fullsized":"2023-09-23T22:13:59.790050+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:53.246678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789109+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:13:59.789109+0000","last_peered":"2023-09-23T22:13:59.789109+0000","last_clean":"2023-09-23T22:13:59.789109+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:13:59.789109+0000","last_undegraded":"2023-09-23T22:13:59.789109+0000","last_fullsized":"2023-09-23T22:13:59.789109+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.791120+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:13:59.791120+0000","last_peered":"2023-09-23T22:13:59.791120+0000","last_clean":"2023-09-23T22:13:59.791120+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:13:59.791120+0000","last_undegraded":"2023-09-23T22:13:59.791120+0000","last_fullsized":"2023-09-23T22:13:59.791120+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785609+0000","last_change":"2023-09-23T22:12:54.880778+0000","last_active":"2023-09-23T22:13:59.785609+0000","last_peered":"2023-09-23T22:13:59.785609+0000","last_clean":"2023-09-23T22:13:59.785609+0000","last_became_active":"2023-09-23T22:12:54.880272+0000","last_became_peered":"2023-09-23T22:12:54.880272+0000","last_unstale":"2023-09-23T22:13:59.785609+0000","last_undegraded":"2023-09-23T22:13:59.785609+0000","last_fullsized":"2023-09-23T22:13:59.785609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:01.030591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792760+0000","last_change":"2023-09-23T22:14:00.792760+0000","last_active":"2023-09-23T22:14:00.792760+0000","last_peered":"2023-09-23T22:14:00.792760+0000","last_clean":"2023-09-23T22:14:00.792760+0000","last_became_active":"2023-09-23T22:14:00.792478+0000","last_became_peered":"2023-09-23T22:14:00.792478+0000","last_unstale":"2023-09-23T22:14:00.792760+0000","last_undegraded":"2023-09-23T22:14:00.792760+0000","last_fullsized":"2023-09-23T22:14:00.792760+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.935338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790875+0000","last_change":"2023-09-23T22:14:00.790875+0000","last_active":"2023-09-23T22:14:00.790875+0000","last_peered":"2023-09-23T22:14:00.790875+0000","last_clean":"2023-09-23T22:14:00.790875+0000","last_became_active":"2023-09-23T22:14:00.790332+0000","last_became_peered":"2023-09-23T22:14:00.790332+0000","last_unstale":"2023-09-23T22:14:00.790875+0000","last_undegraded":"2023-09-23T22:14:00.790875+0000","last_fullsized":"2023-09-23T22:14:00.790875+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:05.904013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786633+0000","last_change":"2023-09-23T22:12:54.871021+0000","last_active":"2023-09-23T22:13:59.786633+0000","last_peered":"2023-09-23T22:13:59.786633+0000","last_clean":"2023-09-23T22:13:59.786633+0000","last_became_active":"2023-09-23T22:12:54.870736+0000","last_became_peered":"2023-09-23T22:12:54.870736+0000","last_unstale":"2023-09-23T22:13:59.786633+0000","last_undegraded":"2023-09-23T22:13:59.786633+0000","last_fullsized":"2023-09-23T22:13:59.786633+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:51.846457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795682+0000","last_change":"2023-09-23T22:14:00.795682+0000","last_active":"2023-09-23T22:14:00.795682+0000","last_peered":"2023-09-23T22:14:00.795682+0000","last_clean":"2023-09-23T22:14:00.795682+0000","last_became_active":"2023-09-23T22:14:00.795377+0000","last_became_peered":"2023-09-23T22:14:00.795377+0000","last_unstale":"2023-09-23T22:14:00.795682+0000","last_undegraded":"2023-09-23T22:14:00.795682+0000","last_fullsized":"2023-09-23T22:14:00.795682+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:33.253920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785440+0000","last_change":"2023-09-23T22:12:54.865320+0000","last_active":"2023-09-23T22:13:59.785440+0000","last_peered":"2023-09-23T22:13:59.785440+0000","last_clean":"2023-09-23T22:13:59.785440+0000","last_became_active":"2023-09-23T22:12:54.865007+0000","last_became_peered":"2023-09-23T22:12:54.865007+0000","last_unstale":"2023-09-23T22:13:59.785440+0000","last_undegraded":"2023-09-23T22:13:59.785440+0000","last_fullsized":"2023-09-23T22:13:59.785440+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:40.963311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787011+0000","last_change":"2023-09-23T22:04:54.384008+0000","last_active":"2023-09-23T22:13:59.787011+0000","last_peered":"2023-09-23T22:13:59.787011+0000","last_clean":"2023-09-23T22:13:59.787011+0000","last_became_active":"2023-09-23T22:04:54.383702+0000","last_became_peered":"2023-09-23T22:04:54.383702+0000","last_unstale":"2023-09-23T22:13:59.787011+0000","last_undegraded":"2023-09-23T22:13:59.787011+0000","last_fullsized":"2023-09-23T22:13:59.787011+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:00.423911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795430+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:14:00.795430+0000","last_peered":"2023-09-23T22:14:00.795430+0000","last_clean":"2023-09-23T22:14:00.795430+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:14:00.795430+0000","last_undegraded":"2023-09-23T22:14:00.795430+0000","last_fullsized":"2023-09-23T22:14:00.795430+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786126+0000","last_change":"2023-09-23T22:12:54.861787+0000","last_active":"2023-09-23T22:13:59.786126+0000","last_peered":"2023-09-23T22:13:59.786126+0000","last_clean":"2023-09-23T22:13:59.786126+0000","last_became_active":"2023-09-23T22:12:54.861377+0000","last_became_peered":"2023-09-23T22:12:54.861377+0000","last_unstale":"2023-09-23T22:13:59.786126+0000","last_undegraded":"2023-09-23T22:13:59.786126+0000","last_fullsized":"2023-09-23T22:13:59.786126+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:02.832226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780468+0000","last_change":"2023-09-23T22:04:54.528019+0000","last_active":"2023-09-23T22:13:59.780468+0000","last_peered":"2023-09-23T22:13:59.780468+0000","last_clean":"2023-09-23T22:13:59.780468+0000","last_became_active":"2023-09-23T22:04:54.527666+0000","last_became_peered":"2023-09-23T22:04:54.527666+0000","last_unstale":"2023-09-23T22:13:59.780468+0000","last_undegraded":"2023-09-23T22:13:59.780468+0000","last_fullsized":"2023-09-23T22:13:59.780468+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:32.774493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786894+0000","last_change":"2023-09-23T22:11:42.694008+0000","last_active":"2023-09-23T22:13:59.786894+0000","last_peered":"2023-09-23T22:13:59.786894+0000","last_clean":"2023-09-23T22:13:59.786894+0000","last_became_active":"2023-09-23T22:11:42.693609+0000","last_became_peered":"2023-09-23T22:11:42.693609+0000","last_unstale":"2023-09-23T22:13:59.786894+0000","last_undegraded":"2023-09-23T22:13:59.786894+0000","last_fullsized":"2023-09-23T22:13:59.786894+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:59.614114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794463+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:14:00.794463+0000","last_peered":"2023-09-23T22:14:00.794463+0000","last_clean":"2023-09-23T22:14:00.794463+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:14:00.794463+0000","last_undegraded":"2023-09-23T22:14:00.794463+0000","last_fullsized":"2023-09-23T22:14:00.794463+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":77,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.784703+0000","last_change":"2023-09-23T22:11:42.701916+0000","last_active":"2023-09-23T22:13:59.784703+0000","last_peered":"2023-09-23T22:13:59.784703+0000","last_clean":"2023-09-23T22:13:59.784703+0000","last_became_active":"2023-09-23T22:11:42.701591+0000","last_became_peered":"2023-09-23T22:11:42.701591+0000","last_unstale":"2023-09-23T22:13:59.784703+0000","last_undegraded":"2023-09-23T22:13:59.784703+0000","last_fullsized":"2023-09-23T22:13:59.784703+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:23.941128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.793040+0000","last_change":"2023-09-23T22:14:00.792449+0000","last_active":"2023-09-23T22:14:00.793040+0000","last_peered":"2023-09-23T22:14:00.793040+0000","last_clean":"2023-09-23T22:14:00.793040+0000","last_became_active":"2023-09-23T22:14:00.792239+0000","last_became_peered":"2023-09-23T22:14:00.792239+0000","last_unstale":"2023-09-23T22:14:00.793040+0000","last_undegraded":"2023-09-23T22:14:00.793040+0000","last_fullsized":"2023-09-23T22:14:00.793040+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:33.488563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783327+0000","last_change":"2023-09-23T22:04:54.387688+0000","last_active":"2023-09-23T22:13:59.783327+0000","last_peered":"2023-09-23T22:13:59.783327+0000","last_clean":"2023-09-23T22:13:59.783327+0000","last_became_active":"2023-09-23T22:04:54.387421+0000","last_became_peered":"2023-09-23T22:04:54.387421+0000","last_unstale":"2023-09-23T22:13:59.783327+0000","last_undegraded":"2023-09-23T22:13:59.783327+0000","last_fullsized":"2023-09-23T22:13:59.783327+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:22.411746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794669+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:14:00.794669+0000","last_peered":"2023-09-23T22:14:00.794669+0000","last_clean":"2023-09-23T22:14:00.794669+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:14:00.794669+0000","last_undegraded":"2023-09-23T22:14:00.794669+0000","last_fullsized":"2023-09-23T22:14:00.794669+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792829+0000","last_change":"2023-09-23T22:14:00.792829+0000","last_active":"2023-09-23T22:14:00.792829+0000","last_peered":"2023-09-23T22:14:00.792829+0000","last_clean":"2023-09-23T22:14:00.792829+0000","last_became_active":"2023-09-23T22:14:00.792462+0000","last_became_peered":"2023-09-23T22:14:00.792462+0000","last_unstale":"2023-09-23T22:14:00.792829+0000","last_undegraded":"2023-09-23T22:14:00.792829+0000","last_fullsized":"2023-09-23T22:14:00.792829+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:55.154145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794351+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:14:00.794351+0000","last_peered":"2023-09-23T22:14:00.794351+0000","last_clean":"2023-09-23T22:14:00.794351+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:14:00.794351+0000","last_undegraded":"2023-09-23T22:14:00.794351+0000","last_fullsized":"2023-09-23T22:14:00.794351+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785117+0000","last_change":"2023-09-23T22:13:27.663039+0000","last_active":"2023-09-23T22:13:59.785117+0000","last_peered":"2023-09-23T22:13:59.785117+0000","last_clean":"2023-09-23T22:13:59.785117+0000","last_became_active":"2023-09-23T22:13:27.662687+0000","last_became_peered":"2023-09-23T22:13:27.662687+0000","last_unstale":"2023-09-23T22:13:59.785117+0000","last_undegraded":"2023-09-23T22:13:59.785117+0000","last_fullsized":"2023-09-23T22:13:59.785117+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:13.432843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788095+0000","last_change":"2023-09-23T22:12:54.866763+0000","last_active":"2023-09-23T22:13:59.788095+0000","last_peered":"2023-09-23T22:13:59.788095+0000","last_clean":"2023-09-23T22:13:59.788095+0000","last_became_active":"2023-09-23T22:12:54.866422+0000","last_became_peered":"2023-09-23T22:12:54.866422+0000","last_unstale":"2023-09-23T22:13:59.788095+0000","last_undegraded":"2023-09-23T22:13:59.788095+0000","last_fullsized":"2023-09-23T22:13:59.788095+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:43.904727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.790220+0000","last_change":"2023-09-23T22:04:54.381132+0000","last_active":"2023-09-23T22:13:59.790220+0000","last_peered":"2023-09-23T22:13:59.790220+0000","last_clean":"2023-09-23T22:13:59.790220+0000","last_became_active":"2023-09-23T22:04:54.380838+0000","last_became_peered":"2023-09-23T22:04:54.380838+0000","last_unstale":"2023-09-23T22:13:59.790220+0000","last_undegraded":"2023-09-23T22:13:59.790220+0000","last_fullsized":"2023-09-23T22:13:59.790220+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:41.334974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780225+0000","last_change":"2023-09-23T22:13:27.661799+0000","last_active":"2023-09-23T22:13:59.780225+0000","last_peered":"2023-09-23T22:13:59.780225+0000","last_clean":"2023-09-23T22:13:59.780225+0000","last_became_active":"2023-09-23T22:13:27.661473+0000","last_became_peered":"2023-09-23T22:13:27.661473+0000","last_unstale":"2023-09-23T22:13:59.780225+0000","last_undegraded":"2023-09-23T22:13:59.780225+0000","last_fullsized":"2023-09-23T22:13:59.780225+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:34.827694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796705+0000","last_change":"2023-09-23T22:14:00.796705+0000","last_active":"2023-09-23T22:14:00.796705+0000","last_peered":"2023-09-23T22:14:00.796705+0000","last_clean":"2023-09-23T22:14:00.796705+0000","last_became_active":"2023-09-23T22:14:00.796252+0000","last_became_peered":"2023-09-23T22:14:00.796252+0000","last_unstale":"2023-09-23T22:14:00.796705+0000","last_undegraded":"2023-09-23T22:14:00.796705+0000","last_fullsized":"2023-09-23T22:14:00.796705+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:58.956676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794963+0000","last_change":"2023-09-23T22:14:00.794963+0000","last_active":"2023-09-23T22:14:00.794963+0000","last_peered":"2023-09-23T22:14:00.794963+0000","last_clean":"2023-09-23T22:14:00.794963+0000","last_became_active":"2023-09-23T22:14:00.794394+0000","last_became_peered":"2023-09-23T22:14:00.794394+0000","last_unstale":"2023-09-23T22:14:00.794963+0000","last_undegraded":"2023-09-23T22:14:00.794963+0000","last_fullsized":"2023-09-23T22:14:00.794963+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:25.119383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780772+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:13:59.780772+0000","last_peered":"2023-09-23T22:13:59.780772+0000","last_clean":"2023-09-23T22:13:59.780772+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:13:59.780772+0000","last_undegraded":"2023-09-23T22:13:59.780772+0000","last_fullsized":"2023-09-23T22:13:59.780772+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790411+0000","last_change":"2023-09-23T22:14:00.790411+0000","last_active":"2023-09-23T22:14:00.790411+0000","last_peered":"2023-09-23T22:14:00.790411+0000","last_clean":"2023-09-23T22:14:00.790411+0000","last_became_active":"2023-09-23T22:14:00.789906+0000","last_became_peered":"2023-09-23T22:14:00.789906+0000","last_unstale":"2023-09-23T22:14:00.790411+0000","last_undegraded":"2023-09-23T22:14:00.790411+0000","last_fullsized":"2023-09-23T22:14:00.790411+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:28:43.440626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.788697+0000","last_change":"2023-09-23T22:14:00.787881+0000","last_active":"2023-09-23T22:14:00.788697+0000","last_peered":"2023-09-23T22:14:00.788697+0000","last_clean":"2023-09-23T22:14:00.788697+0000","last_became_active":"2023-09-23T22:14:00.787498+0000","last_became_peered":"2023-09-23T22:14:00.787498+0000","last_unstale":"2023-09-23T22:14:00.788697+0000","last_undegraded":"2023-09-23T22:14:00.788697+0000","last_fullsized":"2023-09-23T22:14:00.788697+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:19.424721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786273+0000","last_change":"2023-09-23T22:12:54.864958+0000","last_active":"2023-09-23T22:13:59.786273+0000","last_peered":"2023-09-23T22:13:59.786273+0000","last_clean":"2023-09-23T22:13:59.786273+0000","last_became_active":"2023-09-23T22:12:54.864451+0000","last_became_peered":"2023-09-23T22:12:54.864451+0000","last_unstale":"2023-09-23T22:13:59.786273+0000","last_undegraded":"2023-09-23T22:13:59.786273+0000","last_fullsized":"2023-09-23T22:13:59.786273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:32.668061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790097+0000","last_change":"2023-09-23T22:14:00.790097+0000","last_active":"2023-09-23T22:14:00.790097+0000","last_peered":"2023-09-23T22:14:00.790097+0000","last_clean":"2023-09-23T22:14:00.790097+0000","last_became_active":"2023-09-23T22:14:00.789389+0000","last_became_peered":"2023-09-23T22:14:00.789389+0000","last_unstale":"2023-09-23T22:14:00.790097+0000","last_undegraded":"2023-09-23T22:14:00.790097+0000","last_fullsized":"2023-09-23T22:14:00.790097+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:05.673490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792919+0000","last_change":"2023-09-23T22:14:00.792919+0000","last_active":"2023-09-23T22:14:00.792919+0000","last_peered":"2023-09-23T22:14:00.792919+0000","last_clean":"2023-09-23T22:14:00.792919+0000","last_became_active":"2023-09-23T22:14:00.792490+0000","last_became_peered":"2023-09-23T22:14:00.792490+0000","last_unstale":"2023-09-23T22:14:00.792919+0000","last_undegraded":"2023-09-23T22:14:00.792919+0000","last_fullsized":"2023-09-23T22:14:00.792919+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:53.433863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.791655+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:14:00.791655+0000","last_peered":"2023-09-23T22:14:00.791655+0000","last_clean":"2023-09-23T22:14:00.791655+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:14:00.791655+0000","last_undegraded":"2023-09-23T22:14:00.791655+0000","last_fullsized":"2023-09-23T22:14:00.791655+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790313+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:14:00.790313+0000","last_peered":"2023-09-23T22:14:00.790313+0000","last_clean":"2023-09-23T22:14:00.790313+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:14:00.790313+0000","last_undegraded":"2023-09-23T22:14:00.790313+0000","last_fullsized":"2023-09-23T22:14:00.790313+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":86,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787689+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:13:59.787689+0000","last_peered":"2023-09-23T22:13:59.787689+0000","last_clean":"2023-09-23T22:13:59.787689+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:13:59.787689+0000","last_undegraded":"2023-09-23T22:13:59.787689+0000","last_fullsized":"2023-09-23T22:13:59.787689+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13208,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234125,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12896,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":11200,"kb_avail":93732256,"statfs":{"total":95995035648,"available":95981830144,"internally_reserved":0,"allocated":1728512,"data_stored":1203715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11468800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:13:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":0.55500000000000005,"15min":0.55300000000000005},"min":{"1min":0.377,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.59099999999999997,"15min":0.58599999999999997},"min":{"1min":0.42999999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.68999999999999995}]},{"osd":1,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.78700000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.008,"15min":1.008},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":8.5220000000000002},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.68000000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:13:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.61399999999999999,"15min":0.59599999999999997},"min":{"1min":0.38700000000000001,"5min":0.38600000000000001,"15min":0.35699999999999998},"max":{"1min":0.81000000000000005,"5min":1.3109999999999999,"15min":7.8970000000000002},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.624,"5min":0.61299999999999999,"15min":0.59099999999999997},"min":{"1min":0.42799999999999999,"5min":0.41899999999999998,"15min":0.35599999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":2.9039999999999999},"last":0.81299999999999994}]},{"osd":4,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.503,"15min":0.47799999999999998},"min":{"1min":0.32000000000000001,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":0.79000000000000004,"5min":0.81399999999999995,"15min":0.82199999999999995},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.50900000000000001,"15min":0.502},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.80800000000000005},"last":0.59699999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.502,"15min":0.48499999999999999},"min":{"1min":0.379,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":0.70499999999999996,"5min":0.71999999999999997,"15min":0.78700000000000003},"last":0.627},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.57999999999999996,"15min":0.55400000000000005},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29199999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.873},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.55300000000000005,"15min":0.55800000000000005},"min":{"1min":0.41799999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.77600000000000002,"5min":0.82999999999999996,"15min":3.0459999999999998},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.625,"15min":0.58999999999999997},"min":{"1min":0.40799999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.81399999999999995,"5min":0.874,"15min":0.874},"last":0.73099999999999998}]}]},{"osd":6,"up_from":46,"seq":197568495761,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12444,"kb_used_data":5780,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93732708,"statfs":{"total":95995035648,"available":95982292992,"internally_reserved":0,"allocated":5918720,"data_stored":5398215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.54500000000000004,"15min":0.56499999999999995},"min":{"1min":0.34799999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.75600000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.58099999999999996,"15min":0.60499999999999998},"min":{"1min":0.36599999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.69799999999999995,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.5}]},{"osd":1,"last update":"Sat Sep 23 22:13:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.47499999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:13:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.58199999999999996,"15min":0.67600000000000005},"min":{"1min":0.434,"5min":0.379,"15min":0.28699999999999998},"max":{"1min":0.83999999999999997,"5min":1.4299999999999999,"15min":7.6429999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58699999999999997,"15min":0.57299999999999995},"min":{"1min":0.47799999999999998,"5min":0.36099999999999999,"15min":0.35799999999999998},"max":{"1min":0.752,"5min":0.84099999999999997,"15min":4.5670000000000002},"last":0.57099999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:14:05 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.46800000000000003,"15min":0.432},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.252},"max":{"1min":0.68899999999999995,"5min":0.77700000000000002,"15min":1.718},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.50900000000000001,"15min":0.46600000000000003},"min":{"1min":0.32400000000000001,"5min":0.315,"15min":0.26500000000000001},"max":{"1min":0.76800000000000002,"5min":0.86099999999999999,"15min":1.78},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:14:05 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51800000000000002,"15min":0.48299999999999998},"min":{"1min":0.375,"5min":0.315,"15min":0.29499999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":1.5629999999999999},"last":0.439},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.51800000000000002,"15min":0.47999999999999998},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29399999999999998},"max":{"1min":0.68500000000000005,"5min":0.81399999999999995,"15min":1.74},"last":0.41299999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:42 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.56000000000000005,"15min":0.52100000000000002},"min":{"1min":0.378,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":0.77600000000000002,"5min":0.77900000000000003,"15min":1.7709999999999999},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.60599999999999998,"15min":0.57799999999999996},"min":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.35599999999999998},"max":{"1min":0.78500000000000003,"5min":0.90900000000000003,"15min":1.603},"last":0.55000000000000004}]}]},{"osd":1,"up_from":98,"seq":420906795033,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32828,"kb_used_data":5236,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93712324,"statfs":{"total":95995035648,"available":95961419776,"internally_reserved":0,"allocated":5361664,"data_stored":4813549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":28238196},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.311},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:14:28 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.371}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40000000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.504}]},{"osd":6,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.52200000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958568,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29324,"kb_used_data":1732,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93715828,"statfs":{"total":95995035648,"available":95965007872,"internally_reserved":0,"allocated":1773568,"data_stored":1210692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":28237674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:13:58 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.496,"5min":0.496,"15min":0.496},"last":0.39700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.505},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.4020000000000001,"5min":3.4020000000000001,"15min":3.4020000000000001},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.48299999999999998,"15min":0.49199999999999999},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.316},"max":{"1min":0.621,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.57899999999999996,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.49299999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.66600000000000004,"15min":0.65600000000000003},"min":{"1min":0.437,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":3.181,"5min":3.181,"15min":3.181},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.65000000000000002,"15min":0.65700000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89900000000000002,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.628}]},{"osd":5,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.69299999999999995,"15min":0.68999999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.68000000000000005,"15min":0.68300000000000005},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.64900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.68799999999999994,"15min":0.68100000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":3.3159999999999998,"5min":3.3159999999999998,"15min":3.3159999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.69099999999999995,"15min":0.70999999999999996},"min":{"1min":0.436,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.92000000000000004,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.73199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.73299999999999998,"15min":0.73099999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":3.367,"5min":3.367,"15min":3.367},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.748,"15min":0.753},"min":{"1min":0.495,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":3.286,"5min":3.286,"15min":3.286},"last":0.68600000000000005}]}]},{"osd":2,"up_from":103,"seq":442381631507,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28656,"kb_used_data":1128,"kb_used_omap":10,"kb_used_meta":27509,"kb_avail":93716496,"statfs":{"total":95995035648,"available":95965691904,"internally_reserved":0,"allocated":1155072,"data_stored":616429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":28169388},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.38700000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.33900000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.625}]},{"osd":5,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":2.948,"5min":2.948,"15min":2.948},"last":0.59799999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":2.931,"5min":2.931,"15min":2.931},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.65700000000000003}]}]},{"osd":3,"up_from":108,"seq":463856467980,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28572,"kb_used_data":1108,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716580,"statfs":{"total":95995035648,"available":95965777920,"internally_reserved":0,"allocated":1134592,"data_stored":613253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":28110381},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.504}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35299999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54200000000000004}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.623}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64500000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019034,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7772,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93737380,"statfs":{"total":95995035648,"available":95987077120,"internally_reserved":0,"allocated":1134592,"data_stored":613600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.622,"15min":0.626},"min":{"1min":0.38900000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2749999999999999,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":0.505},{"interface":"front","average":{"1min":0.628,"5min":0.63400000000000001,"15min":0.64600000000000002},"min":{"1min":0.42099999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.55100000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.47899999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.63200000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:13:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.57099999999999995,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.37,"15min":0.313},"max":{"1min":0.80000000000000004,"5min":0.86199999999999999,"15min":3.1499999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59599999999999997,"15min":0.56699999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76600000000000001,"5min":0.85099999999999998,"15min":1.0089999999999999},"last":0.443}]},{"osd":5,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48899999999999999,"15min":0.45500000000000002},"min":{"1min":0.29199999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66300000000000003,"5min":0.749,"15min":0.81899999999999995},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.39500000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":0.622,"5min":0.78500000000000003,"15min":3.0470000000000002},"last":0.52100000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:14:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52100000000000002,"15min":0.49399999999999999},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.29599999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.83099999999999996},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56100000000000005,"15min":0.55000000000000004},"min":{"1min":0.41399999999999998,"5min":0.32100000000000001,"15min":0.28899999999999998},"max":{"1min":0.73799999999999999,"5min":0.83299999999999996,"15min":2.2440000000000002},"last":0.60099999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.60099999999999998,"15min":0.624},"min":{"1min":0.46899999999999997,"5min":0.372,"15min":0.313},"max":{"1min":0.82299999999999995,"5min":0.83799999999999997,"15min":3.9609999999999999},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.61299999999999999,"15min":0.58799999999999997},"min":{"1min":0.495,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.94999999999999996},"last":0.61199999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757398,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11664,"kb_used_data":5192,"kb_used_omap":0,"kb_used_meta":6464,"kb_avail":93733488,"statfs":{"total":95995035648,"available":95983091712,"internally_reserved":0,"allocated":5316608,"data_stored":4803667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6619136},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.55500000000000005,"15min":0.56299999999999994},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.502,"5min":0.59799999999999998,"15min":0.63},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.52100000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.374},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.66500000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.59499999999999997,"15min":0.57899999999999996},"min":{"1min":0.46300000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77900000000000003,"5min":0.93999999999999995,"15min":1.028},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.61299999999999999,"15min":0.60299999999999998},"min":{"1min":0.45400000000000001,"5min":0.36599999999999999,"15min":0.33400000000000002},"max":{"1min":0.79200000000000004,"5min":1.6160000000000001,"15min":2.04},"last":0.46100000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47399999999999998,"15min":0.44500000000000001},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69199999999999995,"5min":0.70899999999999996,"15min":0.752},"last":0.625},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53800000000000003,"15min":0.48999999999999999},"min":{"1min":0.42199999999999999,"5min":0.35599999999999998,"15min":0.28999999999999998},"max":{"1min":0.747,"5min":0.751,"15min":0.81100000000000005},"last":0.64200000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:14:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.55000000000000004,"15min":0.50900000000000001},"min":{"1min":0.44600000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.504},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.56699999999999995,"15min":0.53000000000000003},"min":{"1min":0.46400000000000002,"5min":0.34300000000000003,"15min":0.29699999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.61299999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:14:40 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.59599999999999997,"15min":0.57799999999999996},"min":{"1min":0.437,"5min":0.376,"15min":0.32600000000000001},"max":{"1min":0.73199999999999998,"5min":0.81799999999999995,"15min":0.93700000000000006},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.58599999999999997,"15min":0.55900000000000005},"min":{"1min":0.41799999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.84999999999999998},"last":0.70499999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:14:51.912 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:14:51.912 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T22:14:51.913 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T22:14:51.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T22:14:52.276 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:52.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:52 smithi026 ceph-mon[223014]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:52.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:52 smithi026 ceph-mon[223014]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:14:52.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:52 smithi026 ceph-mon[215775]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:52.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:52 smithi026 ceph-mon[215775]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:14:53.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:52 smithi070 ceph-mon[192652]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:53.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:52 smithi070 ceph-mon[192652]: from='client.44507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:14:53.978 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:14:53.978 INFO:teuthology.orchestra.run.smithi026.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T22:14:54.546 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T22:14:54.546 INFO:teuthology.run_tasks:Running task print... 2023-09-23T22:14:54.555 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T22:14:54.555 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T22:14:54.601 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T22:14:54.601 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T22:14:54.601 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T22:14:54.603 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T22:14:54.603 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T22:14:54.603 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T22:14:54.603 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T22:14:54.603 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T22:14:54.603 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:14:54.965 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:54 smithi026 ceph-mon[215775]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1331389835' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:14:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:54 smithi026 ceph-mon[223014]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1331389835' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:14:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:55.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:54 smithi070 ceph-mon[192652]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:55.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1331389835' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T22:14:55.141 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:14:56.576 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:14:56.577 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:14:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:56 smithi026 ceph-mon[223014]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:56 smithi026 ceph-mon[215775]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:56 smithi070 ceph-mon[192652]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:57.230 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":211,"stamp":"2023-09-23T22:14:55.011173+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6483,"num_read_kb":14132,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164156,"kb_used_data":22972,"kb_used_omap":31,"kb_used_meta":141088,"kb_avail":749797060,"statfs":{"total":767960285184,"available":767792189440,"internally_reserved":0,"allocated":23523328,"data_stored":19273120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":144475063},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002444"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787529+0000","last_change":"2023-09-23T22:12:54.868920+0000","last_active":"2023-09-23T22:13:59.787529+0000","last_peered":"2023-09-23T22:13:59.787529+0000","last_clean":"2023-09-23T22:13:59.787529+0000","last_became_active":"2023-09-23T22:12:54.868576+0000","last_became_peered":"2023-09-23T22:12:54.868576+0000","last_unstale":"2023-09-23T22:13:59.787529+0000","last_undegraded":"2023-09-23T22:13:59.787529+0000","last_fullsized":"2023-09-23T22:13:59.787529+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:03.825524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787688+0000","last_change":"2023-09-23T22:13:27.664835+0000","last_active":"2023-09-23T22:13:59.787688+0000","last_peered":"2023-09-23T22:13:59.787688+0000","last_clean":"2023-09-23T22:13:59.787688+0000","last_became_active":"2023-09-23T22:13:27.664333+0000","last_became_peered":"2023-09-23T22:13:27.664333+0000","last_unstale":"2023-09-23T22:13:59.787688+0000","last_undegraded":"2023-09-23T22:13:59.787688+0000","last_fullsized":"2023-09-23T22:13:59.787688+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:11.434650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.784662+0000","last_change":"2023-09-23T22:14:00.784662+0000","last_active":"2023-09-23T22:14:00.784662+0000","last_peered":"2023-09-23T22:14:00.784662+0000","last_clean":"2023-09-23T22:14:00.784662+0000","last_became_active":"2023-09-23T22:14:00.784294+0000","last_became_peered":"2023-09-23T22:14:00.784294+0000","last_unstale":"2023-09-23T22:14:00.784662+0000","last_undegraded":"2023-09-23T22:14:00.784662+0000","last_fullsized":"2023-09-23T22:14:00.784662+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:09.123997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.785463+0000","last_change":"2023-09-23T22:14:00.785463+0000","last_active":"2023-09-23T22:14:00.785463+0000","last_peered":"2023-09-23T22:14:00.785463+0000","last_clean":"2023-09-23T22:14:00.785463+0000","last_became_active":"2023-09-23T22:14:00.785128+0000","last_became_peered":"2023-09-23T22:14:00.785128+0000","last_unstale":"2023-09-23T22:14:00.785463+0000","last_undegraded":"2023-09-23T22:14:00.785463+0000","last_fullsized":"2023-09-23T22:14:00.785463+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:46.343842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792878+0000","last_change":"2023-09-23T22:14:00.792878+0000","last_active":"2023-09-23T22:14:00.792878+0000","last_peered":"2023-09-23T22:14:00.792878+0000","last_clean":"2023-09-23T22:14:00.792878+0000","last_became_active":"2023-09-23T22:14:00.792609+0000","last_became_peered":"2023-09-23T22:14:00.792609+0000","last_unstale":"2023-09-23T22:14:00.792878+0000","last_undegraded":"2023-09-23T22:14:00.792878+0000","last_fullsized":"2023-09-23T22:14:00.792878+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:47.586353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781836+0000","last_change":"2023-09-23T22:04:54.379680+0000","last_active":"2023-09-23T22:13:59.781836+0000","last_peered":"2023-09-23T22:13:59.781836+0000","last_clean":"2023-09-23T22:13:59.781836+0000","last_became_active":"2023-09-23T22:04:54.379383+0000","last_became_peered":"2023-09-23T22:04:54.379383+0000","last_unstale":"2023-09-23T22:13:59.781836+0000","last_undegraded":"2023-09-23T22:13:59.781836+0000","last_fullsized":"2023-09-23T22:13:59.781836+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.282795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794701+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:14:00.794701+0000","last_peered":"2023-09-23T22:14:00.794701+0000","last_clean":"2023-09-23T22:14:00.794701+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:14:00.794701+0000","last_undegraded":"2023-09-23T22:14:00.794701+0000","last_fullsized":"2023-09-23T22:14:00.794701+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.782175+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:13:59.782175+0000","last_peered":"2023-09-23T22:13:59.782175+0000","last_clean":"2023-09-23T22:13:59.782175+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:13:59.782175+0000","last_undegraded":"2023-09-23T22:13:59.782175+0000","last_fullsized":"2023-09-23T22:13:59.782175+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.786689+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:14:00.786689+0000","last_peered":"2023-09-23T22:14:00.786689+0000","last_clean":"2023-09-23T22:14:00.786689+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:14:00.786689+0000","last_undegraded":"2023-09-23T22:14:00.786689+0000","last_fullsized":"2023-09-23T22:14:00.786689+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":40,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.780635+0000","last_change":"2023-09-23T22:11:42.696059+0000","last_active":"2023-09-23T22:14:00.780635+0000","last_peered":"2023-09-23T22:14:00.780635+0000","last_clean":"2023-09-23T22:14:00.780635+0000","last_became_active":"2023-09-23T22:11:42.695756+0000","last_became_peered":"2023-09-23T22:11:42.695756+0000","last_unstale":"2023-09-23T22:14:00.780635+0000","last_undegraded":"2023-09-23T22:14:00.780635+0000","last_fullsized":"2023-09-23T22:14:00.780635+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:34.633172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":50,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788995+0000","last_change":"2023-09-23T22:04:54.402894+0000","last_active":"2023-09-23T22:13:59.788995+0000","last_peered":"2023-09-23T22:13:59.788995+0000","last_clean":"2023-09-23T22:13:59.788995+0000","last_became_active":"2023-09-23T22:04:54.402475+0000","last_became_peered":"2023-09-23T22:04:54.402475+0000","last_unstale":"2023-09-23T22:13:59.788995+0000","last_undegraded":"2023-09-23T22:13:59.788995+0000","last_fullsized":"2023-09-23T22:13:59.788995+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:12.503263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789394+0000","last_change":"2023-09-23T22:12:54.870816+0000","last_active":"2023-09-23T22:13:59.789394+0000","last_peered":"2023-09-23T22:13:59.789394+0000","last_clean":"2023-09-23T22:13:59.789394+0000","last_became_active":"2023-09-23T22:12:54.870450+0000","last_became_peered":"2023-09-23T22:12:54.870450+0000","last_unstale":"2023-09-23T22:13:59.789394+0000","last_undegraded":"2023-09-23T22:13:59.789394+0000","last_fullsized":"2023-09-23T22:13:59.789394+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.764353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":234,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786388+0000","last_change":"2023-09-23T22:13:27.664537+0000","last_active":"2023-09-23T22:13:59.786388+0000","last_peered":"2023-09-23T22:13:59.786388+0000","last_clean":"2023-09-23T22:13:59.786388+0000","last_became_active":"2023-09-23T22:13:27.663897+0000","last_became_peered":"2023-09-23T22:13:27.663897+0000","last_unstale":"2023-09-23T22:13:59.786388+0000","last_undegraded":"2023-09-23T22:13:59.786388+0000","last_fullsized":"2023-09-23T22:13:59.786388+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:37.321580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":222,"num_read_kb":1213,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795827+0000","last_change":"2023-09-23T22:14:00.795827+0000","last_active":"2023-09-23T22:14:00.795827+0000","last_peered":"2023-09-23T22:14:00.795827+0000","last_clean":"2023-09-23T22:14:00.795827+0000","last_became_active":"2023-09-23T22:14:00.795225+0000","last_became_peered":"2023-09-23T22:14:00.795225+0000","last_unstale":"2023-09-23T22:14:00.795827+0000","last_undegraded":"2023-09-23T22:14:00.795827+0000","last_fullsized":"2023-09-23T22:14:00.795827+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:58:56.251393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":705,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781094+0000","last_change":"2023-09-23T22:12:54.874377+0000","last_active":"2023-09-23T22:13:59.781094+0000","last_peered":"2023-09-23T22:13:59.781094+0000","last_clean":"2023-09-23T22:13:59.781094+0000","last_became_active":"2023-09-23T22:12:54.873780+0000","last_became_peered":"2023-09-23T22:12:54.873780+0000","last_unstale":"2023-09-23T22:13:59.781094+0000","last_undegraded":"2023-09-23T22:13:59.781094+0000","last_fullsized":"2023-09-23T22:13:59.781094+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:08.681877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796111+0000","last_change":"2023-09-23T22:14:00.796111+0000","last_active":"2023-09-23T22:14:00.796111+0000","last_peered":"2023-09-23T22:14:00.796111+0000","last_clean":"2023-09-23T22:14:00.796111+0000","last_became_active":"2023-09-23T22:14:00.794856+0000","last_became_peered":"2023-09-23T22:14:00.794856+0000","last_unstale":"2023-09-23T22:14:00.796111+0000","last_undegraded":"2023-09-23T22:14:00.796111+0000","last_fullsized":"2023-09-23T22:14:00.796111+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:16.319602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786418+0000","last_change":"2023-09-23T22:11:42.696594+0000","last_active":"2023-09-23T22:13:59.786418+0000","last_peered":"2023-09-23T22:13:59.786418+0000","last_clean":"2023-09-23T22:13:59.786418+0000","last_became_active":"2023-09-23T22:11:42.696334+0000","last_became_peered":"2023-09-23T22:11:42.696334+0000","last_unstale":"2023-09-23T22:13:59.786418+0000","last_undegraded":"2023-09-23T22:13:59.786418+0000","last_fullsized":"2023-09-23T22:13:59.786418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:36.810008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780034+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:13:59.780034+0000","last_peered":"2023-09-23T22:13:59.780034+0000","last_clean":"2023-09-23T22:13:59.780034+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:13:59.780034+0000","last_undegraded":"2023-09-23T22:13:59.780034+0000","last_fullsized":"2023-09-23T22:13:59.780034+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":544,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786909+0000","last_change":"2023-09-23T22:11:42.693882+0000","last_active":"2023-09-23T22:13:59.786909+0000","last_peered":"2023-09-23T22:13:59.786909+0000","last_clean":"2023-09-23T22:13:59.786909+0000","last_became_active":"2023-09-23T22:11:42.693381+0000","last_became_peered":"2023-09-23T22:11:42.693381+0000","last_unstale":"2023-09-23T22:13:59.786909+0000","last_undegraded":"2023-09-23T22:13:59.786909+0000","last_fullsized":"2023-09-23T22:13:59.786909+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:06.246085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789486+0000","last_change":"2023-09-23T22:13:27.666017+0000","last_active":"2023-09-23T22:13:59.789486+0000","last_peered":"2023-09-23T22:13:59.789486+0000","last_clean":"2023-09-23T22:13:59.789486+0000","last_became_active":"2023-09-23T22:13:27.665706+0000","last_became_peered":"2023-09-23T22:13:27.665706+0000","last_unstale":"2023-09-23T22:13:59.789486+0000","last_undegraded":"2023-09-23T22:13:59.789486+0000","last_fullsized":"2023-09-23T22:13:59.789486+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:16.575957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790941+0000","last_change":"2023-09-23T22:14:00.790941+0000","last_active":"2023-09-23T22:14:00.790941+0000","last_peered":"2023-09-23T22:14:00.790941+0000","last_clean":"2023-09-23T22:14:00.790941+0000","last_became_active":"2023-09-23T22:14:00.790616+0000","last_became_peered":"2023-09-23T22:14:00.790616+0000","last_unstale":"2023-09-23T22:14:00.790941+0000","last_undegraded":"2023-09-23T22:14:00.790941+0000","last_fullsized":"2023-09-23T22:14:00.790941+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:53.552619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":458,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783191+0000","last_change":"2023-09-23T22:12:54.861801+0000","last_active":"2023-09-23T22:13:59.783191+0000","last_peered":"2023-09-23T22:13:59.783191+0000","last_clean":"2023-09-23T22:13:59.783191+0000","last_became_active":"2023-09-23T22:12:54.861358+0000","last_became_peered":"2023-09-23T22:12:54.861358+0000","last_unstale":"2023-09-23T22:13:59.783191+0000","last_undegraded":"2023-09-23T22:13:59.783191+0000","last_fullsized":"2023-09-23T22:13:59.783191+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:24.061357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785893+0000","last_change":"2023-09-23T22:04:54.527318+0000","last_active":"2023-09-23T22:13:59.785893+0000","last_peered":"2023-09-23T22:13:59.785893+0000","last_clean":"2023-09-23T22:13:59.785893+0000","last_became_active":"2023-09-23T22:04:54.527023+0000","last_became_peered":"2023-09-23T22:04:54.527023+0000","last_unstale":"2023-09-23T22:13:59.785893+0000","last_undegraded":"2023-09-23T22:13:59.785893+0000","last_fullsized":"2023-09-23T22:13:59.785893+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:42.048739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.789944+0000","last_change":"2023-09-23T22:14:00.789944+0000","last_active":"2023-09-23T22:14:00.789944+0000","last_peered":"2023-09-23T22:14:00.789944+0000","last_clean":"2023-09-23T22:14:00.789944+0000","last_became_active":"2023-09-23T22:14:00.789559+0000","last_became_peered":"2023-09-23T22:14:00.789559+0000","last_unstale":"2023-09-23T22:14:00.789944+0000","last_undegraded":"2023-09-23T22:14:00.789944+0000","last_fullsized":"2023-09-23T22:14:00.789944+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:30.957719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":451,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783108+0000","last_change":"2023-09-23T22:11:42.695484+0000","last_active":"2023-09-23T22:13:59.783108+0000","last_peered":"2023-09-23T22:13:59.783108+0000","last_clean":"2023-09-23T22:13:59.783108+0000","last_became_active":"2023-09-23T22:11:42.694972+0000","last_became_peered":"2023-09-23T22:11:42.694972+0000","last_unstale":"2023-09-23T22:13:59.783108+0000","last_undegraded":"2023-09-23T22:13:59.783108+0000","last_fullsized":"2023-09-23T22:13:59.783108+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:36.787404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":694,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780660+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:13:59.780660+0000","last_peered":"2023-09-23T22:13:59.780660+0000","last_clean":"2023-09-23T22:13:59.780660+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:13:59.780660+0000","last_undegraded":"2023-09-23T22:13:59.780660+0000","last_fullsized":"2023-09-23T22:13:59.780660+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781830+0000","last_change":"2023-09-23T22:12:54.874146+0000","last_active":"2023-09-23T22:13:59.781830+0000","last_peered":"2023-09-23T22:13:59.781830+0000","last_clean":"2023-09-23T22:13:59.781830+0000","last_became_active":"2023-09-23T22:12:54.873782+0000","last_became_peered":"2023-09-23T22:12:54.873782+0000","last_unstale":"2023-09-23T22:13:59.781830+0000","last_undegraded":"2023-09-23T22:13:59.781830+0000","last_fullsized":"2023-09-23T22:13:59.781830+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:25.933932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.788799+0000","last_change":"2023-09-23T22:13:27.659751+0000","last_active":"2023-09-23T22:14:00.788799+0000","last_peered":"2023-09-23T22:14:00.788799+0000","last_clean":"2023-09-23T22:14:00.788799+0000","last_became_active":"2023-09-23T22:13:27.659422+0000","last_became_peered":"2023-09-23T22:13:27.659422+0000","last_unstale":"2023-09-23T22:14:00.788799+0000","last_undegraded":"2023-09-23T22:14:00.788799+0000","last_fullsized":"2023-09-23T22:14:00.788799+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:25.705909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":378,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.783947+0000","last_change":"2023-09-23T22:14:00.783947+0000","last_active":"2023-09-23T22:14:00.783947+0000","last_peered":"2023-09-23T22:14:00.783947+0000","last_clean":"2023-09-23T22:14:00.783947+0000","last_became_active":"2023-09-23T22:14:00.783319+0000","last_became_peered":"2023-09-23T22:14:00.783319+0000","last_unstale":"2023-09-23T22:14:00.783947+0000","last_undegraded":"2023-09-23T22:14:00.783947+0000","last_fullsized":"2023-09-23T22:14:00.783947+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:02:25.419096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796811+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:14:00.796811+0000","last_peered":"2023-09-23T22:14:00.796811+0000","last_clean":"2023-09-23T22:14:00.796811+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:14:00.796811+0000","last_undegraded":"2023-09-23T22:14:00.796811+0000","last_fullsized":"2023-09-23T22:14:00.796811+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788001+0000","last_change":"2023-09-23T22:11:42.697009+0000","last_active":"2023-09-23T22:13:59.788001+0000","last_peered":"2023-09-23T22:13:59.788001+0000","last_clean":"2023-09-23T22:13:59.788001+0000","last_became_active":"2023-09-23T22:11:42.696588+0000","last_became_peered":"2023-09-23T22:11:42.696588+0000","last_unstale":"2023-09-23T22:13:59.788001+0000","last_undegraded":"2023-09-23T22:13:59.788001+0000","last_fullsized":"2023-09-23T22:13:59.788001+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:57.260787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":750,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780601+0000","last_change":"2023-09-23T22:12:54.877082+0000","last_active":"2023-09-23T22:13:59.780601+0000","last_peered":"2023-09-23T22:13:59.780601+0000","last_clean":"2023-09-23T22:13:59.780601+0000","last_became_active":"2023-09-23T22:12:54.876524+0000","last_became_peered":"2023-09-23T22:12:54.876524+0000","last_unstale":"2023-09-23T22:13:59.780601+0000","last_undegraded":"2023-09-23T22:13:59.780601+0000","last_fullsized":"2023-09-23T22:13:59.780601+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:46.185025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.778046+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:13:59.778046+0000","last_peered":"2023-09-23T22:13:59.778046+0000","last_clean":"2023-09-23T22:13:59.778046+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:13:59.778046+0000","last_undegraded":"2023-09-23T22:13:59.778046+0000","last_fullsized":"2023-09-23T22:13:59.778046+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792797+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:14:00.792797+0000","last_peered":"2023-09-23T22:14:00.792797+0000","last_clean":"2023-09-23T22:14:00.792797+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:14:00.792797+0000","last_undegraded":"2023-09-23T22:14:00.792797+0000","last_fullsized":"2023-09-23T22:14:00.792797+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":555,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.782571+0000","last_change":"2023-09-23T22:12:54.875239+0000","last_active":"2023-09-23T22:13:59.782571+0000","last_peered":"2023-09-23T22:13:59.782571+0000","last_clean":"2023-09-23T22:13:59.782571+0000","last_became_active":"2023-09-23T22:12:54.874675+0000","last_became_peered":"2023-09-23T22:12:54.874675+0000","last_unstale":"2023-09-23T22:13:59.782571+0000","last_undegraded":"2023-09-23T22:13:59.782571+0000","last_fullsized":"2023-09-23T22:13:59.782571+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:21.865417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.791734+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:14:00.791734+0000","last_peered":"2023-09-23T22:14:00.791734+0000","last_clean":"2023-09-23T22:14:00.791734+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:14:00.791734+0000","last_undegraded":"2023-09-23T22:14:00.791734+0000","last_fullsized":"2023-09-23T22:14:00.791734+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795928+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:14:00.795928+0000","last_peered":"2023-09-23T22:14:00.795928+0000","last_clean":"2023-09-23T22:14:00.795928+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:14:00.795928+0000","last_undegraded":"2023-09-23T22:14:00.795928+0000","last_fullsized":"2023-09-23T22:14:00.795928+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.790050+0000","last_change":"2023-09-23T22:12:54.868682+0000","last_active":"2023-09-23T22:13:59.790050+0000","last_peered":"2023-09-23T22:13:59.790050+0000","last_clean":"2023-09-23T22:13:59.790050+0000","last_became_active":"2023-09-23T22:12:54.868339+0000","last_became_peered":"2023-09-23T22:12:54.868339+0000","last_unstale":"2023-09-23T22:13:59.790050+0000","last_undegraded":"2023-09-23T22:13:59.790050+0000","last_fullsized":"2023-09-23T22:13:59.790050+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:53.246678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789109+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:13:59.789109+0000","last_peered":"2023-09-23T22:13:59.789109+0000","last_clean":"2023-09-23T22:13:59.789109+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:13:59.789109+0000","last_undegraded":"2023-09-23T22:13:59.789109+0000","last_fullsized":"2023-09-23T22:13:59.789109+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.791120+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:13:59.791120+0000","last_peered":"2023-09-23T22:13:59.791120+0000","last_clean":"2023-09-23T22:13:59.791120+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:13:59.791120+0000","last_undegraded":"2023-09-23T22:13:59.791120+0000","last_fullsized":"2023-09-23T22:13:59.791120+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785609+0000","last_change":"2023-09-23T22:12:54.880778+0000","last_active":"2023-09-23T22:13:59.785609+0000","last_peered":"2023-09-23T22:13:59.785609+0000","last_clean":"2023-09-23T22:13:59.785609+0000","last_became_active":"2023-09-23T22:12:54.880272+0000","last_became_peered":"2023-09-23T22:12:54.880272+0000","last_unstale":"2023-09-23T22:13:59.785609+0000","last_undegraded":"2023-09-23T22:13:59.785609+0000","last_fullsized":"2023-09-23T22:13:59.785609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:01.030591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792760+0000","last_change":"2023-09-23T22:14:00.792760+0000","last_active":"2023-09-23T22:14:00.792760+0000","last_peered":"2023-09-23T22:14:00.792760+0000","last_clean":"2023-09-23T22:14:00.792760+0000","last_became_active":"2023-09-23T22:14:00.792478+0000","last_became_peered":"2023-09-23T22:14:00.792478+0000","last_unstale":"2023-09-23T22:14:00.792760+0000","last_undegraded":"2023-09-23T22:14:00.792760+0000","last_fullsized":"2023-09-23T22:14:00.792760+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.935338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790875+0000","last_change":"2023-09-23T22:14:00.790875+0000","last_active":"2023-09-23T22:14:00.790875+0000","last_peered":"2023-09-23T22:14:00.790875+0000","last_clean":"2023-09-23T22:14:00.790875+0000","last_became_active":"2023-09-23T22:14:00.790332+0000","last_became_peered":"2023-09-23T22:14:00.790332+0000","last_unstale":"2023-09-23T22:14:00.790875+0000","last_undegraded":"2023-09-23T22:14:00.790875+0000","last_fullsized":"2023-09-23T22:14:00.790875+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:05.904013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786633+0000","last_change":"2023-09-23T22:12:54.871021+0000","last_active":"2023-09-23T22:13:59.786633+0000","last_peered":"2023-09-23T22:13:59.786633+0000","last_clean":"2023-09-23T22:13:59.786633+0000","last_became_active":"2023-09-23T22:12:54.870736+0000","last_became_peered":"2023-09-23T22:12:54.870736+0000","last_unstale":"2023-09-23T22:13:59.786633+0000","last_undegraded":"2023-09-23T22:13:59.786633+0000","last_fullsized":"2023-09-23T22:13:59.786633+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:51.846457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795682+0000","last_change":"2023-09-23T22:14:00.795682+0000","last_active":"2023-09-23T22:14:00.795682+0000","last_peered":"2023-09-23T22:14:00.795682+0000","last_clean":"2023-09-23T22:14:00.795682+0000","last_became_active":"2023-09-23T22:14:00.795377+0000","last_became_peered":"2023-09-23T22:14:00.795377+0000","last_unstale":"2023-09-23T22:14:00.795682+0000","last_undegraded":"2023-09-23T22:14:00.795682+0000","last_fullsized":"2023-09-23T22:14:00.795682+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:33.253920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785440+0000","last_change":"2023-09-23T22:12:54.865320+0000","last_active":"2023-09-23T22:13:59.785440+0000","last_peered":"2023-09-23T22:13:59.785440+0000","last_clean":"2023-09-23T22:13:59.785440+0000","last_became_active":"2023-09-23T22:12:54.865007+0000","last_became_peered":"2023-09-23T22:12:54.865007+0000","last_unstale":"2023-09-23T22:13:59.785440+0000","last_undegraded":"2023-09-23T22:13:59.785440+0000","last_fullsized":"2023-09-23T22:13:59.785440+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:40.963311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787011+0000","last_change":"2023-09-23T22:04:54.384008+0000","last_active":"2023-09-23T22:13:59.787011+0000","last_peered":"2023-09-23T22:13:59.787011+0000","last_clean":"2023-09-23T22:13:59.787011+0000","last_became_active":"2023-09-23T22:04:54.383702+0000","last_became_peered":"2023-09-23T22:04:54.383702+0000","last_unstale":"2023-09-23T22:13:59.787011+0000","last_undegraded":"2023-09-23T22:13:59.787011+0000","last_fullsized":"2023-09-23T22:13:59.787011+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:00.423911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795430+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:14:00.795430+0000","last_peered":"2023-09-23T22:14:00.795430+0000","last_clean":"2023-09-23T22:14:00.795430+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:14:00.795430+0000","last_undegraded":"2023-09-23T22:14:00.795430+0000","last_fullsized":"2023-09-23T22:14:00.795430+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786126+0000","last_change":"2023-09-23T22:12:54.861787+0000","last_active":"2023-09-23T22:13:59.786126+0000","last_peered":"2023-09-23T22:13:59.786126+0000","last_clean":"2023-09-23T22:13:59.786126+0000","last_became_active":"2023-09-23T22:12:54.861377+0000","last_became_peered":"2023-09-23T22:12:54.861377+0000","last_unstale":"2023-09-23T22:13:59.786126+0000","last_undegraded":"2023-09-23T22:13:59.786126+0000","last_fullsized":"2023-09-23T22:13:59.786126+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:02.832226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780468+0000","last_change":"2023-09-23T22:04:54.528019+0000","last_active":"2023-09-23T22:13:59.780468+0000","last_peered":"2023-09-23T22:13:59.780468+0000","last_clean":"2023-09-23T22:13:59.780468+0000","last_became_active":"2023-09-23T22:04:54.527666+0000","last_became_peered":"2023-09-23T22:04:54.527666+0000","last_unstale":"2023-09-23T22:13:59.780468+0000","last_undegraded":"2023-09-23T22:13:59.780468+0000","last_fullsized":"2023-09-23T22:13:59.780468+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:32.774493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786894+0000","last_change":"2023-09-23T22:11:42.694008+0000","last_active":"2023-09-23T22:13:59.786894+0000","last_peered":"2023-09-23T22:13:59.786894+0000","last_clean":"2023-09-23T22:13:59.786894+0000","last_became_active":"2023-09-23T22:11:42.693609+0000","last_became_peered":"2023-09-23T22:11:42.693609+0000","last_unstale":"2023-09-23T22:13:59.786894+0000","last_undegraded":"2023-09-23T22:13:59.786894+0000","last_fullsized":"2023-09-23T22:13:59.786894+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:59.614114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794463+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:14:00.794463+0000","last_peered":"2023-09-23T22:14:00.794463+0000","last_clean":"2023-09-23T22:14:00.794463+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:14:00.794463+0000","last_undegraded":"2023-09-23T22:14:00.794463+0000","last_fullsized":"2023-09-23T22:14:00.794463+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":77,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.784703+0000","last_change":"2023-09-23T22:11:42.701916+0000","last_active":"2023-09-23T22:13:59.784703+0000","last_peered":"2023-09-23T22:13:59.784703+0000","last_clean":"2023-09-23T22:13:59.784703+0000","last_became_active":"2023-09-23T22:11:42.701591+0000","last_became_peered":"2023-09-23T22:11:42.701591+0000","last_unstale":"2023-09-23T22:13:59.784703+0000","last_undegraded":"2023-09-23T22:13:59.784703+0000","last_fullsized":"2023-09-23T22:13:59.784703+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:23.941128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.793040+0000","last_change":"2023-09-23T22:14:00.792449+0000","last_active":"2023-09-23T22:14:00.793040+0000","last_peered":"2023-09-23T22:14:00.793040+0000","last_clean":"2023-09-23T22:14:00.793040+0000","last_became_active":"2023-09-23T22:14:00.792239+0000","last_became_peered":"2023-09-23T22:14:00.792239+0000","last_unstale":"2023-09-23T22:14:00.793040+0000","last_undegraded":"2023-09-23T22:14:00.793040+0000","last_fullsized":"2023-09-23T22:14:00.793040+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:33.488563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783327+0000","last_change":"2023-09-23T22:04:54.387688+0000","last_active":"2023-09-23T22:13:59.783327+0000","last_peered":"2023-09-23T22:13:59.783327+0000","last_clean":"2023-09-23T22:13:59.783327+0000","last_became_active":"2023-09-23T22:04:54.387421+0000","last_became_peered":"2023-09-23T22:04:54.387421+0000","last_unstale":"2023-09-23T22:13:59.783327+0000","last_undegraded":"2023-09-23T22:13:59.783327+0000","last_fullsized":"2023-09-23T22:13:59.783327+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:22.411746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794669+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:14:00.794669+0000","last_peered":"2023-09-23T22:14:00.794669+0000","last_clean":"2023-09-23T22:14:00.794669+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:14:00.794669+0000","last_undegraded":"2023-09-23T22:14:00.794669+0000","last_fullsized":"2023-09-23T22:14:00.794669+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792829+0000","last_change":"2023-09-23T22:14:00.792829+0000","last_active":"2023-09-23T22:14:00.792829+0000","last_peered":"2023-09-23T22:14:00.792829+0000","last_clean":"2023-09-23T22:14:00.792829+0000","last_became_active":"2023-09-23T22:14:00.792462+0000","last_became_peered":"2023-09-23T22:14:00.792462+0000","last_unstale":"2023-09-23T22:14:00.792829+0000","last_undegraded":"2023-09-23T22:14:00.792829+0000","last_fullsized":"2023-09-23T22:14:00.792829+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:55.154145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794351+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:14:00.794351+0000","last_peered":"2023-09-23T22:14:00.794351+0000","last_clean":"2023-09-23T22:14:00.794351+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:14:00.794351+0000","last_undegraded":"2023-09-23T22:14:00.794351+0000","last_fullsized":"2023-09-23T22:14:00.794351+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785117+0000","last_change":"2023-09-23T22:13:27.663039+0000","last_active":"2023-09-23T22:13:59.785117+0000","last_peered":"2023-09-23T22:13:59.785117+0000","last_clean":"2023-09-23T22:13:59.785117+0000","last_became_active":"2023-09-23T22:13:27.662687+0000","last_became_peered":"2023-09-23T22:13:27.662687+0000","last_unstale":"2023-09-23T22:13:59.785117+0000","last_undegraded":"2023-09-23T22:13:59.785117+0000","last_fullsized":"2023-09-23T22:13:59.785117+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:13.432843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788095+0000","last_change":"2023-09-23T22:12:54.866763+0000","last_active":"2023-09-23T22:13:59.788095+0000","last_peered":"2023-09-23T22:13:59.788095+0000","last_clean":"2023-09-23T22:13:59.788095+0000","last_became_active":"2023-09-23T22:12:54.866422+0000","last_became_peered":"2023-09-23T22:12:54.866422+0000","last_unstale":"2023-09-23T22:13:59.788095+0000","last_undegraded":"2023-09-23T22:13:59.788095+0000","last_fullsized":"2023-09-23T22:13:59.788095+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:43.904727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.790220+0000","last_change":"2023-09-23T22:04:54.381132+0000","last_active":"2023-09-23T22:13:59.790220+0000","last_peered":"2023-09-23T22:13:59.790220+0000","last_clean":"2023-09-23T22:13:59.790220+0000","last_became_active":"2023-09-23T22:04:54.380838+0000","last_became_peered":"2023-09-23T22:04:54.380838+0000","last_unstale":"2023-09-23T22:13:59.790220+0000","last_undegraded":"2023-09-23T22:13:59.790220+0000","last_fullsized":"2023-09-23T22:13:59.790220+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:41.334974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780225+0000","last_change":"2023-09-23T22:13:27.661799+0000","last_active":"2023-09-23T22:13:59.780225+0000","last_peered":"2023-09-23T22:13:59.780225+0000","last_clean":"2023-09-23T22:13:59.780225+0000","last_became_active":"2023-09-23T22:13:27.661473+0000","last_became_peered":"2023-09-23T22:13:27.661473+0000","last_unstale":"2023-09-23T22:13:59.780225+0000","last_undegraded":"2023-09-23T22:13:59.780225+0000","last_fullsized":"2023-09-23T22:13:59.780225+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:34.827694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796705+0000","last_change":"2023-09-23T22:14:00.796705+0000","last_active":"2023-09-23T22:14:00.796705+0000","last_peered":"2023-09-23T22:14:00.796705+0000","last_clean":"2023-09-23T22:14:00.796705+0000","last_became_active":"2023-09-23T22:14:00.796252+0000","last_became_peered":"2023-09-23T22:14:00.796252+0000","last_unstale":"2023-09-23T22:14:00.796705+0000","last_undegraded":"2023-09-23T22:14:00.796705+0000","last_fullsized":"2023-09-23T22:14:00.796705+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:58.956676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794963+0000","last_change":"2023-09-23T22:14:00.794963+0000","last_active":"2023-09-23T22:14:00.794963+0000","last_peered":"2023-09-23T22:14:00.794963+0000","last_clean":"2023-09-23T22:14:00.794963+0000","last_became_active":"2023-09-23T22:14:00.794394+0000","last_became_peered":"2023-09-23T22:14:00.794394+0000","last_unstale":"2023-09-23T22:14:00.794963+0000","last_undegraded":"2023-09-23T22:14:00.794963+0000","last_fullsized":"2023-09-23T22:14:00.794963+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:25.119383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780772+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:13:59.780772+0000","last_peered":"2023-09-23T22:13:59.780772+0000","last_clean":"2023-09-23T22:13:59.780772+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:13:59.780772+0000","last_undegraded":"2023-09-23T22:13:59.780772+0000","last_fullsized":"2023-09-23T22:13:59.780772+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790411+0000","last_change":"2023-09-23T22:14:00.790411+0000","last_active":"2023-09-23T22:14:00.790411+0000","last_peered":"2023-09-23T22:14:00.790411+0000","last_clean":"2023-09-23T22:14:00.790411+0000","last_became_active":"2023-09-23T22:14:00.789906+0000","last_became_peered":"2023-09-23T22:14:00.789906+0000","last_unstale":"2023-09-23T22:14:00.790411+0000","last_undegraded":"2023-09-23T22:14:00.790411+0000","last_fullsized":"2023-09-23T22:14:00.790411+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:28:43.440626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.788697+0000","last_change":"2023-09-23T22:14:00.787881+0000","last_active":"2023-09-23T22:14:00.788697+0000","last_peered":"2023-09-23T22:14:00.788697+0000","last_clean":"2023-09-23T22:14:00.788697+0000","last_became_active":"2023-09-23T22:14:00.787498+0000","last_became_peered":"2023-09-23T22:14:00.787498+0000","last_unstale":"2023-09-23T22:14:00.788697+0000","last_undegraded":"2023-09-23T22:14:00.788697+0000","last_fullsized":"2023-09-23T22:14:00.788697+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:19.424721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786273+0000","last_change":"2023-09-23T22:12:54.864958+0000","last_active":"2023-09-23T22:13:59.786273+0000","last_peered":"2023-09-23T22:13:59.786273+0000","last_clean":"2023-09-23T22:13:59.786273+0000","last_became_active":"2023-09-23T22:12:54.864451+0000","last_became_peered":"2023-09-23T22:12:54.864451+0000","last_unstale":"2023-09-23T22:13:59.786273+0000","last_undegraded":"2023-09-23T22:13:59.786273+0000","last_fullsized":"2023-09-23T22:13:59.786273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:32.668061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790097+0000","last_change":"2023-09-23T22:14:00.790097+0000","last_active":"2023-09-23T22:14:00.790097+0000","last_peered":"2023-09-23T22:14:00.790097+0000","last_clean":"2023-09-23T22:14:00.790097+0000","last_became_active":"2023-09-23T22:14:00.789389+0000","last_became_peered":"2023-09-23T22:14:00.789389+0000","last_unstale":"2023-09-23T22:14:00.790097+0000","last_undegraded":"2023-09-23T22:14:00.790097+0000","last_fullsized":"2023-09-23T22:14:00.790097+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:05.673490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792919+0000","last_change":"2023-09-23T22:14:00.792919+0000","last_active":"2023-09-23T22:14:00.792919+0000","last_peered":"2023-09-23T22:14:00.792919+0000","last_clean":"2023-09-23T22:14:00.792919+0000","last_became_active":"2023-09-23T22:14:00.792490+0000","last_became_peered":"2023-09-23T22:14:00.792490+0000","last_unstale":"2023-09-23T22:14:00.792919+0000","last_undegraded":"2023-09-23T22:14:00.792919+0000","last_fullsized":"2023-09-23T22:14:00.792919+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:53.433863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.791655+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:14:00.791655+0000","last_peered":"2023-09-23T22:14:00.791655+0000","last_clean":"2023-09-23T22:14:00.791655+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:14:00.791655+0000","last_undegraded":"2023-09-23T22:14:00.791655+0000","last_fullsized":"2023-09-23T22:14:00.791655+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790313+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:14:00.790313+0000","last_peered":"2023-09-23T22:14:00.790313+0000","last_clean":"2023-09-23T22:14:00.790313+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:14:00.790313+0000","last_undegraded":"2023-09-23T22:14:00.790313+0000","last_fullsized":"2023-09-23T22:14:00.790313+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":86,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787689+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:13:59.787689+0000","last_peered":"2023-09-23T22:13:59.787689+0000","last_clean":"2023-09-23T22:13:59.787689+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:13:59.787689+0000","last_undegraded":"2023-09-23T22:13:59.787689+0000","last_fullsized":"2023-09-23T22:13:59.787689+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13208,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234126,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12896,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":11200,"kb_avail":93732256,"statfs":{"total":95995035648,"available":95981830144,"internally_reserved":0,"allocated":1728512,"data_stored":1203715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11468800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.34000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59599999999999997,"15min":0.58799999999999997},"min":{"1min":0.39900000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":0.73399999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.69999999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.008,"15min":1.008},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":8.5220000000000002},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.54100000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:13:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.61399999999999999,"15min":0.59599999999999997},"min":{"1min":0.38700000000000001,"5min":0.38600000000000001,"15min":0.35699999999999998},"max":{"1min":0.81000000000000005,"5min":1.3109999999999999,"15min":7.8970000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.61299999999999999,"15min":0.59099999999999997},"min":{"1min":0.42799999999999999,"5min":0.41899999999999998,"15min":0.35599999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":2.9039999999999999},"last":0.58599999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.503,"15min":0.47799999999999998},"min":{"1min":0.32000000000000001,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":0.79000000000000004,"5min":0.81399999999999995,"15min":0.82199999999999995},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.50900000000000001,"15min":0.502},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.80800000000000005},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.502,"15min":0.48499999999999999},"min":{"1min":0.379,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":0.70499999999999996,"5min":0.71999999999999997,"15min":0.78700000000000003},"last":0.499},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.57999999999999996,"15min":0.55400000000000005},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29199999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.873},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.55300000000000005,"15min":0.55800000000000005},"min":{"1min":0.41799999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.77600000000000002,"5min":0.82999999999999996,"15min":3.0459999999999998},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.625,"15min":0.58999999999999997},"min":{"1min":0.40799999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.81399999999999995,"5min":0.874,"15min":0.874},"last":0.63700000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495762,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12444,"kb_used_data":5780,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93732708,"statfs":{"total":95995035648,"available":95982292992,"internally_reserved":0,"allocated":5918720,"data_stored":5398215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.54500000000000004,"15min":0.56499999999999995},"min":{"1min":0.34799999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.75600000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.58099999999999996,"15min":0.60499999999999998},"min":{"1min":0.36599999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.69799999999999995,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.65600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:13:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.58899999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.625}]},{"osd":3,"last update":"Sat Sep 23 22:13:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.58199999999999996,"15min":0.67600000000000005},"min":{"1min":0.434,"5min":0.379,"15min":0.28699999999999998},"max":{"1min":0.83999999999999997,"5min":1.4299999999999999,"15min":7.6429999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58699999999999997,"15min":0.57299999999999995},"min":{"1min":0.47799999999999998,"5min":0.36099999999999999,"15min":0.35799999999999998},"max":{"1min":0.752,"5min":0.84099999999999997,"15min":4.5670000000000002},"last":0.69099999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:14:05 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.46800000000000003,"15min":0.432},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.252},"max":{"1min":0.68899999999999995,"5min":0.77700000000000002,"15min":1.718},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.50900000000000001,"15min":0.46600000000000003},"min":{"1min":0.32400000000000001,"5min":0.315,"15min":0.26500000000000001},"max":{"1min":0.76800000000000002,"5min":0.86099999999999999,"15min":1.78},"last":0.71399999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:14:05 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51800000000000002,"15min":0.48299999999999998},"min":{"1min":0.375,"5min":0.315,"15min":0.29499999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":1.5629999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.51800000000000002,"15min":0.47999999999999998},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29399999999999998},"max":{"1min":0.68500000000000005,"5min":0.81399999999999995,"15min":1.74},"last":0.60399999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:42 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.56000000000000005,"15min":0.52100000000000002},"min":{"1min":0.378,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":0.77600000000000002,"5min":0.77900000000000003,"15min":1.7709999999999999},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.60599999999999998,"15min":0.57799999999999996},"min":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.35599999999999998},"max":{"1min":0.78500000000000003,"5min":0.90900000000000003,"15min":1.603},"last":0.748}]}]},{"osd":1,"up_from":98,"seq":420906795034,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32828,"kb_used_data":5236,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93712324,"statfs":{"total":95995035648,"available":95961419776,"internally_reserved":0,"allocated":5361664,"data_stored":4813549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":28238196},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.54100000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:14:28 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.49199999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44900000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.93700000000000006}]},{"osd":5,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.77100000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.75},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.89000000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.86799999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958568,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29324,"kb_used_data":1732,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93715828,"statfs":{"total":95995035648,"available":95965007872,"internally_reserved":0,"allocated":1773568,"data_stored":1210692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":28237674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:13:58 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.496,"5min":0.496,"15min":0.496},"last":0.39700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.505},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.4020000000000001,"5min":3.4020000000000001,"15min":3.4020000000000001},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.48299999999999998,"15min":0.49199999999999999},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.316},"max":{"1min":0.621,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.57899999999999996,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.49299999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.66600000000000004,"15min":0.65600000000000003},"min":{"1min":0.437,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":3.181,"5min":3.181,"15min":3.181},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.65000000000000002,"15min":0.65700000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89900000000000002,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.628}]},{"osd":5,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.69299999999999995,"15min":0.68999999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.68000000000000005,"15min":0.68300000000000005},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.64900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.68799999999999994,"15min":0.68100000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":3.3159999999999998,"5min":3.3159999999999998,"15min":3.3159999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.69099999999999995,"15min":0.70999999999999996},"min":{"1min":0.436,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.92000000000000004,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.73199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.73299999999999998,"15min":0.73099999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":3.367,"5min":3.367,"15min":3.367},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.748,"15min":0.753},"min":{"1min":0.495,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":3.286,"5min":3.286,"15min":3.286},"last":0.68600000000000005}]}]},{"osd":2,"up_from":103,"seq":442381631507,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28656,"kb_used_data":1128,"kb_used_omap":10,"kb_used_meta":27509,"kb_avail":93716496,"statfs":{"total":95995035648,"available":95965691904,"internally_reserved":0,"allocated":1155072,"data_stored":616429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":28169388},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.38700000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.28999999999999998},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.33900000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.625}]},{"osd":5,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":2.948,"5min":2.948,"15min":2.948},"last":0.59799999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":2.931,"5min":2.931,"15min":2.931},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.65700000000000003}]}]},{"osd":3,"up_from":108,"seq":463856467981,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28572,"kb_used_data":1108,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716580,"statfs":{"total":95995035648,"available":95965777920,"internally_reserved":0,"allocated":1134592,"data_stored":613253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":28110381},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.432}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40400000000000003}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29099999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58199999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59699999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.54100000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019035,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7772,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93737380,"statfs":{"total":95995035648,"available":95987077120,"internally_reserved":0,"allocated":1134592,"data_stored":613600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.622,"15min":0.626},"min":{"1min":0.38900000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2749999999999999,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.63400000000000001,"15min":0.64600000000000002},"min":{"1min":0.42099999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.59599999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:14:54 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.55400000000000005,"15min":0.54800000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57699999999999996,"15min":0.57499999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.61599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.57199999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:13:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.57099999999999995,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.37,"15min":0.313},"max":{"1min":0.80000000000000004,"5min":0.86199999999999999,"15min":3.1499999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59599999999999997,"15min":0.56699999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76600000000000001,"5min":0.85099999999999998,"15min":1.0089999999999999},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48899999999999999,"15min":0.45500000000000002},"min":{"1min":0.29199999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66300000000000003,"5min":0.749,"15min":0.81899999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.39500000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":0.622,"5min":0.78500000000000003,"15min":3.0470000000000002},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:14:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52100000000000002,"15min":0.49399999999999999},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.29599999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.83099999999999996},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56100000000000005,"15min":0.55000000000000004},"min":{"1min":0.41399999999999998,"5min":0.32100000000000001,"15min":0.28899999999999998},"max":{"1min":0.73799999999999999,"5min":0.83299999999999996,"15min":2.2440000000000002},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.60099999999999998,"15min":0.624},"min":{"1min":0.46899999999999997,"5min":0.372,"15min":0.313},"max":{"1min":0.82299999999999995,"5min":0.83799999999999997,"15min":3.9609999999999999},"last":0.498},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.61299999999999999,"15min":0.58799999999999997},"min":{"1min":0.495,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.94999999999999996},"last":0.435}]}]},{"osd":5,"up_from":38,"seq":163208757399,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11664,"kb_used_data":5192,"kb_used_omap":0,"kb_used_meta":6464,"kb_avail":93733488,"statfs":{"total":95995035648,"available":95983091712,"internally_reserved":0,"allocated":5316608,"data_stored":4803667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6619136},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.55500000000000005,"15min":0.56299999999999994},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.59799999999999998,"15min":0.63},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.48099999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.51400000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.62},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.59499999999999997,"15min":0.57899999999999996},"min":{"1min":0.46300000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77900000000000003,"5min":0.93999999999999995,"15min":1.028},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.61299999999999999,"15min":0.60299999999999998},"min":{"1min":0.45400000000000001,"5min":0.36599999999999999,"15min":0.33400000000000002},"max":{"1min":0.79200000000000004,"5min":1.6160000000000001,"15min":2.04},"last":0.52900000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47399999999999998,"15min":0.44500000000000001},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69199999999999995,"5min":0.70899999999999996,"15min":0.752},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53800000000000003,"15min":0.48999999999999999},"min":{"1min":0.42199999999999999,"5min":0.35599999999999998,"15min":0.28999999999999998},"max":{"1min":0.747,"5min":0.751,"15min":0.81100000000000005},"last":0.41399999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:14:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.55000000000000004,"15min":0.50900000000000001},"min":{"1min":0.44600000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.5},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.56699999999999995,"15min":0.53000000000000003},"min":{"1min":0.46400000000000002,"5min":0.34300000000000003,"15min":0.29699999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.63400000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:14:40 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.59599999999999997,"15min":0.57799999999999996},"min":{"1min":0.437,"5min":0.376,"15min":0.32600000000000001},"max":{"1min":0.73199999999999998,"5min":0.81799999999999995,"15min":0.93700000000000006},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.58599999999999997,"15min":0.55900000000000005},"min":{"1min":0.41799999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.84999999999999998},"last":0.60899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:14:57.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:14:57.599 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:14:57.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:57 smithi026 ceph-mon[223014]: from='client.44519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:14:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:57 smithi026 ceph-mon[215775]: from='client.44519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:14:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:57 smithi070 ceph-mon[192652]: from='client.44519 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:14:58.817 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:14:58 smithi026 ceph-mon[223014]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:58.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:14:58 smithi026 ceph-mon[215775]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:14:58 smithi070 ceph-mon[192652]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:14:59.202 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:14:59.204 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:14:59.861 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:14:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:14:59] "GET /metrics HTTP/1.1" 200 33945 "" "Prometheus/2.43.0" 2023-09-23T22:14:59.863 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":213,"stamp":"2023-09-23T22:14:59.012167+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6483,"num_read_kb":14132,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164156,"kb_used_data":22972,"kb_used_omap":31,"kb_used_meta":141088,"kb_avail":749797060,"statfs":{"total":767960285184,"available":767792189440,"internally_reserved":0,"allocated":23523328,"data_stored":19273120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":144475063},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002640"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787529+0000","last_change":"2023-09-23T22:12:54.868920+0000","last_active":"2023-09-23T22:13:59.787529+0000","last_peered":"2023-09-23T22:13:59.787529+0000","last_clean":"2023-09-23T22:13:59.787529+0000","last_became_active":"2023-09-23T22:12:54.868576+0000","last_became_peered":"2023-09-23T22:12:54.868576+0000","last_unstale":"2023-09-23T22:13:59.787529+0000","last_undegraded":"2023-09-23T22:13:59.787529+0000","last_fullsized":"2023-09-23T22:13:59.787529+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:03.825524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787688+0000","last_change":"2023-09-23T22:13:27.664835+0000","last_active":"2023-09-23T22:13:59.787688+0000","last_peered":"2023-09-23T22:13:59.787688+0000","last_clean":"2023-09-23T22:13:59.787688+0000","last_became_active":"2023-09-23T22:13:27.664333+0000","last_became_peered":"2023-09-23T22:13:27.664333+0000","last_unstale":"2023-09-23T22:13:59.787688+0000","last_undegraded":"2023-09-23T22:13:59.787688+0000","last_fullsized":"2023-09-23T22:13:59.787688+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:11.434650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.784662+0000","last_change":"2023-09-23T22:14:00.784662+0000","last_active":"2023-09-23T22:14:00.784662+0000","last_peered":"2023-09-23T22:14:00.784662+0000","last_clean":"2023-09-23T22:14:00.784662+0000","last_became_active":"2023-09-23T22:14:00.784294+0000","last_became_peered":"2023-09-23T22:14:00.784294+0000","last_unstale":"2023-09-23T22:14:00.784662+0000","last_undegraded":"2023-09-23T22:14:00.784662+0000","last_fullsized":"2023-09-23T22:14:00.784662+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:09.123997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.785463+0000","last_change":"2023-09-23T22:14:00.785463+0000","last_active":"2023-09-23T22:14:00.785463+0000","last_peered":"2023-09-23T22:14:00.785463+0000","last_clean":"2023-09-23T22:14:00.785463+0000","last_became_active":"2023-09-23T22:14:00.785128+0000","last_became_peered":"2023-09-23T22:14:00.785128+0000","last_unstale":"2023-09-23T22:14:00.785463+0000","last_undegraded":"2023-09-23T22:14:00.785463+0000","last_fullsized":"2023-09-23T22:14:00.785463+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:46.343842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792878+0000","last_change":"2023-09-23T22:14:00.792878+0000","last_active":"2023-09-23T22:14:00.792878+0000","last_peered":"2023-09-23T22:14:00.792878+0000","last_clean":"2023-09-23T22:14:00.792878+0000","last_became_active":"2023-09-23T22:14:00.792609+0000","last_became_peered":"2023-09-23T22:14:00.792609+0000","last_unstale":"2023-09-23T22:14:00.792878+0000","last_undegraded":"2023-09-23T22:14:00.792878+0000","last_fullsized":"2023-09-23T22:14:00.792878+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:47.586353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781836+0000","last_change":"2023-09-23T22:04:54.379680+0000","last_active":"2023-09-23T22:13:59.781836+0000","last_peered":"2023-09-23T22:13:59.781836+0000","last_clean":"2023-09-23T22:13:59.781836+0000","last_became_active":"2023-09-23T22:04:54.379383+0000","last_became_peered":"2023-09-23T22:04:54.379383+0000","last_unstale":"2023-09-23T22:13:59.781836+0000","last_undegraded":"2023-09-23T22:13:59.781836+0000","last_fullsized":"2023-09-23T22:13:59.781836+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:46.282795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794701+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:14:00.794701+0000","last_peered":"2023-09-23T22:14:00.794701+0000","last_clean":"2023-09-23T22:14:00.794701+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:14:00.794701+0000","last_undegraded":"2023-09-23T22:14:00.794701+0000","last_fullsized":"2023-09-23T22:14:00.794701+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.782175+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:13:59.782175+0000","last_peered":"2023-09-23T22:13:59.782175+0000","last_clean":"2023-09-23T22:13:59.782175+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:13:59.782175+0000","last_undegraded":"2023-09-23T22:13:59.782175+0000","last_fullsized":"2023-09-23T22:13:59.782175+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.786689+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:14:00.786689+0000","last_peered":"2023-09-23T22:14:00.786689+0000","last_clean":"2023-09-23T22:14:00.786689+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:14:00.786689+0000","last_undegraded":"2023-09-23T22:14:00.786689+0000","last_fullsized":"2023-09-23T22:14:00.786689+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":40,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.780635+0000","last_change":"2023-09-23T22:11:42.696059+0000","last_active":"2023-09-23T22:14:00.780635+0000","last_peered":"2023-09-23T22:14:00.780635+0000","last_clean":"2023-09-23T22:14:00.780635+0000","last_became_active":"2023-09-23T22:11:42.695756+0000","last_became_peered":"2023-09-23T22:11:42.695756+0000","last_unstale":"2023-09-23T22:14:00.780635+0000","last_undegraded":"2023-09-23T22:14:00.780635+0000","last_fullsized":"2023-09-23T22:14:00.780635+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:25:34.633172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":50,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788995+0000","last_change":"2023-09-23T22:04:54.402894+0000","last_active":"2023-09-23T22:13:59.788995+0000","last_peered":"2023-09-23T22:13:59.788995+0000","last_clean":"2023-09-23T22:13:59.788995+0000","last_became_active":"2023-09-23T22:04:54.402475+0000","last_became_peered":"2023-09-23T22:04:54.402475+0000","last_unstale":"2023-09-23T22:13:59.788995+0000","last_undegraded":"2023-09-23T22:13:59.788995+0000","last_fullsized":"2023-09-23T22:13:59.788995+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:12.503263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789394+0000","last_change":"2023-09-23T22:12:54.870816+0000","last_active":"2023-09-23T22:13:59.789394+0000","last_peered":"2023-09-23T22:13:59.789394+0000","last_clean":"2023-09-23T22:13:59.789394+0000","last_became_active":"2023-09-23T22:12:54.870450+0000","last_became_peered":"2023-09-23T22:12:54.870450+0000","last_unstale":"2023-09-23T22:13:59.789394+0000","last_undegraded":"2023-09-23T22:13:59.789394+0000","last_fullsized":"2023-09-23T22:13:59.789394+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:10.764353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":234,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786388+0000","last_change":"2023-09-23T22:13:27.664537+0000","last_active":"2023-09-23T22:13:59.786388+0000","last_peered":"2023-09-23T22:13:59.786388+0000","last_clean":"2023-09-23T22:13:59.786388+0000","last_became_active":"2023-09-23T22:13:27.663897+0000","last_became_peered":"2023-09-23T22:13:27.663897+0000","last_unstale":"2023-09-23T22:13:59.786388+0000","last_undegraded":"2023-09-23T22:13:59.786388+0000","last_fullsized":"2023-09-23T22:13:59.786388+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:37.321580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":222,"num_read_kb":1213,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795827+0000","last_change":"2023-09-23T22:14:00.795827+0000","last_active":"2023-09-23T22:14:00.795827+0000","last_peered":"2023-09-23T22:14:00.795827+0000","last_clean":"2023-09-23T22:14:00.795827+0000","last_became_active":"2023-09-23T22:14:00.795225+0000","last_became_peered":"2023-09-23T22:14:00.795225+0000","last_unstale":"2023-09-23T22:14:00.795827+0000","last_undegraded":"2023-09-23T22:14:00.795827+0000","last_fullsized":"2023-09-23T22:14:00.795827+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:58:56.251393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":705,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781094+0000","last_change":"2023-09-23T22:12:54.874377+0000","last_active":"2023-09-23T22:13:59.781094+0000","last_peered":"2023-09-23T22:13:59.781094+0000","last_clean":"2023-09-23T22:13:59.781094+0000","last_became_active":"2023-09-23T22:12:54.873780+0000","last_became_peered":"2023-09-23T22:12:54.873780+0000","last_unstale":"2023-09-23T22:13:59.781094+0000","last_undegraded":"2023-09-23T22:13:59.781094+0000","last_fullsized":"2023-09-23T22:13:59.781094+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:08.681877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796111+0000","last_change":"2023-09-23T22:14:00.796111+0000","last_active":"2023-09-23T22:14:00.796111+0000","last_peered":"2023-09-23T22:14:00.796111+0000","last_clean":"2023-09-23T22:14:00.796111+0000","last_became_active":"2023-09-23T22:14:00.794856+0000","last_became_peered":"2023-09-23T22:14:00.794856+0000","last_unstale":"2023-09-23T22:14:00.796111+0000","last_undegraded":"2023-09-23T22:14:00.796111+0000","last_fullsized":"2023-09-23T22:14:00.796111+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:16.319602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786418+0000","last_change":"2023-09-23T22:11:42.696594+0000","last_active":"2023-09-23T22:13:59.786418+0000","last_peered":"2023-09-23T22:13:59.786418+0000","last_clean":"2023-09-23T22:13:59.786418+0000","last_became_active":"2023-09-23T22:11:42.696334+0000","last_became_peered":"2023-09-23T22:11:42.696334+0000","last_unstale":"2023-09-23T22:13:59.786418+0000","last_undegraded":"2023-09-23T22:13:59.786418+0000","last_fullsized":"2023-09-23T22:13:59.786418+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:36.810008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.785043+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:14:00.785043+0000","last_peered":"2023-09-23T22:14:00.785043+0000","last_clean":"2023-09-23T22:14:00.785043+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:14:00.785043+0000","last_undegraded":"2023-09-23T22:14:00.785043+0000","last_fullsized":"2023-09-23T22:14:00.785043+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":544,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786909+0000","last_change":"2023-09-23T22:11:42.693882+0000","last_active":"2023-09-23T22:13:59.786909+0000","last_peered":"2023-09-23T22:13:59.786909+0000","last_clean":"2023-09-23T22:13:59.786909+0000","last_became_active":"2023-09-23T22:11:42.693381+0000","last_became_peered":"2023-09-23T22:11:42.693381+0000","last_unstale":"2023-09-23T22:13:59.786909+0000","last_undegraded":"2023-09-23T22:13:59.786909+0000","last_fullsized":"2023-09-23T22:13:59.786909+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:06.246085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789486+0000","last_change":"2023-09-23T22:13:27.666017+0000","last_active":"2023-09-23T22:13:59.789486+0000","last_peered":"2023-09-23T22:13:59.789486+0000","last_clean":"2023-09-23T22:13:59.789486+0000","last_became_active":"2023-09-23T22:13:27.665706+0000","last_became_peered":"2023-09-23T22:13:27.665706+0000","last_unstale":"2023-09-23T22:13:59.789486+0000","last_undegraded":"2023-09-23T22:13:59.789486+0000","last_fullsized":"2023-09-23T22:13:59.789486+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:16.575957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790941+0000","last_change":"2023-09-23T22:14:00.790941+0000","last_active":"2023-09-23T22:14:00.790941+0000","last_peered":"2023-09-23T22:14:00.790941+0000","last_clean":"2023-09-23T22:14:00.790941+0000","last_became_active":"2023-09-23T22:14:00.790616+0000","last_became_peered":"2023-09-23T22:14:00.790616+0000","last_unstale":"2023-09-23T22:14:00.790941+0000","last_undegraded":"2023-09-23T22:14:00.790941+0000","last_fullsized":"2023-09-23T22:14:00.790941+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:53.552619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":458,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783191+0000","last_change":"2023-09-23T22:12:54.861801+0000","last_active":"2023-09-23T22:13:59.783191+0000","last_peered":"2023-09-23T22:13:59.783191+0000","last_clean":"2023-09-23T22:13:59.783191+0000","last_became_active":"2023-09-23T22:12:54.861358+0000","last_became_peered":"2023-09-23T22:12:54.861358+0000","last_unstale":"2023-09-23T22:13:59.783191+0000","last_undegraded":"2023-09-23T22:13:59.783191+0000","last_fullsized":"2023-09-23T22:13:59.783191+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:24.061357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785893+0000","last_change":"2023-09-23T22:04:54.527318+0000","last_active":"2023-09-23T22:13:59.785893+0000","last_peered":"2023-09-23T22:13:59.785893+0000","last_clean":"2023-09-23T22:13:59.785893+0000","last_became_active":"2023-09-23T22:04:54.527023+0000","last_became_peered":"2023-09-23T22:04:54.527023+0000","last_unstale":"2023-09-23T22:13:59.785893+0000","last_undegraded":"2023-09-23T22:13:59.785893+0000","last_fullsized":"2023-09-23T22:13:59.785893+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:10:42.048739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.789944+0000","last_change":"2023-09-23T22:14:00.789944+0000","last_active":"2023-09-23T22:14:00.789944+0000","last_peered":"2023-09-23T22:14:00.789944+0000","last_clean":"2023-09-23T22:14:00.789944+0000","last_became_active":"2023-09-23T22:14:00.789559+0000","last_became_peered":"2023-09-23T22:14:00.789559+0000","last_unstale":"2023-09-23T22:14:00.789944+0000","last_undegraded":"2023-09-23T22:14:00.789944+0000","last_fullsized":"2023-09-23T22:14:00.789944+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:18:30.957719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":451,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783108+0000","last_change":"2023-09-23T22:11:42.695484+0000","last_active":"2023-09-23T22:13:59.783108+0000","last_peered":"2023-09-23T22:13:59.783108+0000","last_clean":"2023-09-23T22:13:59.783108+0000","last_became_active":"2023-09-23T22:11:42.694972+0000","last_became_peered":"2023-09-23T22:11:42.694972+0000","last_unstale":"2023-09-23T22:13:59.783108+0000","last_undegraded":"2023-09-23T22:13:59.783108+0000","last_fullsized":"2023-09-23T22:13:59.783108+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:36.787404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":694,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780660+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:13:59.780660+0000","last_peered":"2023-09-23T22:13:59.780660+0000","last_clean":"2023-09-23T22:13:59.780660+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:13:59.780660+0000","last_undegraded":"2023-09-23T22:13:59.780660+0000","last_fullsized":"2023-09-23T22:13:59.780660+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.781830+0000","last_change":"2023-09-23T22:12:54.874146+0000","last_active":"2023-09-23T22:13:59.781830+0000","last_peered":"2023-09-23T22:13:59.781830+0000","last_clean":"2023-09-23T22:13:59.781830+0000","last_became_active":"2023-09-23T22:12:54.873782+0000","last_became_peered":"2023-09-23T22:12:54.873782+0000","last_unstale":"2023-09-23T22:13:59.781830+0000","last_undegraded":"2023-09-23T22:13:59.781830+0000","last_fullsized":"2023-09-23T22:13:59.781830+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:25.933932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.788799+0000","last_change":"2023-09-23T22:13:27.659751+0000","last_active":"2023-09-23T22:14:00.788799+0000","last_peered":"2023-09-23T22:14:00.788799+0000","last_clean":"2023-09-23T22:14:00.788799+0000","last_became_active":"2023-09-23T22:13:27.659422+0000","last_became_peered":"2023-09-23T22:13:27.659422+0000","last_unstale":"2023-09-23T22:14:00.788799+0000","last_undegraded":"2023-09-23T22:14:00.788799+0000","last_fullsized":"2023-09-23T22:14:00.788799+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:25.705909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":378,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.783947+0000","last_change":"2023-09-23T22:14:00.783947+0000","last_active":"2023-09-23T22:14:00.783947+0000","last_peered":"2023-09-23T22:14:00.783947+0000","last_clean":"2023-09-23T22:14:00.783947+0000","last_became_active":"2023-09-23T22:14:00.783319+0000","last_became_peered":"2023-09-23T22:14:00.783319+0000","last_unstale":"2023-09-23T22:14:00.783947+0000","last_undegraded":"2023-09-23T22:14:00.783947+0000","last_fullsized":"2023-09-23T22:14:00.783947+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:02:25.419096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796811+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:14:00.796811+0000","last_peered":"2023-09-23T22:14:00.796811+0000","last_clean":"2023-09-23T22:14:00.796811+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:14:00.796811+0000","last_undegraded":"2023-09-23T22:14:00.796811+0000","last_fullsized":"2023-09-23T22:14:00.796811+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788001+0000","last_change":"2023-09-23T22:11:42.697009+0000","last_active":"2023-09-23T22:13:59.788001+0000","last_peered":"2023-09-23T22:13:59.788001+0000","last_clean":"2023-09-23T22:13:59.788001+0000","last_became_active":"2023-09-23T22:11:42.696588+0000","last_became_peered":"2023-09-23T22:11:42.696588+0000","last_unstale":"2023-09-23T22:13:59.788001+0000","last_undegraded":"2023-09-23T22:13:59.788001+0000","last_fullsized":"2023-09-23T22:13:59.788001+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:57.260787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":750,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780601+0000","last_change":"2023-09-23T22:12:54.877082+0000","last_active":"2023-09-23T22:13:59.780601+0000","last_peered":"2023-09-23T22:13:59.780601+0000","last_clean":"2023-09-23T22:13:59.780601+0000","last_became_active":"2023-09-23T22:12:54.876524+0000","last_became_peered":"2023-09-23T22:12:54.876524+0000","last_unstale":"2023-09-23T22:13:59.780601+0000","last_undegraded":"2023-09-23T22:13:59.780601+0000","last_fullsized":"2023-09-23T22:13:59.780601+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:46.185025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.778046+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:13:59.778046+0000","last_peered":"2023-09-23T22:13:59.778046+0000","last_clean":"2023-09-23T22:13:59.778046+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:13:59.778046+0000","last_undegraded":"2023-09-23T22:13:59.778046+0000","last_fullsized":"2023-09-23T22:13:59.778046+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792797+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:14:00.792797+0000","last_peered":"2023-09-23T22:14:00.792797+0000","last_clean":"2023-09-23T22:14:00.792797+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:14:00.792797+0000","last_undegraded":"2023-09-23T22:14:00.792797+0000","last_fullsized":"2023-09-23T22:14:00.792797+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":556,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.785260+0000","last_change":"2023-09-23T22:12:54.875239+0000","last_active":"2023-09-23T22:14:00.785260+0000","last_peered":"2023-09-23T22:14:00.785260+0000","last_clean":"2023-09-23T22:14:00.785260+0000","last_became_active":"2023-09-23T22:12:54.874675+0000","last_became_peered":"2023-09-23T22:12:54.874675+0000","last_unstale":"2023-09-23T22:14:00.785260+0000","last_undegraded":"2023-09-23T22:14:00.785260+0000","last_fullsized":"2023-09-23T22:14:00.785260+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:21.865417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.791734+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:14:00.791734+0000","last_peered":"2023-09-23T22:14:00.791734+0000","last_clean":"2023-09-23T22:14:00.791734+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:14:00.791734+0000","last_undegraded":"2023-09-23T22:14:00.791734+0000","last_fullsized":"2023-09-23T22:14:00.791734+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795928+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:14:00.795928+0000","last_peered":"2023-09-23T22:14:00.795928+0000","last_clean":"2023-09-23T22:14:00.795928+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:14:00.795928+0000","last_undegraded":"2023-09-23T22:14:00.795928+0000","last_fullsized":"2023-09-23T22:14:00.795928+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.790050+0000","last_change":"2023-09-23T22:12:54.868682+0000","last_active":"2023-09-23T22:13:59.790050+0000","last_peered":"2023-09-23T22:13:59.790050+0000","last_clean":"2023-09-23T22:13:59.790050+0000","last_became_active":"2023-09-23T22:12:54.868339+0000","last_became_peered":"2023-09-23T22:12:54.868339+0000","last_unstale":"2023-09-23T22:13:59.790050+0000","last_undegraded":"2023-09-23T22:13:59.790050+0000","last_fullsized":"2023-09-23T22:13:59.790050+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:53.246678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.789109+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:13:59.789109+0000","last_peered":"2023-09-23T22:13:59.789109+0000","last_clean":"2023-09-23T22:13:59.789109+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:13:59.789109+0000","last_undegraded":"2023-09-23T22:13:59.789109+0000","last_fullsized":"2023-09-23T22:13:59.789109+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.791120+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:13:59.791120+0000","last_peered":"2023-09-23T22:13:59.791120+0000","last_clean":"2023-09-23T22:13:59.791120+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:13:59.791120+0000","last_undegraded":"2023-09-23T22:13:59.791120+0000","last_fullsized":"2023-09-23T22:13:59.791120+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785609+0000","last_change":"2023-09-23T22:12:54.880778+0000","last_active":"2023-09-23T22:13:59.785609+0000","last_peered":"2023-09-23T22:13:59.785609+0000","last_clean":"2023-09-23T22:13:59.785609+0000","last_became_active":"2023-09-23T22:12:54.880272+0000","last_became_peered":"2023-09-23T22:12:54.880272+0000","last_unstale":"2023-09-23T22:13:59.785609+0000","last_undegraded":"2023-09-23T22:13:59.785609+0000","last_fullsized":"2023-09-23T22:13:59.785609+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:01.030591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792760+0000","last_change":"2023-09-23T22:14:00.792760+0000","last_active":"2023-09-23T22:14:00.792760+0000","last_peered":"2023-09-23T22:14:00.792760+0000","last_clean":"2023-09-23T22:14:00.792760+0000","last_became_active":"2023-09-23T22:14:00.792478+0000","last_became_peered":"2023-09-23T22:14:00.792478+0000","last_unstale":"2023-09-23T22:14:00.792760+0000","last_undegraded":"2023-09-23T22:14:00.792760+0000","last_fullsized":"2023-09-23T22:14:00.792760+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.935338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790875+0000","last_change":"2023-09-23T22:14:00.790875+0000","last_active":"2023-09-23T22:14:00.790875+0000","last_peered":"2023-09-23T22:14:00.790875+0000","last_clean":"2023-09-23T22:14:00.790875+0000","last_became_active":"2023-09-23T22:14:00.790332+0000","last_became_peered":"2023-09-23T22:14:00.790332+0000","last_unstale":"2023-09-23T22:14:00.790875+0000","last_undegraded":"2023-09-23T22:14:00.790875+0000","last_fullsized":"2023-09-23T22:14:00.790875+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:05.904013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786633+0000","last_change":"2023-09-23T22:12:54.871021+0000","last_active":"2023-09-23T22:13:59.786633+0000","last_peered":"2023-09-23T22:13:59.786633+0000","last_clean":"2023-09-23T22:13:59.786633+0000","last_became_active":"2023-09-23T22:12:54.870736+0000","last_became_peered":"2023-09-23T22:12:54.870736+0000","last_unstale":"2023-09-23T22:13:59.786633+0000","last_undegraded":"2023-09-23T22:13:59.786633+0000","last_fullsized":"2023-09-23T22:13:59.786633+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:51.846457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795682+0000","last_change":"2023-09-23T22:14:00.795682+0000","last_active":"2023-09-23T22:14:00.795682+0000","last_peered":"2023-09-23T22:14:00.795682+0000","last_clean":"2023-09-23T22:14:00.795682+0000","last_became_active":"2023-09-23T22:14:00.795377+0000","last_became_peered":"2023-09-23T22:14:00.795377+0000","last_unstale":"2023-09-23T22:14:00.795682+0000","last_undegraded":"2023-09-23T22:14:00.795682+0000","last_fullsized":"2023-09-23T22:14:00.795682+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:33.253920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785440+0000","last_change":"2023-09-23T22:12:54.865320+0000","last_active":"2023-09-23T22:13:59.785440+0000","last_peered":"2023-09-23T22:13:59.785440+0000","last_clean":"2023-09-23T22:13:59.785440+0000","last_became_active":"2023-09-23T22:12:54.865007+0000","last_became_peered":"2023-09-23T22:12:54.865007+0000","last_unstale":"2023-09-23T22:13:59.785440+0000","last_undegraded":"2023-09-23T22:13:59.785440+0000","last_fullsized":"2023-09-23T22:13:59.785440+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:40.963311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787011+0000","last_change":"2023-09-23T22:04:54.384008+0000","last_active":"2023-09-23T22:13:59.787011+0000","last_peered":"2023-09-23T22:13:59.787011+0000","last_clean":"2023-09-23T22:13:59.787011+0000","last_became_active":"2023-09-23T22:04:54.383702+0000","last_became_peered":"2023-09-23T22:04:54.383702+0000","last_unstale":"2023-09-23T22:13:59.787011+0000","last_undegraded":"2023-09-23T22:13:59.787011+0000","last_fullsized":"2023-09-23T22:13:59.787011+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:00.423911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.795430+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:14:00.795430+0000","last_peered":"2023-09-23T22:14:00.795430+0000","last_clean":"2023-09-23T22:14:00.795430+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:14:00.795430+0000","last_undegraded":"2023-09-23T22:14:00.795430+0000","last_fullsized":"2023-09-23T22:14:00.795430+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786126+0000","last_change":"2023-09-23T22:12:54.861787+0000","last_active":"2023-09-23T22:13:59.786126+0000","last_peered":"2023-09-23T22:13:59.786126+0000","last_clean":"2023-09-23T22:13:59.786126+0000","last_became_active":"2023-09-23T22:12:54.861377+0000","last_became_peered":"2023-09-23T22:12:54.861377+0000","last_unstale":"2023-09-23T22:13:59.786126+0000","last_undegraded":"2023-09-23T22:13:59.786126+0000","last_fullsized":"2023-09-23T22:13:59.786126+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:02.832226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780468+0000","last_change":"2023-09-23T22:04:54.528019+0000","last_active":"2023-09-23T22:13:59.780468+0000","last_peered":"2023-09-23T22:13:59.780468+0000","last_clean":"2023-09-23T22:13:59.780468+0000","last_became_active":"2023-09-23T22:04:54.527666+0000","last_became_peered":"2023-09-23T22:04:54.527666+0000","last_unstale":"2023-09-23T22:13:59.780468+0000","last_undegraded":"2023-09-23T22:13:59.780468+0000","last_fullsized":"2023-09-23T22:13:59.780468+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:26:32.774493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786894+0000","last_change":"2023-09-23T22:11:42.694008+0000","last_active":"2023-09-23T22:13:59.786894+0000","last_peered":"2023-09-23T22:13:59.786894+0000","last_clean":"2023-09-23T22:13:59.786894+0000","last_became_active":"2023-09-23T22:11:42.693609+0000","last_became_peered":"2023-09-23T22:11:42.693609+0000","last_unstale":"2023-09-23T22:13:59.786894+0000","last_undegraded":"2023-09-23T22:13:59.786894+0000","last_fullsized":"2023-09-23T22:13:59.786894+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:59.614114+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794463+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:14:00.794463+0000","last_peered":"2023-09-23T22:14:00.794463+0000","last_clean":"2023-09-23T22:14:00.794463+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:14:00.794463+0000","last_undegraded":"2023-09-23T22:14:00.794463+0000","last_fullsized":"2023-09-23T22:14:00.794463+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":77,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.784703+0000","last_change":"2023-09-23T22:11:42.701916+0000","last_active":"2023-09-23T22:13:59.784703+0000","last_peered":"2023-09-23T22:13:59.784703+0000","last_clean":"2023-09-23T22:13:59.784703+0000","last_became_active":"2023-09-23T22:11:42.701591+0000","last_became_peered":"2023-09-23T22:11:42.701591+0000","last_unstale":"2023-09-23T22:13:59.784703+0000","last_undegraded":"2023-09-23T22:13:59.784703+0000","last_fullsized":"2023-09-23T22:13:59.784703+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:23.941128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.793040+0000","last_change":"2023-09-23T22:14:00.792449+0000","last_active":"2023-09-23T22:14:00.793040+0000","last_peered":"2023-09-23T22:14:00.793040+0000","last_clean":"2023-09-23T22:14:00.793040+0000","last_became_active":"2023-09-23T22:14:00.792239+0000","last_became_peered":"2023-09-23T22:14:00.792239+0000","last_unstale":"2023-09-23T22:14:00.793040+0000","last_undegraded":"2023-09-23T22:14:00.793040+0000","last_fullsized":"2023-09-23T22:14:00.793040+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:33.488563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.783327+0000","last_change":"2023-09-23T22:04:54.387688+0000","last_active":"2023-09-23T22:13:59.783327+0000","last_peered":"2023-09-23T22:13:59.783327+0000","last_clean":"2023-09-23T22:13:59.783327+0000","last_became_active":"2023-09-23T22:04:54.387421+0000","last_became_peered":"2023-09-23T22:04:54.387421+0000","last_unstale":"2023-09-23T22:13:59.783327+0000","last_undegraded":"2023-09-23T22:13:59.783327+0000","last_fullsized":"2023-09-23T22:13:59.783327+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:22.411746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794669+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:14:00.794669+0000","last_peered":"2023-09-23T22:14:00.794669+0000","last_clean":"2023-09-23T22:14:00.794669+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:14:00.794669+0000","last_undegraded":"2023-09-23T22:14:00.794669+0000","last_fullsized":"2023-09-23T22:14:00.794669+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792829+0000","last_change":"2023-09-23T22:14:00.792829+0000","last_active":"2023-09-23T22:14:00.792829+0000","last_peered":"2023-09-23T22:14:00.792829+0000","last_clean":"2023-09-23T22:14:00.792829+0000","last_became_active":"2023-09-23T22:14:00.792462+0000","last_became_peered":"2023-09-23T22:14:00.792462+0000","last_unstale":"2023-09-23T22:14:00.792829+0000","last_undegraded":"2023-09-23T22:14:00.792829+0000","last_fullsized":"2023-09-23T22:14:00.792829+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:19:55.154145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794351+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:14:00.794351+0000","last_peered":"2023-09-23T22:14:00.794351+0000","last_clean":"2023-09-23T22:14:00.794351+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:14:00.794351+0000","last_undegraded":"2023-09-23T22:14:00.794351+0000","last_fullsized":"2023-09-23T22:14:00.794351+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.785117+0000","last_change":"2023-09-23T22:13:27.663039+0000","last_active":"2023-09-23T22:13:59.785117+0000","last_peered":"2023-09-23T22:13:59.785117+0000","last_clean":"2023-09-23T22:13:59.785117+0000","last_became_active":"2023-09-23T22:13:27.662687+0000","last_became_peered":"2023-09-23T22:13:27.662687+0000","last_unstale":"2023-09-23T22:13:59.785117+0000","last_undegraded":"2023-09-23T22:13:59.785117+0000","last_fullsized":"2023-09-23T22:13:59.785117+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:13.432843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.788095+0000","last_change":"2023-09-23T22:12:54.866763+0000","last_active":"2023-09-23T22:13:59.788095+0000","last_peered":"2023-09-23T22:13:59.788095+0000","last_clean":"2023-09-23T22:13:59.788095+0000","last_became_active":"2023-09-23T22:12:54.866422+0000","last_became_peered":"2023-09-23T22:12:54.866422+0000","last_unstale":"2023-09-23T22:13:59.788095+0000","last_undegraded":"2023-09-23T22:13:59.788095+0000","last_fullsized":"2023-09-23T22:13:59.788095+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:43.904727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.790220+0000","last_change":"2023-09-23T22:04:54.381132+0000","last_active":"2023-09-23T22:13:59.790220+0000","last_peered":"2023-09-23T22:13:59.790220+0000","last_clean":"2023-09-23T22:13:59.790220+0000","last_became_active":"2023-09-23T22:04:54.380838+0000","last_became_peered":"2023-09-23T22:04:54.380838+0000","last_unstale":"2023-09-23T22:13:59.790220+0000","last_undegraded":"2023-09-23T22:13:59.790220+0000","last_fullsized":"2023-09-23T22:13:59.790220+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:41.334974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780225+0000","last_change":"2023-09-23T22:13:27.661799+0000","last_active":"2023-09-23T22:13:59.780225+0000","last_peered":"2023-09-23T22:13:59.780225+0000","last_clean":"2023-09-23T22:13:59.780225+0000","last_became_active":"2023-09-23T22:13:27.661473+0000","last_became_peered":"2023-09-23T22:13:27.661473+0000","last_unstale":"2023-09-23T22:13:59.780225+0000","last_undegraded":"2023-09-23T22:13:59.780225+0000","last_fullsized":"2023-09-23T22:13:59.780225+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:34.827694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":73,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.796705+0000","last_change":"2023-09-23T22:14:00.796705+0000","last_active":"2023-09-23T22:14:00.796705+0000","last_peered":"2023-09-23T22:14:00.796705+0000","last_clean":"2023-09-23T22:14:00.796705+0000","last_became_active":"2023-09-23T22:14:00.796252+0000","last_became_peered":"2023-09-23T22:14:00.796252+0000","last_unstale":"2023-09-23T22:14:00.796705+0000","last_undegraded":"2023-09-23T22:14:00.796705+0000","last_fullsized":"2023-09-23T22:14:00.796705+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:58.956676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.794963+0000","last_change":"2023-09-23T22:14:00.794963+0000","last_active":"2023-09-23T22:14:00.794963+0000","last_peered":"2023-09-23T22:14:00.794963+0000","last_clean":"2023-09-23T22:14:00.794963+0000","last_became_active":"2023-09-23T22:14:00.794394+0000","last_became_peered":"2023-09-23T22:14:00.794394+0000","last_unstale":"2023-09-23T22:14:00.794963+0000","last_undegraded":"2023-09-23T22:14:00.794963+0000","last_fullsized":"2023-09-23T22:14:00.794963+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:25.119383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.780772+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:13:59.780772+0000","last_peered":"2023-09-23T22:13:59.780772+0000","last_clean":"2023-09-23T22:13:59.780772+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:13:59.780772+0000","last_undegraded":"2023-09-23T22:13:59.780772+0000","last_fullsized":"2023-09-23T22:13:59.780772+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790411+0000","last_change":"2023-09-23T22:14:00.790411+0000","last_active":"2023-09-23T22:14:00.790411+0000","last_peered":"2023-09-23T22:14:00.790411+0000","last_clean":"2023-09-23T22:14:00.790411+0000","last_became_active":"2023-09-23T22:14:00.789906+0000","last_became_peered":"2023-09-23T22:14:00.789906+0000","last_unstale":"2023-09-23T22:14:00.790411+0000","last_undegraded":"2023-09-23T22:14:00.790411+0000","last_fullsized":"2023-09-23T22:14:00.790411+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:28:43.440626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.788697+0000","last_change":"2023-09-23T22:14:00.787881+0000","last_active":"2023-09-23T22:14:00.788697+0000","last_peered":"2023-09-23T22:14:00.788697+0000","last_clean":"2023-09-23T22:14:00.788697+0000","last_became_active":"2023-09-23T22:14:00.787498+0000","last_became_peered":"2023-09-23T22:14:00.787498+0000","last_unstale":"2023-09-23T22:14:00.788697+0000","last_undegraded":"2023-09-23T22:14:00.788697+0000","last_fullsized":"2023-09-23T22:14:00.788697+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:19.424721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.786273+0000","last_change":"2023-09-23T22:12:54.864958+0000","last_active":"2023-09-23T22:13:59.786273+0000","last_peered":"2023-09-23T22:13:59.786273+0000","last_clean":"2023-09-23T22:13:59.786273+0000","last_became_active":"2023-09-23T22:12:54.864451+0000","last_became_peered":"2023-09-23T22:12:54.864451+0000","last_unstale":"2023-09-23T22:13:59.786273+0000","last_undegraded":"2023-09-23T22:13:59.786273+0000","last_fullsized":"2023-09-23T22:13:59.786273+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:32.668061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790097+0000","last_change":"2023-09-23T22:14:00.790097+0000","last_active":"2023-09-23T22:14:00.790097+0000","last_peered":"2023-09-23T22:14:00.790097+0000","last_clean":"2023-09-23T22:14:00.790097+0000","last_became_active":"2023-09-23T22:14:00.789389+0000","last_became_peered":"2023-09-23T22:14:00.789389+0000","last_unstale":"2023-09-23T22:14:00.790097+0000","last_undegraded":"2023-09-23T22:14:00.790097+0000","last_fullsized":"2023-09-23T22:14:00.790097+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:05.673490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.792919+0000","last_change":"2023-09-23T22:14:00.792919+0000","last_active":"2023-09-23T22:14:00.792919+0000","last_peered":"2023-09-23T22:14:00.792919+0000","last_clean":"2023-09-23T22:14:00.792919+0000","last_became_active":"2023-09-23T22:14:00.792490+0000","last_became_peered":"2023-09-23T22:14:00.792490+0000","last_unstale":"2023-09-23T22:14:00.792919+0000","last_undegraded":"2023-09-23T22:14:00.792919+0000","last_fullsized":"2023-09-23T22:14:00.792919+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:53.433863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.791655+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:14:00.791655+0000","last_peered":"2023-09-23T22:14:00.791655+0000","last_clean":"2023-09-23T22:14:00.791655+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:14:00.791655+0000","last_undegraded":"2023-09-23T22:14:00.791655+0000","last_fullsized":"2023-09-23T22:14:00.791655+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T22:14:00.790313+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:14:00.790313+0000","last_peered":"2023-09-23T22:14:00.790313+0000","last_clean":"2023-09-23T22:14:00.790313+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:14:00.790313+0000","last_undegraded":"2023-09-23T22:14:00.790313+0000","last_fullsized":"2023-09-23T22:14:00.790313+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":86,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T22:13:59.787689+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:13:59.787689+0000","last_peered":"2023-09-23T22:13:59.787689+0000","last_clean":"2023-09-23T22:13:59.787689+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:13:59.787689+0000","last_undegraded":"2023-09-23T22:13:59.787689+0000","last_fullsized":"2023-09-23T22:13:59.787689+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13208,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234126,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12896,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":11200,"kb_avail":93732256,"statfs":{"total":95995035648,"available":95981830144,"internally_reserved":0,"allocated":1728512,"data_stored":1203715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11468800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:51 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.34000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.59599999999999997,"15min":0.58799999999999997},"min":{"1min":0.39900000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.609,"5min":1.609,"15min":1.609},"last":0.73399999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.69999999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.008,"15min":1.008},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":8.5220000000000002,"5min":8.5220000000000002,"15min":8.5220000000000002},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.329,"5min":1.329,"15min":1.329},"last":0.54100000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:13:31 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.61399999999999999,"15min":0.59599999999999997},"min":{"1min":0.38700000000000001,"5min":0.38600000000000001,"15min":0.35699999999999998},"max":{"1min":0.81000000000000005,"5min":1.3109999999999999,"15min":7.8970000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.624,"5min":0.61299999999999999,"15min":0.59099999999999997},"min":{"1min":0.42799999999999999,"5min":0.41899999999999998,"15min":0.35599999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":2.9039999999999999},"last":0.58599999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.503,"15min":0.47799999999999998},"min":{"1min":0.32000000000000001,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":0.79000000000000004,"5min":0.81399999999999995,"15min":0.82199999999999995},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.50900000000000001,"15min":0.502},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.80800000000000005},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.502,"15min":0.48499999999999999},"min":{"1min":0.379,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":0.70499999999999996,"5min":0.71999999999999997,"15min":0.78700000000000003},"last":0.499},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.57999999999999996,"15min":0.55400000000000005},"min":{"1min":0.42999999999999999,"5min":0.29799999999999999,"15min":0.29199999999999998},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.873},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:14:32 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.55300000000000005,"15min":0.55800000000000005},"min":{"1min":0.41799999999999998,"5min":0.315,"15min":0.315},"max":{"1min":0.77600000000000002,"5min":0.82999999999999996,"15min":3.0459999999999998},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.625,"15min":0.58999999999999997},"min":{"1min":0.40799999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.81399999999999995,"5min":0.874,"15min":0.874},"last":0.63700000000000001}]}]},{"osd":6,"up_from":46,"seq":197568495762,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12444,"kb_used_data":5780,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93732708,"statfs":{"total":95995035648,"available":95982292992,"internally_reserved":0,"allocated":5918720,"data_stored":5398215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.54500000000000004,"15min":0.56499999999999995},"min":{"1min":0.34799999999999998,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.75600000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.58099999999999996,"15min":0.60499999999999998},"min":{"1min":0.36599999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.69799999999999995,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.65600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:13:55 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.58899999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.625}]},{"osd":3,"last update":"Sat Sep 23 22:13:05 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.58199999999999996,"15min":0.67600000000000005},"min":{"1min":0.434,"5min":0.379,"15min":0.28699999999999998},"max":{"1min":0.83999999999999997,"5min":1.4299999999999999,"15min":7.6429999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58699999999999997,"15min":0.57299999999999995},"min":{"1min":0.47799999999999998,"5min":0.36099999999999999,"15min":0.35799999999999998},"max":{"1min":0.752,"5min":0.84099999999999997,"15min":4.5670000000000002},"last":0.69099999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:14:05 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.46800000000000003,"15min":0.432},"min":{"1min":0.33600000000000002,"5min":0.309,"15min":0.252},"max":{"1min":0.68899999999999995,"5min":0.77700000000000002,"15min":1.718},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.50900000000000001,"15min":0.46600000000000003},"min":{"1min":0.32400000000000001,"5min":0.315,"15min":0.26500000000000001},"max":{"1min":0.76800000000000002,"5min":0.86099999999999999,"15min":1.78},"last":0.71399999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:14:05 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51800000000000002,"15min":0.48299999999999998},"min":{"1min":0.375,"5min":0.315,"15min":0.29499999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":1.5629999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.51800000000000002,"15min":0.47999999999999998},"min":{"1min":0.314,"5min":0.29899999999999999,"15min":0.29399999999999998},"max":{"1min":0.68500000000000005,"5min":0.81399999999999995,"15min":1.74},"last":0.60399999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:42 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.56000000000000005,"15min":0.52100000000000002},"min":{"1min":0.378,"5min":0.33200000000000002,"15min":0.29899999999999999},"max":{"1min":0.77600000000000002,"5min":0.77900000000000003,"15min":1.7709999999999999},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.60599999999999998,"15min":0.57799999999999996},"min":{"1min":0.42799999999999999,"5min":0.38300000000000001,"15min":0.35599999999999998},"max":{"1min":0.78500000000000003,"5min":0.90900000000000003,"15min":1.603},"last":0.748}]}]},{"osd":1,"up_from":98,"seq":420906795035,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32828,"kb_used_data":5236,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93712324,"statfs":{"total":95995035648,"available":95961419776,"internally_reserved":0,"allocated":5361664,"data_stored":4813549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":28238196},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":0.26300000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:14:28 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"last":0.30199999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.0700000000000001,"5min":1.0700000000000001,"15min":1.0700000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.0129999999999999,"5min":1.0129999999999999,"15min":1.0129999999999999},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.63800000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.0349999999999999,"5min":1.0349999999999999,"15min":1.0349999999999999},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:13:54 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"min":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"max":{"1min":0.998,"5min":0.998,"15min":0.998},"last":0.495}]}]},{"osd":0,"up_from":93,"seq":399431958569,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29324,"kb_used_data":1732,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93715828,"statfs":{"total":95995035648,"available":95965007872,"internally_reserved":0,"allocated":1773568,"data_stored":1210692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":28237674},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:13:58 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"last":0.311},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.496,"5min":0.496,"15min":0.496},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:14:30 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.4020000000000001,"5min":3.4020000000000001,"15min":3.4020000000000001},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:13:47 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":0.48299999999999998,"15min":0.49199999999999999},"min":{"1min":0.34399999999999997,"5min":0.316,"15min":0.316},"max":{"1min":0.621,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.49099999999999999,"15min":0.503},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.57899999999999996,"5min":0.76800000000000002,"15min":0.76800000000000002},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.66600000000000004,"15min":0.65600000000000003},"min":{"1min":0.437,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":3.181,"5min":3.181,"15min":3.181},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.65000000000000002,"15min":0.65700000000000003},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.89900000000000002,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.63700000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.75900000000000001,"5min":0.69299999999999995,"15min":0.68999999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.71299999999999997,"5min":0.68000000000000005,"15min":0.68300000000000005},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":1.6799999999999999,"5min":1.6799999999999999,"15min":1.6799999999999999},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.68799999999999994,"15min":0.68100000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":3.3159999999999998,"5min":3.3159999999999998,"15min":3.3159999999999998},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.69099999999999995,"15min":0.70999999999999996},"min":{"1min":0.436,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.92000000000000004,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:14:48 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":0.73299999999999998,"15min":0.73099999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":3.367,"5min":3.367,"15min":3.367},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.748,"15min":0.753},"min":{"1min":0.495,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":3.286,"5min":3.286,"15min":3.286},"last":0.59999999999999998}]}]},{"osd":2,"up_from":103,"seq":442381631508,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28656,"kb_used_data":1128,"kb_used_omap":10,"kb_used_meta":27509,"kb_avail":93716496,"statfs":{"total":95995035648,"available":95965691904,"internally_reserved":0,"allocated":1155072,"data_stored":616429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":28169388},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.245},{"interface":"front","average":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.224},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.30399999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27800000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.69299999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.92600000000000005,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":2.948,"5min":2.948,"15min":2.948},"last":0.72199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:14:26 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":2.931,"5min":2.931,"15min":2.931},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.57599999999999996}]}]},{"osd":3,"up_from":108,"seq":463856467982,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28572,"kb_used_data":1108,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716580,"statfs":{"total":95995035648,"available":95965777920,"internally_reserved":0,"allocated":1134592,"data_stored":613253,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":28110381},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35099999999999998}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36199999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39600000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.432}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47799999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019035,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7772,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93737380,"statfs":{"total":95995035648,"available":95987077120,"internally_reserved":0,"allocated":1134592,"data_stored":613600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.622,"15min":0.626},"min":{"1min":0.38900000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.2749999999999999,"5min":1.8180000000000001,"15min":1.8180000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.628,"5min":0.63400000000000001,"15min":0.64600000000000002},"min":{"1min":0.42099999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.2889999999999999,"5min":1.2889999999999999,"15min":1.2889999999999999},"last":0.59599999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:14:54 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.55400000000000005,"15min":0.54800000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.403,"5min":1.403,"15min":1.403},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57699999999999996,"15min":0.57499999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.61599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.57199999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:13:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.57099999999999995,"15min":0.56100000000000005},"min":{"1min":0.38500000000000001,"5min":0.37,"15min":0.313},"max":{"1min":0.80000000000000004,"5min":0.86199999999999999,"15min":3.1499999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59599999999999997,"15min":0.56699999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.76600000000000001,"5min":0.85099999999999998,"15min":1.0089999999999999},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48899999999999999,"15min":0.45500000000000002},"min":{"1min":0.29199999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66300000000000003,"5min":0.749,"15min":0.81899999999999995},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.54800000000000004,"15min":0.54400000000000004},"min":{"1min":0.39500000000000002,"5min":0.33300000000000002,"15min":0.318},"max":{"1min":0.622,"5min":0.78500000000000003,"15min":3.0470000000000002},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:14:16 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.52100000000000002,"15min":0.49399999999999999},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.29599999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.83099999999999996},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56100000000000005,"15min":0.55000000000000004},"min":{"1min":0.41399999999999998,"5min":0.32100000000000001,"15min":0.28899999999999998},"max":{"1min":0.73799999999999999,"5min":0.83299999999999996,"15min":2.2440000000000002},"last":0.54300000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:14:49 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.60099999999999998,"15min":0.624},"min":{"1min":0.46899999999999997,"5min":0.372,"15min":0.313},"max":{"1min":0.82299999999999995,"5min":0.83799999999999997,"15min":3.9609999999999999},"last":0.498},{"interface":"front","average":{"1min":0.65300000000000002,"5min":0.61299999999999999,"15min":0.58799999999999997},"min":{"1min":0.495,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.94999999999999996},"last":0.435}]}]},{"osd":5,"up_from":38,"seq":163208757399,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11664,"kb_used_data":5192,"kb_used_omap":0,"kb_used_meta":6464,"kb_avail":93733488,"statfs":{"total":95995035648,"available":95983091712,"internally_reserved":0,"allocated":5316608,"data_stored":4803667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6619136},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:14:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.55500000000000005,"15min":0.56299999999999994},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.502,"5min":0.59799999999999998,"15min":0.63},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.48099999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:13:56 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.51400000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:14:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.62},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.44400000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.59499999999999997,"15min":0.57899999999999996},"min":{"1min":0.46300000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77900000000000003,"5min":0.93999999999999995,"15min":1.028},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.61299999999999999,"15min":0.60299999999999998},"min":{"1min":0.45400000000000001,"5min":0.36599999999999999,"15min":0.33400000000000002},"max":{"1min":0.79200000000000004,"5min":1.6160000000000001,"15min":2.04},"last":0.52900000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:13:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.47399999999999998,"15min":0.44500000000000001},"min":{"1min":0.34000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69199999999999995,"5min":0.70899999999999996,"15min":0.752},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.53800000000000003,"15min":0.48999999999999999},"min":{"1min":0.42199999999999999,"5min":0.35599999999999998,"15min":0.28999999999999998},"max":{"1min":0.747,"5min":0.751,"15min":0.81100000000000005},"last":0.41399999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:14:21 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.55000000000000004,"15min":0.50900000000000001},"min":{"1min":0.44600000000000001,"5min":0.318,"15min":0.318},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.5},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.56699999999999995,"15min":0.53000000000000003},"min":{"1min":0.46400000000000002,"5min":0.34300000000000003,"15min":0.29699999999999999},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.63400000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:14:40 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.59599999999999997,"15min":0.57799999999999996},"min":{"1min":0.437,"5min":0.376,"15min":0.32600000000000001},"max":{"1min":0.73199999999999998,"5min":0.81799999999999995,"15min":0.93700000000000006},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.58599999999999997,"15min":0.55900000000000005},"min":{"1min":0.41799999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.84999999999999998},"last":0.60899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:14:59.868 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T22:14:59.868 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T22:15:00.232 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:00.714 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:00 smithi026 ceph-mon[215775]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:00.714 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:00 smithi026 ceph-mon[215775]: from='client.44525 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:15:00.714 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:00 smithi026 ceph-mon[223014]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:00.714 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:00 smithi026 ceph-mon[223014]: from='client.44525 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:15:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:00 smithi070 ceph-mon[192652]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:00 smithi070 ceph-mon[192652]: from='client.44525 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:15:01.812 INFO:teuthology.orchestra.run.smithi026.stdout:epoch 109 2023-09-23T22:15:01.812 INFO:teuthology.orchestra.run.smithi026.stdout:fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 2023-09-23T22:15:01.812 INFO:teuthology.orchestra.run.smithi026.stdout:created 2023-09-23T21:58:20.282373+0000 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:modified 2023-09-23T22:14:00.774068+0000 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:crush_version 22 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:full_ratio 0.95 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:backfillfull_ratio 0.9 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:nearfull_ratio 0.85 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:require_min_compat_client luminous 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:min_compat_client luminous 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:require_osd_release quincy 2023-09-23T22:15:01.813 INFO:teuthology.orchestra.run.smithi026.stdout:stretch_mode_enabled false 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 83 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 66 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:max_osd 8 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 up in weight 1 up_from 93 up_thru 108 down_at 90 last_clean_interval [8,89) [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654] [v2:172.21.15.26:6804/507489654,v1:172.21.15.26:6805/507489654] exists,up fdf0a195-9329-458c-b103-38db9a4408a8 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 up in weight 1 up_from 98 up_thru 108 down_at 95 last_clean_interval [13,94) [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792] [v2:172.21.15.26:6812/693231792,v1:172.21.15.26:6813/693231792] exists,up 9c0507d8-f63e-47b2-841d-394196258d6b 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 up in weight 1 up_from 103 up_thru 108 down_at 100 last_clean_interval [18,99) [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869] [v2:172.21.15.26:6820/1775189869,v1:172.21.15.26:6821/1775189869] exists,up 4b241a88-8860-4616-8af5-383823ce290f 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 up in weight 1 up_from 108 up_thru 108 down_at 105 last_clean_interval [25,104) [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469] [v2:172.21.15.26:6828/1956095469,v1:172.21.15.26:6829/1956095469] exists,up 8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea 2023-09-23T22:15:01.814 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 up in weight 1 up_from 30 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884] [v2:172.21.15.70:6802/1461751884,v1:172.21.15.70:6803/1461751884] exists,up 6d4a4d17-075d-4538-9109-7a1dde1b422d 2023-09-23T22:15:01.815 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 up in weight 1 up_from 38 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994] [v2:172.21.15.70:6810/4236773994,v1:172.21.15.70:6811/4236773994] exists,up b25ee343-9def-42dc-8b45-a3c85bf14805 2023-09-23T22:15:01.815 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6 up in weight 1 up_from 46 up_thru 105 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869] [v2:172.21.15.70:6818/572290869,v1:172.21.15.70:6819/572290869] exists,up f7ecfbb9-ea1a-480b-b31e-114e2d7b1721 2023-09-23T22:15:01.815 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 up in weight 1 up_from 54 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624] [v2:172.21.15.70:6826/265778624,v1:172.21.15.70:6827/265778624] exists,up e9057258-2f16-4805-a17d-b672eacc4cb0 2023-09-23T22:15:01.815 INFO:teuthology.orchestra.run.smithi026.stdout:pg_upmap_items 4.2 [1,2] 2023-09-23T22:15:01.815 INFO:teuthology.orchestra.run.smithi026.stdout:pg_upmap_items 4.4 [1,5] 2023-09-23T22:15:01.815 INFO:teuthology.orchestra.run.smithi026.stdout:pg_upmap_items 4.1d [1,2] 2023-09-23T22:15:01.815 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/575035623 expires 2023-09-24T22:08:38.898443+0000 2023-09-23T22:15:01.815 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/472809172 expires 2023-09-24T22:08:38.898443+0000 2023-09-23T22:15:01.815 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/3973672550 expires 2023-09-24T21:58:49.560739+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3419561437 expires 2023-09-24T22:03:52.071180+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/771693589 expires 2023-09-24T22:07:17.644537+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:6835/1647959066 expires 2023-09-24T22:07:55.974675+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/2583870620 expires 2023-09-24T22:08:38.898443+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/1692711239 expires 2023-09-24T22:07:17.644537+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/202130526 expires 2023-09-24T22:07:55.974675+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/3973672550 expires 2023-09-24T21:58:49.560739+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3568300881 expires 2023-09-24T21:58:49.560739+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3618866901 expires 2023-09-24T22:07:17.644537+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/3676980533 expires 2023-09-24T22:07:55.974675+0000 2023-09-23T22:15:01.816 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/1888936544 expires 2023-09-24T22:07:55.974675+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/574101533 expires 2023-09-24T22:08:38.898443+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/4237902526 expires 2023-09-24T22:08:38.898443+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:6834/1647959066 expires 2023-09-24T22:07:55.974675+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/985150005 expires 2023-09-24T21:59:16.628901+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/250294092 expires 2023-09-24T21:58:49.560739+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3643922551 expires 2023-09-24T21:59:16.628901+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/562032249 expires 2023-09-24T22:07:55.974675+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/1211771677 expires 2023-09-24T22:08:38.898443+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/3849079523 expires 2023-09-24T22:03:52.071180+0000 2023-09-23T22:15:01.817 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3996051568 expires 2023-09-24T22:07:17.644537+0000 2023-09-23T22:15:01.818 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3678729612 expires 2023-09-24T21:59:16.628901+0000 2023-09-23T22:15:01.818 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/1887156644 expires 2023-09-24T22:03:52.071180+0000 2023-09-23T22:15:01.818 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3021908149 expires 2023-09-24T21:59:16.628901+0000 2023-09-23T22:15:01.818 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/577898280 expires 2023-09-24T22:03:52.071180+0000 2023-09-23T22:15:01.818 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/2951021001 expires 2023-09-24T22:08:38.898443+0000 2023-09-23T22:15:01.818 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/3435183798 expires 2023-09-24T22:07:55.974675+0000 2023-09-23T22:15:01.818 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/3350176691 expires 2023-09-24T22:03:52.071180+0000 2023-09-23T22:15:01.818 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/3849079523 expires 2023-09-24T22:03:52.071180+0000 2023-09-23T22:15:01.818 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/985150005 expires 2023-09-24T21:59:16.628901+0000 2023-09-23T22:15:01.819 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6800/3418308803 expires 2023-09-24T22:07:17.644537+0000 2023-09-23T22:15:01.819 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/3418308803 expires 2023-09-24T22:07:17.644537+0000 2023-09-23T22:15:01.819 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.70:0/1453808871 expires 2023-09-24T22:07:55.974675+0000 2023-09-23T22:15:01.819 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:0/2705141594 expires 2023-09-24T21:58:49.560739+0000 2023-09-23T22:15:01.819 INFO:teuthology.orchestra.run.smithi026.stdout:blocklist 172.21.15.26:6801/2951021001 expires 2023-09-24T22:08:38.898443+0000 2023-09-23T22:15:02.411 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 93 up_thru 108 down_at 90 last_clean_interval [8,89) [v2:172.21.15.26:6802/507489654,v1:172.21.15.26:6803/507489654] [v2:172.21.15.26:6804/507489654,v1:172.21.15.26:6805/507489654] exists,up fdf0a195-9329-458c-b103-38db9a4408a8', 'osd.1 up in weight 1 up_from 98 up_thru 108 down_at 95 last_clean_interval [13,94) [v2:172.21.15.26:6810/693231792,v1:172.21.15.26:6811/693231792] [v2:172.21.15.26:6812/693231792,v1:172.21.15.26:6813/693231792] exists,up 9c0507d8-f63e-47b2-841d-394196258d6b', 'osd.2 up in weight 1 up_from 103 up_thru 108 down_at 100 last_clean_interval [18,99) [v2:172.21.15.26:6818/1775189869,v1:172.21.15.26:6819/1775189869] [v2:172.21.15.26:6820/1775189869,v1:172.21.15.26:6821/1775189869] exists,up 4b241a88-8860-4616-8af5-383823ce290f', 'osd.3 up in weight 1 up_from 108 up_thru 108 down_at 105 last_clean_interval [25,104) [v2:172.21.15.26:6826/1956095469,v1:172.21.15.26:6827/1956095469] [v2:172.21.15.26:6828/1956095469,v1:172.21.15.26:6829/1956095469] exists,up 8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea', 'osd.4 up in weight 1 up_from 30 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6800/1461751884,v1:172.21.15.70:6801/1461751884] [v2:172.21.15.70:6802/1461751884,v1:172.21.15.70:6803/1461751884] exists,up 6d4a4d17-075d-4538-9109-7a1dde1b422d', 'osd.5 up in weight 1 up_from 38 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6808/4236773994,v1:172.21.15.70:6809/4236773994] [v2:172.21.15.70:6810/4236773994,v1:172.21.15.70:6811/4236773994] exists,up b25ee343-9def-42dc-8b45-a3c85bf14805', 'osd.6 up in weight 1 up_from 46 up_thru 105 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6816/572290869,v1:172.21.15.70:6817/572290869] [v2:172.21.15.70:6818/572290869,v1:172.21.15.70:6819/572290869] exists,up f7ecfbb9-ea1a-480b-b31e-114e2d7b1721', 'osd.7 up in weight 1 up_from 54 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.70:6824/265778624,v1:172.21.15.70:6825/265778624] [v2:172.21.15.70:6826/265778624,v1:172.21.15.70:6827/265778624] exists,up e9057258-2f16-4805-a17d-b672eacc4cb0'] 2023-09-23T22:15:02.411 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T22:15:02.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:02 smithi026 ceph-mon[215775]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:02.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4077889324' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:15:02.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:02 smithi026 ceph-mon[223014]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:02.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4077889324' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:15:02.776 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:03.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:02 smithi070 ceph-mon[192652]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:03.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4077889324' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:15:04.224 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:15:04.224 INFO:teuthology.orchestra.run.smithi026.stdout: "name": "mon.a", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "cluster": "ceph", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "admin_socket_mode": "", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_client_required": "cephx, none", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_cluster_required": "cephx", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_debug": "true", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_service_required": "cephx", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T22:15:04.225 INFO:teuthology.orchestra.run.smithi026.stdout: "auth_supported": "", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_aio": "true", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_aio_reap_max": "16", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_async_discard": "false", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_block_size": "4096", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_debug_aio": "false", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_enable_discard": "false", 2023-09-23T22:15:04.226 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_flock_retry": "3", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_inject_crash": "0", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_ioring": "false", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_ioring_hipri": "false", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bdev_type": "", 2023-09-23T22:15:04.227 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_allocator": "hybrid", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_buffered_io": "true", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T22:15:04.228 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_replay_recovery": "false", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluefs_sync_write": "false", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T22:15:04.229 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_allocator": "hybrid", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_create": "true", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_db_create": "false", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_db_path": "", 2023-09-23T22:15:04.230 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_db_size": "0", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_path": "", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_size": "107374182400", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_wal_create": "false", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_wal_path": "", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bluefs": "true", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T22:15:04.231 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_autotune": "true", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T22:15:04.232 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_size": "0", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cache_type": "2q", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_clone_cow": "true", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T22:15:04.233 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_mode": "none", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_freelist": "false", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T22:15:04.234 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T22:15:04.235 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fail_eio": "false", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T22:15:04.236 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T22:15:04.237 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T22:15:04.238 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_blob_size": "0", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T22:15:04.239 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_spdk_mem": "512", 2023-09-23T22:15:04.240 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_tracing": "false", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T22:15:04.241 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T22:15:04.242 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_cluster_require_version": "2", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_require_signatures": "false", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_require_version": "2", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_service_require_signatures": "false", 2023-09-23T22:15:04.243 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_service_require_version": "2", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "cephx_sign_messages": "true", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "chdir": "", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_acl_type": "", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_asio_thread_count": "2", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_cache_mid": "0.750000", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_cache_size": "16384", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_caps_release_delay": "5", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_check_pool_perm": "true", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_debug_force_sync_read": "false", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_debug_getattr_caps": "false", 2023-09-23T22:15:04.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_die_on_failed_remount": "false", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_dirsize_rbytes": "true", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_force_lazyio": "false", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_fs": "", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_inject_release_failure": "false", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_max_inline_size": "4096", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mds_namespace": "", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_metadata": "", 2023-09-23T22:15:04.245 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mount_gid": "-1", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mount_timeout": "300", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mount_uid": "-1", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_mountpoint": "/", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_notify_timeout": "10", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc": "true", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_max_objects": "1000", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_size": "209715200", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T22:15:04.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_permissions": "true", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_quota": "true", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_quota_df": "true", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_readahead_max_bytes": "0", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_readahead_max_periods": "4", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_readahead_min": "131072", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_reconnect_stale": "false", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_shutdown_timeout": "30", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_snapdir": ".snap", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_tick_interval": "1", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_trace": "", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_use_faked_inos": "false", 2023-09-23T22:15:04.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_use_random_mds": "false", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_graylog": "false", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_graylog_port": "12201", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_monitors": "default=true", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_syslog": "false", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "clog_to_syslog_level": "info", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "cluster_addr": "-", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "cluster_network": "", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "cluster_network_interface": "", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "compressor_zlib_isal": "false", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "compressor_zlib_level": "5", 2023-09-23T22:15:04.248 INFO:teuthology.orchestra.run.smithi026.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "compressor_zstd_level": "1", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "crimson_seastar_smp": "1", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "crush_location": "", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "crush_location_hook": "", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "crush_location_hook_timeout": "10", 2023-09-23T22:15:04.249 INFO:teuthology.orchestra.run.smithi026.stdout: "daemonize": "false", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "daos_pool": "tank", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_alienstore": "0/5", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_asok": "1/5", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_asok_assert_abort": "false", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_auth": "1/5", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_bdev": "1/3", 2023-09-23T22:15:04.250 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_bluefs": "1/5", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_bluestore": "1/5", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_buffer": "0/1", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_cephsqlite": "0/5", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_client": "0/5", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_compressor": "1/5", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_context": "0/1", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_crush": "1/1", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_crypto": "1/5", 2023-09-23T22:15:04.251 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_cyanstore": "0/5", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_dpdk": "1/5", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_eventtrace": "1/5", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_filer": "0/1", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_filestore": "1/3", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_finisher": "1/1", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_fuse": "1/5", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T22:15:04.252 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_javaclient": "1/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_journal": "1/3", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_journaler": "0/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_kstore": "1/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_leveldb": "4/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_lockdep": "0/1", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mclock": "1/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds": "1/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_balancer": "1/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_locker": "1/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_log": "1/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mds_migrator": "1/5", 2023-09-23T22:15:04.253 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_memstore": "1/5", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mgr": "2/5", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mgrc": "1/5", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_mon": "20/20", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_monc": "0/10", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_ms": "1/1", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_none": "0/5", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_objclass": "0/5", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_objectcacher": "0/5", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_objecter": "0/1", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_optracker": "0/5", 2023-09-23T22:15:04.254 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_osd": "1/5", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_paxos": "20/20", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_perfcounter": "1/5", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_prioritycache": "1/5", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rados": "0/5", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rbd": "0/5", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rbd_replay": "0/5", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_refs": "0/0", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_reserver": "1/1", 2023-09-23T22:15:04.255 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw": "1/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_access": "1/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_flight": "1/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rgw_sync": "1/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_rocksdb": "4/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore": "0/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_backref": "0/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_cache": "0/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_device": "0/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_epm": "0/5", 2023-09-23T22:15:04.256 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_journal": "0/5", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_lba": "0/5", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_odata": "0/5", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_omap": "0/5", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_onode": "0/5", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_t": "0/5", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_seastore_tm": "0/5", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_striper": "0/1", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_test": "0/5", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_throttle": "1/1", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_timer": "0/1", 2023-09-23T22:15:04.257 INFO:teuthology.orchestra.run.smithi026.stdout: "debug_tp": "0/5", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "device_failure_prediction_mode": "none", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "err_to_graylog": "false", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "err_to_journald": "false", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "err_to_stderr": "true", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "err_to_syslog": "false", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event_tracing": "false", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T22:15:04.258 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_http_port": "9926", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_prio_limit": "5", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_sort_metrics": "true", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "exporter_stats_period": "5", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "fake_statfs_for_testing": "0", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "fatal_signal_handlers": "true", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "filer_max_purge_ops": "10", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "filer_max_truncate_ops": "128", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_blackhole": "false", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_btrfs_snap": "true", 2023-09-23T22:15:04.259 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_caller_concurrency": "10", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_debug_omap_check": "false", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_debug_verify_split": "false", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_dump_file": "", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fadvise": "true", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fail_eio": "true", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T22:15:04.260 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fd_cache_size": "128", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fiemap": "false", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_inject_stall": "0", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_journal_parallel": "false", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_journal_trailing": "false", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_journal_writeahead": "false", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_kill_at": "0", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T22:15:04.261 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T22:15:04.262 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_merge_threshold": "-10", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_odsync_write": "false", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_omap_backend_path": "", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_op_threads": "2", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_punch_hole": "false", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T22:15:04.263 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_queue_max_ops": "50", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_seek_data_hole": "false", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_sloppy_crc": "false", 2023-09-23T22:15:04.264 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_splice": "false", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_split_multiple": "2", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_split_rand_factor": "20", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_update_to": "1000", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T22:15:04.265 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_xfs_extsize": "false", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "filestore_zfs_snap": "false", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "fio_dir": "/tmp/fio", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "fsid": "2a3049e2-5a5c-11ee-8db4-212e2dc638e7", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_allow_other": "true", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T22:15:04.266 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_big_writes": "true", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_debug": "false", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_default_permissions": "false", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_disable_pagecache": "false", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_max_write": "0", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_multithreaded": "true", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_require_active_mds": "true", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_set_user_groups": "true", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_splice_move": "true", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_splice_read": "true", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_splice_write": "true", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T22:15:04.267 INFO:teuthology.orchestra.run.smithi026.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "gss_target_name": "ceph", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "heartbeat_file": "", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "heartbeat_inject_failure": "0", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "heartbeat_interval": "5", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "host": "smithi026", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T22:15:04.268 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "inject_early_sigterm": "false", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "jaeger_agent_port": "6799", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "jaeger_tracing_enable": "false", 2023-09-23T22:15:04.269 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_aio": "true", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_align_min_size": "65536", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_block_align": "true", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_block_size": "4096", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_dio": "true", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_discard": "false", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_force_aio": "false", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_ignore_corruption": "false", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_max_write_entries": "100", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_replay_from": "0", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T22:15:04.270 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_write_header_frequency": "0", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "journal_zero_on_create": "false", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "journaler_prefetch_periods": "10", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "journaler_prezero_periods": "5", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "journaler_write_head_interval": "15", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "key": "", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "keyfile": "", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_backend": "rocksdb", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T22:15:04.271 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_max_bytes": "67108864", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_max_ops": "512", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_onode_map_size": "1024", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "kstore_sync_transaction": "false", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_block_size": "65536", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_bloom_size": "0", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_cache_size": "536870912", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T22:15:04.272 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_compression": "false", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_log": "", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_max_open_files": "0", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_paranoid": "false", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "librados_thread_count": "2", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "lockdep": "false", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "lockdep_force_backtrace": "false", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "log_coarse_timestamps": "true", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "log_flush_on_exit": "false", 2023-09-23T22:15:04.273 INFO:teuthology.orchestra.run.smithi026.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_graylog_port": "12201", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_max_new": "1000", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_max_recent": "10000", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_stderr_prefix": "", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_file": "true", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_graylog": "false", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_journald": "false", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_stderr": "false", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "log_to_syslog": "false", 2023-09-23T22:15:04.274 INFO:teuthology.orchestra.run.smithi026.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_action_on_write_error": "1", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_alternate_name_max": "8192", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_asio_thread_count": "2", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_export_pin": "true", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_interval": "10", 2023-09-23T22:15:04.275 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_max": "-1", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_max_until": "-1", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_merge_size": "50", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_mode": "0", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T22:15:04.276 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_split_bits": "3", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_split_size": "10000", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_mid": "0.700000", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T22:15:04.277 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_trim_interval": "1", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T22:15:04.278 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_debug_auth_pins": "false", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_debug_frag": "false", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_debug_scatterstat": "false", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_debug_subtrees": "false", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_default_dir_hash": "2", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_defer_session_stale": "true", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T22:15:04.279 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dir_max_entries": "0", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dir_prefetch": "true", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_early_reply": "true", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_enable_op_tracker": "true", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_enforce_unique_name": "true", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T22:15:04.280 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T22:15:04.281 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_health_dummy": "false", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_join_fs": "", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_journal_format": "1", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_create_at": "0", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_export_at": "0", 2023-09-23T22:15:04.282 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_import_at": "0", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_journal_at": "0", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_link_at": "0", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_openc_at": "0", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_rename_at": "0", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_max_events": "-1", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_max_segments": "128", 2023-09-23T22:15:04.283 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_pause": "false", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_segment_size": "0", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_completed_requests": "100000", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_export_size": "20971520", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_file_recover": "32", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_purge_files": "64", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_purge_ops": "8192", 2023-09-23T22:15:04.284 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_metrics_update_interval": "2", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_min_caps_per_client": "100", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_numa_node": "-1", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_op_history_duration": "600", 2023-09-23T22:15:04.285 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_op_history_size": "20", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_op_log_threshold": "5", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_ping_grace": "15", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_ping_interval": "5", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_max_caps": "30000", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T22:15:04.286 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_replay_interval": "1.000000", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_root_ino_gid": "0", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_root_ino_uid": "0", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T22:15:04.287 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_shutdown_check": "0", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_skip_ino": "0", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_snap_min_uid": "0", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_snap_rstat": "false", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_symlink_recovery": "true", 2023-09-23T22:15:04.288 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_thrash_exports": "0", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_thrash_fragments": "0", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_tick_interval": "5.000000", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_valgrind_exit": "false", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_verify_backtrace": "1", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_verify_scatter": "false", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "mds_wipe_sessions": "false", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "mempool_debug": "false", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T22:15:04.289 INFO:teuthology.orchestra.run.smithi026.stdout: "memstore_page_set": "false", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "memstore_page_size": "65536", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_client_bytes": "134217728", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_client_messages": "512", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_disabled_modules": "", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T22:15:04.290 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_mds_messages": "128", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_mon_messages": "128", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_osd_messages": "8192", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_pool": "true", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_standby_modules": "true", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_stats_period": "5", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_stats_threshold": "5", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_tick_period": "2", 2023-09-23T22:15:04.291 INFO:teuthology.orchestra.run.smithi026.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_allow_pool_delete": "true", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_bytes": "104857600", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T22:15:04.292 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T22:15:04.293 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_journald": "true", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_compact_on_start": "false", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_compact_on_trim": "true", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T22:15:04.294 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_cpu_threads": "4", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_data_avail_crit": "5", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_data_avail_warn": "5", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_dump_json": "false", 2023-09-23T22:15:04.295 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_extra_checks": "false", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T22:15:04.296 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_election_default_strategy": "3", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_election_timeout": "5.000000", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_enable_op_tracker": "true", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_fake_pool_delete": "false", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_force_quorum_join": "false", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_log_update_period": "5", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_max_detail": "50", 2023-09-23T22:15:04.297 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_to_clog": "true", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_host": "[v2:172.21.15.26:3300/0,v1:172.21.15.26:6789/0] [v2:172.21.15.70:3300/0,v1:172.21.15.70:6789/0] [v2:172.21.15.26:3301/0,v1:172.21.15.26:6790/0]", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_host_override": "", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_initial_members": "", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T22:15:04.298 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_lease": "5.000000", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_log_full_interval": "50", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_log_max": "10000", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_log_max_summary": "50", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_log_epochs": "500", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T22:15:04.299 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_osd": "10000", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_memory_autotune": "true", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_memory_target": "2147483648", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T22:15:04.300 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_digest_period": "5", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_complaint_time": "30", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_history_duration": "600", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_history_size": "20", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_op_log_threshold": "5", 2023-09-23T22:15:04.301 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_cache_size": "500", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T22:15:04.302 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T22:15:04.303 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T22:15:04.303 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T22:15:04.303 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T22:15:04.303 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T22:15:04.303 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T22:15:04.303 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T22:15:04.303 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T22:15:04.303 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T22:15:04.303 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_report_timeout": "900", 2023-09-23T22:15:04.304 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T22:15:04.305 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_reweight_max_osds": "4", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T22:15:04.306 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_interval": "86400", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_max_keys": "100", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_scrub_timeout": "300", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_session_timeout": "300", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_smart_report_timeout": "5", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stretch_pool_size": "4", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_debug": "false", 2023-09-23T22:15:04.307 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_tick_interval": "5", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T22:15:04.308 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_older_version": "true", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T22:15:04.309 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "monmap": "", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T22:15:04.310 INFO:teuthology.orchestra.run.smithi026.stdout: "motr_tracing_enabled": "false", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_op_threads": "3", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_cm": "false", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_device_name": "", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T22:15:04.311 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T22:15:04.312 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T22:15:04.312 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T22:15:04.312 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_sl": "3", 2023-09-23T22:15:04.312 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T22:15:04.312 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_rdma_type": "ib", 2023-09-23T22:15:04.312 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_async_reap_threshold": "5", 2023-09-23T22:15:04.312 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_before_connect": "false", 2023-09-23T22:15:04.312 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_ipv4": "true", 2023-09-23T22:15:04.312 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_ipv6": "false", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_msgr1": "true", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_msgr2": "true", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_port_max": "7568", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_port_min": "6800", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_retry_count": "3", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_bind_retry_delay": "5", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_client": "false", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_mds": "false", 2023-09-23T22:15:04.313 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_mgr": "false", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_mon": "false", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_blackhole_osd": "false", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_client_mode": "crc secure", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_cluster_type": "", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_compress_secure": "false", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_crc_data": "true", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_crc_header": "true", 2023-09-23T22:15:04.314 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_bug": "true", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_old_message": "true", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T22:15:04.315 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_hugepages": "", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_lro": "true", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_pmd": "", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_port_id": "0", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_dump_on_send": "false", 2023-09-23T22:15:04.316 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_delay_type": "", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_network_congestion": "0", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_inject_socket_failures": "0", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_max_accept_failures": "4", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_max_backoff": "15.000000", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T22:15:04.317 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_osd_compress_mode": "none", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_pq_min_cost": "65536", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_public_type": "", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_service_mode": "crc secure", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_tcp_nodelay": "true", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T22:15:04.318 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "ms_type": "async+posix", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "no_config_file": "false", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_inflight_ops": "1024", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "objecter_timeout": "10.000000", 2023-09-23T22:15:04.319 INFO:teuthology.orchestra.run.smithi026.stdout: "objectstore_blackhole": "false", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "openssl_engine_opts": "", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_max_ops": "4", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_agent_slop": "0.020000", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T22:15:04.320 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_asio_thread_count": "2", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backfill_scan_max": "512", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backfill_scan_min": "64", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backoff_on_peering": "false", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_beacon_report_interval": "300", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_bench_duration": "30", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T22:15:04.321 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_blkin_trace_all": "false", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T22:15:04.322 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_class_update_on_start": "true", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_client_message_cap": "256", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_client_op_priority": "63", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_client_watch_timeout": "30", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_command_max_records": "256", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_compact_on_start": "false", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_crush_update_on_start": "true", 2023-09-23T22:15:04.323 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T22:15:04.324 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_op_order": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_shutdown": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T22:15:04.325 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_default_notify_timeout": "30", 2023-09-23T22:15:04.326 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_enable_op_tracker": "true", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_info": "true", 2023-09-23T22:15:04.327 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_shutdown": "true", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_function_tracing": "false", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_grace": "20", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_interval": "6", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T22:15:04.328 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_stale": "600", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_journal_size": "5120", 2023-09-23T22:15:04.329 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_kill_backfill_at": "0", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_cache_size": "50", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_dedup": "true", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_message_max": "40", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_attr_name_len": "100", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_attr_size": "0", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_backfills": "1", 2023-09-23T22:15:04.330 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_markdown_count": "5", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_markdown_period": "600", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_object_name_len": "2048", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_object_size": "134217728", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_pgls": "1024", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_push_cost": "8388608", 2023-09-23T22:15:04.331 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_push_objects": "10", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_scrubs": "1", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_max_write_size": "90", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T22:15:04.332 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_profile": "balanced", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T22:15:04.333 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_base": "805306368", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_target": "4294967296", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_target_autotune": "false", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_min_recovery_priority": "0", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T22:15:04.334 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_report_interval": "5", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_num_cache_shards": "32", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_numa_node": "-1", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_objecter_finishers": "1", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_objectstore": "bluestore", 2023-09-23T22:15:04.335 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_objectstore_fuse": "false", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_objectstore_tracing": "false", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_history_duration": "600", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_history_size": "20", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_log_threshold": "5", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_shards": "0", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T22:15:04.336 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_op_thread_timeout": "15", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_open_classes_on_start": "true", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_os_flags": "0", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_peering_op_priority": "255", 2023-09-23T22:15:04.337 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_delete_priority": "5", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T22:15:04.338 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_crimson": "false", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T22:15:04.339 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_flags": "0", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_min_size": "0", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_size": "3", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_default_type": "replicated", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T22:15:04.340 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_cost": "20971520", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_active": "0", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T22:15:04.341 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_op_priority": "3", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_priority": "5", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_repair_during_recovery": "false", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T22:15:04.342 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T22:15:04.343 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T22:15:04.343 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T22:15:04.343 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T22:15:04.343 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T22:15:04.343 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T22:15:04.343 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T22:15:04.343 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_cost": "52428800", 2023-09-23T22:15:04.343 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T22:15:04.343 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_end_hour": "0", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_priority": "5", 2023-09-23T22:15:04.344 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_skip_data_digest": "false", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_smart_report_timeout": "5", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_priority": "5", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T22:15:04.345 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_target_transaction_size": "30", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T22:15:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_tracing": "false", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_use_stale_snap": "false", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_kill_at": "0", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_max_join_drift": "10", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_min": "500", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_min_wait": "0.050000", 2023-09-23T22:15:04.347 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_service_trim_max": "500", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_service_trim_min": "250", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_stash_full_interval": "25", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_trim_max": "500", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "paxos_trim_min": "250", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "perf": "true", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "pid_file": "", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "public_addr": "-", 2023-09-23T22:15:04.348 INFO:teuthology.orchestra.run.smithi026.stdout: "public_addrv": "", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "public_bind_addr": "-", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "public_network": "172.21.0.0/20", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "public_network_interface": "", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "qat_compressor_enabled": "false", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "rados_mon_op_timeout": "0", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "rados_osd_op_timeout": "0", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "rados_tracing": "false", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_atime_update_interval": "60", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T22:15:04.349 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache": "true", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_policy": "writearound", 2023-09-23T22:15:04.350 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_size": "33554432", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_compression_hint": "none", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_clone_format": "auto", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_data_pool": "", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_features": "61", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_format": "2", 2023-09-23T22:15:04.351 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_map_options": "", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_order": "22", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_pool": "rbd", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_stripe_count": "0", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_io_scheduler": "simple", 2023-09-23T22:15:04.352 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_order": "24", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_pool": "", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_journal_splay_width": "4", 2023-09-23T22:15:04.353 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T22:15:04.354 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T22:15:04.355 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T22:15:04.356 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_op_threads": "1", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_plugins": "", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T22:15:04.357 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T22:15:04.358 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T22:15:04.359 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_tracing": "false", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_validate_names": "true", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "rbd_validate_pool": "true", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "restapi_base_url": "", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "restapi_log_level": "", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_admin_entry": "admin", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_backend_store": "rados", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_barbican_url": "", 2023-09-23T22:15:04.360 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_beast_enable_async": "true", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cache_enabled": "true", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_config_store": "rados", 2023-09-23T22:15:04.361 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_content_length_compat": "false", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_cross_domain_policy": "", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T22:15:04.362 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T22:15:04.363 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T22:15:04.364 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T22:15:04.365 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T22:15:04.366 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_log_window": "30", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T22:15:04.367 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T22:15:04.368 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dns_name": "", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dns_s3website_name": "", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_ops_log": "false", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_static_website": "false", 2023-09-23T22:15:04.369 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enable_usage_log": "false", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_expose_bucket": "false", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_extended_http_attrs": "", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_filter": "none", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_objs": "32", 2023-09-23T22:15:04.370 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_init_timeout": "300", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T22:15:04.371 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_password": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_project": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_token": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_admin_user": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_api_version": "2", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T22:15:04.372 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_url": "", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T22:15:04.373 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_max_objs": "32", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_max_worker": "3", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T22:15:04.374 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_log_http_headers": "", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_attr_size": "0", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T22:15:04.375 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_listing_results": "1000", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_notify_retries": "10", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T22:15:04.376 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T22:15:04.377 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_num_control_oids": "8", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_numa_node": "-1", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T22:15:04.378 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_op_tracing": "false", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_opa_token": "", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_opa_url": "", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T22:15:04.379 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ops_log_rados": "false", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_print_continue": "true", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T22:15:04.380 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_rados_tracing": "false", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_realm": "", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_realm_id": "", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_region": "", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T22:15:04.381 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_request_uri": "", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_resolve_cname": "false", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_run_sync_thread": "true", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T22:15:04.382 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_script_uri": "", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_service_provider_name": "", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_client_id": "", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_client_secret": "", 2023-09-23T22:15:04.383 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_entry": "sts", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_key": "sts", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_auth_url": "", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_custom_header": "", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_need_stats": "true", 2023-09-23T22:15:04.384 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_tenant_name": "", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_url": "", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_lease_period": "120", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T22:15:04.385 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_thread_pool_size": "512", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_comment": "", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_createby": "", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_encoding": "", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_flag": "false", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_origin": "", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T22:15:04.386 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_torrent_tracker": "", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_usage_max_shards": "32", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_use_opa_authz": "false", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T22:15:04.387 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_user_unique_email": "true", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_verify_ssl": "true", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zone": "", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zone_id": "", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zonegroup": "", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zonegroup_id": "", 2023-09-23T22:15:04.388 INFO:teuthology.orchestra.run.smithi026.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_block_size": "4096", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T22:15:04.389 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_partition_filters": "false", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_perf": "false", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T22:15:04.390 INFO:teuthology.orchestra.run.smithi026.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "run_dir": "/var/run/ceph", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_block_create": "true", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_device_size": "53687091200", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T22:15:04.391 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_main_device_type": "SSD", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "seastore_segment_size": "67108864", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "set_keepcaps": "false", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "setgroup": "ceph", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "setuser": "ceph", 2023-09-23T22:15:04.392 INFO:teuthology.orchestra.run.smithi026.stdout: "setuser_match_path": "", 2023-09-23T22:15:04.393 INFO:teuthology.orchestra.run.smithi026.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T22:15:04.393 INFO:teuthology.orchestra.run.smithi026.stdout: "thp": "false", 2023-09-23T22:15:04.393 INFO:teuthology.orchestra.run.smithi026.stdout: "threadpool_default_timeout": "60", 2023-09-23T22:15:04.393 INFO:teuthology.orchestra.run.smithi026.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T22:15:04.393 INFO:teuthology.orchestra.run.smithi026.stdout: "throttler_perf_counter": "true" 2023-09-23T22:15:04.393 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:15:04.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:04 smithi026 ceph-mon[215775]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:04.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:04 smithi026 ceph-mon[223014]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:04.846 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T22:15:05.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:04 smithi070 ceph-mon[192652]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:05.207 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:06.598 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a: {} 2023-09-23T22:15:06.609 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b: {} 2023-09-23T22:15:06.618 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c: {} 2023-09-23T22:15:06.619 INFO:teuthology.orchestra.run.smithi026.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T22:15:06.620 INFO:teuthology.orchestra.run.smithi026.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T22:15:06.620 INFO:teuthology.orchestra.run.smithi026.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T22:15:06.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:06 smithi026 ceph-mon[223014]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:06.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1229033338' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:15:06.650 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:06 smithi026 ceph-mon[215775]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:06.650 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1229033338' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:15:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:06 smithi070 ceph-mon[192652]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1229033338' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:15:07.245 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T22:15:07.246 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T22:15:07.246 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:15:07.248 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:15:07.248 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T22:15:07.248 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T22:15:07.248 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T22:15:07.249 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:15:07.249 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:15:07.249 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:15:07.249 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.000000 2023-09-23T22:15:07.249 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 0 2023-09-23T22:15:07.249 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T22:15:07.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:07.250 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T22:15:07.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:15:07.250 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T22:15:07.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:15:07.251 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T22:15:07.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:15:07.293 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:07.382 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:07.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:07.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:07.489 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:07.538 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:07.728 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:07.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:07.772 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:07.801 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:07.831 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:07.881 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:07.931 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:07.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:07.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:07.938 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:08.034 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:08.038 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:08.067 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:08.111 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:08.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:08.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:08.218 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:08.228 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:08.245 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:08.286 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:08.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:08.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:08.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:08.423 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:08.496 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:08.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:08.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:08.599 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:08.630 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:08.975 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:08 smithi026 ceph-mon[215775]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:08.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:08 smithi026 ceph-mon[223014]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:08.983 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:08.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:08.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:09.086 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:09.115 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:08 smithi070 ceph-mon[192652]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:09.941 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:09.941 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:15:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:15:09] "GET /metrics HTTP/1.1" 200 33945 "" "Prometheus/2.43.0" 2023-09-23T22:15:09.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:10.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:10.415 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:10.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:10.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:10.519 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:10.550 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:10.902 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:10.904 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:10 smithi026 ceph-mon[215775]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:10.904 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:10 smithi026 ceph-mon[223014]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:10.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:10.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:11.006 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:11.040 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:11.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:10 smithi070 ceph-mon[192652]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:11.763 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:15:11.763 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:15:11.764 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:15:11.764 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:15:11.764 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:15:11.764 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:15:11.764 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:15:11.764 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:15:11.764 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:15:11.765 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:15:11.765 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:15:11.765 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:15:11.765 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:15:11.765 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:15:11.765 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:15:11.765 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:15:11.929 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/627270649' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:11.930 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/627270649' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:11.943 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:11.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:11.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:12.047 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:12.076 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/627270649' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:12.080 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:12.129 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:12.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:12.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:12.236 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:12.263 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:12.306 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:12.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:12.413 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:12.460 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:12.613 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:12.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:12.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:12.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:12 smithi026 ceph-mon[223014]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:12.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3737820216' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:15:12.661 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:12 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:15:12.661 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:12 smithi026 ceph-mon[215775]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:12.661 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3737820216' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:15:12.661 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:12 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:15:12.672 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:15:12.718 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:12.748 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:12 smithi070 ceph-mon[192652]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3737820216' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:15:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:12 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:15:13.548 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:13.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:13.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:13.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:13.682 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:13.733 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:13.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:13.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:13.819 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.7 primary-affinity to 0 (802) 2023-09-23T22:15:13.823 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_0' created 2023-09-23T22:15:13.838 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:13.866 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:14.026 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:15:14.026 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:15:14.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:15:14.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[223014]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:15:14.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[223014]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T22:15:14.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1109215261' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T22:15:14.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T22:15:14.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[223014]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[223014]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[223014]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:15:14.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[215775]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:15:14.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[215775]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T22:15:14.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1109215261' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T22:15:14.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T22:15:14.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[215775]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[215775]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:13 smithi026 ceph-mon[215775]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:13 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:15:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:13 smithi070 ceph-mon[192652]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:13 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:15:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:13 smithi070 ceph-mon[192652]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T22:15:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1109215261' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T22:15:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:13 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]: dispatch 2023-09-23T22:15:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:13 smithi070 ceph-mon[192652]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:13 smithi070 ceph-mon[192652]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:13 smithi070 ceph-mon[192652]: failed to encode map e110 with expected crc 2023-09-23T22:15:14.520 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:14.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:14.524 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:14.625 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:14.654 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:15.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:15.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2003577377' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:15:15.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-09-23T22:15:15.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2003577377' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T22:15:15.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T22:15:15.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[215775]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2003577377' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2003577377' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T22:15:15.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:14 smithi026 ceph-mon[223014]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:15.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2003577377' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T22:15:15.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.0}]': finished 2023-09-23T22:15:15.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2003577377' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T22:15:15.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T22:15:15.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:14 smithi070 ceph-mon[192652]: failed to encode map e111 with expected crc 2023-09-23T22:15:15.204 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:15.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:15.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:15.308 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:15.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T22:15:15.339 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:15.386 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:15.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:15.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:15.491 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:15.518 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:15.620 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:15.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:15.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:15.670 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:15:15.703 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:15.769 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:15.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:15:15.800 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:15.852 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:15.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:15.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:15.920 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[223014]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[223014]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[223014]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[223014]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[223014]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[223014]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[223014]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[223014]: pgmap v224: 89 pgs: 10 unknown, 79 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[215775]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[215775]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.921 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[215775]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T22:15:15.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[215775]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[215775]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[215775]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[215775]: failed to encode map e112 with expected crc 2023-09-23T22:15:15.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:15 smithi026 ceph-mon[215775]: pgmap v224: 89 pgs: 10 unknown, 79 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:15.957 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:15.985 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:16.029 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:16.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:16.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:16.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:15 smithi070 ceph-mon[192652]: failed to encode map e112 with expected crc 2023-09-23T22:15:16.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:15 smithi070 ceph-mon[192652]: failed to encode map e112 with expected crc 2023-09-23T22:15:16.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:15 smithi070 ceph-mon[192652]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T22:15:16.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:15 smithi070 ceph-mon[192652]: failed to encode map e112 with expected crc 2023-09-23T22:15:16.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:15 smithi070 ceph-mon[192652]: failed to encode map e112 with expected crc 2023-09-23T22:15:16.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:15 smithi070 ceph-mon[192652]: failed to encode map e112 with expected crc 2023-09-23T22:15:16.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:15 smithi070 ceph-mon[192652]: failed to encode map e112 with expected crc 2023-09-23T22:15:16.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:15 smithi070 ceph-mon[192652]: pgmap v224: 89 pgs: 10 unknown, 79 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:16.137 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:16.170 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:16.318 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:16.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:16.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:16.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:16.449 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:16.493 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:16.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:16.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:16.597 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:16.627 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:16.674 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:16.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:16.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:16.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:16.798 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:16.806 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:16.849 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:16.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:16.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:16.952 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:16.955 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:15:16.956 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:16.989 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:17.054 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:16 smithi026 ceph-mon[223014]: failed to encode map e113 with expected crc 2023-09-23T22:15:17.054 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:16 smithi026 ceph-mon[223014]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T22:15:17.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:16 smithi026 ceph-mon[215775]: failed to encode map e113 with expected crc 2023-09-23T22:15:17.055 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:16 smithi026 ceph-mon[215775]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T22:15:17.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:16 smithi070 ceph-mon[192652]: failed to encode map e113 with expected crc 2023-09-23T22:15:17.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:16 smithi070 ceph-mon[192652]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T22:15:17.126 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:17.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:17.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:17.229 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:17.257 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:17.463 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:17.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:17.567 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:17.596 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:17.642 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:17.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:17.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:17.747 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:17.777 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:18.233 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:18.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:18.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:18.338 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:18.373 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:18.828 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T22:15:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:18 smithi026 ceph-mon[223014]: pgmap v226: 89 pgs: 2 unknown, 87 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:18 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:15:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3083043228' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:15:18.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:18 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:15:18.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:18 smithi026 ceph-mon[215775]: pgmap v226: 89 pgs: 2 unknown, 87 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:18.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:18 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:15:18.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3083043228' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:15:18.857 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:18 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:15:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:18 smithi070 ceph-mon[192652]: pgmap v226: 89 pgs: 2 unknown, 87 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:18 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:15:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3083043228' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:15:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:18 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:15:18.940 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:18.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:18.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:19.044 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:19.071 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:19.074 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:19.219 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:19.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:19.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:19.323 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:19.351 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:19.396 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:19.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:19.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:19.501 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:19.532 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:19.535 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:15:19.535 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:15:19.826 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:15:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:15:19] "GET /metrics HTTP/1.1" 200 33945 "" "Prometheus/2.43.0" 2023-09-23T22:15:19.991 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:15:19.992 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:15:19.992 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:15:19.992 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 0 2023-09-23T22:15:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:19 smithi070 ceph-mon[192652]: failed to encode map e114 with expected crc 2023-09-23T22:15:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:15:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:19 smithi070 ceph-mon[192652]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T22:15:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:19 smithi070 ceph-mon[192652]: failed to encode map e114 with expected crc 2023-09-23T22:15:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:19 smithi070 ceph-mon[192652]: failed to encode map e114 with expected crc 2023-09-23T22:15:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:19 smithi070 ceph-mon[192652]: pgmap v228: 89 pgs: 89 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:20.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[223014]: failed to encode map e114 with expected crc 2023-09-23T22:15:20.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:15:20.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[223014]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T22:15:20.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[223014]: failed to encode map e114 with expected crc 2023-09-23T22:15:20.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[223014]: failed to encode map e114 with expected crc 2023-09-23T22:15:20.214 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[223014]: pgmap v228: 89 pgs: 89 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:20.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[215775]: failed to encode map e114 with expected crc 2023-09-23T22:15:20.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:15:20.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[215775]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T22:15:20.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[215775]: failed to encode map e114 with expected crc 2023-09-23T22:15:20.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[215775]: failed to encode map e114 with expected crc 2023-09-23T22:15:20.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:19 smithi026 ceph-mon[215775]: pgmap v228: 89 pgs: 89 active+clean; 4.6 MiB data, 161 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:20.301 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:20.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:20.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:20.405 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:20.431 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:20.473 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:20.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:20.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:20.578 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:20.594 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:15:20.599 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:15:20.604 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:20.619 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:15:20.641 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:15:20.652 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:20.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:20.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:20.664 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:15:20.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:15:20.720 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:15:20.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:20.758 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:15:20.778 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:15:20.780 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:15:20.780 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:15:20.780 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:15:20.780 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:15:20.780 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:15:20.780 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:15:20.780 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:15:20.781 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:15:20.786 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/734094406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:21.233 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T22:15:21.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/734094406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:21.236 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:15:21.236 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:15:21.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/734094406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:21.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T22:15:21.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:15:21.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_286012 2023-09-23T22:15:21.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:15:21.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:15:21.852 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:21.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:21.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:21.914 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:21.977 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:22.011 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:22.085 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:22.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:22.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:22.094 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:22.192 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:22.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:22.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:22.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:21 smithi026 ceph-mon[223014]: pgmap v229: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:22.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:22.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:22.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:21 smithi026 ceph-mon[215775]: pgmap v229: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:22.255 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:22.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 3502 3486 217.858 217.875 0.00367819 0.00457181 2023-09-23T22:15:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:22.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:22.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:21 smithi070 ceph-mon[192652]: pgmap v229: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T22:15:22.581 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:22.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:22.584 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:22.685 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:22.740 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:22.935 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:22.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:22.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:23.041 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:23.119 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:23.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 6501 6485 202.635 187.438 0.00316642 0.00492568 2023-09-23T22:15:23.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2637357369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:15:23.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:23 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:15:23.597 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:15:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2637357369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:15:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:23 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:15:23.900 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:15:23.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2637357369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:15:23.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:23 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:15:24.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 9439 9423 196.293 183.625 0.00490827 0.00508803 2023-09-23T22:15:24.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:24 smithi070 ceph-mon[192652]: pgmap v230: 89 pgs: 89 active+clean; 27 MiB data, 245 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s wr, 44 op/s 2023-09-23T22:15:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:24 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:15:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:24 smithi070 ceph-mon[192652]: failed to encode map e115 with expected crc 2023-09-23T22:15:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:24 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:15:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:24 smithi070 ceph-mon[192652]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T22:15:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[215775]: pgmap v230: 89 pgs: 89 active+clean; 27 MiB data, 245 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s wr, 44 op/s 2023-09-23T22:15:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:15:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[215775]: failed to encode map e115 with expected crc 2023-09-23T22:15:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:15:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[215775]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T22:15:24.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[223014]: pgmap v230: 89 pgs: 89 active+clean; 27 MiB data, 245 MiB used, 715 GiB / 715 GiB avail; 2.8 MiB/s wr, 44 op/s 2023-09-23T22:15:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:15:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[223014]: failed to encode map e115 with expected crc 2023-09-23T22:15:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:15:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[223014]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T22:15:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:24.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:24.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:25.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 12870 12854 200.824 214.438 0.0031949 0.00497333 2023-09-23T22:15:25.628 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:25.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:25.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:25.736 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:25.794 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:26.262 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:26.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:26.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:26.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 14737 14721 183.994 116.688 0.00146619 0.00540259 2023-09-23T22:15:26.365 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:26.409 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:26.481 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:26.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:26.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:26.588 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:26.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:26 smithi026 ceph-mon[215775]: pgmap v232: 89 pgs: 89 active+clean; 327 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s 2023-09-23T22:15:26.644 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:26 smithi070 ceph-mon[192652]: pgmap v232: 89 pgs: 89 active+clean; 327 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s 2023-09-23T22:15:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:26 smithi026 ceph-mon[223014]: pgmap v232: 89 pgs: 89 active+clean; 327 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 40 MiB/s wr, 644 op/s 2023-09-23T22:15:27.257 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:27.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:27.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:27.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 16552 16536 172.234 113.438 0.00292622 0.00579892 2023-09-23T22:15:27.362 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:27.388 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:27.461 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:27.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:27.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:27.569 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:27.609 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:27.613 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:27.676 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:27.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:27.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:27.781 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:27.821 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:27.894 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:27.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:27.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:28.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:28.083 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:28.195 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:28.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:28.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 18473 18457 164.779 120.062 0.00774704 0.00606335 2023-09-23T22:15:28.306 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:28.339 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:28.354 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:7768 5.c e115/111 5:3a4db92a:::benchmark_data_smithi026_286012_object7767:head v 115'504, mlcod=115'502)", 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:23.733456+0000", 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 4.6127168479999998, 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018657876, 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.355 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.733456+0000", 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.733456+0000", 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.733456+0000", 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.733478+0000", 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2299000000000001e-05 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.733488+0000", 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0152e-05 2023-09-23T22:15:28.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.733494+0000", 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1199999999999999e-06 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.733545+0000", 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0544999999999998e-05 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.733565+0000", 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0491000000000001e-05 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.734056+0000", 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049060399999999995 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.752114+0000", 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018058353999999999 2023-09-23T22:15:28.359 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:7770 5.2 e115/111 5:408ee55a:::benchmark_data_smithi026_286012_object7769:head v 115'499, mlcod=115'497)", 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:23.735117+0000", 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 4.6110557779999999, 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020454511000000002, 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.735117+0000", 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.735117+0000", 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.735117+0000", 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:15:28.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.735186+0000", 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9428e-05 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.752982+0000", 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017795406999999999 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.752998+0000", 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5971999999999999e-05 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.754217+0000", 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012193239999999999 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.754238+0000", 2023-09-23T22:15:28.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0659999999999999e-05 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.755515+0000", 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001277776 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:23.755572+0000", 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6523000000000001e-05 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.364 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:13282 5.3 5:cb9b2ed2:::benchmark_data_smithi026_286012_object13281:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:25.409490+0000", 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.9366828630000001, 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.046280125999999998, 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 13282 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.365 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.409490+0000", 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.409490+0000", 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.409490+0000", 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.409508+0000", 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8156999999999999e-05 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.409516+0000", 2023-09-23T22:15:28.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4630000000000001e-06 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.409522+0000", 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.806e-06 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.409566+0000", 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4443999999999997e-05 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.409733+0000", 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016686899999999999 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.409876+0000", 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014238200000000001 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.411284+0000", 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001408554 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.411421+0000", 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013697199999999999 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412064+0000", 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00064235799999999999 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455738+0000", 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043674350000000001 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455770+0000", 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2536999999999997e-05 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.371 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:13288 5.3 5:cfc1ef6e:::benchmark_data_smithi026_286012_object13287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:25.410356+0000", 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.9358167370000001, 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.046412024000000003, 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 13288 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.410356+0000", 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.410356+0000", 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.410356+0000", 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:15:28.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.410372+0000", 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6685e-05 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.410379+0000", 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5819999999999998e-06 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.410383+0000", 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6949999999999999e-06 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.410531+0000", 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147021 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.410638+0000", 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010736899999999999 2023-09-23T22:15:28.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.410775+0000", 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136697 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.411515+0000", 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00074079300000000001 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455811+0000", 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.044295656000000003 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.456678+0000", 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00086708300000000005 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.456735+0000", 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7108999999999998e-05 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.456769+0000", 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3300999999999998e-05 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:13282 5.3 e115/111 ondisk, result = 0)", 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:25.412005+0000", 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.934167789, 2023-09-23T22:15:28.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043772526999999999, 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412005+0000", 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412005+0000", 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412005+0000", 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412009+0000", 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1590000000000003e-06 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412017+0000", 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9780000000000005e-06 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412022+0000", 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6519999999999997e-06 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412056+0000", 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4381999999999997e-05 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412061+0000", 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.651e-06 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455778+0000", 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043717356999999998 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.382 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:13288 5.3 e115/111 ondisk, result = 0)", 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:25.412170+0000", 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.934003454, 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043643908000000002, 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412170+0000", 2023-09-23T22:15:28.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412170+0000", 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412169+0000", 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412173+0000", 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5839999999999999e-06 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455754+0000", 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043580700999999999 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455763+0000", 2023-09-23T22:15:28.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8770000000000005e-06 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455798+0000", 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4572e-05 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455806+0000", 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6190000000000002e-06 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455814+0000", 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.055e-06 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:13288 5.3 e115/111 ondisk, result = 0)", 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:25.412198+0000", 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.9339756120000002, 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.044576497, 2023-09-23T22:15:28.387 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412198+0000", 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412198+0000", 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412197+0000", 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.412200+0000", 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6910000000000002e-06 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455869+0000", 2023-09-23T22:15:28.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043669486 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.455879+0000", 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8360000000000004e-06 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.456666+0000", 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000786405 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.456673+0000", 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5079999999999999e-06 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:25.456774+0000", 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101065 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:14604 5.5 5:a9716e1b:::benchmark_data_smithi026_286012_object14603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:26.071033+0000", 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.2751406489999999, 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.073708241999999993, 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 14604 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.392 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.071033+0000", 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.071033+0000", 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.071031+0000", 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.071060+0000", 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9158000000000001e-05 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.071073+0000", 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3112000000000001e-05 2023-09-23T22:15:28.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.071083+0000", 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5969999999999998e-06 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.071202+0000", 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119336 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.071360+0000", 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000157388 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.071502+0000", 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014227900000000001 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.073140+0000", 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001637658 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.073364+0000", 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000224739 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.144623+0000", 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.071258890000000005 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.144680+0000", 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6545999999999997e-05 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.144741+0000", 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1099000000000006e-05 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.398 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:14646 5.5 5:a05ad919:::benchmark_data_smithi026_286012_object14645:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:26.083697+0000", 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.2624758370000002, 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.061733859000000002, 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 14646 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.083697+0000", 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.083697+0000", 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.083696+0000", 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:15:28.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.083732+0000", 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5519999999999999e-05 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.083748+0000", 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6636000000000001e-05 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.083757+0000", 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8629999999999997e-06 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.083814+0000", 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7305000000000001e-05 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.083954+0000", 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013947100000000001 2023-09-23T22:15:28.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.084117+0000", 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000163008 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.084865+0000", 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00074771799999999997 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.085118+0000", 2023-09-23T22:15:28.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025307999999999998 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.145333+0000", 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060215537 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.145390+0000", 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6511000000000002e-05 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.145431+0000", 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1600000000000002e-05 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:15662 5.5 5:af41903e:::benchmark_data_smithi026_286012_object15661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:26.757050+0000", 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.5891232280000001, 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.079822539999999997, 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.405 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15662 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.757050+0000", 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.757050+0000", 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.757048+0000", 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.757073+0000", 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5009999999999999e-05 2023-09-23T22:15:28.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.757086+0000", 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3158e-05 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.757095+0000", 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4689999999999997e-06 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.757154+0000", 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9335e-05 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.757288+0000", 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013438100000000001 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.757439+0000", 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015044900000000001 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.758881+0000", 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014425589999999999 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.759318+0000", 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000436993 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.836770+0000", 2023-09-23T22:15:28.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.077451220000000001 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.836826+0000", 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6326000000000002e-05 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.836873+0000", 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6618e-05 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.411 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:15693 5.5 5:a3683b3e:::benchmark_data_smithi026_286012_object15692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:26.771648+0000", 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.5745252949999999, 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065461134000000004, 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15693 2023-09-23T22:15:28.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.771648+0000", 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.771648+0000", 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.771646+0000", 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.771666+0000", 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9732999999999999e-05 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.771679+0000", 2023-09-23T22:15:28.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2621e-05 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.771685+0000", 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1090000000000003e-06 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.771738+0000", 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3217999999999998e-05 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.771846+0000", 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108113 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.771951+0000", 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010432199999999999 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.772640+0000", 2023-09-23T22:15:28.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00068931199999999998 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.773038+0000", 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039783599999999999 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.837013+0000", 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.063975224999999997 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.837066+0000", 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3511000000000003e-05 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.837109+0000", 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2657999999999997e-05 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:15717 5.5 5:ad512859:::benchmark_data_smithi026_286012_object15716:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:26.843405+0000", 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.502768229, 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.11179993000000001, 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15717 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.843405+0000", 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.843405+0000", 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.843404+0000", 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.843425+0000", 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1118999999999999e-05 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.843437+0000", 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1742999999999999e-05 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.843445+0000", 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0660000000000004e-06 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.843830+0000", 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038482700000000002 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.843959+0000", 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012939700000000001 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.844207+0000", 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024758400000000002 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.954989+0000", 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.110782652 2023-09-23T22:15:28.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955034+0000", 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4996000000000001e-05 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955104+0000", 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9308999999999997e-05 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955151+0000", 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7231000000000002e-05 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955205+0000", 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4144000000000002e-05 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.425 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:15717 5.5 e115/111 ondisk, result = 0)", 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:26.845757+0000", 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.5004163799999999, 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10928025099999999, 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845757+0000", 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845757+0000", 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845756+0000", 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845765+0000", 2023-09-23T22:15:28.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6300000000000004e-06 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845775+0000", 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0886999999999999e-05 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845785+0000", 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.132e-06 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.954946+0000", 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.109161859 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.954971+0000", 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4301999999999999e-05 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955037+0000", 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6309999999999994e-05 2023-09-23T22:15:28.429 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:15717 5.5 e115/111 ondisk, result = 0)", 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:26.845814+0000", 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.5003592139999999, 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.109398517, 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:15:28.430 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845814+0000", 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845814+0000", 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845813+0000", 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845818+0000", 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.092e-06 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845827+0000", 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1230000000000006e-06 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.845833+0000", 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0950000000000001e-06 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955090+0000", 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.109257321 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955098+0000", 2023-09-23T22:15:28.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7489999999999998e-06 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955213+0000", 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114658 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:15724 5.e e115/111 5:7c90e698:::benchmark_data_smithi026_286012_object15723:head v 115'991, mlcod=115'989)", 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:26.846534+0000", 2023-09-23T22:15:28.434 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.499639696, 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.109622071, 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.846534+0000", 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.846534+0000", 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.846533+0000", 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.846593+0000", 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9911000000000003e-05 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.954917+0000", 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10832417 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.954935+0000", 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8122999999999999e-05 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955042+0000", 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107138 2023-09-23T22:15:28.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955094+0000", 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1480999999999997e-05 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.956127+0000", 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010327769999999999 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.956156+0000", 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9090000000000001e-05 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:15725 5.3 5:c57e1787:::benchmark_data_smithi026_286012_object15724:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:26.846964+0000", 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.4992093550000001, 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10967712, 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.439 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 15725 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.846964+0000", 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.846964+0000", 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.846963+0000", 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.846982+0000", 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8921999999999999e-05 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.954905+0000", 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.107922817 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.954938+0000", 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3729999999999997e-05 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955092+0000", 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015370499999999999 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955236+0000", 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014447200000000001 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.955384+0000", 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014763800000000001 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.956260+0000", 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00087639800000000002 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.956531+0000", 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027031400000000001 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.956589+0000", 2023-09-23T22:15:28.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8588000000000002e-05 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.956621+0000", 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1247000000000001e-05 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:26.956641+0000", 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0364e-05 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.445 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:16599 5.3 5:c7d2e08d:::benchmark_data_smithi026_286012_object16598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:27.305931+0000", 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.040241835, 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.098987157000000006, 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.446 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 16599 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.305931+0000", 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.305931+0000", 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.305930+0000", 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.305952+0000", 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2238999999999998e-05 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.305965+0000", 2023-09-23T22:15:28.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2952000000000001e-05 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.305974+0000", 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7879999999999995e-06 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.306029+0000", 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5006999999999997e-05 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.306176+0000", 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147322 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.306332+0000", 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015572399999999999 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.307365+0000", 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001033649 2023-09-23T22:15:28.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.308764+0000", 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013989989999999999 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.404837+0000", 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.096072483 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.404885+0000", 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8128999999999999e-05 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.404919+0000", 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3442000000000003e-05 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:16606 5.5 5:ab94b6a3:::benchmark_data_smithi026_286012_object16605:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.452 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:27.309149+0000", 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.0370246279999999, 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.081119905000000006, 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 16606 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.309149+0000", 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.309149+0000", 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.309147+0000", 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:15:28.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.309173+0000", 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5846e-05 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.309188+0000", 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4803e-05 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.309198+0000", 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0305e-05 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.309890+0000", 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000691813 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.310001+0000", 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011092899999999999 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.310150+0000", 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014888799999999999 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.311068+0000", 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00091787699999999999 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.312821+0000", 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001753454 2023-09-23T22:15:28.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.390160+0000", 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.077338718000000001 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.390223+0000", 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3391000000000003e-05 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.390269+0000", 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5191000000000002e-05 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:16609 5.3 5:c84c8b5f:::benchmark_data_smithi026_286012_object16608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:27.313981+0000", 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.0321920710000001, 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.091187878999999999, 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.459 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 16609 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.313981+0000", 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.313981+0000", 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.313979+0000", 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.314001+0000", 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2005999999999999e-05 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.314018+0000", 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7039999999999999e-05 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.314027+0000", 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.884e-06 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.314090+0000", 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3138000000000004e-05 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.314262+0000", 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017179399999999999 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.314425+0000", 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016256199999999999 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.316585+0000", 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021600479999999999 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.318023+0000", 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00143827 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.405081+0000", 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.087058142000000005 2023-09-23T22:15:28.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.405132+0000", 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0784e-05 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:27.405169+0000", 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7024999999999999e-05 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:18300 5.3 5:cc0be44a:::benchmark_data_smithi026_286012_object18299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.192508+0000", 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.153665306, 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018708455999999998, 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 18300 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.466 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.192508+0000", 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.192508+0000", 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.192506+0000", 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.192530+0000", 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3051e-05 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.192547+0000", 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7430000000000001e-05 2023-09-23T22:15:28.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.192556+0000", 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9160000000000007e-06 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.192603+0000", 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7445999999999999e-05 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:15:28.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.192754+0000", 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015018400000000001 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.192943+0000", 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018901599999999999 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.197230+0000", 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042878480000000004 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.197319+0000", 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8201000000000005e-05 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.211123+0000", 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013804573000000001 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:15:28.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.211176+0000", 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2929000000000001e-05 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.211216+0000", 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0287999999999998e-05 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:15:28.472 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:15:28.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:28 smithi026 conmon[215751]: 2023-09-23T22:15:28.242+0000 7f4580f04700 -1 mon.a@0(leader).osd e115 definitely_dead 0 2023-09-23T22:15:28.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:28.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:28.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:28.575 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:28.622 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:28.631 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.0. 2023-09-23T22:15:28.669 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:28.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:28.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:28.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:28 smithi026 ceph-mon[223014]: pgmap v233: 89 pgs: 89 active+clean; 759 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T22:15:28.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:28 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/404216732' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T22:15:28.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:28 smithi026 ceph-mon[223014]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T22:15:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:28 smithi026 ceph-mon[215775]: pgmap v233: 89 pgs: 89 active+clean; 759 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T22:15:28.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:28 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/404216732' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T22:15:28.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:28 smithi026 ceph-mon[215775]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T22:15:28.774 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:28.806 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:28.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:28 smithi070 ceph-mon[192652]: pgmap v233: 89 pgs: 89 active+clean; 759 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T22:15:28.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:28 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/404216732' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T22:15:28.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:28 smithi070 ceph-mon[192652]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T22:15:29.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:15:29.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 19284 19268 150.517 50.6875 0.00170649 0.00609131 2023-09-23T22:15:29.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:29 smithi070 ceph-mon[192652]: failed to encode map e116 with expected crc 2023-09-23T22:15:29.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:29 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/404216732' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T22:15:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:29 smithi070 ceph-mon[192652]: osdmap e116: 8 total, 8 up, 7 in 2023-09-23T22:15:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:29 smithi070 ceph-mon[192652]: failed to encode map e116 with expected crc 2023-09-23T22:15:29.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:15:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:15:29] "GET /metrics HTTP/1.1" 200 36355 "" "Prometheus/2.43.0" 2023-09-23T22:15:29.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:29 smithi026 ceph-mon[223014]: failed to encode map e116 with expected crc 2023-09-23T22:15:29.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:29 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/404216732' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T22:15:29.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:29 smithi026 ceph-mon[223014]: osdmap e116: 8 total, 8 up, 7 in 2023-09-23T22:15:29.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:29 smithi026 ceph-mon[223014]: failed to encode map e116 with expected crc 2023-09-23T22:15:29.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:29 smithi026 ceph-mon[215775]: failed to encode map e116 with expected crc 2023-09-23T22:15:29.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:29 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/404216732' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T22:15:29.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:29 smithi026 ceph-mon[215775]: osdmap e116: 8 total, 8 up, 7 in 2023-09-23T22:15:29.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:29 smithi026 ceph-mon[215775]: failed to encode map e116 with expected crc 2023-09-23T22:15:30.078 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:30.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:30.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:30.147 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:15:30.184 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:30.214 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:30.261 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:30.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:30.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:30.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 19284 19268 133.793 0 - 0.00609131 2023-09-23T22:15:30.366 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:30.394 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:30.436 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:30.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:30.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:30.542 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:30.569 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:30.610 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:30.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:30.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:30.716 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:30.777 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:30.864 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:30.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:30.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:30 smithi070 ceph-mon[192652]: pgmap v235: 89 pgs: 8 peering, 81 active+clean; 856 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-23T22:15:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:30 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:15:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:30 smithi070 ceph-mon[192652]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:30 smithi070 ceph-mon[192652]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:30 smithi070 ceph-mon[192652]: osdmap e117: 8 total, 8 up, 7 in 2023-09-23T22:15:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:30 smithi070 ceph-mon[192652]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:30 smithi070 ceph-mon[192652]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:30 smithi070 ceph-mon[192652]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:30 smithi070 ceph-mon[192652]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[223014]: pgmap v235: 89 pgs: 8 peering, 81 active+clean; 856 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-23T22:15:30.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:15:30.921 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[223014]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[223014]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[223014]: osdmap e117: 8 total, 8 up, 7 in 2023-09-23T22:15:30.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[223014]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[223014]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[223014]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[223014]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[215775]: pgmap v235: 89 pgs: 8 peering, 81 active+clean; 856 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 106 MiB/s wr, 1.70k op/s 2023-09-23T22:15:30.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:15:30.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[215775]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[215775]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[215775]: osdmap e117: 8 total, 8 up, 7 in 2023-09-23T22:15:30.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[215775]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[215775]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[215775]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:30 smithi026 ceph-mon[215775]: failed to encode map e117 with expected crc 2023-09-23T22:15:30.971 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:31.021 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:31.214 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:31.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:31.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:31.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 20378 20362 127.251 34.1875 0.00591443 0.00785229 2023-09-23T22:15:31.310 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:15:31.324 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:31.345 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:15:31.364 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:31.379 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:31.402 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:15:31.440 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:31.448 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:15:31.488 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:15:31.507 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:31.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:31.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:31.524 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:15:31.560 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:15:31.598 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:15:31.610 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:15:31.610 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:15:31.610 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:15:31.611 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:15:31.611 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:15:31.611 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:15:31.611 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:15:31.611 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:15:31.619 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:31.681 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:31.782 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:31.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[223014]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[223014]: osdmap e118: 8 total, 8 up, 7 in 2023-09-23T22:15:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[223014]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[223014]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[223014]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[223014]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[223014]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.883 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3969549707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[215775]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[215775]: osdmap e118: 8 total, 8 up, 7 in 2023-09-23T22:15:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[215775]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[215775]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.883 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[215775]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[215775]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[215775]: failed to encode map e118 with expected crc 2023-09-23T22:15:31.884 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3969549707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:31.895 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:31.944 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:32.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:31 smithi070 ceph-mon[192652]: failed to encode map e118 with expected crc 2023-09-23T22:15:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:31 smithi070 ceph-mon[192652]: osdmap e118: 8 total, 8 up, 7 in 2023-09-23T22:15:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:31 smithi070 ceph-mon[192652]: failed to encode map e118 with expected crc 2023-09-23T22:15:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:31 smithi070 ceph-mon[192652]: failed to encode map e118 with expected crc 2023-09-23T22:15:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:31 smithi070 ceph-mon[192652]: failed to encode map e118 with expected crc 2023-09-23T22:15:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:31 smithi070 ceph-mon[192652]: failed to encode map e118 with expected crc 2023-09-23T22:15:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:31 smithi070 ceph-mon[192652]: failed to encode map e118 with expected crc 2023-09-23T22:15:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3969549707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:32.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 22159 22143 125.802 111.312 0.0130759 0.00794392 2023-09-23T22:15:32.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[215775]: pgmap v238: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 8 peering, 74 active+clean; 1.2 GiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 143 MiB/s wr, 2.28k op/s; 2338/56649 objects degraded (4.127%); 95 B/s, 4 objects/s recovering 2023-09-23T22:15:32.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 2338/56649 objects degraded (4.127%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:32.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[215775]: failed to encode map e119 with expected crc 2023-09-23T22:15:32.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[215775]: failed to encode map e119 with expected crc 2023-09-23T22:15:32.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[215775]: osdmap e119: 8 total, 8 up, 7 in 2023-09-23T22:15:32.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[215775]: failed to encode map e119 with expected crc 2023-09-23T22:15:32.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[223014]: pgmap v238: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 8 peering, 74 active+clean; 1.2 GiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 143 MiB/s wr, 2.28k op/s; 2338/56649 objects degraded (4.127%); 95 B/s, 4 objects/s recovering 2023-09-23T22:15:32.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 2338/56649 objects degraded (4.127%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:32.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[223014]: failed to encode map e119 with expected crc 2023-09-23T22:15:32.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[223014]: failed to encode map e119 with expected crc 2023-09-23T22:15:32.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[223014]: osdmap e119: 8 total, 8 up, 7 in 2023-09-23T22:15:32.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:32 smithi026 ceph-mon[223014]: failed to encode map e119 with expected crc 2023-09-23T22:15:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:32 smithi070 ceph-mon[192652]: pgmap v238: 89 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 8 peering, 74 active+clean; 1.2 GiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 143 MiB/s wr, 2.28k op/s; 2338/56649 objects degraded (4.127%); 95 B/s, 4 objects/s recovering 2023-09-23T22:15:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:32 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 2338/56649 objects degraded (4.127%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:32 smithi070 ceph-mon[192652]: failed to encode map e119 with expected crc 2023-09-23T22:15:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:32 smithi070 ceph-mon[192652]: failed to encode map e119 with expected crc 2023-09-23T22:15:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:32 smithi070 ceph-mon[192652]: osdmap e119: 8 total, 8 up, 7 in 2023-09-23T22:15:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:32 smithi070 ceph-mon[192652]: failed to encode map e119 with expected crc 2023-09-23T22:15:33.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 24050 24034 125.166 118.188 0.00589621 0.00798374 2023-09-23T22:15:33.359 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:33.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:33.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:33.466 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:33.515 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[215775]: failed to encode map e120 with expected crc 2023-09-23T22:15:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[215775]: osdmap e120: 8 total, 8 up, 7 in 2023-09-23T22:15:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[215775]: failed to encode map e120 with expected crc 2023-09-23T22:15:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[215775]: failed to encode map e120 with expected crc 2023-09-23T22:15:33.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[215775]: failed to encode map e120 with expected crc 2023-09-23T22:15:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[223014]: failed to encode map e120 with expected crc 2023-09-23T22:15:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[223014]: osdmap e120: 8 total, 8 up, 7 in 2023-09-23T22:15:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[223014]: failed to encode map e120 with expected crc 2023-09-23T22:15:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[223014]: failed to encode map e120 with expected crc 2023-09-23T22:15:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:33 smithi026 ceph-mon[223014]: failed to encode map e120 with expected crc 2023-09-23T22:15:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:33 smithi070 ceph-mon[192652]: failed to encode map e120 with expected crc 2023-09-23T22:15:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:33 smithi070 ceph-mon[192652]: osdmap e120: 8 total, 8 up, 7 in 2023-09-23T22:15:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:33 smithi070 ceph-mon[192652]: failed to encode map e120 with expected crc 2023-09-23T22:15:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:33 smithi070 ceph-mon[192652]: failed to encode map e120 with expected crc 2023-09-23T22:15:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:33 smithi070 ceph-mon[192652]: failed to encode map e120 with expected crc 2023-09-23T22:15:34.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 25580 25564 122.893 95.625 0.0104187 0.00813169 2023-09-23T22:15:34.288 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:34.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:34.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:34.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:34.449 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:34.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:34 smithi026 ceph-mon[223014]: pgmap v241: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 8 peering, 72 active+clean; 1.2 GiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s; 4854/59388 objects degraded (8.173%); 143 B/s, 12 objects/s recovering 2023-09-23T22:15:34.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:34 smithi026 ceph-mon[215775]: pgmap v241: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 8 peering, 72 active+clean; 1.2 GiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s; 4854/59388 objects degraded (8.173%); 143 B/s, 12 objects/s recovering 2023-09-23T22:15:35.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:34 smithi070 ceph-mon[192652]: pgmap v241: 89 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 8 peering, 72 active+clean; 1.2 GiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 96 MiB/s wr, 1.53k op/s; 4854/59388 objects degraded (8.173%); 143 B/s, 12 objects/s recovering 2023-09-23T22:15:35.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 27106 27090 120.927 95.375 0.00631645 0.00826369 2023-09-23T22:15:35.586 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:35.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:35.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:35.696 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:35.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:35.853 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:15:35.853 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:15:35.853 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-09-23T22:15:35.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 0 2023-09-23T22:15:35.905 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:35 smithi026 ceph-mon[215775]: pgmap v242: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s; 7291/70332 objects degraded (10.367%); 1249/70332 objects misplaced (1.776%); 14 MiB/s, 229 objects/s recovering 2023-09-23T22:15:35.906 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:35 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-09-23T22:15:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:35 smithi026 ceph-mon[223014]: pgmap v242: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s; 7291/70332 objects degraded (10.367%); 1249/70332 objects misplaced (1.776%); 14 MiB/s, 229 objects/s recovering 2023-09-23T22:15:35.906 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:35 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-09-23T22:15:36.018 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:36.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:36.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:36.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:15:36.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:35 smithi070 ceph-mon[192652]: pgmap v242: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.4 GiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s; 7291/70332 objects degraded (10.367%); 1249/70332 objects misplaced (1.776%); 14 MiB/s, 229 objects/s recovering 2023-09-23T22:15:36.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:35 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 4 pgs peering) 2023-09-23T22:15:36.123 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:36.164 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:36.241 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:36.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:36.286 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 28603 28587 119.098 93.5625 0.00911091 0.00838949 2023-09-23T22:15:36.352 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:36.406 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:36.430 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:15:36.430 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:15:36.501 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:36.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:36.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:36.609 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:36.681 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:36.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:36 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/979925744' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:15:36.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:15:36.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:36.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:36.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:36 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/979925744' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:15:36.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:15:36.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:36.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:36.756 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:15:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:36 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/979925744' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:15:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:15:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:37.287 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 30213 30197 117.937 100.625 0.00997815 0.00847361 2023-09-23T22:15:37.694 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:37.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:37.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:37.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:37.854 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:37.916 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:37.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:15:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:15:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[215775]: osdmap e121: 8 total, 8 up, 7 in 2023-09-23T22:15:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[215775]: failed to encode map e121 with expected crc 2023-09-23T22:15:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[215775]: failed to encode map e121 with expected crc 2023-09-23T22:15:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[215775]: pgmap v244: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 7285/79872 objects degraded (9.121%); 1249/79872 objects misplaced (1.564%); 12 MiB/s, 202 objects/s recovering 2023-09-23T22:15:37.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7291/70332 objects degraded (10.367%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:15:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:15:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[223014]: osdmap e121: 8 total, 8 up, 7 in 2023-09-23T22:15:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[223014]: failed to encode map e121 with expected crc 2023-09-23T22:15:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[223014]: failed to encode map e121 with expected crc 2023-09-23T22:15:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[223014]: pgmap v244: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 7285/79872 objects degraded (9.121%); 1249/79872 objects misplaced (1.564%); 12 MiB/s, 202 objects/s recovering 2023-09-23T22:15:37.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:37 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7291/70332 objects degraded (10.367%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:38.022 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:38.074 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:37 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:15:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:37 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:15:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:37 smithi070 ceph-mon[192652]: osdmap e121: 8 total, 8 up, 7 in 2023-09-23T22:15:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:37 smithi070 ceph-mon[192652]: failed to encode map e121 with expected crc 2023-09-23T22:15:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:37 smithi070 ceph-mon[192652]: failed to encode map e121 with expected crc 2023-09-23T22:15:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:37 smithi070 ceph-mon[192652]: pgmap v244: 89 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.6 GiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s; 7285/79872 objects degraded (9.121%); 1249/79872 objects misplaced (1.564%); 12 MiB/s, 202 objects/s recovering 2023-09-23T22:15:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:37 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7291/70332 objects degraded (10.367%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:38.287 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 31606 31590 116.121 87.0625 0.00489171 0.00860621 2023-09-23T22:15:38.375 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:15:38.511 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:38.550 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:38.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:38.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:38.652 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:38.662 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:38.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:38.812 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:38.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:38.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:38.924 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:38.957 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:39.035 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:15:39.035 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:15:39.035 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:39.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:15:39.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:15:39.037 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:15:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:39.287 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 32538 32522 112.906 58.25 0.00225649 0.00865901 2023-09-23T22:15:39.721 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:39.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:39.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:39.825 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:39.843 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:15:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:15:39] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T22:15:39.857 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:39 smithi070 ceph-mon[192652]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:39 smithi070 ceph-mon[192652]: osdmap e122: 8 total, 8 up, 7 in 2023-09-23T22:15:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:39 smithi070 ceph-mon[192652]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:39 smithi070 ceph-mon[192652]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:39 smithi070 ceph-mon[192652]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:39 smithi070 ceph-mon[192652]: pgmap v246: 89 pgs: 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s; 6032/83985 objects degraded (7.182%); 2502/83985 objects misplaced (2.979%); 12 MiB/s, 190 objects/s recovering 2023-09-23T22:15:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[223014]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[223014]: osdmap e122: 8 total, 8 up, 7 in 2023-09-23T22:15:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[223014]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[223014]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[223014]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[223014]: pgmap v246: 89 pgs: 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s; 6032/83985 objects degraded (7.182%); 2502/83985 objects misplaced (2.979%); 12 MiB/s, 190 objects/s recovering 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[215775]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[215775]: osdmap e122: 8 total, 8 up, 7 in 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[215775]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[215775]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[215775]: failed to encode map e122 with expected crc 2023-09-23T22:15:40.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:40.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:40.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:39 smithi026 ceph-mon[215775]: pgmap v246: 89 pgs: 3 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 1.7 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s; 6032/83985 objects degraded (7.182%); 2502/83985 objects misplaced (2.979%); 12 MiB/s, 190 objects/s recovering 2023-09-23T22:15:40.288 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 33152 33136 108.978 38.375 0.00697515 0.00917095 2023-09-23T22:15:40.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:15:41.044 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:41.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:40 smithi070 ceph-mon[192652]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:40 smithi070 ceph-mon[192652]: osdmap e123: 8 total, 8 up, 7 in 2023-09-23T22:15:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:40 smithi070 ceph-mon[192652]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:40 smithi070 ceph-mon[192652]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:40 smithi070 ceph-mon[192652]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:40 smithi070 ceph-mon[192652]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.152 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:41.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:41.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[215775]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[215775]: osdmap e123: 8 total, 8 up, 7 in 2023-09-23T22:15:41.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[215775]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[215775]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[215775]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[215775]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[223014]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[223014]: osdmap e123: 8 total, 8 up, 7 in 2023-09-23T22:15:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[223014]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[223014]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[223014]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:40 smithi026 ceph-mon[223014]: failed to encode map e123 with expected crc 2023-09-23T22:15:41.244 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:41.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:41.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:41.288 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:15:41.288927+0000 min lat: 0.00108301 max lat: 2.08369 avg lat: 0.00921875 2023-09-23T22:15:41.288 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:15:41.288 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 34528 34512 107.829 86 0.00263123 0.00921875 2023-09-23T22:15:41.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:41.408 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:41.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:41.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:41.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:41.624 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:41.683 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:41.731 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:41.846 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:41.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:41.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:41.956 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:42.033 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:42.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:41 smithi026 ceph-mon[223014]: pgmap v248: 89 pgs: 2 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 4836/98586 objects degraded (4.905%); 2092/98586 objects misplaced (2.122%); 17 MiB/s, 268 objects/s recovering 2023-09-23T22:15:42.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:41 smithi026 ceph-mon[215775]: pgmap v248: 89 pgs: 2 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 4836/98586 objects degraded (4.905%); 2092/98586 objects misplaced (2.122%); 17 MiB/s, 268 objects/s recovering 2023-09-23T22:15:42.289 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 36009 35993 107.098 92.5625 0.00806856 0.00932785 2023-09-23T22:15:42.335 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:42.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:42.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:41 smithi070 ceph-mon[192652]: pgmap v248: 89 pgs: 2 active+recovery_wait, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.0 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 4836/98586 objects degraded (4.905%); 2092/98586 objects misplaced (2.122%); 17 MiB/s, 268 objects/s recovering 2023-09-23T22:15:42.445 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:42.511 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:42.786 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:42 smithi026 conmon[215751]: 2023-09-23T22:15:42.561+0000 7f4580f04700 -1 mon.a@0(leader).osd e123 definitely_dead 0 2023-09-23T22:15:42.925 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.0. 2023-09-23T22:15:42.932 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:15:42.932 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:15:42.933 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:15:42.933 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:15:42.933 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:15:42.933 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:15:42.956 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:15:42.958 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:42.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:42.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:15:42.977 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:15:42.977 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:15:42.977 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:15:42.977 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:15:42.977 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:15:42.977 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:15:42.978 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:15:42.978 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:15:43.062 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:43.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2071349646' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:15:43.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:42 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:15:43.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:42 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4836/98586 objects degraded (4.905%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:43.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1111462406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:43.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2071349646' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:15:43.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:42 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:15:43.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:42 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4836/98586 objects degraded (4.905%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:43.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1111462406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:43.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:43.137 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:43.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:43.140 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:43.241 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:43.267 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:43.289 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 37009 36993 105.071 62.5 0.00204904 0.00935426 2023-09-23T22:15:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2071349646' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:15:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:42 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:15:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:42 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4836/98586 objects degraded (4.905%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1111462406' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:44.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T22:15:44.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[223014]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T22:15:44.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[223014]: failed to encode map e124 with expected crc 2023-09-23T22:15:44.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[223014]: failed to encode map e124 with expected crc 2023-09-23T22:15:44.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[223014]: pgmap v250: 89 pgs: 2 remapped+peering, 3 peering, 2 active+recovery_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.1 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 3573/100791 objects degraded (3.545%); 839/100791 objects misplaced (0.832%); 25 MiB/s, 404 objects/s recovering 2023-09-23T22:15:44.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T22:15:44.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[215775]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T22:15:44.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[215775]: failed to encode map e124 with expected crc 2023-09-23T22:15:44.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[215775]: failed to encode map e124 with expected crc 2023-09-23T22:15:44.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:43 smithi026 ceph-mon[215775]: pgmap v250: 89 pgs: 2 remapped+peering, 3 peering, 2 active+recovery_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.1 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 3573/100791 objects degraded (3.545%); 839/100791 objects misplaced (0.832%); 25 MiB/s, 404 objects/s recovering 2023-09-23T22:15:44.289 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 37067 37051 100.66 3.625 0.00184964 0.00961294 2023-09-23T22:15:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:43 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T22:15:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:43 smithi070 ceph-mon[192652]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T22:15:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:43 smithi070 ceph-mon[192652]: failed to encode map e124 with expected crc 2023-09-23T22:15:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:43 smithi070 ceph-mon[192652]: failed to encode map e124 with expected crc 2023-09-23T22:15:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:43 smithi070 ceph-mon[192652]: pgmap v250: 89 pgs: 2 remapped+peering, 3 peering, 2 active+recovery_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 2.1 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 73 MiB/s wr, 1.16k op/s; 3573/100791 objects degraded (3.545%); 839/100791 objects misplaced (0.832%); 25 MiB/s, 404 objects/s recovering 2023-09-23T22:15:44.509 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:44.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:44.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:44.613 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:44.639 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:44.683 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:44.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:44.688 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:44.697 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-09-23T22:15:44.709 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:44.789 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:44.815 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:44.859 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:44.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:44.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T22:15:44.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.908 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.909 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.909 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:44 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:44.967 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:45.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T22:15:45.035 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T22:15:45.036 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.037 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.037 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.037 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.037 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.037 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.037 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.037 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:44 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.037 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:45.289 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 37578 37562 97.7969 31.9375 0.00532929 0.0102195 2023-09-23T22:15:45.356 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:45.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:45.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:45.462 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:45.507 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:45.589 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:45.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:45.671 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:15:45.671 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:15:45.671 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:15:45.671 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:15:45.671 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:15:45.696 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:45.737 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:45.814 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:45.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:45.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:45.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:45.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[215775]: failed to encode map e125 with expected crc 2023-09-23T22:15:45.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[215775]: failed to encode map e126 with expected crc 2023-09-23T22:15:45.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[215775]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T22:15:45.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[215775]: failed to encode map e126 with expected crc 2023-09-23T22:15:45.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[215775]: failed to encode map e126 with expected crc 2023-09-23T22:15:45.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[215775]: failed to encode map e126 with expected crc 2023-09-23T22:15:45.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[215775]: failed to encode map e126 with expected crc 2023-09-23T22:15:45.923 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[215775]: pgmap v253: 89 pgs: 2 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 72 active+clean, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 2 remapped+peering; 2.2 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 5795/107589 objects degraded (5.386%); 833/107589 objects misplaced (0.774%); 37 MiB/s, 599 objects/s recovering 2023-09-23T22:15:45.975 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:46.074 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:46.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:45 smithi070 ceph-mon[192652]: failed to encode map e125 with expected crc 2023-09-23T22:15:46.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:45 smithi070 ceph-mon[192652]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:45 smithi070 ceph-mon[192652]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T22:15:46.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:45 smithi070 ceph-mon[192652]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:45 smithi070 ceph-mon[192652]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:45 smithi070 ceph-mon[192652]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:45 smithi070 ceph-mon[192652]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:45 smithi070 ceph-mon[192652]: pgmap v253: 89 pgs: 2 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 72 active+clean, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 2 remapped+peering; 2.2 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 5795/107589 objects degraded (5.386%); 833/107589 objects misplaced (0.774%); 37 MiB/s, 599 objects/s recovering 2023-09-23T22:15:46.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:46.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:46.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:46.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[223014]: failed to encode map e125 with expected crc 2023-09-23T22:15:46.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[223014]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[223014]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T22:15:46.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[223014]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[223014]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[223014]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[223014]: failed to encode map e126 with expected crc 2023-09-23T22:15:46.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:45 smithi026 ceph-mon[223014]: pgmap v253: 89 pgs: 2 active+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 72 active+clean, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 2 remapped+peering; 2.2 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 95 MiB/s wr, 1.52k op/s; 5795/107589 objects degraded (5.386%); 833/107589 objects misplaced (0.774%); 37 MiB/s, 599 objects/s recovering 2023-09-23T22:15:46.260 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:46.289 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 39140 39124 97.7897 97.625 0.00535447 0.0102215 2023-09-23T22:15:47.032 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:47.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:47.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:47.142 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:47.169 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:47.209 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:47.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:47.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:47.289 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 40339 40323 96.9106 74.9375 0.00175013 0.0102671 2023-09-23T22:15:47.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:47.357 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:47.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:15:47.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:15:47.687 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:47.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:47.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:47.834 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:47.902 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:47.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:47.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:48.009 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:48.087 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:48 smithi070 ceph-mon[192652]: pgmap v254: 89 pgs: 2 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 71 active+clean, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 2 remapped+peering; 2.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 5646/112392 objects degraded (5.023%); 833/112392 objects misplaced (0.741%); 15 MiB/s, 248 objects/s recovering 2023-09-23T22:15:48.087 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 5646/112392 objects degraded (5.023%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:48.095 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:48.214 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:48.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:48.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:48.291 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 41580 41564 96.187 77.5625 0.00564872 0.0103908 2023-09-23T22:15:48.328 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:48.368 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:48.436 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:48.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:48.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:48.468 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:48.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:48 smithi026 ceph-mon[223014]: pgmap v254: 89 pgs: 2 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 71 active+clean, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 2 remapped+peering; 2.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 5646/112392 objects degraded (5.023%); 833/112392 objects misplaced (0.741%); 15 MiB/s, 248 objects/s recovering 2023-09-23T22:15:48.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 5646/112392 objects degraded (5.023%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:48 smithi026 ceph-mon[215775]: pgmap v254: 89 pgs: 2 active+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 71 active+clean, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 8 peering, 2 remapped+peering; 2.3 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 48 MiB/s wr, 766 op/s; 5646/112392 objects degraded (5.023%); 833/112392 objects misplaced (0.741%); 15 MiB/s, 248 objects/s recovering 2023-09-23T22:15:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 5646/112392 objects degraded (5.023%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:48.513 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:48.545 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:48.595 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:48.668 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:48.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:48.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:48.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:48.864 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:48.992 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:49.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:49.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:49.110 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:49.119 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:15:49.147 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:49.230 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:49.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:49.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:49.291 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 42889 42873 95.6735 81.8125 0.00676162 0.0104462 2023-09-23T22:15:49.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:49 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3190225960' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:15:49.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:49 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3190225960' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:15:49.341 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:49.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:49 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3190225960' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:15:49.417 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:49.698 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:15:49.698 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:15:49.698 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T22:15:49.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T22:15:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:15:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:15:49] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T22:15:50.292 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 44142 44126 95.0747 78.3125 0.00763273 0.0105125 2023-09-23T22:15:50.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:50 smithi070 ceph-mon[192652]: pgmap v255: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 75 active+clean, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 7841/115131 objects degraded (6.811%); 835/115131 objects misplaced (0.725%); 15 MiB/s, 242 objects/s recovering 2023-09-23T22:15:50.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:50 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:15:50.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3190225960' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:15:50.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:50 smithi070 ceph-mon[192652]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T22:15:50.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:50 smithi070 ceph-mon[192652]: failed to encode map e127 with expected crc 2023-09-23T22:15:50.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:50 smithi070 ceph-mon[192652]: failed to encode map e127 with expected crc 2023-09-23T22:15:50.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:50 smithi070 ceph-mon[192652]: failed to encode map e127 with expected crc 2023-09-23T22:15:50.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[223014]: pgmap v255: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 75 active+clean, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 7841/115131 objects degraded (6.811%); 835/115131 objects misplaced (0.725%); 15 MiB/s, 242 objects/s recovering 2023-09-23T22:15:50.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:15:50.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3190225960' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:15:50.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[223014]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T22:15:50.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[223014]: failed to encode map e127 with expected crc 2023-09-23T22:15:50.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[223014]: failed to encode map e127 with expected crc 2023-09-23T22:15:50.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[223014]: failed to encode map e127 with expected crc 2023-09-23T22:15:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[215775]: pgmap v255: 89 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped, 4 active+recovery_wait+degraded, 75 active+clean, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 7841/115131 objects degraded (6.811%); 835/115131 objects misplaced (0.725%); 15 MiB/s, 242 objects/s recovering 2023-09-23T22:15:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:15:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3190225960' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:15:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[215775]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T22:15:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[215775]: failed to encode map e127 with expected crc 2023-09-23T22:15:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[215775]: failed to encode map e127 with expected crc 2023-09-23T22:15:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:50 smithi026 ceph-mon[215775]: failed to encode map e127 with expected crc 2023-09-23T22:15:50.812 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:50.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:50.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:50.918 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:50.970 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:51.046 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:51.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:51.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:51.157 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:51.187 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:51.239 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:51.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:51.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:51.291 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 45073 45057 93.845 58.1875 0.00252782 0.0106229 2023-09-23T22:15:51.344 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:51.415 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:51.616 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:51.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:51.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:51.723 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:51.767 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:51.849 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:51.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:51.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:51.956 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:52.062 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:52.234 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:52.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:52 smithi026 ceph-mon[215775]: pgmap v257: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 75 active+clean, 8 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 7839/130920 objects degraded (5.988%); 227/130920 objects misplaced (0.173%); 11 MiB/s, 179 objects/s recovering 2023-09-23T22:15:52.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:52 smithi026 ceph-mon[223014]: pgmap v257: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 75 active+clean, 8 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 7839/130920 objects degraded (5.988%); 227/130920 objects misplaced (0.173%); 11 MiB/s, 179 objects/s recovering 2023-09-23T22:15:52.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 46296 46280 93.2828 76.4375 0.00389337 0.0107023 2023-09-23T22:15:52.360 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:52.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:52.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:52.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:52 smithi070 ceph-mon[192652]: pgmap v257: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 75 active+clean, 8 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 7839/130920 objects degraded (5.988%); 227/130920 objects misplaced (0.173%); 11 MiB/s, 179 objects/s recovering 2023-09-23T22:15:52.388 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:15:52.415 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:15:52.443 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:15:52.466 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:52.469 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:15:52.491 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:15:52.493 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:52.515 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:15:52.536 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:52.538 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:15:52.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:52.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:52.560 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:15:52.561 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:15:52.561 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:15:52.561 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:15:52.561 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:15:52.561 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:15:52.562 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:15:52.562 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:15:52.562 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:15:52.584 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:15:52.584 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:15:52.642 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:52.668 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:52.711 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:52.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:52.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:52.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:52.851 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:53.121 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:53 smithi026 ceph-mon[223014]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T22:15:53.292 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 46463 46447 90.6943 10.4375 0.0107054 0.0110199 2023-09-23T22:15:53.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:53 smithi070 ceph-mon[192652]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T22:15:53.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:53 smithi070 ceph-mon[192652]: failed to encode map e128 with expected crc 2023-09-23T22:15:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:53 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1862167825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7839/130920 objects degraded (5.988%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:53 smithi026 ceph-mon[223014]: failed to encode map e128 with expected crc 2023-09-23T22:15:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:53 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1862167825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7839/130920 objects degraded (5.988%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:53.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:53 smithi026 ceph-mon[215775]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T22:15:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:53 smithi026 ceph-mon[215775]: failed to encode map e128 with expected crc 2023-09-23T22:15:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:53 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1862167825' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:15:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7839/130920 objects degraded (5.988%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:53.726 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:15:54.295 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 47576 47560 90.0539 69.5625 0.00848236 0.0110993 2023-09-23T22:15:54.382 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:54.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:54.387 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:15:54.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:54.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[215775]: pgmap v259: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 76 active+clean, 8 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 7832/132849 objects degraded (5.895%); 227/132849 objects misplaced (0.171%); 4.7 MiB/s, 76 objects/s recovering 2023-09-23T22:15:54.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[215775]: failed to encode map e129 with expected crc 2023-09-23T22:15:54.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[215775]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T22:15:54.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[215775]: failed to encode map e129 with expected crc 2023-09-23T22:15:54.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[215775]: failed to encode map e129 with expected crc 2023-09-23T22:15:54.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:54.477 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:54.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[223014]: pgmap v259: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 76 active+clean, 8 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 7832/132849 objects degraded (5.895%); 227/132849 objects misplaced (0.171%); 4.7 MiB/s, 76 objects/s recovering 2023-09-23T22:15:54.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[223014]: failed to encode map e129 with expected crc 2023-09-23T22:15:54.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[223014]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T22:15:54.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[223014]: failed to encode map e129 with expected crc 2023-09-23T22:15:54.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[223014]: failed to encode map e129 with expected crc 2023-09-23T22:15:54.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:54.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:54.489 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:54.582 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:54 smithi070 ceph-mon[192652]: pgmap v259: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 76 active+clean, 8 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 7832/132849 objects degraded (5.895%); 227/132849 objects misplaced (0.171%); 4.7 MiB/s, 76 objects/s recovering 2023-09-23T22:15:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:54 smithi070 ceph-mon[192652]: failed to encode map e129 with expected crc 2023-09-23T22:15:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:54 smithi070 ceph-mon[192652]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T22:15:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:54 smithi070 ceph-mon[192652]: failed to encode map e129 with expected crc 2023-09-23T22:15:54.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:54 smithi070 ceph-mon[192652]: failed to encode map e129 with expected crc 2023-09-23T22:15:54.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:15:54.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:15:54.717 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:54.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:54.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:54.822 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:54.848 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:54.889 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:54.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:54.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:54.995 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:55.043 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:55.292 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 48651 48635 89.3813 67.1875 0.021309 0.0111813 2023-09-23T22:15:55.315 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:55.360 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:55.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:55.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:55.475 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:15:55.525 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:55.561 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:55.768 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:15:55.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:55.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:15:55.874 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:55.925 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:56.008 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:56.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:56.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:56.118 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:56.175 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:56 smithi070 ceph-mon[192652]: pgmap v261: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 78 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 6694/139629 objects degraded (4.794%); 624/139629 objects misplaced (0.447%); 14 MiB/s, 226 objects/s recovering 2023-09-23T22:15:56.175 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:56.230 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:15:56.250 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:56.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:56.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:56.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:56 smithi026 ceph-mon[215775]: pgmap v261: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 78 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 6694/139629 objects degraded (4.794%); 624/139629 objects misplaced (0.447%); 14 MiB/s, 226 objects/s recovering 2023-09-23T22:15:56.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:56 smithi026 ceph-mon[223014]: pgmap v261: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 78 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 6694/139629 objects degraded (4.794%); 624/139629 objects misplaced (0.447%); 14 MiB/s, 226 objects/s recovering 2023-09-23T22:15:56.292 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 49954 49938 89.1541 81.4375 0.0146576 0.011212 2023-09-23T22:15:56.356 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:15:56.407 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:56.475 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:15:56.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:56.479 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:15:56.580 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:56.608 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:57.154 INFO:teuthology.orchestra.run.smithi026.stdout:moved 4 / 267 (1.49813%) 2023-09-23T22:15:57.154 INFO:teuthology.orchestra.run.smithi026.stdout:avg 33.375 2023-09-23T22:15:57.154 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 4.49826 -> 4.09077 (expected baseline 5.40399) 2023-09-23T22:15:57.154 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 25 -> 26 pgs (0.749064 -> 0.779026 * mean) 2023-09-23T22:15:57.154 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.4 with 41 -> 39 pgs (1.22846 -> 1.16854 * mean) 2023-09-23T22:15:57.154 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:15:57.154 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T22:15:57.155 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 0.05 2023-09-23T22:15:57.155 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:15:57.155 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0128 2023-09-23T22:15:57.155 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0141 2023-09-23T22:15:57.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5 weight 1.0000 -> 0.9500 2023-09-23T22:15:57.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4 weight 1.0000 -> 0.9500 2023-09-23T22:15:57.292 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 50789 50773 88.1273 52.1875 0.00469736 0.0113202 2023-09-23T22:15:57.327 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:57.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:57.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:57.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:15:57.466 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:57 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260'}"}]: dispatch 2023-09-23T22:15:57.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:57 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260'}"}]: dispatch 2023-09-23T22:15:57.470 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:57.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:57 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260'}"}]: dispatch 2023-09-23T22:15:58.292 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 50955 50939 86.0261 10.375 0.00277058 0.0113005 2023-09-23T22:15:58.350 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T22:15:58.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T22:15:58.402 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:15:58.404 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: from='client.44624 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:15:58.404 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: pgmap v262: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 79 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 6682/144855 objects degraded (4.613%); 624/144855 objects misplaced (0.431%); 11 MiB/s, 172 objects/s recovering 2023-09-23T22:15:58.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: failed to encode map e130 with expected crc 2023-09-23T22:15:58.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260'}"}]': finished 2023-09-23T22:15:58.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T22:15:58.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6682/144855 objects degraded (4.613%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:58.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T22:15:58.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[223014]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: from='client.44624 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: pgmap v262: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 79 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 6682/144855 objects degraded (4.613%); 624/144855 objects misplaced (0.431%); 11 MiB/s, 172 objects/s recovering 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: failed to encode map e130 with expected crc 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260'}"}]': finished 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6682/144855 objects degraded (4.613%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:58 smithi026 ceph-mon[215775]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:58.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:15:58.508 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:58.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: from='client.44624 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:15:58.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: pgmap v262: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 79 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 6682/144855 objects degraded (4.613%); 624/144855 objects misplaced (0.431%); 11 MiB/s, 172 objects/s recovering 2023-09-23T22:15:58.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: failed to encode map e130 with expected crc 2023-09-23T22:15:58.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'62260','5':'62260'}"}]': finished 2023-09-23T22:15:58.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T22:15:58.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6682/144855 objects degraded (4.613%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:15:58.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.533 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T22:15:58.533 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.533 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:58 smithi070 ceph-mon[192652]: failed to encode map e131 with expected crc 2023-09-23T22:15:58.534 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:58.577 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:58.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:58.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:58.660 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:58.683 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:15:58.732 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:58.882 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:15:58.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:58.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:15:58.991 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:15:59.031 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:59.123 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:15:59.123 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:15:59.144 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:15:59.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:59.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:15:59.249 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:59.275 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:59.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 51184 51168 84.1391 14.3125 0.00269051 0.0117541 2023-09-23T22:15:59.320 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:59.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:59.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:59.425 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:15:59.451 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:59.496 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:15:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:59.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:15:59.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:15:59.603 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:15:59.631 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:59.704 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:15:59.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:59.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:15:59.775 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:15:59.811 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:15:59.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:59 smithi070 ceph-mon[192652]: failed to encode map e132 with expected crc 2023-09-23T22:15:59.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:59 smithi070 ceph-mon[192652]: failed to encode map e132 with expected crc 2023-09-23T22:15:59.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:59 smithi070 ceph-mon[192652]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T22:15:59.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:59 smithi070 ceph-mon[192652]: failed to encode map e132 with expected crc 2023-09-23T22:15:59.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:15:59 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/297819116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:15:59.864 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:15:59.945 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:15:59.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:15:59.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:15:59.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[223014]: failed to encode map e132 with expected crc 2023-09-23T22:15:59.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[223014]: failed to encode map e132 with expected crc 2023-09-23T22:15:59.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[223014]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T22:15:59.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[223014]: failed to encode map e132 with expected crc 2023-09-23T22:15:59.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/297819116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:15:59.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[215775]: failed to encode map e132 with expected crc 2023-09-23T22:15:59.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[215775]: failed to encode map e132 with expected crc 2023-09-23T22:15:59.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[215775]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T22:15:59.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[215775]: failed to encode map e132 with expected crc 2023-09-23T22:15:59.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:15:59 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/297819116' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:15:59.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:15:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:15:59] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2023-09-23T22:16:00.062 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:00.117 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:00.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 52084 52068 83.4227 56.25 0.0116665 0.0119814 2023-09-23T22:16:00.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:16:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:00 smithi070 ceph-mon[192652]: pgmap v266: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 79 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 6682/147396 objects degraded (4.533%); 624/147396 objects misplaced (0.423%); 7.8 MiB/s, 127 objects/s recovering 2023-09-23T22:16:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:00 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:00 smithi070 ceph-mon[192652]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:00 smithi070 ceph-mon[192652]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/297819116' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:16:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:00 smithi070 ceph-mon[192652]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T22:16:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:00 smithi070 ceph-mon[192652]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:00 smithi070 ceph-mon[192652]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[215775]: pgmap v266: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 79 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 6682/147396 objects degraded (4.533%); 624/147396 objects misplaced (0.423%); 7.8 MiB/s, 127 objects/s recovering 2023-09-23T22:16:00.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:00.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[215775]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[215775]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/297819116' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:16:00.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[215775]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T22:16:00.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[215775]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[215775]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[223014]: pgmap v266: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 79 active+clean, 7 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 6682/147396 objects degraded (4.533%); 624/147396 objects misplaced (0.423%); 7.8 MiB/s, 127 objects/s recovering 2023-09-23T22:16:00.966 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:00.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[223014]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[223014]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/297819116' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:16:00.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[223014]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T22:16:00.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[223014]: failed to encode map e133 with expected crc 2023-09-23T22:16:00.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:00 smithi026 ceph-mon[223014]: failed to encode map e133 with expected crc 2023-09-23T22:16:01.120 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:01.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:01.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:01.224 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:01.249 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:01.290 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:01.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:16:01.294292+0000 min lat: 0.00108301 max lat: 2.08369 avg lat: 0.0120091 2023-09-23T22:16:01.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:16:01.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 53183 53167 83.0541 68.6875 0.00421316 0.0120091 2023-09-23T22:16:01.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:01.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:01.397 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:01.432 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:01.510 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:01.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:01.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:01.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:01.700 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:01.889 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:01.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:01.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:01.999 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:02.041 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:02.114 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:02.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:02.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:02.227 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:02.282 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:02.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 54249 54233 82.6533 66.625 0.00789989 0.0120937 2023-09-23T22:16:02.367 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:02.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:02.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:02.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:02.533 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:02.814 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:02.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:02.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:02 smithi026 ceph-mon[223014]: pgmap v268: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 7517/155715 objects degraded (4.827%); 836/155715 objects misplaced (0.537%); 15 MiB/s, 238 objects/s recovering 2023-09-23T22:16:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:02 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7517/155715 objects degraded (4.827%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:02.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:02 smithi026 ceph-mon[215775]: pgmap v268: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 7517/155715 objects degraded (4.827%); 836/155715 objects misplaced (0.537%); 15 MiB/s, 238 objects/s recovering 2023-09-23T22:16:02.820 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:02 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7517/155715 objects degraded (4.827%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:02 smithi070 ceph-mon[192652]: pgmap v268: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 7517/155715 objects degraded (4.827%); 836/155715 objects misplaced (0.537%); 15 MiB/s, 238 objects/s recovering 2023-09-23T22:16:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:02 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7517/155715 objects degraded (4.827%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:02.917 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:02.920 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:03.013 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:03.268 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:03.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:03.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:03.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 55528 55512 82.5884 79.9375 0.0119801 0.0121025 2023-09-23T22:16:03.378 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:03.418 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:03.422 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:16:03.470 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:16:03.482 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:03.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:03.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:03.518 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:16:03.554 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:16:03.573 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:16:03.587 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:03.603 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:16:03.658 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:16:03.686 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:03.700 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:16:03.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2151961378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:03.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2151961378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:03.763 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:16:03.772 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:16:03.772 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:16:03.772 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:16:03.772 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:16:03.772 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:16:03.772 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:16:03.772 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:16:03.772 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:16:03.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2151961378' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:04.219 INFO:teuthology.orchestra.run.smithi026.stdout:moved 0 / 267 (0%) 2023-09-23T22:16:04.219 INFO:teuthology.orchestra.run.smithi026.stdout:avg 33.375 2023-09-23T22:16:04.219 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 4.09077 -> 4.09077 (expected baseline 5.40399) 2023-09-23T22:16:04.219 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-09-23T22:16:04.220 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.4 with 39 -> 39 pgs (1.16854 -> 1.16854 * mean) 2023-09-23T22:16:04.220 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:16:04.220 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T22:16:04.220 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 1 2023-09-23T22:16:04.220 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:16:04.220 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0144 2023-09-23T22:16:04.220 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.1444 2023-09-23T22:16:04.227 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T22:16:04.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 56586 56570 82.2055 66.125 0.00965333 0.0121592 2023-09-23T22:16:04.624 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:04 smithi026 ceph-mon[215775]: pgmap v269: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 7517/157404 objects degraded (4.776%); 836/157404 objects misplaced (0.531%); 10 MiB/s, 162 objects/s recovering 2023-09-23T22:16:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:04 smithi070 ceph-mon[192652]: pgmap v269: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 7517/157404 objects degraded (4.776%); 836/157404 objects misplaced (0.531%); 10 MiB/s, 162 objects/s recovering 2023-09-23T22:16:04.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:04 smithi026 ceph-mon[223014]: pgmap v269: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 44 MiB/s wr, 710 op/s; 7517/157404 objects degraded (4.776%); 836/157404 objects misplaced (0.531%); 10 MiB/s, 162 objects/s recovering 2023-09-23T22:16:05.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 57442 57426 81.5531 53.5 0.00616843 0.0122569 2023-09-23T22:16:05.543 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:05.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:05.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:05.652 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:05.730 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:05.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:05 smithi026 ceph-mon[223014]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:05.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:05 smithi026 ceph-mon[215775]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:05.850 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:05.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:05.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:05.956 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:05.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:05 smithi070 ceph-mon[192652]: from='client.44636 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:05.983 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:06.025 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:06.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:06.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:06.129 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:06.170 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:06.258 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:06.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:06.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:06.295 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 58463 58447 81.1588 63.8125 0.00939293 0.0123175 2023-09-23T22:16:06.370 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:06.427 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:06.506 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T22:16:06.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T22:16:06.613 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:06.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:06.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:06.717 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:06.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:06 smithi026 ceph-mon[223014]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 7517/165513 objects degraded (4.542%); 288/165513 objects misplaced (0.174%); 13 MiB/s, 201 objects/s recovering 2023-09-23T22:16:06.718 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:06 smithi026 ceph-mon[215775]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 7517/165513 objects degraded (4.542%); 288/165513 objects misplaced (0.174%); 13 MiB/s, 201 objects/s recovering 2023-09-23T22:16:06.765 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:06.831 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:06.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:06.841 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:06.936 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:06.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:06 smithi070 ceph-mon[192652]: pgmap v270: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 58 MiB/s wr, 924 op/s; 7517/165513 objects degraded (4.542%); 288/165513 objects misplaced (0.174%); 13 MiB/s, 201 objects/s recovering 2023-09-23T22:16:06.985 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:06.991 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:16:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:16:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:16:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:16:06.992 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:16:07.061 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:07.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:07.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:07.167 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:07.210 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:07.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 59563 59547 80.889 68.75 0.00976878 0.0123577 2023-09-23T22:16:07.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:07.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:07.425 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:07.514 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:07.594 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:16:07.604 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:07.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:07.718 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:07.752 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:07.811 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:07.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:07.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:07.916 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:07.942 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:07 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7517/165513 objects degraded (4.542%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:07.942 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:07 smithi026 ceph-mon[223014]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T22:16:07.942 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:07 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7517/165513 objects degraded (4.542%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:07.942 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:07 smithi026 ceph-mon[215775]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T22:16:07.945 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:07 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7517/165513 objects degraded (4.542%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:07 smithi070 ceph-mon[192652]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T22:16:08.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 60138 60122 79.9326 35.9375 0.00324301 0.0123563 2023-09-23T22:16:08.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[215775]: pgmap v271: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s; 7517/171603 objects degraded (4.380%); 288/171603 objects misplaced (0.168%); 11 MiB/s, 178 objects/s recovering 2023-09-23T22:16:08.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[215775]: failed to encode map e134 with expected crc 2023-09-23T22:16:08.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[215775]: failed to encode map e134 with expected crc 2023-09-23T22:16:08.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[215775]: failed to encode map e134 with expected crc 2023-09-23T22:16:08.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[215775]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T22:16:08.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[223014]: pgmap v271: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s; 7517/171603 objects degraded (4.380%); 288/171603 objects misplaced (0.168%); 11 MiB/s, 178 objects/s recovering 2023-09-23T22:16:08.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[223014]: failed to encode map e134 with expected crc 2023-09-23T22:16:08.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[223014]: failed to encode map e134 with expected crc 2023-09-23T22:16:08.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[223014]: failed to encode map e134 with expected crc 2023-09-23T22:16:08.665 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:08 smithi026 ceph-mon[223014]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T22:16:08.878 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:08.927 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:16:08.940 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:08.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:08.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:09.062 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:09.112 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:08 smithi070 ceph-mon[192652]: pgmap v271: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s; 7517/171603 objects degraded (4.380%); 288/171603 objects misplaced (0.168%); 11 MiB/s, 178 objects/s recovering 2023-09-23T22:16:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:08 smithi070 ceph-mon[192652]: failed to encode map e134 with expected crc 2023-09-23T22:16:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:08 smithi070 ceph-mon[192652]: failed to encode map e134 with expected crc 2023-09-23T22:16:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:08 smithi070 ceph-mon[192652]: failed to encode map e134 with expected crc 2023-09-23T22:16:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:08 smithi070 ceph-mon[192652]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T22:16:09.175 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:09.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 60794 60778 79.1215 41 0.0409646 0.012634 2023-09-23T22:16:09.385 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:09.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:09.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:09.492 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:09.556 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:09.768 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:09.769 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:09 smithi026 ceph-mon[215775]: failed to encode map e135 with expected crc 2023-09-23T22:16:09.769 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:09 smithi026 ceph-mon[215775]: failed to encode map e135 with expected crc 2023-09-23T22:16:09.769 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:09.769 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:09.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:09 smithi026 ceph-mon[223014]: failed to encode map e135 with expected crc 2023-09-23T22:16:09.769 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:09 smithi026 ceph-mon[223014]: failed to encode map e135 with expected crc 2023-09-23T22:16:09.770 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:09.770 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:09.770 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:16:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:16:09] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-09-23T22:16:09.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:09.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:09.872 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:09.913 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:09.988 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:09.995 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:09 smithi070 ceph-mon[192652]: failed to encode map e135 with expected crc 2023-09-23T22:16:09.995 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:09 smithi070 ceph-mon[192652]: failed to encode map e135 with expected crc 2023-09-23T22:16:09.995 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:09.995 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:09.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:09.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:10.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:10.177 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:10.296 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 61894 61878 78.9071 68.75 0.0140305 0.0126676 2023-09-23T22:16:10.878 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:10.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:10.984 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:10 smithi026 ceph-mon[223014]: pgmap v274: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 7517/174312 objects degraded (4.312%); 288/174312 objects misplaced (0.165%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:16:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/517673131' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:10.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:10 smithi026 ceph-mon[215775]: pgmap v274: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 7517/174312 objects degraded (4.312%); 288/174312 objects misplaced (0.165%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:16:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/517673131' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:11.020 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:16:11.028 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:11.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:10 smithi070 ceph-mon[192652]: pgmap v274: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 80 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 48 MiB/s wr, 774 op/s; 7517/174312 objects degraded (4.312%); 288/174312 objects misplaced (0.165%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:16:11.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/517673131' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:11.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:11.296 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 62929 62913 78.6227 64.6875 0.00840052 0.0127147 2023-09-23T22:16:11.503 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:11.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:11.612 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:11.667 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:11.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:11.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[223014]: failed to encode map e136 with expected crc 2023-09-23T22:16:11.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[223014]: failed to encode map e136 with expected crc 2023-09-23T22:16:11.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:16:11.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[223014]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T22:16:11.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:11.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[215775]: failed to encode map e136 with expected crc 2023-09-23T22:16:11.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[215775]: failed to encode map e136 with expected crc 2023-09-23T22:16:11.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:16:11.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:11 smithi026 ceph-mon[215775]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T22:16:11.766 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:11.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:11.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:11.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:11.979 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:12.111 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:12.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:11 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:12.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:11 smithi070 ceph-mon[192652]: failed to encode map e136 with expected crc 2023-09-23T22:16:12.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:11 smithi070 ceph-mon[192652]: failed to encode map e136 with expected crc 2023-09-23T22:16:12.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:16:12.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:11 smithi070 ceph-mon[192652]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T22:16:12.194 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:12.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:12.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:12.296 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 64102 64086 78.5184 73.3125 0.010409 0.01273 2023-09-23T22:16:12.299 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:12.354 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:12.432 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:12.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:12.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:12.541 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:12.607 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:12.747 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:12.747 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:16:12.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:12.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:12.785 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:16:12.823 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:16:12.849 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:12.881 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:16:12.894 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:12.934 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:16:12.964 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:16:12.966 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:12.972 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:12 smithi070 ceph-mon[192652]: pgmap v276: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 6364/184830 objects degraded (3.443%); 746/184830 objects misplaced (0.404%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:16:12.972 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:12 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6364/184830 objects degraded (3.443%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:12.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:12.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:12 smithi026 ceph-mon[223014]: pgmap v276: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 6364/184830 objects degraded (3.443%); 746/184830 objects misplaced (0.404%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:16:12.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:12 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6364/184830 objects degraded (3.443%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:12.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:12 smithi026 ceph-mon[215775]: pgmap v276: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 6364/184830 objects degraded (3.443%); 746/184830 objects misplaced (0.404%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:16:12.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:12 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6364/184830 objects degraded (3.443%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:13.004 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:16:13.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:16:13.062 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:16:13.062 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:16:13.062 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:16:13.062 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:16:13.062 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:16:13.062 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:16:13.063 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:16:13.063 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:16:13.074 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:13.111 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:13.157 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:13.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:13.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:13.165 INFO:teuthology.orchestra.run.smithi026.stdout:moved 0 / 267 (0%) 2023-09-23T22:16:13.165 INFO:teuthology.orchestra.run.smithi026.stdout:avg 33.375 2023-09-23T22:16:13.165 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 4.09077 -> 4.09077 (expected baseline 5.40399) 2023-09-23T22:16:13.165 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-09-23T22:16:13.165 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.4 with 39 -> 39 pgs (1.16854 -> 1.16854 * mean) 2023-09-23T22:16:13.165 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:16:13.165 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T22:16:13.165 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 0.05 2023-09-23T22:16:13.165 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:16:13.166 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0170 2023-09-23T22:16:13.166 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.1705 2023-09-23T22:16:13.166 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T22:16:13.262 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:13.296 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 65117 65101 78.2282 63.4375 0.00507291 0.0127782 2023-09-23T22:16:13.309 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:13.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:13.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:13.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:13.502 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:13.555 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:13.635 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:13.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:13.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:13.741 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:13.767 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:13.821 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:13.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:13.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:13.934 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2293903103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:13.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2293903103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:14.001 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2293903103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:14.151 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:16:14.296 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 66137 66121 77.9549 63.75 0.00282553 0.0128185 2023-09-23T22:16:14.694 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:14 smithi026 ceph-mon[215775]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 6364/186477 objects degraded (3.413%); 746/186477 objects misplaced (0.400%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:16:14.695 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:14 smithi026 ceph-mon[215775]: from='client.54526 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:14.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:14 smithi026 ceph-mon[223014]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 6364/186477 objects degraded (3.413%); 746/186477 objects misplaced (0.400%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:16:14.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:14 smithi026 ceph-mon[223014]: from='client.54526 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:15.002 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T22:16:15.003 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T22:16:15.086 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:15.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:15.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:15.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:14 smithi070 ceph-mon[192652]: pgmap v277: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 52 MiB/s wr, 825 op/s; 6364/186477 objects degraded (3.413%); 746/186477 objects misplaced (0.400%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:16:15.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:14 smithi070 ceph-mon[192652]: from='client.54526 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:15.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:15.238 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:15.298 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 67012 66996 77.5211 54.6875 0.0200767 0.012894 2023-09-23T22:16:15.315 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:15.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:15.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:15.394 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:15.422 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:15.473 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:15.552 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:15.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:15.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:15.662 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:15.706 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:15.786 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:15.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:15.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:15 smithi070 ceph-mon[192652]: pgmap v278: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 6364/194295 objects degraded (3.275%); 165/194295 objects misplaced (0.085%); 11 MiB/s, 171 objects/s recovering 2023-09-23T22:16:15.893 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:15.954 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:15 smithi026 ceph-mon[215775]: pgmap v278: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 6364/194295 objects degraded (3.275%); 165/194295 objects misplaced (0.085%); 11 MiB/s, 171 objects/s recovering 2023-09-23T22:16:15.954 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:15 smithi026 ceph-mon[223014]: pgmap v278: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 6364/194295 objects degraded (3.275%); 165/194295 objects misplaced (0.085%); 11 MiB/s, 171 objects/s recovering 2023-09-23T22:16:15.965 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:16.070 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:16:16.070 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:16:16.299 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 68093 68077 77.3399 67.5625 0.00888927 0.0129243 2023-09-23T22:16:17.043 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:16:17.298 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 68853 68837 76.8071 47.5 0.00337303 0.012919 2023-09-23T22:16:17.837 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:17.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:17.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:17.944 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:17.999 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:18.061 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[215775]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[215775]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[215775]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T22:16:18.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[215775]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[215775]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[215775]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[215775]: pgmap v280: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 6364/200094 objects degraded (3.181%); 165/200094 objects misplaced (0.082%); 10 MiB/s, 159 objects/s recovering 2023-09-23T22:16:18.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6364/194295 objects degraded (3.275%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:18.062 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[223014]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.063 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[223014]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.063 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[223014]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T22:16:18.063 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[223014]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.063 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[223014]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.063 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[223014]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.063 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[223014]: pgmap v280: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 6364/200094 objects degraded (3.181%); 165/200094 objects misplaced (0.082%); 10 MiB/s, 159 objects/s recovering 2023-09-23T22:16:18.063 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:17 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6364/194295 objects degraded (3.275%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:18.118 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:17 smithi070 ceph-mon[192652]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:17 smithi070 ceph-mon[192652]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:17 smithi070 ceph-mon[192652]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T22:16:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:17 smithi070 ceph-mon[192652]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:17 smithi070 ceph-mon[192652]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:17 smithi070 ceph-mon[192652]: failed to encode map e137 with expected crc 2023-09-23T22:16:18.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:17 smithi070 ceph-mon[192652]: pgmap v280: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 81 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 6364/200094 objects degraded (3.181%); 165/200094 objects misplaced (0.082%); 10 MiB/s, 159 objects/s recovering 2023-09-23T22:16:18.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:17 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6364/194295 objects degraded (3.275%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:18.145 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:16:18.193 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:18.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:18.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:18.299 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 69317 69301 75.9684 29 0.00834348 0.0131559 2023-09-23T22:16:18.315 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:18.420 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:18.591 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:18.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:18.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:18.700 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:18.794 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:18.913 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:18.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:18.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:18.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[215775]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T22:16:18.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[215775]: failed to encode map e138 with expected crc 2023-09-23T22:16:18.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[215775]: failed to encode map e138 with expected crc 2023-09-23T22:16:18.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[215775]: failed to encode map e138 with expected crc 2023-09-23T22:16:18.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[215775]: failed to encode map e138 with expected crc 2023-09-23T22:16:18.943 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[223014]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T22:16:18.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[223014]: failed to encode map e138 with expected crc 2023-09-23T22:16:18.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[223014]: failed to encode map e138 with expected crc 2023-09-23T22:16:18.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[223014]: failed to encode map e138 with expected crc 2023-09-23T22:16:18.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:18 smithi026 ceph-mon[223014]: failed to encode map e138 with expected crc 2023-09-23T22:16:18.973 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:19.029 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:19.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:18 smithi070 ceph-mon[192652]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T22:16:19.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:18 smithi070 ceph-mon[192652]: failed to encode map e138 with expected crc 2023-09-23T22:16:19.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:18 smithi070 ceph-mon[192652]: failed to encode map e138 with expected crc 2023-09-23T22:16:19.061 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:18 smithi070 ceph-mon[192652]: failed to encode map e138 with expected crc 2023-09-23T22:16:19.062 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:18 smithi070 ceph-mon[192652]: failed to encode map e138 with expected crc 2023-09-23T22:16:19.065 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:19.132 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:19.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:19.240 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:19.299 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 70223 70207 75.6349 56.625 0.0126054 0.013217 2023-09-23T22:16:19.314 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:19.801 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:16:19.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:16:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:16:19] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-09-23T22:16:19.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:19 smithi026 ceph-mon[223014]: pgmap v282: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 81 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s; 5266/202497 objects degraded (2.601%); 1067/202497 objects misplaced (0.527%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:16:19.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/489987559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:16:19.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:16:19.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:19 smithi026 ceph-mon[215775]: pgmap v282: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 81 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s; 5266/202497 objects degraded (2.601%); 1067/202497 objects misplaced (0.527%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:16:19.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/489987559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:16:19.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:16:20.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:19 smithi070 ceph-mon[192652]: pgmap v282: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 81 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 46 MiB/s wr, 735 op/s; 5266/202497 objects degraded (2.601%); 1067/202497 objects misplaced (0.527%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:16:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/489987559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:16:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:16:20.299 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 71277 71261 75.4694 65.875 0.0127785 0.0132459 2023-09-23T22:16:20.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:20.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:16:20.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[223014]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T22:16:20.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[223014]: failed to encode map e139 with expected crc 2023-09-23T22:16:20.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[223014]: failed to encode map e139 with expected crc 2023-09-23T22:16:20.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:20.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:16:20.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[215775]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T22:16:20.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[215775]: failed to encode map e139 with expected crc 2023-09-23T22:16:20.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:20 smithi026 ceph-mon[215775]: failed to encode map e139 with expected crc 2023-09-23T22:16:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:20 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:20 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:16:21.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:20 smithi070 ceph-mon[192652]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T22:16:21.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:20 smithi070 ceph-mon[192652]: failed to encode map e139 with expected crc 2023-09-23T22:16:21.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:20 smithi070 ceph-mon[192652]: failed to encode map e139 with expected crc 2023-09-23T22:16:21.287 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:21.300 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:16:21.300455+0000 min lat: 0.00108301 max lat: 2.08369 avg lat: 0.0132814 2023-09-23T22:16:21.300 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:16:21.300 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 72292 72276 75.2681 63.4375 0.0157292 0.0132814 2023-09-23T22:16:21.314 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:21.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:21.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:21.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:21.522 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:21.685 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:21.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:21.793 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:21.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:21 smithi026 ceph-mon[215775]: pgmap v284: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 4154/213147 objects degraded (1.949%); 1650/213147 objects misplaced (0.774%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T22:16:21.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:21 smithi026 ceph-mon[223014]: pgmap v284: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 4154/213147 objects degraded (1.949%); 1650/213147 objects misplaced (0.774%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T22:16:21.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:22.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:21 smithi070 ceph-mon[192652]: pgmap v284: 89 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s; 4154/213147 objects degraded (1.949%); 1650/213147 objects misplaced (0.774%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T22:16:22.300 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 73288 73272 75.0546 62.25 0.0162318 0.0133187 2023-09-23T22:16:22.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:22 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4154/213147 objects degraded (1.949%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:23.093 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:23.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:23.100 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:22 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4154/213147 objects degraded (1.949%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:23.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:22 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4154/213147 objects degraded (1.949%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:23.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:23.254 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:23.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 74271 74255 74.8291 61.4375 0.0105094 0.0133587 2023-09-23T22:16:23.342 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:23.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:23.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:23.451 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:23.500 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:16:23.526 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:24.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:23 smithi070 ceph-mon[192652]: pgmap v285: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 4154/214863 objects degraded (1.933%); 1250/214863 objects misplaced (0.582%); 11 MiB/s, 178 objects/s recovering 2023-09-23T22:16:24.156 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:24.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:24.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:23 smithi026 ceph-mon[215775]: pgmap v285: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 4154/214863 objects degraded (1.933%); 1250/214863 objects misplaced (0.582%); 11 MiB/s, 178 objects/s recovering 2023-09-23T22:16:24.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:23 smithi026 ceph-mon[223014]: pgmap v285: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 49 MiB/s wr, 783 op/s; 4154/214863 objects degraded (1.933%); 1250/214863 objects misplaced (0.582%); 11 MiB/s, 178 objects/s recovering 2023-09-23T22:16:24.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:24.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 75206 75190 74.568 58.4375 0.0235519 0.0134058 2023-09-23T22:16:24.338 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:24.640 INFO:teuthology.orchestra.run.smithi026.stdout:moved 0 / 267 (0%) 2023-09-23T22:16:24.640 INFO:teuthology.orchestra.run.smithi026.stdout:avg 33.375 2023-09-23T22:16:24.640 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 4.09077 -> 4.09077 (expected baseline 5.40399) 2023-09-23T22:16:24.640 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-09-23T22:16:24.640 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.4 with 39 -> 39 pgs (1.16854 -> 1.16854 * mean) 2023-09-23T22:16:24.641 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:16:24.641 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T22:16:24.641 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 3 2023-09-23T22:16:24.641 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:16:24.641 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0196 2023-09-23T22:16:24.641 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.1959 2023-09-23T22:16:24.653 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T22:16:24.665 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:24.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:24.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:24.698 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:24.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:24.890 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:24.911 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:16:24.911 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:16:25.045 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:16:25.092 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:16:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:24 smithi070 ceph-mon[192652]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:25.135 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:16:25.173 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:16:25.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:16:25.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:25.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:24 smithi026 ceph-mon[223014]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:25.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:25.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:24 smithi026 ceph-mon[215775]: from='client.44660 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:25.244 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:16:25.288 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:16:25.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 76074 76058 74.2506 54.25 0.00909221 0.0134629 2023-09-23T22:16:25.324 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:16:25.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:16:25.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:16:25.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:16:25.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:16:25.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:16:25.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:16:25.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:16:25.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:16:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/622357336' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:25 smithi070 ceph-mon[192652]: pgmap v286: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 4154/221364 objects degraded (1.877%); 1250/221364 objects misplaced (0.565%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T22:16:26.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/622357336' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:26.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:25 smithi026 ceph-mon[215775]: pgmap v286: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 4154/221364 objects degraded (1.877%); 1250/221364 objects misplaced (0.565%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T22:16:26.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/622357336' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:26.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:25 smithi026 ceph-mon[223014]: pgmap v286: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 4154/221364 objects degraded (1.877%); 1250/221364 objects misplaced (0.565%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T22:16:26.306 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 77114 77098 74.1082 65 0.0132451 0.0134891 2023-09-23T22:16:26.641 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:26.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:26.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:26.745 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:26.771 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:26.812 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:26.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:26.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:26.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:26.956 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:27.048 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:27.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:27.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:27.107 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T22:16:27.107 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T22:16:27.161 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:27.198 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:27.271 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:27.328 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 77929 77913 73.7571 50.9375 0.00639985 0.013542 2023-09-23T22:16:27.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:27.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:27.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:27.513 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:27.827 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:16:27.846 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:27.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:27.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:27.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:28.075 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:28.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:28 smithi026 ceph-mon[223014]: pgmap v287: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 4154/227487 objects degraded (1.826%); 928/227487 objects misplaced (0.408%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T22:16:28.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4154/227487 objects degraded (1.826%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:28.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:28 smithi026 ceph-mon[215775]: pgmap v287: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 4154/227487 objects degraded (1.826%); 928/227487 objects misplaced (0.408%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T22:16:28.115 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4154/227487 objects degraded (1.826%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:28.306 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 78858 78842 73.5229 58.0625 0.0148113 0.0135963 2023-09-23T22:16:28.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:28 smithi070 ceph-mon[192652]: pgmap v287: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s; 4154/227487 objects degraded (1.826%); 928/227487 objects misplaced (0.408%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T22:16:28.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4154/227487 objects degraded (1.826%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:28.791 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:28.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:28.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:28.902 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:28.953 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:29.013 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:29.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:29.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:29.066 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:29.121 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:29.202 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:29.213 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:29.306 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 79580 79564 73.1051 45.125 0.00875177 0.0136635 2023-09-23T22:16:29.363 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:29.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:29.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:29.471 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:29.505 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:29.671 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:29.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:29.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:29.806 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:29.838 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:16:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:16:29] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.43.0" 2023-09-23T22:16:29.855 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:29.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:29.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:29.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:16:29.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:30.030 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:30.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[223014]: pgmap v288: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s; 4154/230118 objects degraded (1.805%); 470/230118 objects misplaced (0.204%); 11 MiB/s, 170 objects/s recovering 2023-09-23T22:16:30.149 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:16:30.306 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 79719 79703 72.1717 8.6875 0.0267417 0.0138502 2023-09-23T22:16:30.322 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:30.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:30.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:30.426 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[223014]: failed to encode map e140 with expected crc 2023-09-23T22:16:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[223014]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T22:16:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[223014]: failed to encode map e140 with expected crc 2023-09-23T22:16:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[223014]: failed to encode map e140 with expected crc 2023-09-23T22:16:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4009934136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:30.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:30.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[215775]: pgmap v288: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s; 4154/230118 objects degraded (1.805%); 470/230118 objects misplaced (0.204%); 11 MiB/s, 170 objects/s recovering 2023-09-23T22:16:30.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[215775]: failed to encode map e140 with expected crc 2023-09-23T22:16:30.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[215775]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T22:16:30.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[215775]: failed to encode map e140 with expected crc 2023-09-23T22:16:30.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[215775]: failed to encode map e140 with expected crc 2023-09-23T22:16:30.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4009934136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:30.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:30 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:30.487 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:30.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:30 smithi070 ceph-mon[192652]: pgmap v288: 89 pgs: 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 58 MiB/s wr, 920 op/s; 4154/230118 objects degraded (1.805%); 470/230118 objects misplaced (0.204%); 11 MiB/s, 170 objects/s recovering 2023-09-23T22:16:30.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:30 smithi070 ceph-mon[192652]: failed to encode map e140 with expected crc 2023-09-23T22:16:30.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:30 smithi070 ceph-mon[192652]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T22:16:30.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:30 smithi070 ceph-mon[192652]: failed to encode map e140 with expected crc 2023-09-23T22:16:30.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:30 smithi070 ceph-mon[192652]: failed to encode map e140 with expected crc 2023-09-23T22:16:30.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4009934136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:30.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:30 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:31.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 80158 80142 71.5311 27.4375 0.00792472 0.0139749 2023-09-23T22:16:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:16:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[215775]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[215775]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:16:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[215775]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T22:16:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[215775]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[215775]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:16:31.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[223014]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[223014]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:16:31.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[223014]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T22:16:31.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[223014]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:31 smithi026 ceph-mon[223014]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:31 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:16:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:31 smithi070 ceph-mon[192652]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:31 smithi070 ceph-mon[192652]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:31 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:16:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:31 smithi070 ceph-mon[192652]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T22:16:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:31 smithi070 ceph-mon[192652]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:31 smithi070 ceph-mon[192652]: failed to encode map e141 with expected crc 2023-09-23T22:16:31.911 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:31.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:31.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:32.019 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:32.077 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:32.174 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:32.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:32.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:32.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[215775]: pgmap v291: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 4154/238944 objects degraded (1.738%); 46/238944 objects misplaced (0.019%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T22:16:32.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[215775]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T22:16:32.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[215775]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[215775]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[215775]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[215775]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[223014]: pgmap v291: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 4154/238944 objects degraded (1.738%); 46/238944 objects misplaced (0.019%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T22:16:32.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[223014]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T22:16:32.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[223014]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[223014]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[223014]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:32 smithi026 ceph-mon[223014]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.283 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:32.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 81046 81030 71.3053 55.5 0.028427 0.0140174 2023-09-23T22:16:32.338 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:32 smithi070 ceph-mon[192652]: pgmap v291: 89 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 83 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 4154/238944 objects degraded (1.738%); 46/238944 objects misplaced (0.019%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T22:16:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:32 smithi070 ceph-mon[192652]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T22:16:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:32 smithi070 ceph-mon[192652]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:32 smithi070 ceph-mon[192652]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:32 smithi070 ceph-mon[192652]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:32 smithi070 ceph-mon[192652]: failed to encode map e142 with expected crc 2023-09-23T22:16:32.476 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:32.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:32.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:32.591 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:32.593 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:32.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:32.823 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:32.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:32.940 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:33.003 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:33.261 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4154/238944 objects degraded (1.738%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:33.262 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4154/238944 objects degraded (1.738%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:33.267 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:16:33.267 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:16:33.267 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:16:33.267 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:16:33.268 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:16:33.292 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:33.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:33.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:33.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 81917 81901 71.071 54.4375 0.0246939 0.0140644 2023-09-23T22:16:33.399 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:33.434 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4154/238944 objects degraded (1.738%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:33.436 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:33.510 INFO:teuthology.orchestra.run.smithi026.stdout:moved 0 / 267 (0%) 2023-09-23T22:16:33.511 INFO:teuthology.orchestra.run.smithi026.stdout:avg 33.375 2023-09-23T22:16:33.511 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 4.09077 -> 4.09077 (expected baseline 5.40399) 2023-09-23T22:16:33.511 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 26 -> 26 pgs (0.779026 -> 0.779026 * mean) 2023-09-23T22:16:33.512 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.4 with 39 -> 39 pgs (1.16854 -> 1.16854 * mean) 2023-09-23T22:16:33.512 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:16:33.512 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T22:16:33.512 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 3 2023-09-23T22:16:33.512 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:16:33.512 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0220 2023-09-23T22:16:33.512 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.2200 2023-09-23T22:16:33.517 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:33.524 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T22:16:33.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:33.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:33.629 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:33.689 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:34.289 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:16:34.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 82986 82970 71.0126 66.8125 0.00288069 0.0140719 2023-09-23T22:16:34.717 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:34.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:34.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:34.823 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:34.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:34 smithi070 ceph-mon[192652]: pgmap v293: 89 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s; 3080/240045 objects degraded (1.283%); 522/240045 objects misplaced (0.217%); 15 MiB/s, 246 objects/s recovering 2023-09-23T22:16:34.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:34 smithi070 ceph-mon[192652]: from='client.54547 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:34.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:34 smithi026 ceph-mon[215775]: pgmap v293: 89 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s; 3080/240045 objects degraded (1.283%); 522/240045 objects misplaced (0.217%); 15 MiB/s, 246 objects/s recovering 2023-09-23T22:16:34.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:34 smithi026 ceph-mon[215775]: from='client.54547 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:34.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:34 smithi026 ceph-mon[223014]: pgmap v293: 89 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s; 3080/240045 objects degraded (1.283%); 522/240045 objects misplaced (0.217%); 15 MiB/s, 246 objects/s recovering 2023-09-23T22:16:34.897 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:34 smithi026 ceph-mon[223014]: from='client.54547 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:16:34.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:35.002 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:16:35.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 15 83776 83761 70.721 49.4375 0.0179921 0.0141346 2023-09-23T22:16:35.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:35.525 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:35.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:35.626 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:35.666 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:36.130 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:36.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:36.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:36.238 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:36.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 84660 84644 70.5139 55.1875 0.0140187 0.0141765 2023-09-23T22:16:36.318 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:36.452 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:36.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:36.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:36.501 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:36.525 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:36.561 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:36.593 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:36.654 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:36.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:36.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:36 smithi026 ceph-mon[223014]: pgmap v294: 89 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 3080/245133 objects degraded (1.256%); 522/245133 objects misplaced (0.213%); 11 MiB/s, 170 objects/s recovering 2023-09-23T22:16:36.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:36 smithi026 ceph-mon[215775]: pgmap v294: 89 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 3080/245133 objects degraded (1.256%); 522/245133 objects misplaced (0.213%); 11 MiB/s, 170 objects/s recovering 2023-09-23T22:16:36.763 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:36.847 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:36 smithi070 ceph-mon[192652]: pgmap v294: 89 pgs: 1 active+recovery_wait+remapped, 1 peering, 1 active+recovering+undersized+remapped, 84 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 52 MiB/s wr, 833 op/s; 3080/245133 objects degraded (1.256%); 522/245133 objects misplaced (0.213%); 11 MiB/s, 170 objects/s recovering 2023-09-23T22:16:37.197 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:16:37.219 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:37.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:37.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:37.238 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:16:37.283 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:16:37.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 85562 85546 70.3278 56.375 0.0168047 0.0142137 2023-09-23T22:16:37.329 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:37.331 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:16:37.376 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:16:37.388 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:37.415 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:16:37.451 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:37.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:37.463 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:37.468 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:16:37.520 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:16:37.530 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:16:37.530 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:16:37.531 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:16:37.531 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:16:37.531 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:16:37.531 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:16:37.531 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:16:37.531 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:16:37.563 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:37.617 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:37.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:37 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3574271054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:37.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:37 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3080/245133 objects degraded (1.256%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:37.659 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:37.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:37.766 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:37.814 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:37.913 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:37.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:37.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:37.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:37 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3574271054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:37.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:37 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3080/245133 objects degraded (1.256%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:37.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:37 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3574271054' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:37.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:37 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3080/245133 objects degraded (1.256%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:38.027 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:38.105 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:38.309 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 86410 86394 70.1018 53 0.0994657 0.0142529 2023-09-23T22:16:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:38 smithi026 ceph-mon[223014]: pgmap v295: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 85 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 3080/250602 objects degraded (1.229%); 522/250602 objects misplaced (0.208%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T22:16:38.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:16:38.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:38 smithi026 ceph-mon[215775]: pgmap v295: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 85 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 3080/250602 objects degraded (1.229%); 522/250602 objects misplaced (0.208%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T22:16:38.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:16:39.063 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:39.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:38 smithi070 ceph-mon[192652]: pgmap v295: 89 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 85 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 54 MiB/s wr, 862 op/s; 3080/250602 objects degraded (1.229%); 522/250602 objects misplaced (0.208%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T22:16:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:16:39.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:39.197 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:39.242 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:39.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:39.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:39.310 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 86973 86957 69.6542 35.1875 0.00382529 0.0142585 2023-09-23T22:16:39.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:39.379 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:39.936 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:39.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:39.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:39.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[215775]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T22:16:39.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[215775]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[215775]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[215775]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[215775]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[215775]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:39.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:39.944 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:16:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:16:39] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.43.0" 2023-09-23T22:16:39.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[223014]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T22:16:39.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[223014]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[223014]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[223014]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[223014]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[223014]: failed to encode map e143 with expected crc 2023-09-23T22:16:39.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:39.945 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:40.043 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:39 smithi070 ceph-mon[192652]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T22:16:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:39 smithi070 ceph-mon[192652]: failed to encode map e143 with expected crc 2023-09-23T22:16:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:39 smithi070 ceph-mon[192652]: failed to encode map e143 with expected crc 2023-09-23T22:16:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:39 smithi070 ceph-mon[192652]: failed to encode map e143 with expected crc 2023-09-23T22:16:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:39 smithi070 ceph-mon[192652]: failed to encode map e143 with expected crc 2023-09-23T22:16:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:39 smithi070 ceph-mon[192652]: failed to encode map e143 with expected crc 2023-09-23T22:16:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:40.139 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:40.286 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:40.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:40.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:40.309 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 87506 87490 69.1943 33.3125 0.0314977 0.014447 2023-09-23T22:16:40.396 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:40.459 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:40.700 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:40.706 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:40 smithi070 ceph-mon[192652]: pgmap v297: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 85 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 37 MiB/s wr, 585 op/s; 3080/253002 objects degraded (1.217%); 62/253002 objects misplaced (0.025%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T22:16:40.706 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:40 smithi070 ceph-mon[192652]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T22:16:40.706 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:40 smithi070 ceph-mon[192652]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.706 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:40 smithi070 ceph-mon[192652]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.707 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:40 smithi070 ceph-mon[192652]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.707 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:40 smithi070 ceph-mon[192652]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:40.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:40.808 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:40.881 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:16:40.881 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:16:40.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[215775]: pgmap v297: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 85 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 37 MiB/s wr, 585 op/s; 3080/253002 objects degraded (1.217%); 62/253002 objects misplaced (0.025%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T22:16:40.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[215775]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T22:16:40.908 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[215775]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[215775]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[215775]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.909 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[215775]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[223014]: pgmap v297: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 85 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 37 MiB/s wr, 585 op/s; 3080/253002 objects degraded (1.217%); 62/253002 objects misplaced (0.025%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T22:16:40.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[223014]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T22:16:40.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[223014]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[223014]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[223014]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:40 smithi026 ceph-mon[223014]: failed to encode map e144 with expected crc 2023-09-23T22:16:40.929 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:41.231 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 6, 7, 0] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:16:41.231 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:16:41.232 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [1, 2, 3, 4, 5, 6, 7, 0] 2023-09-23T22:16:41.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T22:16:41.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:16:41.314736+0000 min lat: 0.00108301 max lat: 2.08369 avg lat: 0.014484 2023-09-23T22:16:41.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:16:41.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 88389 88373 69.0157 55.1875 0.0138668 0.014484 2023-09-23T22:16:41.589 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:41.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:41.706 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:41.730 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:16:41.753 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:41.822 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:16:41.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/153658061' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:16:41.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:41 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:16:41.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/153658061' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:16:41.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:41 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:16:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/153658061' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:16:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:41 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:16:42.135 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:42.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:42.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:42.247 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:42.305 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:42.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 89211 89195 68.7979 51.375 0.00835681 0.0145286 2023-09-23T22:16:42.399 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:42.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:42.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:42.476 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:16:42.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:42.581 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: pgmap v299: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s; 15/262509 objects degraded (0.006%); 2927/262509 objects misplaced (1.115%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T22:16:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: failed to encode map e145 with expected crc 2023-09-23T22:16:42.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: failed to encode map e145 with expected crc 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: failed to encode map e145 with expected crc 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15/262509 objects degraded (0.006%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: pgmap v299: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s; 15/262509 objects degraded (0.006%); 2927/262509 objects misplaced (1.115%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: failed to encode map e145 with expected crc 2023-09-23T22:16:42.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:16:42.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T22:16:42.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: failed to encode map e145 with expected crc 2023-09-23T22:16:42.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: failed to encode map e145 with expected crc 2023-09-23T22:16:42.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:42.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:42.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:42 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15/262509 objects degraded (0.006%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:16:42.987 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:16:43.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: pgmap v299: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 86 active+clean, 1 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s; 15/262509 objects degraded (0.006%); 2927/262509 objects misplaced (1.115%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T22:16:43.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: failed to encode map e145 with expected crc 2023-09-23T22:16:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:16:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T22:16:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: failed to encode map e145 with expected crc 2023-09-23T22:16:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: failed to encode map e145 with expected crc 2023-09-23T22:16:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:43.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:42 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15/262509 objects degraded (0.006%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:16:43.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 89980 89964 68.5451 48.0625 0.010563 0.0145849 2023-09-23T22:16:43.396 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:43.405 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:43.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:43.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:43.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:43.569 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:43.613 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:43.649 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:43.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:43.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:43.753 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:43.801 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:43 smithi070 ceph-mon[192652]: pgmap v301: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 15/263214 objects degraded (0.006%); 2927/263214 objects misplaced (1.112%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T22:16:43.892 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:43.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:43.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:43.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:43 smithi026 ceph-mon[223014]: pgmap v301: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 15/263214 objects degraded (0.006%); 2927/263214 objects misplaced (1.112%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T22:16:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:43 smithi026 ceph-mon[215775]: pgmap v301: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 700 op/s; 15/263214 objects degraded (0.006%); 2927/263214 objects misplaced (1.112%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T22:16:43.998 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:44.100 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:44.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 90870 90854 68.3894 55.625 0.0197684 0.014617 2023-09-23T22:16:44.442 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:44.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:44.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:44.561 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:44.617 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:44.702 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:44.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:44.718 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:44.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:44.874 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:45.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 91549 91533 68.0805 42.4375 0.0113172 0.0146549 2023-09-23T22:16:45.434 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:16:45.999 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:46.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:46.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:46.118 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:46.234 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:46 smithi026 ceph-mon[215775]: pgmap v302: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 15/268758 objects degraded (0.006%); 2927/268758 objects misplaced (1.089%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:16:46.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:46 smithi026 ceph-mon[223014]: pgmap v302: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 15/268758 objects degraded (0.006%); 2927/268758 objects misplaced (1.089%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:16:46.237 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:46.310 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:46.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 92275 92259 67.8134 45.375 0.00953465 0.0147413 2023-09-23T22:16:46.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:46 smithi070 ceph-mon[192652]: pgmap v302: 89 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 5.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 54 MiB/s wr, 865 op/s; 15/268758 objects degraded (0.006%); 2927/268758 objects misplaced (1.089%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:16:46.474 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:16:46.475 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:16:46.696 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:46.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:46.816 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:46.873 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:47.036 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:47.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:47.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:47.144 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:47.222 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:47.242 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:16:47.304 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:16:47.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 92967 92951 67.5275 43.25 0.047668 0.0148023 2023-09-23T22:16:47.353 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:16:47.354 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:16:47.396 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:16:47.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:16:47.435 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:47.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:47.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:47.496 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:16:47.539 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:16:47.548 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:47.575 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:16:47.577 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:16:47.577 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:16:47.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:16:47.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:16:47.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:16:47.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:16:47.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:16:47.578 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:16:47.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:47.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:47 smithi026 conmon[215751]: 2023-09-23T22:16:47.497+0000 7f4580f04700 -1 mon.a@0(leader).osd e145 definitely_dead 0 2023-09-23T22:16:47.916 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:16:48.262 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.6. 2023-09-23T22:16:48.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 93719 93703 67.2915 47 0.0137871 0.0148515 2023-09-23T22:16:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[215775]: pgmap v303: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 86 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 15/274563 objects degraded (0.005%); 2740/274563 objects misplaced (0.998%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:16:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3724494680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2441071557' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:16:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:16:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[215775]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T22:16:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15/274563 objects degraded (0.005%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:16:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[223014]: pgmap v303: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 86 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 15/274563 objects degraded (0.005%); 2740/274563 objects misplaced (0.998%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:16:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3724494680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2441071557' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:16:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:16:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[223014]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T22:16:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15/274563 objects degraded (0.005%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:16:48.524 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:48 smithi070 ceph-mon[192652]: pgmap v303: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 86 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 15/274563 objects degraded (0.005%); 2740/274563 objects misplaced (0.998%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:16:48.524 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3724494680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:48.524 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2441071557' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:16:48.524 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:48 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T22:16:48.524 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:48 smithi070 ceph-mon[192652]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T22:16:48.524 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15/274563 objects degraded (0.005%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:16:49.301 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:49.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:49.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 93734 93718 66.5377 0.9375 0.00318632 0.0148504 2023-09-23T22:16:49.404 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:49.419 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:49 smithi070 ceph-mon[192652]: failed to encode map e146 with expected crc 2023-09-23T22:16:49.419 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:49 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T22:16:49.419 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:49 smithi070 ceph-mon[192652]: osdmap e146: 8 total, 8 up, 7 in 2023-09-23T22:16:49.419 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:49 smithi070 ceph-mon[192652]: failed to encode map e146 with expected crc 2023-09-23T22:16:49.442 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:49 smithi026 ceph-mon[215775]: failed to encode map e146 with expected crc 2023-09-23T22:16:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:49 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T22:16:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:49 smithi026 ceph-mon[215775]: osdmap e146: 8 total, 8 up, 7 in 2023-09-23T22:16:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:49 smithi026 ceph-mon[215775]: failed to encode map e146 with expected crc 2023-09-23T22:16:49.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:49 smithi026 ceph-mon[223014]: failed to encode map e146 with expected crc 2023-09-23T22:16:49.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:49 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T22:16:49.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:49 smithi026 ceph-mon[223014]: osdmap e146: 8 total, 8 up, 7 in 2023-09-23T22:16:49.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:49 smithi026 ceph-mon[223014]: failed to encode map e146 with expected crc 2023-09-23T22:16:49.587 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:49.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:49.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:49.643 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:16:49.644 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:16:49.644 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:16:49.644 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:16:49.644 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:16:49.692 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:49.720 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:49.914 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:49.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:49.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:16:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:16:49] "GET /metrics HTTP/1.1" 200 36437 "" "Prometheus/2.43.0" 2023-09-23T22:16:50.017 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:50.043 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:50.085 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:50.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:50.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:50.188 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:50.213 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:50.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 93735 93719 65.7897 0.0625 2.00964 0.0148716 2023-09-23T22:16:50.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: pgmap v305: 89 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 5.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 15/275979 objects degraded (0.005%); 2740/275979 objects misplaced (0.993%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:16:50.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: osdmap e147: 8 total, 8 up, 7 in 2023-09-23T22:16:50.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:50 smithi070 ceph-mon[192652]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.385 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:50.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:50.463 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:16:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: pgmap v305: 89 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 5.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 15/275979 objects degraded (0.005%); 2740/275979 objects misplaced (0.993%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:16:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: osdmap e147: 8 total, 8 up, 7 in 2023-09-23T22:16:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[223014]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: pgmap v305: 89 pgs: 4 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 82 active+clean; 5.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 35 MiB/s wr, 560 op/s; 15/275979 objects degraded (0.005%); 2740/275979 objects misplaced (0.993%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: osdmap e147: 8 total, 8 up, 7 in 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:50 smithi026 ceph-mon[215775]: failed to encode map e147 with expected crc 2023-09-23T22:16:50.490 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:50.543 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:50.590 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:50.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:50.696 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:50.723 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:50.773 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:50.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:50.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:50.854 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:16:50.880 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:50.908 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:51.007 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:51.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:51.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:51.111 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:51.147 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:51.205 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:51.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:51.310 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:51.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 90 16 93805 93789 65.1075 4.375 0.00449867 0.0150909 2023-09-23T22:16:51.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: osdmap e148: 8 total, 8 up, 7 in 2023-09-23T22:16:51.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:51 smithi070 ceph-mon[192652]: 5.3 starting backfill to osd.3 from (0'0,123'2381] MIN to 145'5864 2023-09-23T22:16:51.338 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:51.388 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:51.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:51.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: osdmap e148: 8 total, 8 up, 7 in 2023-09-23T22:16:51.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[223014]: 5.3 starting backfill to osd.3 from (0'0,123'2381] MIN to 145'5864 2023-09-23T22:16:51.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: osdmap e148: 8 total, 8 up, 7 in 2023-09-23T22:16:51.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: failed to encode map e148 with expected crc 2023-09-23T22:16:51.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:51 smithi026 ceph-mon[215775]: 5.3 starting backfill to osd.3 from (0'0,123'2381] MIN to 145'5864 2023-09-23T22:16:51.492 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:51.520 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:51.563 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:51.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:51.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:51.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:51.695 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:51.736 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:51.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:51.840 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:51.865 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:52.018 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:52.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:52.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:52.122 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:52.151 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:52.160 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:16:52.245 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:52.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:52.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:52.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 91 16 93805 93789 64.3922 0 - 0.0150909 2023-09-23T22:16:52.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:52.352 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: pgmap v308: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 4 peering, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s; 4847/281349 objects degraded (1.723%); 8247/281349 objects misplaced (2.931%); 5.7 MiB/s, 93 objects/s recovering 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: failed to encode map e149 with expected crc 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: osdmap e149: 8 total, 8 up, 7 in 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: failed to encode map e149 with expected crc 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: failed to encode map e149 with expected crc 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2451748006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:16:52.353 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:52.380 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:52.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: pgmap v308: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 4 peering, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s; 4847/281349 objects degraded (1.723%); 8247/281349 objects misplaced (2.931%); 5.7 MiB/s, 93 objects/s recovering 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: failed to encode map e149 with expected crc 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: osdmap e149: 8 total, 8 up, 7 in 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: failed to encode map e149 with expected crc 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: failed to encode map e149 with expected crc 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2451748006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:16:52.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: pgmap v308: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 4 peering, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 77 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 699 op/s; 4847/281349 objects degraded (1.723%); 8247/281349 objects misplaced (2.931%); 5.7 MiB/s, 93 objects/s recovering 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: failed to encode map e149 with expected crc 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: osdmap e149: 8 total, 8 up, 7 in 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: failed to encode map e149 with expected crc 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: failed to encode map e149 with expected crc 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2451748006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:16:52.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:16:52.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:52.528 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:52.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:52.562 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:14264 5.1 5:85e28b51:::benchmark_data_smithi026_286012_object14263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.815534+0000", 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 86.744614209000005, 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.163901033, 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.563 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 14264 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815534+0000", 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815534+0000", 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815532+0000", 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815586+0000", 2023-09-23T22:16:52.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3952000000000001e-05 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815601+0000", 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5305999999999999e-05 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815607+0000", 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1090000000000003e-06 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815652+0000", 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4972000000000002e-05 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815745+0000", 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2986999999999996e-05 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815837+0000", 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1716000000000001e-05 2023-09-23T22:16:52.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816913+0000", 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001076453 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.818802+0000", 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001888729 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979332+0000", 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16052960099999999 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979388+0000", 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6750999999999999e-05 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979435+0000", 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6319000000000002e-05 2023-09-23T22:16:52.569 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:14265 5.1 5:89dc4bcb:::benchmark_data_smithi026_286012_object14264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.815655+0000", 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 86.744493109000004, 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16393613000000001, 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.570 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 14265 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815655+0000", 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815655+0000", 2023-09-23T22:16:52.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815654+0000", 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815736+0000", 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2218000000000005e-05 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815756+0000", 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9692000000000001e-05 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815762+0000", 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9569999999999999e-06 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816347+0000", 2023-09-23T22:16:52.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00058496199999999996 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816461+0000", 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113621 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816559+0000", 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8647999999999995e-05 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.817091+0000", 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000531585 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.818834+0000", 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001743639 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979500+0000", 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16066592499999999 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979547+0000", 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6186999999999997e-05 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979591+0000", 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4190999999999998e-05 2023-09-23T22:16:52.576 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:14299 5.1 5:8374d921:::benchmark_data_smithi026_286012_object14298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.828867+0000", 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 86.731281025000001, 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15088543700000001, 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.577 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 14299 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.828867+0000", 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.828867+0000", 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.828866+0000", 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.828913+0000", 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7284000000000003e-05 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.828924+0000", 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0601999999999999e-05 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.828928+0000", 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9480000000000001e-06 2023-09-23T22:16:52.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.828976+0000", 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7936999999999999e-05 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.829101+0000", 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000125227 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.829268+0000", 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016704999999999999 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.829913+0000", 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00064559199999999995 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.830496+0000", 2023-09-23T22:16:52.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00058290100000000004 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979666+0000", 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.149170049 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979715+0000", 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8996999999999999e-05 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979752+0000", 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6798000000000002e-05 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:14303 5.1 5:897f45e6:::benchmark_data_smithi026_286012_object14302:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.832643+0000", 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 86.727504433000007, 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14723971999999999, 2023-09-23T22:16:52.584 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 14303 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.832643+0000", 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.832643+0000", 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.832643+0000", 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.832732+0000", 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9823999999999999e-05 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.832744+0000", 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1476e-05 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.832748+0000", 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6299999999999997e-06 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.832780+0000", 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1868e-05 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.832860+0000", 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9315999999999998e-05 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.832956+0000", 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6139999999999998e-05 2023-09-23T22:16:52.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.833465+0000", 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00050933599999999999 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.833860+0000", 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039463200000000001 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979793+0000", 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14593277299999999 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979845+0000", 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2521000000000001e-05 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979883+0000", 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7997000000000002e-05 2023-09-23T22:16:52.590 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:15724 5.e 5:7c90e698:::benchmark_data_smithi026_286012_object15723:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:26.846230+0000", 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 85.713917539999997, 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.110275359, 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.591 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 15724 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.846230+0000", 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.846230+0000", 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.846230+0000", 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.846266+0000", 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6069000000000003e-05 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.846273+0000", 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0310000000000001e-06 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.846277+0000", 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6750000000000001e-06 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.846328+0000", 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0637000000000003e-05 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.846435+0000", 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010697200000000001 2023-09-23T22:16:52.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.846561+0000", 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012544300000000001 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.847672+0000", 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001111922 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.847753+0000", 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0332000000000003e-05 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.956425+0000", 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.108671752 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.956463+0000", 2023-09-23T22:16:52.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8676e-05 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.956506+0000", 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2370999999999998e-05 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.598 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:15728 5.e 5:7d2c8d46:::benchmark_data_smithi026_286012_object15727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:26.850252+0000", 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 85.709896220000005, 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.106420046, 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 15728 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.850252+0000", 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.850252+0000", 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.850251+0000", 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.850301+0000", 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0716e-05 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.850312+0000", 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0261999999999999e-05 2023-09-23T22:16:52.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.850316+0000", 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4510000000000002e-06 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.850357+0000", 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1105e-05 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.850443+0000", 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5377999999999995e-05 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.850538+0000", 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5728000000000002e-05 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.851130+0000", 2023-09-23T22:16:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059137299999999996 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.851448+0000", 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031784899999999999 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.956619+0000", 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.105171035 2023-09-23T22:16:52.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.956647+0000", 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8342000000000001e-05 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:26.956672+0000", 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4617000000000002e-05 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.605 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:16583 5.e 5:7c0090b6:::benchmark_data_smithi026_286012_object16582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:27.299983+0000", 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 85.260165174999997, 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.088374467999999998, 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.606 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 16583 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.299983+0000", 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.299983+0000", 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.299981+0000", 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300042+0000", 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0782999999999997e-05 2023-09-23T22:16:52.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300053+0000", 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1076e-05 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300058+0000", 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9119999999999997e-06 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300106+0000", 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7938000000000001e-05 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300210+0000", 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103893 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300342+0000", 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0001325 2023-09-23T22:16:52.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.301805+0000", 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014622159999999999 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.301892+0000", 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7762000000000004e-05 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.388273+0000", 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.086381026 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.388323+0000", 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9937e-05 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.388357+0000", 2023-09-23T22:16:52.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3695999999999997e-05 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:16585 5.e 5:73d8ecec:::benchmark_data_smithi026_286012_object16584:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:27.300075+0000", 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 85.260073164999994, 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.088395306000000007, 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 16585 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300075+0000", 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300075+0000", 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300074+0000", 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300104+0000", 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9740999999999999e-05 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300109+0000", 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.079e-06 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.300112+0000", 2023-09-23T22:16:52.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.038e-06 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.301063+0000", 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000950606 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.301171+0000", 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010872800000000001 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.301293+0000", 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121804 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.302070+0000", 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00077700100000000004 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.302982+0000", 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00091163799999999999 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.388396+0000", 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.085414285000000006 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.388436+0000", 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0024999999999997e-05 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.388470+0000", 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3782000000000003e-05 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:16602 5.1 5:8bf32fac:::benchmark_data_smithi026_286012_object16601:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:27.306202+0000", 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 85.253946107000004, 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083214897999999995, 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.621 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 16602 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.306202+0000", 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.306202+0000", 2023-09-23T22:16:52.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.306201+0000", 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.306248+0000", 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7590999999999999e-05 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.306258+0000", 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5699999999999999e-06 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.306274+0000", 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5675000000000002e-05 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.306349+0000", 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4833000000000007e-05 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.306461+0000", 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011268399999999999 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.306590+0000", 2023-09-23T22:16:52.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128687 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.307694+0000", 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011040500000000001 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.388208+0000", 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.080514286000000004 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.389319+0000", 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001110984 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.389373+0000", 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4128000000000001e-05 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:27.389417+0000", 2023-09-23T22:16:52.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3192999999999998e-05 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19249 5.e 5:741020f8:::benchmark_data_smithi026_286012_object19248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.612049+0000", 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 83.948098701999996, 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0748528140000002, 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:16:52.628 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19249 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612049+0000", 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612049+0000", 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612048+0000", 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612100+0000", 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2673999999999997e-05 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612113+0000", 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2835e-05 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612118+0000", 2023-09-23T22:16:52.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9479999999999999e-06 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612163+0000", 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4447999999999999e-05 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612311+0000", 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000148877 2023-09-23T22:16:52.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612464+0000", 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000152452 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.613528+0000", 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010644700000000001 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.613796+0000", 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026775600000000002 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.625193+0000", 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011396692999999999 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.625207+0000", 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4555e-05 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.622896+0000", 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.997689041 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.622909+0000", 2023-09-23T22:16:52.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2702e-05 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.686798+0000", 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0638887100000001 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.686902+0000", 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104128 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.636 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37003 5.e 5:74658dc9:::benchmark_data_smithi026_286012_object37002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.939383+0000", 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 69.620764315000002, 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0753498450000001, 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.637 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37003 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939383+0000", 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939383+0000", 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939382+0000", 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939413+0000", 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0855000000000002e-05 2023-09-23T22:16:52.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939426+0000", 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3127e-05 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939431+0000", 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0690000000000001e-06 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939491+0000", 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0344999999999999e-05 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939504+0000", 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2612999999999999e-05 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.006978+0000", 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0674743799999999 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.007104+0000", 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012563799999999999 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.007235+0000", 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000130636 2023-09-23T22:16:52.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.009090+0000", 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018550039999999999 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.009127+0000", 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7441999999999998e-05 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.013897+0000", 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047696400000000003 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.013966+0000", 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9115000000000005e-05 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.014733+0000", 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00076754000000000004 2023-09-23T22:16:52.644 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:39429 5.1 5:8ca29b3b:::benchmark_data_smithi026_286012_object39428:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:46.457274+0000", 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 66.102873892000005, 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.13378586200000001, 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.645 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 39429 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.457274+0000", 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.457274+0000", 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.457272+0000", 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.457304+0000", 2023-09-23T22:16:52.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1562e-05 2023-09-23T22:16:52.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.457316+0000", 2023-09-23T22:16:52.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2395000000000001e-05 2023-09-23T22:16:52.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.457323+0000", 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3849999999999999e-06 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.459908+0000", 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025847750000000001 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.460058+0000", 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000150035 2023-09-23T22:16:52.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.460192+0000", 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013397300000000001 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.461979+0000", 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017876700000000001 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.465766+0000", 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037869729999999999 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.590968+0000", 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12520176499999999 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.591021+0000", 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2849999999999997e-05 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.591060+0000", 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8896999999999997e-05 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:39451 5.1 5:88598b40:::benchmark_data_smithi026_286012_object39450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:46.469245+0000", 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 66.090902811999996, 2023-09-23T22:16:52.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12383136, 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 39451 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469245+0000", 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469245+0000", 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469243+0000", 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469298+0000", 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4809000000000003e-05 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469311+0000", 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3396e-05 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469317+0000", 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2879999999999998e-06 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469578+0000", 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026021699999999998 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469693+0000", 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115018 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469824+0000", 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013143599999999999 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.472701+0000", 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028774880000000001 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.476701+0000", 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003999781 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.592988+0000", 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.116287051 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.593039+0000", 2023-09-23T22:16:52.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0977000000000003e-05 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.593076+0000", 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7042999999999998e-05 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.660 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:39452 5.1 5:8f5696bf:::benchmark_data_smithi026_286012_object39451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:46.469483+0000", 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 66.090664859, 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.123905353, 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 39452 2023-09-23T22:16:52.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469483+0000", 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469483+0000", 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469482+0000", 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469527+0000", 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4931000000000001e-05 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469537+0000", 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.944e-06 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.469543+0000", 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7470000000000002e-06 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.471146+0000", 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016028850000000001 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.471264+0000", 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118714 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.471385+0000", 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012057099999999999 2023-09-23T22:16:52.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.475200+0000", 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038149719999999998 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.476732+0000", 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015318560000000001 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.593314+0000", 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.116582645 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.593357+0000", 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2589000000000002e-05 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.593388+0000", 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1188e-05 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:39453 5.e 5:7e19ff0a:::benchmark_data_smithi026_286012_object39452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:46.470361+0000", 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 66.089786462999996, 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.173090996, 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.668 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 39453 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.470361+0000", 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.470361+0000", 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.470360+0000", 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.470399+0000", 2023-09-23T22:16:52.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8155999999999999e-05 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.470409+0000", 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0383999999999999e-05 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.470415+0000", 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5879999999999997e-06 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.470466+0000", 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1558999999999999e-05 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.470609+0000", 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014243099999999999 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.470758+0000", 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000149689 2023-09-23T22:16:52.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.472327+0000", 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001568884 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.475250+0000", 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029224099999999999 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.642477+0000", 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16722741699999999 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.642526+0000", 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9203999999999998e-05 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.643452+0000", 2023-09-23T22:16:52.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00092616900000000002 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:39464 5.e 5:79abbd3c:::benchmark_data_smithi026_286012_object39463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:46.477127+0000", 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 66.083020622999996, 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.166903724, 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 39464 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.477127+0000", 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.477127+0000", 2023-09-23T22:16:52.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.477125+0000", 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.477155+0000", 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9456999999999999e-05 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.477171+0000", 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6118999999999999e-05 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.477177+0000", 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8499999999999999e-06 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.477235+0000", 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8281e-05 2023-09-23T22:16:52.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.477356+0000", 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012091699999999999 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.477524+0000", 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016756900000000001 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.481133+0000", 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036089619999999998 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.482536+0000", 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014035950000000001 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.643508+0000", 2023-09-23T22:16:52.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16097148999999999 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.643572+0000", 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4121999999999994e-05 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:46.644031+0000", 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045906099999999997 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.681 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:44135 5.e 5:7102c131:::benchmark_data_smithi026_286012_object44134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:50.285039+0000", 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 62.275108733000003, 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.086274459999999997, 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.682 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 44135 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.285039+0000", 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.285039+0000", 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.285038+0000", 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.285067+0000", 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8833e-05 2023-09-23T22:16:52.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.285075+0000", 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0879999999999995e-06 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.285080+0000", 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9269999999999996e-06 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.286672+0000", 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001592265 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.286810+0000", 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000137866 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.286987+0000", 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017653300000000001 2023-09-23T22:16:52.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.290146+0000", 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031592009999999999 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.292957+0000", 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028105209999999999 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.370439+0000", 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.077481942999999998 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.370491+0000", 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2766000000000003e-05 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:50.371314+0000", 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000822239 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.688 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:56567 5.1 5:84f01361:::benchmark_data_smithi026_286012_object56566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:04.279264+0000", 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 48.280883580000001, 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.089646010999999998, 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 56567 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.279264+0000", 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.279264+0000", 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.279263+0000", 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.279318+0000", 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5139000000000002e-05 2023-09-23T22:16:52.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.279331+0000", 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2359e-05 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.279336+0000", 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2209999999999996e-06 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.279364+0000", 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7915e-05 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.279460+0000", 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6359000000000004e-05 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.279554+0000", 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3386000000000003e-05 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.281206+0000", 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001652487 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.281366+0000", 2023-09-23T22:16:52.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016045400000000001 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.368789+0000", 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.087422522000000003 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.368867+0000", 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8190999999999997e-05 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.368910+0000", 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3028000000000002e-05 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:56569 5.1 5:88bb6a9e:::benchmark_data_smithi026_286012_object56568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:04.282503+0000", 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 48.277644901999999, 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.086546697000000006, 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.696 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 56569 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.282503+0000", 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.282503+0000", 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.282502+0000", 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.282557+0000", 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4947000000000001e-05 2023-09-23T22:16:52.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.282571+0000", 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4012000000000001e-05 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.282578+0000", 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0770000000000002e-06 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.282623+0000", 2023-09-23T22:16:52.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5336000000000001e-05 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.282750+0000", 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012723000000000001 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.282890+0000", 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013999699999999999 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.284908+0000", 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020176450000000002 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.285360+0000", 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000452201 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.368960+0000", 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.083599845000000006 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.369012+0000", 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2447000000000001e-05 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.369050+0000", 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7190999999999997e-05 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:80550 5.e 5:7e2288f7:::benchmark_data_smithi026_286012_object80549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:31.764562+0000", 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 20.795586081, 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.108282273, 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 80550 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:16:52.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764562+0000", 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764562+0000", 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764561+0000", 2023-09-23T22:16:52.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764663+0000", 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102361 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764677+0000", 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.403e-05 2023-09-23T22:16:52.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764684+0000", 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3489999999999997e-06 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764741+0000", 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7661000000000002e-05 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764883+0000", 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014220500000000001 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.765041+0000", 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015731299999999999 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.767889+0000", 2023-09-23T22:16:52.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028486509999999998 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770212+0000", 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023227920000000002 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.872778+0000", 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10256546900000001 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:16:52.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.872814+0000", 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5976999999999998e-05 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.872844+0000", 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0341999999999999e-05 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:16:52.709 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:16:52.710 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:16:52.711 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:52.740 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:52.788 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:52.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:52.891 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:52.917 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:52.959 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:52.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:52.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:53.063 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:53.091 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:53.250 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:16:53.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 92 16 93805 93789 63.6925 0 - 0.0150909 2023-09-23T22:16:53.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: failed to encode map e149 with expected crc 2023-09-23T22:16:53.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: failed to encode map e149 with expected crc 2023-09-23T22:16:53.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:53.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:16:53.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: osdmap e150: 8 total, 8 up, 7 in 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4847/281349 objects degraded (1.723%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: failed to encode map e149 with expected crc 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: failed to encode map e149 with expected crc 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:16:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: osdmap e150: 8 total, 8 up, 7 in 2023-09-23T22:16:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4847/281349 objects degraded (1.723%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:16:53.605 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:16:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: failed to encode map e149 with expected crc 2023-09-23T22:16:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: failed to encode map e149 with expected crc 2023-09-23T22:16:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:16:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:16:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: osdmap e150: 8 total, 8 up, 7 in 2023-09-23T22:16:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: failed to encode map e150 with expected crc 2023-09-23T22:16:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4847/281349 objects degraded (1.723%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:16:53.822 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:53.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:53.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:53.926 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:53.957 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:53.973 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:54.008 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:54.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:54.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:54.113 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:54.141 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:54.241 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:54.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:54.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 93 16 93805 93789 63.0078 0 - 0.0150909 2023-09-23T22:16:54.346 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:54.376 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:54.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:54 smithi026 ceph-mon[223014]: failed to encode map e150 with expected crc 2023-09-23T22:16:54.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:54 smithi026 ceph-mon[223014]: pgmap v311: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 4 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 28 MiB/s wr, 450 op/s; 4832/281388 objects degraded (1.717%); 8247/281388 objects misplaced (2.931%); 5.7 MiB/s, 100 objects/s recovering 2023-09-23T22:16:54.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:54.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:54.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:54 smithi026 ceph-mon[215775]: failed to encode map e150 with expected crc 2023-09-23T22:16:54.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:54 smithi026 ceph-mon[215775]: pgmap v311: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 4 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 28 MiB/s wr, 450 op/s; 4832/281388 objects degraded (1.717%); 8247/281388 objects misplaced (2.931%); 5.7 MiB/s, 100 objects/s recovering 2023-09-23T22:16:54.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:54.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:54.423 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:54.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:54.527 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:54.556 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:54.603 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:54.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:54.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:54 smithi070 ceph-mon[192652]: failed to encode map e150 with expected crc 2023-09-23T22:16:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:54 smithi070 ceph-mon[192652]: pgmap v311: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 4 peering, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 78 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 28 MiB/s wr, 450 op/s; 4832/281388 objects degraded (1.717%); 8247/281388 objects misplaced (2.931%); 5.7 MiB/s, 100 objects/s recovering 2023-09-23T22:16:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:16:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:16:54.707 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:54.735 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:54.783 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:54.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:54.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:54.887 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:54.914 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:54.960 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:54.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:54.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:55.063 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:55.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:55.137 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:55.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:55.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:55.217 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:16:55.217 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:16:55.218 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:16:55.218 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:16:55.240 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:55.267 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:55.314 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:55.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:55.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 94 16 93805 93789 62.3377 0 - 0.0150909 2023-09-23T22:16:55.418 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:55.445 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:55.550 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:55.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:55.554 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:55.579 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 7, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:16:55.579 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:16:55.579 INFO:tasks.thrashosds.thrasher:Reweighting osd 0 to 0.2247423920525452 2023-09-23T22:16:55.580 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 0.2247423920525452 2023-09-23T22:16:55.655 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:16:55.689 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:55.902 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:16:55.960 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:16:55.967 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:16:55.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:55.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:16:56.073 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:56.100 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:56.102 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:56.121 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:16:56.150 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:56.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:56 smithi026 ceph-mon[215775]: pgmap v312: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 19 MiB/s wr, 307 op/s; 10721/281439 objects degraded (3.809%); 8247/281439 objects misplaced (2.930%); 3.9 MiB/s, 67 objects/s recovering 2023-09-23T22:16:56.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/750740580' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:56.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:56.156 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:16:56.181 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:16:56.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:16:56.257 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:56.284 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:56 smithi070 ceph-mon[192652]: pgmap v312: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 19 MiB/s wr, 307 op/s; 10721/281439 objects degraded (3.809%); 8247/281439 objects misplaced (2.930%); 3.9 MiB/s, 67 objects/s recovering 2023-09-23T22:16:56.284 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/750740580' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:56.286 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:56.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 95 16 93805 93789 61.6817 0 - 0.0150909 2023-09-23T22:16:56.336 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:56.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:56.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:56.353 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:16:56.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:56.472 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:56 smithi026 ceph-mon[223014]: pgmap v312: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 19 MiB/s wr, 307 op/s; 10721/281439 objects degraded (3.809%); 8247/281439 objects misplaced (2.930%); 3.9 MiB/s, 67 objects/s recovering 2023-09-23T22:16:56.472 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/750740580' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:16:56.475 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:56.676 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:56.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:56.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:56.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:56.809 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:56.962 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:56.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:56.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:57.066 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:16:57.095 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:57.143 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:16:57.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:57.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:16:57.247 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:57.275 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:57.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 96 16 93805 93789 61.0393 0 - 0.0150909 2023-09-23T22:16:57.321 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:57.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:57.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:57.425 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:16:57.457 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:57.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:16:57.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:57.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:16:57.612 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:57.639 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:57.691 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:16:57.736 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:57.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:57.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:57.842 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:57.873 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:57.925 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:57.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:57.934 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:16:57.934 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:16:57.935 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:16:57.935 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:16:57.935 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:16:58.028 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:16:58.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:58.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:58 smithi026 ceph-mon[223014]: pgmap v313: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10 MiB/s wr, 163 op/s; 10721/281439 objects degraded (3.809%); 7498/281439 objects misplaced (2.664%); 10 MiB/s, 165 objects/s recovering 2023-09-23T22:16:58.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10721/281439 objects degraded (3.809%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:58 smithi026 ceph-mon[215775]: pgmap v313: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10 MiB/s wr, 163 op/s; 10721/281439 objects degraded (3.809%); 7498/281439 objects misplaced (2.664%); 10 MiB/s, 165 objects/s recovering 2023-09-23T22:16:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10721/281439 objects degraded (3.809%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:58.295 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:16:58.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:58.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:16:58.317 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 97 16 93805 93789 60.4102 0 - 0.0150909 2023-09-23T22:16:58.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:58 smithi070 ceph-mon[192652]: pgmap v313: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10 MiB/s wr, 163 op/s; 10721/281439 objects degraded (3.809%); 7498/281439 objects misplaced (2.664%); 10 MiB/s, 165 objects/s recovering 2023-09-23T22:16:58.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10721/281439 objects degraded (3.809%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:16:58.400 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:16:58.427 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:58.700 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:16:58.701 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:16:58.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:58.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:16:58.806 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:16:58.833 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:58.889 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:16:58.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:58.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:16:58.996 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:59.027 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:59.077 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:16:59.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:59.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:16:59.093 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:16:59.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:16:59.183 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 0.224742 (3988) 2023-09-23T22:16:59.214 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:59.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 98 16 93805 93789 59.7939 0 - 0.0150909 2023-09-23T22:16:59.470 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:16:59 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/47782403' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.2247423920525452}]: dispatch 2023-09-23T22:16:59.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:16:59 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/47782403' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.2247423920525452}]: dispatch 2023-09-23T22:16:59.491 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:16:59.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:16:59.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:16:59.502 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:16:59 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/47782403' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.2247423920525452}]: dispatch 2023-09-23T22:16:59.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:16:59.626 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:16:59.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:16:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:16:59] "GET /metrics HTTP/1.1" 200 36427 "" "Prometheus/2.43.0" 2023-09-23T22:17:00.159 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:00.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:00.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:00.264 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:00.295 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:00.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 99 16 93805 93789 59.1901 0 - 0.0150909 2023-09-23T22:17:00.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[223014]: pgmap v314: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 240 KiB/s wr, 3 op/s; 10721/281439 objects degraded (3.809%); 7498/281439 objects misplaced (2.664%); 5.9 MiB/s, 96 objects/s recovering 2023-09-23T22:17:00.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[223014]: failed to encode map e151 with expected crc 2023-09-23T22:17:00.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[223014]: failed to encode map e151 with expected crc 2023-09-23T22:17:00.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/47782403' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.2247423920525452}]': finished 2023-09-23T22:17:00.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[223014]: osdmap e151: 8 total, 8 up, 7 in 2023-09-23T22:17:00.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[215775]: pgmap v314: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 240 KiB/s wr, 3 op/s; 10721/281439 objects degraded (3.809%); 7498/281439 objects misplaced (2.664%); 5.9 MiB/s, 96 objects/s recovering 2023-09-23T22:17:00.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[215775]: failed to encode map e151 with expected crc 2023-09-23T22:17:00.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[215775]: failed to encode map e151 with expected crc 2023-09-23T22:17:00.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/47782403' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.2247423920525452}]': finished 2023-09-23T22:17:00.434 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:00 smithi026 ceph-mon[215775]: osdmap e151: 8 total, 8 up, 7 in 2023-09-23T22:17:00.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:00 smithi070 ceph-mon[192652]: pgmap v314: 89 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 240 KiB/s wr, 3 op/s; 10721/281439 objects degraded (3.809%); 7498/281439 objects misplaced (2.664%); 5.9 MiB/s, 96 objects/s recovering 2023-09-23T22:17:00.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:00 smithi070 ceph-mon[192652]: failed to encode map e151 with expected crc 2023-09-23T22:17:00.471 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:00 smithi070 ceph-mon[192652]: failed to encode map e151 with expected crc 2023-09-23T22:17:00.471 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/47782403' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.2247423920525452}]': finished 2023-09-23T22:17:00.471 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:00 smithi070 ceph-mon[192652]: osdmap e151: 8 total, 8 up, 7 in 2023-09-23T22:17:00.497 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:00.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:00.604 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:00.641 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:00.691 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:00.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:00.693 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:00.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:00.823 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:00.872 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:00.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:00.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:00.975 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19259 5.7 5:e3ec1517:::benchmark_data_smithi026_286012_object19258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.612249+0000", 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 92.364447149, 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0598800580000001, 2023-09-23T22:17:00.981 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19259 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612249+0000", 2023-09-23T22:17:00.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612249+0000", 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612247+0000", 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:00.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612300+0000", 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2468e-05 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612319+0000", 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9431000000000001e-05 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612325+0000", 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4820000000000002e-06 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612377+0000", 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1932999999999999e-05 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:00.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612500+0000", 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122898 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612640+0000", 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000140168 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.613682+0000", 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010420690000000001 2023-09-23T22:17:00.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.617494+0000", 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038125429999999998 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.617503+0000", 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3459999999999994e-06 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627652+0000", 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.010149754 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627667+0000", 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4209000000000001e-05 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672028+0000", 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0443611349999999 2023-09-23T22:17:00.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672129+0000", 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010098 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19268 5.7 5:e4a7ba5f:::benchmark_data_smithi026_286012_object19267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.635055+0000", 2023-09-23T22:17:00.989 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 92.341641225000004, 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.042290714, 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19268 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:00.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635055+0000", 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635055+0000", 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635053+0000", 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635096+0000", 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2938999999999997e-05 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635110+0000", 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3417e-05 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635116+0000", 2023-09-23T22:17:00.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.139e-06 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635172+0000", 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6567999999999998e-05 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635186+0000", 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3509999999999999e-05 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627702+0000", 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99251634700000002 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627708+0000", 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3789999999999997e-06 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:00.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672153+0000", 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.044445216 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672284+0000", 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013079899999999999 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672410+0000", 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012686899999999999 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.673809+0000", 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013983579999999999 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.674260+0000", 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045079799999999998 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.676705+0000", 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024456840000000001 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.676749+0000", 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4087e-05 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:00.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.677345+0000", 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059599999999999996 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:36982 5.7 5:ee40b8fd:::benchmark_data_smithi026_286012_object36981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.905294+0000", 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 78.071402312999993, 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0459705229999998, 2023-09-23T22:17:00.998 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36982 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905294+0000", 2023-09-23T22:17:00.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905294+0000", 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905292+0000", 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905348+0000", 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5515e-05 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905359+0000", 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1467e-05 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905366+0000", 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.229e-06 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905414+0000", 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8909000000000002e-05 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905562+0000", 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014794 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905720+0000", 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015746799999999999 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.907741+0000", 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020212910000000001 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.912362+0000", 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046209500000000004 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.918820+0000", 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064581580000000003 2023-09-23T22:17:01.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.918832+0000", 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1997000000000001e-05 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915724+0000", 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99689173499999995 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915741+0000", 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7453999999999999e-05 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951176+0000", 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0354348980000001 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951264+0000", 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7725000000000005e-05 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:36996 5.7 5:e8cdce36:::benchmark_data_smithi026_286012_object36995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.923128+0000", 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 78.053567760999996, 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0387094659999998, 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.007 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36996 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923128+0000", 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923128+0000", 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923127+0000", 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923158+0000", 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1696999999999998e-05 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923168+0000", 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8609999999999993e-06 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.010 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923175+0000", 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3940000000000001e-06 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923656+0000", 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048170500000000002 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923667+0000", 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0723999999999999e-05 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915782+0000", 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99211468700000005 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915789+0000", 2023-09-23T22:17:01.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7309999999999993e-06 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951301+0000", 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.035511493 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951393+0000", 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2347000000000002e-05 2023-09-23T22:17:01.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951500+0000", 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010628600000000001 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956154+0000", 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046545290000000001 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961149+0000", 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049946900000000004 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961196+0000", 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6995e-05 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961241+0000", 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5599000000000002e-05 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961838+0000", 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059618699999999998 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37004 5.7 5:eeedc2c2:::benchmark_data_smithi026_286012_object37003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T22:17:01.016 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.940630+0000", 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 78.036066269000003, 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0219609119999999, 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37004 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940630+0000", 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940630+0000", 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.018 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940622+0000", 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999924 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940657+0000", 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5079000000000001e-05 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940672+0000", 2023-09-23T22:17:01.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5036e-05 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940678+0000", 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2940000000000002e-06 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940726+0000", 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8319000000000003e-05 2023-09-23T22:17:01.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940734+0000", 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1199999999999996e-06 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915819+0000", 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97508563500000001 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915826+0000", 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4749999999999998e-06 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.952797+0000", 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0369709229999999 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.952925+0000", 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128796 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.953102+0000", 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000177011 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.958840+0000", 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057373930000000004 2023-09-23T22:17:01.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961876+0000", 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003036029 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962106+0000", 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022989700000000001 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962141+0000", 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5138999999999997e-05 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962590+0000", 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044962099999999998 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.025 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37005 5.7 5:ebf3df01:::benchmark_data_smithi026_286012_object37004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.942342+0000", 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 78.034354234000006, 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0208824060000001, 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37005 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.026 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942342+0000", 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942342+0000", 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942340+0000", 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942383+0000", 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2715e-05 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942409+0000", 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6746000000000001e-05 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942421+0000", 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1542999999999999e-05 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942572+0000", 2023-09-23T22:17:01.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015078199999999999 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942607+0000", 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5651999999999997e-05 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915850+0000", 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97324232600000005 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915855+0000", 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8389999999999997e-06 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.954081+0000", 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.038225908 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.954178+0000", 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7398000000000005e-05 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.954284+0000", 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106477 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.958847+0000", 2023-09-23T22:17:01.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045624619999999998 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961899+0000", 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030523680000000002 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962660+0000", 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00076089199999999999 2023-09-23T22:17:01.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962727+0000", 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7230000000000005e-05 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963224+0000", 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049668100000000001 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.034 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37018 5.7 5:e9bfc706:::benchmark_data_smithi026_286012_object37017:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.022925+0000", 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 76.953771266999993, 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94092104499999996, 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.035 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37018 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022925+0000", 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022925+0000", 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022923+0000", 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022979+0000", 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5467000000000003e-05 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022992+0000", 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3006999999999999e-05 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022999+0000", 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0439999999999998e-06 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.023072+0000", 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3577000000000004e-05 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.023087+0000", 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4313e-05 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.955164+0000", 2023-09-23T22:17:01.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93207778100000005 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.955263+0000", 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8320999999999996e-05 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.955401+0000", 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013850399999999999 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.959959+0000", 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045578490000000001 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961930+0000", 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001971048 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963292+0000", 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013615999999999999 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963355+0000", 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3847e-05 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963846+0000", 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049014800000000004 2023-09-23T22:17:01.042 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37021 5.7 5:e7df05ef:::benchmark_data_smithi026_286012_object37020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.024001+0000", 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 76.952694936, 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94048630600000005, 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.043 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37021 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024001+0000", 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024001+0000", 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024000+0000", 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024054+0000", 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4146e-05 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024071+0000", 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6731999999999998e-05 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024078+0000", 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2099999999999996e-06 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024135+0000", 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7099999999999999e-05 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024146+0000", 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.084e-05 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956202+0000", 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93205636999999997 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956349+0000", 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147093 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956522+0000", 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017295900000000001 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.959972+0000", 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034492390000000002 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961978+0000", 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020065759999999999 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963918+0000", 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001940034 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963991+0000", 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2616999999999999e-05 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964487+0000", 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049637900000000003 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37045 5.7 5:e066772a:::benchmark_data_smithi026_286012_object37044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.042677+0000", 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 76.934018366999993, 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92228575899999998, 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.051 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37045 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042677+0000", 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042677+0000", 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042676+0000", 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042737+0000", 2023-09-23T22:17:01.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0795999999999999e-05 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042751+0000", 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4453e-05 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042759+0000", 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1729999999999998e-06 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042834+0000", 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5841999999999997e-05 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042847+0000", 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.255e-05 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957481+0000", 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91463394099999995 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957625+0000", 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000144027 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957796+0000", 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017132399999999999 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961086+0000", 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003289882 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962006+0000", 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00091965299999999999 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964538+0000", 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025323949999999998 2023-09-23T22:17:01.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964578+0000", 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9359e-05 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964963+0000", 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038571299999999998 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.058 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59924 5.7 5:e199b184:::benchmark_data_smithi026_286012_object59923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.616936+0000", 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 53.359759705999998, 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0339116370000001, 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.059 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59924 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616936+0000", 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616936+0000", 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616935+0000", 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616983+0000", 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8554999999999998e-05 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616996+0000", 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3098000000000001e-05 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.617003+0000", 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8950000000000001e-06 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.618902+0000", 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001899022 2023-09-23T22:17:01.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.618913+0000", 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0487e-05 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.629818+0000", 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.01090558 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.629940+0000", 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121614 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.630050+0000", 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110517 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636053+0000", 2023-09-23T22:17:01.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060029760000000001 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650747+0000", 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014693632999999999 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650777+0000", 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9669999999999999e-05 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650811+0000", 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4023999999999998e-05 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650848+0000", 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7058999999999999e-05 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59932 5.7 5:eebeefdc:::benchmark_data_smithi026_286012_object59931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.623830+0000", 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 53.352866122999998, 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027128673, 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.067 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59932 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623830+0000", 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623830+0000", 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623829+0000", 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623852+0000", 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3107000000000001e-05 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623863+0000", 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1671999999999999e-05 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623868+0000", 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1880000000000001e-06 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623915+0000", 2023-09-23T22:17:01.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6591999999999997e-05 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623926+0000", 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0712e-05 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.632001+0000", 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0080748070000001 2023-09-23T22:17:01.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.632141+0000", 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014057300000000001 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.632325+0000", 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000183525 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638341+0000", 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060166400000000002 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650882+0000", 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012540689000000001 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650908+0000", 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5919e-05 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650934+0000", 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6418999999999999e-05 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650958+0000", 2023-09-23T22:17:01.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4048000000000001e-05 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59934 5.7 5:e79a8a57:::benchmark_data_smithi026_286012_object59933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.625976+0000", 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 53.350719410000004, 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.025169207, 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.075 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59934 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.625976+0000", 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.625976+0000", 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.625975+0000", 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626005+0000", 2023-09-23T22:17:01.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0679999999999998e-05 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626023+0000", 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7546000000000001e-05 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626031+0000", 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9349999999999994e-06 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626091+0000", 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9787999999999997e-05 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626100+0000", 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6150000000000003e-06 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.633572+0000", 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0074717230000001 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.633653+0000", 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1277000000000001e-05 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.633745+0000", 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1607999999999994e-05 2023-09-23T22:17:01.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638348+0000", 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046025989999999998 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650976+0000", 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012628758 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651066+0000", 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9389999999999996e-05 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651119+0000", 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3155999999999997e-05 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651146+0000", 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6772999999999999e-05 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59943 5.7 5:e02a7111:::benchmark_data_smithi026_286012_object59942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.635287+0000", 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 53.341408948999998, 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0159653, 2023-09-23T22:17:01.083 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59943 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635287+0000", 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635287+0000", 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635285+0000", 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635349+0000", 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3595000000000004e-05 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635365+0000", 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6218e-05 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635371+0000", 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1619999999999996e-06 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635435+0000", 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3384000000000002e-05 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635446+0000", 2023-09-23T22:17:01.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0754000000000001e-05 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.634774+0000", 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99932857799999997 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.634895+0000", 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012116999999999999 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.635052+0000", 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000156162 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640622+0000", 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055706200000000001 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.089 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651186+0000", 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010564146999999999 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651210+0000", 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3373000000000001e-05 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651235+0000", 2023-09-23T22:17:01.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5253999999999999e-05 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651252+0000", 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7295999999999998e-05 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.091 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59963 5.7 5:ec56653b:::benchmark_data_smithi026_286012_object59962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.657440+0000", 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 53.319256240999998, 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99390371200000005, 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59963 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657440+0000", 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657440+0000", 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657438+0000", 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657481+0000", 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2404999999999998e-05 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657495+0000", 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4557e-05 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657502+0000", 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6989999999999997e-06 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657564+0000", 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1988000000000003e-05 2023-09-23T22:17:01.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657573+0000", 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4689999999999997e-06 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636095+0000", 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97852269199999997 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636182+0000", 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6561e-05 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636276+0000", 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4087999999999998e-05 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640638+0000", 2023-09-23T22:17:01.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043621459999999999 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651280+0000", 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010641651 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651303+0000", 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2943999999999999e-05 2023-09-23T22:17:01.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651326+0000", 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3354999999999999e-05 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651343+0000", 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7377999999999999e-05 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.099 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59987 5.7 5:ea604450:::benchmark_data_smithi026_286012_object59986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.678386+0000", 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 53.298310162, 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97304345299999995, 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.100 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59987 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678386+0000", 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678386+0000", 2023-09-23T22:17:01.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678384+0000", 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678451+0000", 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6938000000000002e-05 2023-09-23T22:17:01.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678463+0000", 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.146e-05 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678470+0000", 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5079999999999999e-06 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678519+0000", 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8720000000000001e-05 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678530+0000", 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0726e-05 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.637191+0000", 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95866113100000006 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.637310+0000", 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119374 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.637455+0000", 2023-09-23T22:17:01.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014488599999999999 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.643133+0000", 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056781310000000003 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651368+0000", 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082348999999999999 2023-09-23T22:17:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651390+0000", 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2087999999999999e-05 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651413+0000", 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2354e-05 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651429+0000", 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6606999999999999e-05 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:60017 5.7 5:e7911ea8:::benchmark_data_smithi026_286012_object60016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.703829+0000", 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 53.272867325, 2023-09-23T22:17:01.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94768691699999996, 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60017 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703829+0000", 2023-09-23T22:17:01.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703829+0000", 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703827+0000", 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:17:01.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703929+0000", 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102562 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703949+0000", 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9378999999999999e-05 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703957+0000", 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1659999999999994e-06 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.704024+0000", 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7266000000000003e-05 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.704036+0000", 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1711e-05 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638386+0000", 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93434993 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638486+0000", 2023-09-23T22:17:01.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9834000000000007e-05 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638579+0000", 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3580999999999997e-05 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.643145+0000", 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045657620000000001 2023-09-23T22:17:01.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651454+0000", 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083090219999999992 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651477+0000", 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2541999999999999e-05 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651499+0000", 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2277999999999999e-05 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651515+0000", 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6566e-05 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.116 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:60025 5.7 5:ee51d15d:::benchmark_data_smithi026_286012_object60024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.711079+0000", 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 53.265616778000002, 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94052496900000004, 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.117 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60025 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711079+0000", 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711079+0000", 2023-09-23T22:17:01.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711078+0000", 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711136+0000", 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7855000000000001e-05 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711150+0000", 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4219000000000001e-05 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711156+0000", 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5379999999999999e-06 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711209+0000", 2023-09-23T22:17:01.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2714999999999999e-05 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711221+0000", 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1578e-05 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.639471+0000", 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92825038299999996 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.639589+0000", 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011770200000000001 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.639740+0000", 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015135499999999999 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.645463+0000", 2023-09-23T22:17:01.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057232120000000001 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651540+0000", 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060769939999999996 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651563+0000", 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3059000000000001e-05 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651586+0000", 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2154000000000002e-05 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651604+0000", 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8445999999999999e-05 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.124 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:60041 5.7 5:e6579469:::benchmark_data_smithi026_286012_object60040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.728814+0000", 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 53.247882050000001, 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92288071299999996, 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60041 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728814+0000", 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728814+0000", 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728813+0000", 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728906+0000", 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3043999999999999e-05 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728918+0000", 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2201000000000001e-05 2023-09-23T22:17:01.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728923+0000", 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1590000000000001e-06 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728988+0000", 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5283000000000005e-05 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728999+0000", 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0393e-05 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640683+0000", 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.911684351 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640810+0000", 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012665500000000001 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640977+0000", 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016692500000000001 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.645469+0000", 2023-09-23T22:17:01.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044926890000000002 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651632+0000", 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061627959999999999 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651654+0000", 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2266e-05 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651677+0000", 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2198999999999999e-05 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651695+0000", 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7923000000000001e-05 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:93693 5.f 5:f82a8543:::benchmark_data_smithi026_286012_object93692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:48.289235+0000", 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 12.687460516, 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0721697720000001, 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 93693 2023-09-23T22:17:01.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289235+0000", 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289235+0000", 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289234+0000", 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289280+0000", 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6313000000000002e-05 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289294+0000", 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3896999999999999e-05 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289299+0000", 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4210000000000003e-06 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289365+0000", 2023-09-23T22:17:01.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6038999999999996e-05 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289378+0000", 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2371000000000001e-05 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152914+0000", 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.863536533 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152926+0000", 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1677e-05 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.346430+0000", 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1935035489999999 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.346618+0000", 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000188378 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.346836+0000", 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021775200000000001 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358797+0000", 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011961090000000001 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358858+0000", 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1081999999999996e-05 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358963+0000", 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105529 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.359038+0000", 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4793e-05 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.361405+0000", 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002366908 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.141 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:93720 5.f 5:f1e7c9aa:::benchmark_data_smithi026_286012_object93719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:48.318037+0000", 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 12.658658983, 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0455209810000001, 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 93720 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:01.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318037+0000", 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318037+0000", 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:01.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318035+0000", 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318080+0000", 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4404999999999999e-05 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318094+0000", 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4014e-05 2023-09-23T22:17:01.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318100+0000", 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6380000000000001e-06 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318152+0000", 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2349000000000003e-05 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318161+0000", 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9260000000000006e-06 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152958+0000", 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.834797067 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152965+0000", 2023-09-23T22:17:01.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3849999999999996e-06 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.350144+0000", 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.197178563 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.350310+0000", 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000166584 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.350530+0000", 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000219235 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358894+0000", 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083640529999999998 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358901+0000", 2023-09-23T22:17:01.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1829999999999997e-06 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.361482+0000", 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002581046 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.361552+0000", 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9646000000000004e-05 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:01.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:01.150 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:17:01.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.363558+0000", 2023-09-23T22:17:01.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020062909999999999 2023-09-23T22:17:01.150 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.150 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.150 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.150 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:01.150 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:01.150 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:17:01.151 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:01.168 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:01.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:01.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:01.189 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:17:01.273 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:01.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:17:01.318746+0000 min lat: 0.00108301 max lat: 2.1739 avg lat: 0.0150909 2023-09-23T22:17:01.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:17:01.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 100 16 93805 93789 58.5983 0 - 0.0150909 2023-09-23T22:17:01.333 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:01 smithi070 ceph-mon[192652]: failed to encode map e152 with expected crc 2023-09-23T22:17:01.333 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:01 smithi070 ceph-mon[192652]: osdmap e152: 8 total, 8 up, 7 in 2023-09-23T22:17:01.333 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:01 smithi070 ceph-mon[192652]: failed to encode map e152 with expected crc 2023-09-23T22:17:01.333 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:01 smithi070 ceph-mon[192652]: failed to encode map e152 with expected crc 2023-09-23T22:17:01.333 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/378685689' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:01.333 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:01 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:01.338 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:01.416 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:01.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[215775]: failed to encode map e152 with expected crc 2023-09-23T22:17:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[215775]: osdmap e152: 8 total, 8 up, 7 in 2023-09-23T22:17:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[215775]: failed to encode map e152 with expected crc 2023-09-23T22:17:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[215775]: failed to encode map e152 with expected crc 2023-09-23T22:17:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/378685689' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[223014]: failed to encode map e152 with expected crc 2023-09-23T22:17:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[223014]: osdmap e152: 8 total, 8 up, 7 in 2023-09-23T22:17:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[223014]: failed to encode map e152 with expected crc 2023-09-23T22:17:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[223014]: failed to encode map e152 with expected crc 2023-09-23T22:17:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/378685689' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:01 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:01.521 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:01.550 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:01.600 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:01.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:01.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:01.705 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:01.736 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:01.785 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:01.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:01.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:01.889 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:01.919 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:01.967 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:01.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:01.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:02.073 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:02.101 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:02.170 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: pgmap v317: 89 pgs: 5 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 136 KiB/s wr, 2 op/s; 4831/281439 objects degraded (1.717%); 12353/281439 objects misplaced (4.389%); 14 MiB/s, 221 objects/s recovering 2023-09-23T22:17:02.170 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:17:02.170 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:02.170 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.171 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.171 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:17:02.171 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: osdmap e153: 8 total, 8 up, 7 in 2023-09-23T22:17:02.171 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.171 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: 5.8 starting backfill to osd.5 from (0'0,123'2239] MIN to 148'5766 2023-09-23T22:17:02.171 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.171 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: 5.7 starting backfill to osd.4 from (0'0,123'2373] MIN to 148'5965 2023-09-23T22:17:02.171 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:02 smithi070 ceph-mon[192652]: 5.3 continuing backfill to osd.3 from (0'0,148'5874] MIN to 148'5874 2023-09-23T22:17:02.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 101 16 93805 93789 58.0183 0 - 0.0150909 2023-09-23T22:17:02.327 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:02.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:02.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:02.387 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:17:02.431 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:02.463 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: pgmap v317: 89 pgs: 5 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 136 KiB/s wr, 2 op/s; 4831/281439 objects degraded (1.717%); 12353/281439 objects misplaced (4.389%); 14 MiB/s, 221 objects/s recovering 2023-09-23T22:17:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:17:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:17:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: osdmap e153: 8 total, 8 up, 7 in 2023-09-23T22:17:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: 5.8 starting backfill to osd.5 from (0'0,123'2239] MIN to 148'5766 2023-09-23T22:17:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: 5.7 starting backfill to osd.4 from (0'0,123'2373] MIN to 148'5965 2023-09-23T22:17:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[215775]: 5.3 continuing backfill to osd.3 from (0'0,148'5874] MIN to 148'5874 2023-09-23T22:17:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: pgmap v317: 89 pgs: 5 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 76 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 136 KiB/s wr, 2 op/s; 4831/281439 objects degraded (1.717%); 12353/281439 objects misplaced (4.389%); 14 MiB/s, 221 objects/s recovering 2023-09-23T22:17:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:17:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:17:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: osdmap e153: 8 total, 8 up, 7 in 2023-09-23T22:17:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: 5.8 starting backfill to osd.5 from (0'0,123'2239] MIN to 148'5766 2023-09-23T22:17:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: failed to encode map e153 with expected crc 2023-09-23T22:17:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: 5.7 starting backfill to osd.4 from (0'0,123'2373] MIN to 148'5965 2023-09-23T22:17:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:02 smithi026 ceph-mon[223014]: 5.3 continuing backfill to osd.3 from (0'0,148'5874] MIN to 148'5874 2023-09-23T22:17:02.525 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:02.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:02.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:02.632 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:02.660 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:02.707 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:02.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:02.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:02.811 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:02.829 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:17:02.839 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:02.885 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:02.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:02.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:02.988 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:03.022 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:03.080 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:03.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:03.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:03.187 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:03.217 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:03.273 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:03 smithi070 ceph-mon[192652]: failed to encode map e154 with expected crc 2023-09-23T22:17:03.273 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:03 smithi070 ceph-mon[192652]: osdmap e154: 8 total, 8 up, 7 in 2023-09-23T22:17:03.273 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:03 smithi070 ceph-mon[192652]: failed to encode map e154 with expected crc 2023-09-23T22:17:03.273 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4831/281439 objects degraded (1.717%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:17:03.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 102 16 93805 93789 57.4496 0 - 0.0150909 2023-09-23T22:17:03.428 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:03.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:03.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:03.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:03 smithi026 ceph-mon[223014]: failed to encode map e154 with expected crc 2023-09-23T22:17:03.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:03 smithi026 ceph-mon[223014]: osdmap e154: 8 total, 8 up, 7 in 2023-09-23T22:17:03.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:03 smithi026 ceph-mon[223014]: failed to encode map e154 with expected crc 2023-09-23T22:17:03.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4831/281439 objects degraded (1.717%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:17:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:03 smithi026 ceph-mon[215775]: failed to encode map e154 with expected crc 2023-09-23T22:17:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:03 smithi026 ceph-mon[215775]: osdmap e154: 8 total, 8 up, 7 in 2023-09-23T22:17:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:03 smithi026 ceph-mon[215775]: failed to encode map e154 with expected crc 2023-09-23T22:17:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4831/281439 objects degraded (1.717%), 2 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:17:03.532 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:03.562 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:03.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:03.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:03.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:03.711 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:03.738 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:03.786 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:03.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:03.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:03.889 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:03.918 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:03.969 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:03.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:04.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:04.100 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:04.147 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:04.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:04.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:04.251 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:04.279 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:04.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 103 16 93805 93789 56.892 0 - 0.0150909 2023-09-23T22:17:04.329 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:04.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:04.375 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:17:04.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:04.463 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:04.514 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:04.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:04.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:04.617 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:04.645 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:04.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:04 smithi026 ceph-mon[223014]: pgmap v320: 89 pgs: 5 peering, 2 active+recovering+undersized+remapped, 3 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 75 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10816/281439 objects degraded (3.843%); 12353/281439 objects misplaced (4.389%); 16 MiB/s, 255 objects/s recovering 2023-09-23T22:17:04.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:04 smithi026 ceph-mon[223014]: failed to encode map e155 with expected crc 2023-09-23T22:17:04.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:04 smithi026 ceph-mon[223014]: osdmap e155: 8 total, 8 up, 7 in 2023-09-23T22:17:04.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:04 smithi026 ceph-mon[215775]: pgmap v320: 89 pgs: 5 peering, 2 active+recovering+undersized+remapped, 3 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 75 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10816/281439 objects degraded (3.843%); 12353/281439 objects misplaced (4.389%); 16 MiB/s, 255 objects/s recovering 2023-09-23T22:17:04.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:04 smithi026 ceph-mon[215775]: failed to encode map e155 with expected crc 2023-09-23T22:17:04.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:04 smithi026 ceph-mon[215775]: osdmap e155: 8 total, 8 up, 7 in 2023-09-23T22:17:04.763 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:04.781 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:04.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:04 smithi070 ceph-mon[192652]: pgmap v320: 89 pgs: 5 peering, 2 active+recovering+undersized+remapped, 3 remapped+peering, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 75 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10816/281439 objects degraded (3.843%); 12353/281439 objects misplaced (4.389%); 16 MiB/s, 255 objects/s recovering 2023-09-23T22:17:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:04 smithi070 ceph-mon[192652]: failed to encode map e155 with expected crc 2023-09-23T22:17:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:04 smithi070 ceph-mon[192652]: osdmap e155: 8 total, 8 up, 7 in 2023-09-23T22:17:04.886 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:04.915 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:05.019 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:05.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:05.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:05.124 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:05.160 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:05.216 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:05.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:05.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:05.253 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 7, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:17:05.254 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:17:05.254 INFO:tasks.thrashosds.thrasher:Reweighting osd 0 to 0.5115731408381224 2023-09-23T22:17:05.254 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 0.5115731408381224 2023-09-23T22:17:05.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 104 16 93805 93789 56.3451 0 - 0.0150909 2023-09-23T22:17:05.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:05.350 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:05.353 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:17:05.353 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:17:05.353 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:17:05.354 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:17:05.405 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:05.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:05.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:05.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:05.536 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:05.588 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:05.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:05.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:05.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:05 smithi026 ceph-mon[215775]: failed to encode map e155 with expected crc 2023-09-23T22:17:05.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:05 smithi026 ceph-mon[215775]: failed to encode map e156 with expected crc 2023-09-23T22:17:05.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:05 smithi026 ceph-mon[215775]: osdmap e156: 8 total, 8 up, 7 in 2023-09-23T22:17:05.692 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:05.721 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:05 smithi070 ceph-mon[192652]: failed to encode map e155 with expected crc 2023-09-23T22:17:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:05 smithi070 ceph-mon[192652]: failed to encode map e156 with expected crc 2023-09-23T22:17:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:05 smithi070 ceph-mon[192652]: osdmap e156: 8 total, 8 up, 7 in 2023-09-23T22:17:05.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:05 smithi026 ceph-mon[223014]: failed to encode map e155 with expected crc 2023-09-23T22:17:05.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:05 smithi026 ceph-mon[223014]: failed to encode map e156 with expected crc 2023-09-23T22:17:05.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:05 smithi026 ceph-mon[223014]: osdmap e156: 8 total, 8 up, 7 in 2023-09-23T22:17:06.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 105 16 93805 93789 55.8086 0 - 0.0150909 2023-09-23T22:17:06.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:17:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[215775]: pgmap v322: 89 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 16690/281439 objects degraded (5.930%); 17999/281439 objects misplaced (6.395%); 17 MiB/s, 265 objects/s recovering 2023-09-23T22:17:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[215775]: failed to encode map e156 with expected crc 2023-09-23T22:17:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[215775]: failed to encode map e156 with expected crc 2023-09-23T22:17:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[215775]: failed to encode map e156 with expected crc 2023-09-23T22:17:06.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T22:17:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[223014]: pgmap v322: 89 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 16690/281439 objects degraded (5.930%); 17999/281439 objects misplaced (6.395%); 17 MiB/s, 265 objects/s recovering 2023-09-23T22:17:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[223014]: failed to encode map e156 with expected crc 2023-09-23T22:17:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[223014]: failed to encode map e156 with expected crc 2023-09-23T22:17:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[223014]: failed to encode map e156 with expected crc 2023-09-23T22:17:06.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:06 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T22:17:07.052 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:17:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:06 smithi070 ceph-mon[192652]: pgmap v322: 89 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 16690/281439 objects degraded (5.930%); 17999/281439 objects misplaced (6.395%); 17 MiB/s, 265 objects/s recovering 2023-09-23T22:17:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:06 smithi070 ceph-mon[192652]: failed to encode map e156 with expected crc 2023-09-23T22:17:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:06 smithi070 ceph-mon[192652]: failed to encode map e156 with expected crc 2023-09-23T22:17:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:06 smithi070 ceph-mon[192652]: failed to encode map e156 with expected crc 2023-09-23T22:17:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:06 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T22:17:07.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 106 16 93805 93789 55.2822 0 - 0.0150909 2023-09-23T22:17:07.377 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:17:07.403 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:17:07.423 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:17:07.442 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:17:07.443 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:17:07.443 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:17:07.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:17:07.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:17:07.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:17:07.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:17:07.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:17:07.444 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:17:07.473 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:07.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:07.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:07.577 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:07.620 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:07.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:07 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 16690/281439 objects degraded (5.930%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:17:07.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:07 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 16690/281439 objects degraded (5.930%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:17:08.083 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:08.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:08.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:07 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 16690/281439 objects degraded (5.930%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:17:08.187 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:08.217 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:08.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 106.393 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 93805 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 55.1053 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 43.4779 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 217.875 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 881 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 695.646 2023-09-23T22:17:08.319 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 3486 2023-09-23T22:17:08.320 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T22:17:08.320 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.018136 2023-09-23T22:17:08.320 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.240625 2023-09-23T22:17:08.320 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 19.5363 2023-09-23T22:17:08.320 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00108301 2023-09-23T22:17:08.323 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:17:08.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:08 smithi026 ceph-mon[215775]: pgmap v324: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 16690/281439 objects degraded (5.930%); 22652/281439 objects misplaced (8.049%); 16 MiB/s, 245 objects/s recovering 2023-09-23T22:17:08.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:08 smithi026 ceph-mon[223014]: pgmap v324: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 16690/281439 objects degraded (5.930%); 22652/281439 objects misplaced (8.049%); 16 MiB/s, 245 objects/s recovering 2023-09-23T22:17:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:08 smithi070 ceph-mon[192652]: pgmap v324: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 remapped+peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 16690/281439 objects degraded (5.930%); 22652/281439 objects misplaced (8.049%); 16 MiB/s, 245 objects/s recovering 2023-09-23T22:17:09.180 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:09.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:09.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:09.296 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:09.365 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:09.463 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:09.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:09.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:09.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:09.639 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:09.682 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:09.742 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:09.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:09.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:09.846 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:09.923 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:17:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:17:09] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-09-23T22:17:09.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:09.930 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:09.936 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:09.995 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:17:09.995 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:17:10.023 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:10.090 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:10.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:10.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:10.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:10.263 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:10.358 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:10.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:10.465 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:10.520 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:10.616 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:10.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:10.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:10.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:10 smithi026 ceph-mon[215775]: pgmap v325: 89 pgs: 1 activating+undersized+degraded+remapped, 2 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 KiB/s wr, 3 op/s; 22457/281403 objects degraded (7.980%); 22652/281403 objects misplaced (8.050%); 14 MiB/s, 211 objects/s recovering 2023-09-23T22:17:10.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2484810663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:10.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:10.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:10 smithi026 ceph-mon[223014]: pgmap v325: 89 pgs: 1 activating+undersized+degraded+remapped, 2 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 KiB/s wr, 3 op/s; 22457/281403 objects degraded (7.980%); 22652/281403 objects misplaced (8.050%); 14 MiB/s, 211 objects/s recovering 2023-09-23T22:17:10.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2484810663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:10.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:10.696 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:17:10.720 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:10.804 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:10.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:10 smithi070 ceph-mon[192652]: pgmap v325: 89 pgs: 1 activating+undersized+degraded+remapped, 2 peering, 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 56 KiB/s wr, 3 op/s; 22457/281403 objects degraded (7.980%); 22652/281403 objects misplaced (8.050%); 14 MiB/s, 211 objects/s recovering 2023-09-23T22:17:10.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2484810663' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:10.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:10.912 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:10.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:10.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:11.019 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:11.091 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:11.309 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:17:11.669 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:17:11.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:17:11.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:17:11.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[215775]: osdmap e157: 8 total, 8 up, 7 in 2023-09-23T22:17:11.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[215775]: failed to encode map e157 with expected crc 2023-09-23T22:17:11.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[215775]: failed to encode map e157 with expected crc 2023-09-23T22:17:11.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:17:11.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:17:11.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[223014]: osdmap e157: 8 total, 8 up, 7 in 2023-09-23T22:17:11.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[223014]: failed to encode map e157 with expected crc 2023-09-23T22:17:11.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:11 smithi026 ceph-mon[223014]: failed to encode map e157 with expected crc 2023-09-23T22:17:12.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:11 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:17:12.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:17:12.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:11 smithi070 ceph-mon[192652]: osdmap e157: 8 total, 8 up, 7 in 2023-09-23T22:17:12.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:11 smithi070 ceph-mon[192652]: failed to encode map e157 with expected crc 2023-09-23T22:17:12.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:11 smithi070 ceph-mon[192652]: failed to encode map e157 with expected crc 2023-09-23T22:17:12.348 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:17:12.524 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:12.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:12.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:12.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:12.698 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:12.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:12 smithi026 ceph-mon[215775]: pgmap v327: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 148 B/s rd, 56 KiB/s wr, 141 op/s; 22381/278562 objects degraded (8.034%); 21443/278562 objects misplaced (7.698%); 23 MiB/s, 363 objects/s recovering 2023-09-23T22:17:12.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:12 smithi026 ceph-mon[215775]: failed to encode map e157 with expected crc 2023-09-23T22:17:12.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:12 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 22381/278562 objects degraded (8.034%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:12.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:12 smithi026 ceph-mon[223014]: pgmap v327: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 148 B/s rd, 56 KiB/s wr, 141 op/s; 22381/278562 objects degraded (8.034%); 21443/278562 objects misplaced (7.698%); 23 MiB/s, 363 objects/s recovering 2023-09-23T22:17:12.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:12 smithi026 ceph-mon[223014]: failed to encode map e157 with expected crc 2023-09-23T22:17:12.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:12 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 22381/278562 objects degraded (8.034%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:13.069 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:13.069 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:12 smithi070 ceph-mon[192652]: pgmap v327: 89 pgs: 1 active+remapped+backfilling, 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 peering, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 148 B/s rd, 56 KiB/s wr, 141 op/s; 22381/278562 objects degraded (8.034%); 21443/278562 objects misplaced (7.698%); 23 MiB/s, 363 objects/s recovering 2023-09-23T22:17:13.070 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:12 smithi070 ceph-mon[192652]: failed to encode map e157 with expected crc 2023-09-23T22:17:13.070 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:12 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 22381/278562 objects degraded (8.034%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:13.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:13.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:13.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:13.230 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:13.253 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:13.381 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:13.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:13.578 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:13.641 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:13.710 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:13.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:13.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:13.818 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:13.863 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:13.950 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:13.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:13.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:14.054 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:14.099 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:14.217 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:14.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:14.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:14.321 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:14.376 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:14.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:14.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:14.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:14.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:14.637 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:14.688 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:14.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:14.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:14.702 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 0.511573 (82f6) 2023-09-23T22:17:14.792 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:14.803 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:14 smithi070 ceph-mon[192652]: pgmap v328: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 127 B/s rd, 160 KiB/s wr, 239 op/s; 16614/275817 objects degraded (6.024%); 27030/275817 objects misplaced (9.800%); 18 MiB/s, 293 objects/s recovering 2023-09-23T22:17:14.804 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:14 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3797575221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5115731408381224}]: dispatch 2023-09-23T22:17:14.804 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:14 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5115731408381224}]: dispatch 2023-09-23T22:17:14.830 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:14.886 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:14.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:14 smithi026 ceph-mon[215775]: pgmap v328: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 127 B/s rd, 160 KiB/s wr, 239 op/s; 16614/275817 objects degraded (6.024%); 27030/275817 objects misplaced (9.800%); 18 MiB/s, 293 objects/s recovering 2023-09-23T22:17:14.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:14 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3797575221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5115731408381224}]: dispatch 2023-09-23T22:17:14.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:14 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5115731408381224}]: dispatch 2023-09-23T22:17:14.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:14.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:14.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:14 smithi026 ceph-mon[223014]: pgmap v328: 89 pgs: 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 127 B/s rd, 160 KiB/s wr, 239 op/s; 16614/275817 objects degraded (6.024%); 27030/275817 objects misplaced (9.800%); 18 MiB/s, 293 objects/s recovering 2023-09-23T22:17:14.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:14 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3797575221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5115731408381224}]: dispatch 2023-09-23T22:17:14.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:14 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 0.5115731408381224}]: dispatch 2023-09-23T22:17:14.990 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:15.017 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:15.087 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:17:15.115 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:15.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:15.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:15.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:15.249 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:15.298 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:15.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:15.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:15.338 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:17:15.338 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:17:15.402 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:15.434 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:15.588 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:15.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:15.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:15.693 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:15.757 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:15.861 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:15.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:15.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:15.903 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:15.970 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:15.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:15 smithi026 ceph-mon[223014]: failed to encode map e158 with expected crc 2023-09-23T22:17:15.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:15 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.5115731408381224}]': finished 2023-09-23T22:17:15.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:15 smithi026 ceph-mon[223014]: osdmap e158: 8 total, 8 up, 7 in 2023-09-23T22:17:15.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:15 smithi026 ceph-mon[215775]: failed to encode map e158 with expected crc 2023-09-23T22:17:15.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:15 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.5115731408381224}]': finished 2023-09-23T22:17:15.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:15 smithi026 ceph-mon[215775]: osdmap e158: 8 total, 8 up, 7 in 2023-09-23T22:17:16.001 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:15 smithi070 ceph-mon[192652]: failed to encode map e158 with expected crc 2023-09-23T22:17:16.001 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:15 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 0.5115731408381224}]': finished 2023-09-23T22:17:16.002 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:15 smithi070 ceph-mon[192652]: osdmap e158: 8 total, 8 up, 7 in 2023-09-23T22:17:16.004 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:16.089 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:16.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:16.092 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:16.193 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:16.224 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:16.249 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:17:16.268 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:16.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:16.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:16.272 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:17:16.296 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:17:16.317 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:17:16.337 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:17:16.359 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:17:16.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:16.378 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:17:16.401 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:17:16.403 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:17:16.403 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:17:16.403 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:17:16.403 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:17:16.404 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:17:16.404 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:17:16.404 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:17:16.404 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:17:16.405 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:16.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[215775]: pgmap v330: 89 pgs: 2 peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 127 B/s rd, 160 KiB/s wr, 345 op/s; 16398/273273 objects degraded (6.001%); 20971/273273 objects misplaced (7.674%); 11 MiB/s, 174 objects/s recovering 2023-09-23T22:17:16.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[215775]: osdmap e159: 8 total, 8 up, 7 in 2023-09-23T22:17:16.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[215775]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[215775]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[215775]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[215775]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[215775]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1404586582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:16.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[223014]: pgmap v330: 89 pgs: 2 peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 127 B/s rd, 160 KiB/s wr, 345 op/s; 16398/273273 objects degraded (6.001%); 20971/273273 objects misplaced (7.674%); 11 MiB/s, 174 objects/s recovering 2023-09-23T22:17:16.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[223014]: osdmap e159: 8 total, 8 up, 7 in 2023-09-23T22:17:16.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[223014]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[223014]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[223014]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[223014]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[223014]: failed to encode map e159 with expected crc 2023-09-23T22:17:16.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1404586582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:16.965 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:16.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:16.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:17.071 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:17.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:16 smithi070 ceph-mon[192652]: pgmap v330: 89 pgs: 2 peering, 1 active+remapped+backfilling, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 127 B/s rd, 160 KiB/s wr, 345 op/s; 16398/273273 objects degraded (6.001%); 20971/273273 objects misplaced (7.674%); 11 MiB/s, 174 objects/s recovering 2023-09-23T22:17:17.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:16 smithi070 ceph-mon[192652]: osdmap e159: 8 total, 8 up, 7 in 2023-09-23T22:17:17.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:16 smithi070 ceph-mon[192652]: failed to encode map e159 with expected crc 2023-09-23T22:17:17.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:16 smithi070 ceph-mon[192652]: failed to encode map e159 with expected crc 2023-09-23T22:17:17.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:16 smithi070 ceph-mon[192652]: failed to encode map e159 with expected crc 2023-09-23T22:17:17.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:16 smithi070 ceph-mon[192652]: failed to encode map e159 with expected crc 2023-09-23T22:17:17.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:16 smithi070 ceph-mon[192652]: failed to encode map e159 with expected crc 2023-09-23T22:17:17.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1404586582' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:17.125 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:17.241 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:17.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:17.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:17.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:17.383 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:17.489 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:17:17.500 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:17.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:17.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:17.606 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:17.685 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:17.771 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:17.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:17.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:17.882 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:17.974 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[223014]: failed to encode map e160 with expected crc 2023-09-23T22:17:17.974 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[223014]: failed to encode map e160 with expected crc 2023-09-23T22:17:17.974 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[223014]: osdmap e160: 8 total, 8 up, 7 in 2023-09-23T22:17:17.974 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[223014]: failed to encode map e160 with expected crc 2023-09-23T22:17:17.974 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 16398/273273 objects degraded (6.001%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:17.975 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[215775]: failed to encode map e160 with expected crc 2023-09-23T22:17:17.975 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[215775]: failed to encode map e160 with expected crc 2023-09-23T22:17:17.975 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[215775]: osdmap e160: 8 total, 8 up, 7 in 2023-09-23T22:17:17.975 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[215775]: failed to encode map e160 with expected crc 2023-09-23T22:17:17.975 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:17 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 16398/273273 objects degraded (6.001%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:17.984 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:18.084 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:18.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:18.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:18.106 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:17:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:17 smithi070 ceph-mon[192652]: failed to encode map e160 with expected crc 2023-09-23T22:17:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:17 smithi070 ceph-mon[192652]: failed to encode map e160 with expected crc 2023-09-23T22:17:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:17 smithi070 ceph-mon[192652]: osdmap e160: 8 total, 8 up, 7 in 2023-09-23T22:17:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:17 smithi070 ceph-mon[192652]: failed to encode map e160 with expected crc 2023-09-23T22:17:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:17 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 16398/273273 objects degraded (6.001%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:18.194 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:18.260 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:18.361 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:18.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:18.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:18.484 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:18.560 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:18.659 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:18.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:18.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:18.771 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:18.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:18 smithi026 ceph-mon[215775]: pgmap v333: 89 pgs: 3 peering, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 149 KiB/s wr, 503 op/s; 16398/269577 objects degraded (6.083%); 16504/269577 objects misplaced (6.122%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:17:18.819 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:18 smithi026 ceph-mon[215775]: failed to encode map e160 with expected crc 2023-09-23T22:17:18.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:18 smithi026 ceph-mon[223014]: pgmap v333: 89 pgs: 3 peering, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 149 KiB/s wr, 503 op/s; 16398/269577 objects degraded (6.083%); 16504/269577 objects misplaced (6.122%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:17:18.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:18 smithi026 ceph-mon[223014]: failed to encode map e160 with expected crc 2023-09-23T22:17:18.822 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:18.947 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:18.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:18.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:19.052 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:19.105 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:19.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:18 smithi070 ceph-mon[192652]: pgmap v333: 89 pgs: 3 peering, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 149 KiB/s wr, 503 op/s; 16398/269577 objects degraded (6.083%); 16504/269577 objects misplaced (6.122%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:17:19.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:18 smithi070 ceph-mon[192652]: failed to encode map e160 with expected crc 2023-09-23T22:17:19.225 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:19.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:19.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:19.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:19.393 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:19.496 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:19.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:19.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:19.604 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:19.687 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:19.773 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:17:19.776 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:19.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:19.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:19.882 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:17:19.886 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:19.930 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:17:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:17:19] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-09-23T22:17:19.930 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:19.931 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2706814618' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:17:19.931 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:19 smithi026 ceph-mon[223014]: pgmap v334: 89 pgs: 3 peering, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 16398/267228 objects degraded (6.136%); 16254/267228 objects misplaced (6.082%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:17:19.931 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2706814618' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:17:19.931 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:19 smithi026 ceph-mon[215775]: pgmap v334: 89 pgs: 3 peering, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 16398/267228 objects degraded (6.136%); 16254/267228 objects misplaced (6.082%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:17:20.002 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:20.008 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2706814618' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:17:20.008 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:19 smithi070 ceph-mon[192652]: pgmap v334: 89 pgs: 3 peering, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 16398/267228 objects degraded (6.136%); 16254/267228 objects misplaced (6.082%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:17:20.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:20.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:20.110 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:20.185 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:20.292 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:20.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:20.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:20.397 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:20.424 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:20.471 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:20.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:20.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:20.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:20.651 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:20.763 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:20.763 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 7, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:17:20.763 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:17:20.764 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T22:17:20.764 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:17:20.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:20.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:20.869 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:20.918 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:21.013 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:21.024 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:17:21.024 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:17:21.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.024 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101259 5.5 e161/152 5:ada33536:::benchmark_data_smithi026_286012_object7451:head v 161'6478, mlcod=0'0)", 2023-09-23T22:17:21.024 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:21.006941+0000", 2023-09-23T22:17:21.024 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.017333258000000001, 2023-09-23T22:17:21.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017363264999999999, 2023-09-23T22:17:21.024 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:21.024 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.006941+0000", 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.006941+0000", 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.006940+0000", 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.006962+0000", 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1282000000000001e-05 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:21.026 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.006982+0000", 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0044e-05 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.006989+0000", 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7910000000000005e-06 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.012978+0000", 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059886330000000002 2023-09-23T22:17:21.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.012999+0000", 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0577e-05 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101260 5.5 e161/152 5:ad5f4cad:::benchmark_data_smithi026_286012_object7452:head v 161'6479, mlcod=0'0)", 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:21.010172+0000", 2023-09-23T22:17:21.028 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.014102300999999999, 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01417364, 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.010172+0000", 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:21.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.010172+0000", 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.010171+0000", 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.010180+0000", 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7830000000000004e-06 2023-09-23T22:17:21.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.010195+0000", 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4873e-05 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.010202+0000", 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7240000000000003e-06 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.018658+0000", 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084559979999999993 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.018684+0000", 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6021e-05 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:21.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101262 5.5 e161/152 5:a9a421ed:::benchmark_data_smithi026_286012_object7454:head v 161'6480, mlcod=0'0)", 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:21.015941+0000", 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0083330339999999996, 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0084387660000000003, 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.015941+0000", 2023-09-23T22:17:21.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.015941+0000", 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.015941+0000", 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:17:21.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.015954+0000", 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3118e-05 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.015964+0000", 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0471e-05 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.015970+0000", 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2219999999999999e-06 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.023975+0000", 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0080050509999999991 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.024000+0000", 2023-09-23T22:17:21.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5021999999999999e-05 2023-09-23T22:17:21.037 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:21.037 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:17:21.037 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:21.037 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:17:21.037 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T22:17:21.037 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 3 2023-09-23T22:17:21.037 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:17:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:21.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:21.038 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:20 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:21.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2706814618' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:17:21.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:20 smithi070 ceph-mon[192652]: osdmap e161: 8 total, 8 up, 7 in 2023-09-23T22:17:21.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:20 smithi070 ceph-mon[192652]: failed to encode map e161 with expected crc 2023-09-23T22:17:21.039 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:20 smithi070 ceph-mon[192652]: failed to encode map e161 with expected crc 2023-09-23T22:17:21.045 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:21.139 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:21.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:21.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2706814618' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:17:21.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[223014]: osdmap e161: 8 total, 8 up, 7 in 2023-09-23T22:17:21.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[223014]: failed to encode map e161 with expected crc 2023-09-23T22:17:21.229 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[223014]: failed to encode map e161 with expected crc 2023-09-23T22:17:21.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:21.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2706814618' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:17:21.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[215775]: osdmap e161: 8 total, 8 up, 7 in 2023-09-23T22:17:21.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[215775]: failed to encode map e161 with expected crc 2023-09-23T22:17:21.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:20 smithi026 ceph-mon[215775]: failed to encode map e161 with expected crc 2023-09-23T22:17:21.235 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:21.451 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:21.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:21.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:21.570 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:21.616 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:21.701 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:21.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:21.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:21.809 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:21.860 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:21 smithi070 ceph-mon[192652]: pgmap v336: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 16241/262077 objects degraded (6.197%); 16037/262077 objects misplaced (6.119%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T22:17:21.975 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:21.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:21.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:22.080 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:22.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:21 smithi026 ceph-mon[215775]: pgmap v336: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 16241/262077 objects degraded (6.197%); 16037/262077 objects misplaced (6.119%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T22:17:22.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:21 smithi026 ceph-mon[223014]: pgmap v336: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 621 op/s; 16241/262077 objects degraded (6.197%); 16037/262077 objects misplaced (6.119%); 7.9 MiB/s, 125 objects/s recovering 2023-09-23T22:17:22.110 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:22.174 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:22.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:22.279 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:22.347 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:22.514 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:22.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:17:22.537 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:22.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:22.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:22.647 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:22.713 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:22.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:22 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 16241/262077 objects degraded (6.197%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:22.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:22 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 16241/262077 objects degraded (6.197%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:22.837 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:22.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:22.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:22.897 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:17:22.949 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:23.011 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:22 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 16241/262077 objects degraded (6.197%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:23.024 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:23.105 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:23.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:23.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:23.151 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:17:23.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:23.269 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:23.433 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:17:23.436 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:23.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:23.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:23.474 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:17:23.511 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:17:23.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:23.550 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:17:23.585 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:17:23.595 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:23.750 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:17:23.750 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:17:23.750 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:17:23.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:17:23.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:17:23.750 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:17:23.751 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:17:23.751 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:17:23.751 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:17:23.751 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:17:23.751 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:17:23.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:23 smithi070 ceph-mon[192652]: pgmap v337: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 16241/260190 objects degraded (6.242%); 15887/260190 objects misplaced (6.106%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:17:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/993168456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:23.939 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:23.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:23.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:23 smithi026 ceph-mon[223014]: pgmap v337: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 16241/260190 objects degraded (6.242%); 15887/260190 objects misplaced (6.106%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:17:23.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/993168456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:23 smithi026 ceph-mon[215775]: pgmap v337: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 592 op/s; 16241/260190 objects degraded (6.242%); 15887/260190 objects misplaced (6.106%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:17:23.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/993168456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:24.051 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:24.120 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:24.452 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:24.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:24.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:24.557 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:24.611 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:24.793 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:24.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:24.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:24.896 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:24.925 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:25.052 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:25.052 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:25.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:25.139 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:25.141 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:25.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:25.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:25.253 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:25.361 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:25.521 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:25.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:25.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:25.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:25.677 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:17:25.693 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:17:25.711 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:25.846 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:25.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:25.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:25 smithi070 ceph-mon[192652]: pgmap v338: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 16060/256491 objects degraded (6.261%); 15708/256491 objects misplaced (6.124%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T22:17:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:25 smithi070 ceph-mon[192652]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:17:25.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:25.961 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:17:25.961 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:17:26.048 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:26.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:25 smithi026 ceph-mon[223014]: pgmap v338: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 16060/256491 objects degraded (6.261%); 15708/256491 objects misplaced (6.124%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T22:17:26.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:25 smithi026 ceph-mon[223014]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:17:26.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:25 smithi026 ceph-mon[215775]: pgmap v338: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 521 op/s; 16060/256491 objects degraded (6.261%); 15708/256491 objects misplaced (6.124%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T22:17:26.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:25 smithi026 ceph-mon[215775]: from='client.54598 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:17:26.696 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:26.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:26.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:26.796 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:17:26.807 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:26.843 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:26.898 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:26.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:26.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:27.001 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:27.027 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:27.078 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:27.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:27.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:27.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:27.214 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":338,"stamp":"2023-09-23T22:17:25.067212+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5606021067,"num_objects":85497,"num_object_clones":0,"num_object_copies":256491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16060,"num_objects_misplaced":15708,"num_objects_unfound":0,"num_objects_dirty":85497,"num_whiteouts":0,"num_read":6469,"num_read_kb":13097,"num_write":103289,"num_write_kb":6018662,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18365,"num_bytes_recovered":1200102861,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101403,"ondisk_log_size":101403,"up":267,"acting":261,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":287,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16361916,"kb_used_data":15735428,"kb_used_omap":31,"kb_used_meta":626272,"kb_avail":639854148,"statfs":{"total":671965249536,"available":655210647552,"internally_reserved":0,"allocated":16113078272,"data_stored":16107542458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":641303479},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,5,14,6,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-285868032,"num_objects":-4362,"num_object_clones":0,"num_object_copies":-13086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-338,"num_objects_misplaced":-796,"num_objects_unfound":0,"num_objects_dirty":-4362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4362,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":796,"num_bytes_recovered":52166656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.357612"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":138,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718672+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:17:16.718672+0000","last_peered":"2023-09-23T22:17:16.718672+0000","last_clean":"2023-09-23T22:17:16.718672+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:17:16.718672+0000","last_undegraded":"2023-09-23T22:17:16.718672+0000","last_fullsized":"2023-09-23T22:17:16.718672+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":139,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.921505+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:17:16.921505+0000","last_peered":"2023-09-23T22:17:16.921505+0000","last_clean":"2023-09-23T22:17:16.921505+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:17:16.921505+0000","last_undegraded":"2023-09-23T22:17:16.921505+0000","last_fullsized":"2023-09-23T22:17:16.921505+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":99,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715963+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:17:16.715963+0000","last_peered":"2023-09-23T22:17:16.715963+0000","last_clean":"2023-09-23T22:17:16.715963+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:17:16.715963+0000","last_undegraded":"2023-09-23T22:17:16.715963+0000","last_fullsized":"2023-09-23T22:17:16.715963+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":80,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.921745+0000","last_change":"2023-09-23T22:17:00.187112+0000","last_active":"2023-09-23T22:17:16.921745+0000","last_peered":"2023-09-23T22:17:16.921745+0000","last_clean":"2023-09-23T22:17:16.921745+0000","last_became_active":"2023-09-23T22:17:00.186765+0000","last_became_peered":"2023-09-23T22:17:00.186765+0000","last_unstale":"2023-09-23T22:17:16.921745+0000","last_undegraded":"2023-09-23T22:17:16.921745+0000","last_fullsized":"2023-09-23T22:17:16.921745+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:09.875994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":170,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.920205+0000","last_change":"2023-09-23T22:17:00.187215+0000","last_active":"2023-09-23T22:17:16.920205+0000","last_peered":"2023-09-23T22:17:16.920205+0000","last_clean":"2023-09-23T22:17:16.920205+0000","last_became_active":"2023-09-23T22:17:00.186736+0000","last_became_peered":"2023-09-23T22:17:00.186736+0000","last_unstale":"2023-09-23T22:17:16.920205+0000","last_undegraded":"2023-09-23T22:17:16.920205+0000","last_fullsized":"2023-09-23T22:17:16.920205+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:52:04.601668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":86,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714956+0000","last_change":"2023-09-23T22:17:00.191966+0000","last_active":"2023-09-23T22:17:16.714956+0000","last_peered":"2023-09-23T22:17:16.714956+0000","last_clean":"2023-09-23T22:17:16.714956+0000","last_became_active":"2023-09-23T22:17:00.191320+0000","last_became_peered":"2023-09-23T22:17:00.191320+0000","last_unstale":"2023-09-23T22:17:16.714956+0000","last_undegraded":"2023-09-23T22:17:16.714956+0000","last_fullsized":"2023-09-23T22:17:16.714956+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:24.282286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":82,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.713672+0000","last_change":"2023-09-23T22:17:00.192696+0000","last_active":"2023-09-23T22:17:16.713672+0000","last_peered":"2023-09-23T22:17:16.713672+0000","last_clean":"2023-09-23T22:17:16.713672+0000","last_became_active":"2023-09-23T22:17:00.192183+0000","last_became_peered":"2023-09-23T22:17:00.192183+0000","last_unstale":"2023-09-23T22:17:16.713672+0000","last_undegraded":"2023-09-23T22:17:16.713672+0000","last_fullsized":"2023-09-23T22:17:16.713672+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:14.848683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":134,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.920253+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:17:16.920253+0000","last_peered":"2023-09-23T22:17:16.920253+0000","last_clean":"2023-09-23T22:17:16.920253+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:17:16.920253+0000","last_undegraded":"2023-09-23T22:17:16.920253+0000","last_fullsized":"2023-09-23T22:17:16.920253+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":121,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714812+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:17:16.714812+0000","last_peered":"2023-09-23T22:17:16.714812+0000","last_clean":"2023-09-23T22:17:16.714812+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:17:16.714812+0000","last_undegraded":"2023-09-23T22:17:16.714812+0000","last_fullsized":"2023-09-23T22:17:16.714812+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":84,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.921069+0000","last_change":"2023-09-23T22:16:49.154341+0000","last_active":"2023-09-23T22:17:16.921069+0000","last_peered":"2023-09-23T22:17:16.921069+0000","last_clean":"2023-09-23T22:17:16.921069+0000","last_became_active":"2023-09-23T22:16:49.154041+0000","last_became_peered":"2023-09-23T22:16:49.154041+0000","last_unstale":"2023-09-23T22:17:16.921069+0000","last_undegraded":"2023-09-23T22:17:16.921069+0000","last_fullsized":"2023-09-23T22:17:16.921069+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:05.733262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":71,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.921429+0000","last_change":"2023-09-23T22:15:14.829731+0000","last_active":"2023-09-23T22:17:16.921429+0000","last_peered":"2023-09-23T22:17:16.921429+0000","last_clean":"2023-09-23T22:17:16.921429+0000","last_became_active":"2023-09-23T22:15:14.829437+0000","last_became_peered":"2023-09-23T22:15:14.829437+0000","last_unstale":"2023-09-23T22:17:16.921429+0000","last_undegraded":"2023-09-23T22:17:16.921429+0000","last_fullsized":"2023-09-23T22:17:16.921429+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:45.456558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":117,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715747+0000","last_change":"2023-09-23T22:17:00.185122+0000","last_active":"2023-09-23T22:17:16.715747+0000","last_peered":"2023-09-23T22:17:16.715747+0000","last_clean":"2023-09-23T22:17:16.715747+0000","last_became_active":"2023-09-23T22:17:00.184659+0000","last_became_peered":"2023-09-23T22:17:00.184659+0000","last_unstale":"2023-09-23T22:17:16.715747+0000","last_undegraded":"2023-09-23T22:17:16.715747+0000","last_fullsized":"2023-09-23T22:17:16.715747+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:20.305756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":145,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718861+0000","last_change":"2023-09-23T22:16:49.265991+0000","last_active":"2023-09-23T22:17:16.718861+0000","last_peered":"2023-09-23T22:17:16.718861+0000","last_clean":"2023-09-23T22:17:16.718861+0000","last_became_active":"2023-09-23T22:16:49.251169+0000","last_became_peered":"2023-09-23T22:16:49.251169+0000","last_unstale":"2023-09-23T22:17:16.718861+0000","last_undegraded":"2023-09-23T22:17:16.718861+0000","last_fullsized":"2023-09-23T22:17:16.718861+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.502218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":129,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718672+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:17:16.718672+0000","last_peered":"2023-09-23T22:17:16.718672+0000","last_clean":"2023-09-23T22:17:16.718672+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:17:16.718672+0000","last_undegraded":"2023-09-23T22:17:16.718672+0000","last_fullsized":"2023-09-23T22:17:16.718672+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":72,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718698+0000","last_change":"2023-09-23T22:15:14.826143+0000","last_active":"2023-09-23T22:17:16.718698+0000","last_peered":"2023-09-23T22:17:16.718698+0000","last_clean":"2023-09-23T22:17:16.718698+0000","last_became_active":"2023-09-23T22:15:14.825780+0000","last_became_peered":"2023-09-23T22:15:14.825780+0000","last_unstale":"2023-09-23T22:17:16.718698+0000","last_undegraded":"2023-09-23T22:17:16.718698+0000","last_fullsized":"2023-09-23T22:17:16.718698+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:22.608676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":125,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.921630+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:17:16.921630+0000","last_peered":"2023-09-23T22:17:16.921630+0000","last_clean":"2023-09-23T22:17:16.921630+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:17:16.921630+0000","last_undegraded":"2023-09-23T22:17:16.921630+0000","last_fullsized":"2023-09-23T22:17:16.921630+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":80,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.922991+0000","last_change":"2023-09-23T22:17:00.187345+0000","last_active":"2023-09-23T22:17:16.922991+0000","last_peered":"2023-09-23T22:17:16.922991+0000","last_clean":"2023-09-23T22:17:16.922991+0000","last_became_active":"2023-09-23T22:17:00.186004+0000","last_became_peered":"2023-09-23T22:17:00.186004+0000","last_unstale":"2023-09-23T22:17:16.922991+0000","last_undegraded":"2023-09-23T22:17:16.922991+0000","last_fullsized":"2023-09-23T22:17:16.922991+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:23.380689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":120,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714597+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:17:16.714597+0000","last_peered":"2023-09-23T22:17:16.714597+0000","last_clean":"2023-09-23T22:17:16.714597+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:17:16.714597+0000","last_undegraded":"2023-09-23T22:17:16.714597+0000","last_fullsized":"2023-09-23T22:17:16.714597+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":84,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.722610+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:17:16.722610+0000","last_peered":"2023-09-23T22:17:16.722610+0000","last_clean":"2023-09-23T22:17:16.722610+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:17:16.722610+0000","last_undegraded":"2023-09-23T22:17:16.722610+0000","last_fullsized":"2023-09-23T22:17:16.722610+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":79,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.921874+0000","last_change":"2023-09-23T22:17:00.183212+0000","last_active":"2023-09-23T22:17:16.921874+0000","last_peered":"2023-09-23T22:17:16.921874+0000","last_clean":"2023-09-23T22:17:16.921874+0000","last_became_active":"2023-09-23T22:17:00.182870+0000","last_became_peered":"2023-09-23T22:17:00.182870+0000","last_unstale":"2023-09-23T22:17:16.921874+0000","last_undegraded":"2023-09-23T22:17:16.921874+0000","last_fullsized":"2023-09-23T22:17:16.921874+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:52.600734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":117,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.920083+0000","last_change":"2023-09-23T22:17:00.203378+0000","last_active":"2023-09-23T22:17:16.920083+0000","last_peered":"2023-09-23T22:17:16.920083+0000","last_clean":"2023-09-23T22:17:16.920083+0000","last_became_active":"2023-09-23T22:17:00.186864+0000","last_became_peered":"2023-09-23T22:17:00.186864+0000","last_unstale":"2023-09-23T22:17:16.920083+0000","last_undegraded":"2023-09-23T22:17:16.920083+0000","last_fullsized":"2023-09-23T22:17:16.920083+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:21.950488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":125,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.921559+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:17:16.921559+0000","last_peered":"2023-09-23T22:17:16.921559+0000","last_clean":"2023-09-23T22:17:16.921559+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:17:16.921559+0000","last_undegraded":"2023-09-23T22:17:16.921559+0000","last_fullsized":"2023-09-23T22:17:16.921559+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"161'6258","reported_seq":7480,"reported_epoch":161,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:17:22.380661+0000","last_change":"2023-09-23T22:17:07.675910+0000","last_active":"2023-09-23T22:17:22.380661+0000","last_peered":"2023-09-23T22:17:22.380661+0000","last_clean":"2023-09-23T22:16:48.081574+0000","last_became_active":"2023-09-23T22:17:07.425647+0000","last_became_peered":"2023-09-23T22:17:07.425647+0000","last_unstale":"2023-09-23T22:17:22.380661+0000","last_undegraded":"2023-09-23T22:17:22.380661+0000","last_fullsized":"2023-09-23T22:17:07.283635+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6258,"log_dups_size":0,"ondisk_log_size":6258,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:28.012738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346554368,"num_objects":5288,"num_object_clones":0,"num_object_copies":15864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5187,"num_objects_unfound":0,"num_objects_dirty":5288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6258,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":578,"num_bytes_recovered":37879808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":5187},{"shards":"4,7","objects":101}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":114,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.720370+0000","last_change":"2023-09-23T22:17:00.191723+0000","last_active":"2023-09-23T22:17:16.720370+0000","last_peered":"2023-09-23T22:17:16.720370+0000","last_clean":"2023-09-23T22:17:16.720370+0000","last_became_active":"2023-09-23T22:17:00.186206+0000","last_became_peered":"2023-09-23T22:17:00.186206+0000","last_unstale":"2023-09-23T22:17:16.720370+0000","last_undegraded":"2023-09-23T22:17:16.720370+0000","last_fullsized":"2023-09-23T22:17:16.720370+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:01.962508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":113,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.713294+0000","last_change":"2023-09-23T22:16:49.158580+0000","last_active":"2023-09-23T22:17:16.713294+0000","last_peered":"2023-09-23T22:17:16.713294+0000","last_clean":"2023-09-23T22:17:16.713294+0000","last_became_active":"2023-09-23T22:16:49.158266+0000","last_became_peered":"2023-09-23T22:16:49.158266+0000","last_unstale":"2023-09-23T22:17:16.713294+0000","last_undegraded":"2023-09-23T22:17:16.713294+0000","last_fullsized":"2023-09-23T22:17:16.713294+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:03.791658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"161'6207","reported_seq":11514,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:20.837395+0000","last_change":"2023-09-23T22:17:16.903782+0000","last_active":"2023-09-23T22:17:20.837395+0000","last_peered":"2023-09-23T22:17:20.837395+0000","last_clean":"2023-09-23T22:17:20.837395+0000","last_became_active":"2023-09-23T22:17:16.895165+0000","last_became_peered":"2023-09-23T22:17:16.895165+0000","last_unstale":"2023-09-23T22:17:20.837395+0000","last_undegraded":"2023-09-23T22:17:20.837395+0000","last_fullsized":"2023-09-23T22:17:20.837395+0000","mapping_epoch":159,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:31.886804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348979200,"num_objects":5325,"num_object_clones":0,"num_object_copies":15975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6207,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2600,"num_bytes_recovered":170393600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":72,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.720168+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:17:16.720168+0000","last_peered":"2023-09-23T22:17:16.720168+0000","last_clean":"2023-09-23T22:17:16.720168+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:17:16.720168+0000","last_undegraded":"2023-09-23T22:17:16.720168+0000","last_fullsized":"2023-09-23T22:17:16.720168+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":128,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.722567+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:17:16.722567+0000","last_peered":"2023-09-23T22:17:16.722567+0000","last_clean":"2023-09-23T22:17:16.722567+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:17:16.722567+0000","last_undegraded":"2023-09-23T22:17:16.722567+0000","last_fullsized":"2023-09-23T22:17:16.722567+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"161'6200","reported_seq":12433,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:20.817061+0000","last_change":"2023-09-23T22:17:05.192208+0000","last_active":"2023-09-23T22:17:20.817061+0000","last_peered":"2023-09-23T22:17:20.817061+0000","last_clean":"2023-09-23T22:17:20.817061+0000","last_became_active":"2023-09-23T22:17:05.187152+0000","last_became_peered":"2023-09-23T22:17:05.187152+0000","last_unstale":"2023-09-23T22:17:20.817061+0000","last_undegraded":"2023-09-23T22:17:20.817061+0000","last_fullsized":"2023-09-23T22:17:20.817061+0000","mapping_epoch":155,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6200,"log_dups_size":0,"ondisk_log_size":6200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:30.709178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348258304,"num_objects":5314,"num_object_clones":0,"num_object_copies":15942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6200,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3068,"num_bytes_recovered":201064448,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":86,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714192+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:17:16.714192+0000","last_peered":"2023-09-23T22:17:16.714192+0000","last_clean":"2023-09-23T22:17:16.714192+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:17:16.714192+0000","last_undegraded":"2023-09-23T22:17:16.714192+0000","last_fullsized":"2023-09-23T22:17:16.714192+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":75,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715617+0000","last_change":"2023-09-23T22:16:49.161676+0000","last_active":"2023-09-23T22:17:16.715617+0000","last_peered":"2023-09-23T22:17:16.715617+0000","last_clean":"2023-09-23T22:17:16.715617+0000","last_became_active":"2023-09-23T22:16:49.161247+0000","last_became_peered":"2023-09-23T22:16:49.161247+0000","last_unstale":"2023-09-23T22:17:16.715617+0000","last_undegraded":"2023-09-23T22:17:16.715617+0000","last_fullsized":"2023-09-23T22:17:16.715617+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:30.708371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"161'6321","reported_seq":6380,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:20.882951+0000","last_change":"2023-09-23T22:15:14.826723+0000","last_active":"2023-09-23T22:17:20.882951+0000","last_peered":"2023-09-23T22:17:20.882951+0000","last_clean":"2023-09-23T22:17:20.882951+0000","last_became_active":"2023-09-23T22:15:14.826454+0000","last_became_peered":"2023-09-23T22:15:14.826454+0000","last_unstale":"2023-09-23T22:17:20.882951+0000","last_undegraded":"2023-09-23T22:17:20.882951+0000","last_fullsized":"2023-09-23T22:17:20.882951+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6321,"log_dups_size":0,"ondisk_log_size":6321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:11.166077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352124928,"num_objects":5373,"num_object_clones":0,"num_object_copies":16119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6321,"num_write_kb":374208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":131,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714842+0000","last_change":"2023-09-23T22:16:49.164833+0000","last_active":"2023-09-23T22:17:16.714842+0000","last_peered":"2023-09-23T22:17:16.714842+0000","last_clean":"2023-09-23T22:17:16.714842+0000","last_became_active":"2023-09-23T22:16:49.163833+0000","last_became_peered":"2023-09-23T22:16:49.163833+0000","last_unstale":"2023-09-23T22:17:16.714842+0000","last_undegraded":"2023-09-23T22:17:16.714842+0000","last_fullsized":"2023-09-23T22:17:16.714842+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.891395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714586+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:17:16.714586+0000","last_peered":"2023-09-23T22:17:16.714586+0000","last_clean":"2023-09-23T22:17:16.714586+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:17:16.714586+0000","last_undegraded":"2023-09-23T22:17:16.714586+0000","last_fullsized":"2023-09-23T22:17:16.714586+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"161'6378","reported_seq":6438,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:20.852483+0000","last_change":"2023-09-23T22:15:14.824965+0000","last_active":"2023-09-23T22:17:20.852483+0000","last_peered":"2023-09-23T22:17:20.852483+0000","last_clean":"2023-09-23T22:17:20.852483+0000","last_became_active":"2023-09-23T22:15:14.824626+0000","last_became_peered":"2023-09-23T22:15:14.824626+0000","last_unstale":"2023-09-23T22:17:20.852483+0000","last_undegraded":"2023-09-23T22:17:20.852483+0000","last_fullsized":"2023-09-23T22:17:20.852483+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6378,"log_dups_size":0,"ondisk_log_size":6378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.613455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354746392,"num_objects":5414,"num_object_clones":0,"num_object_copies":16242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5414,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6378,"num_write_kb":377281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":162,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.921923+0000","last_change":"2023-09-23T22:17:00.195681+0000","last_active":"2023-09-23T22:17:16.921923+0000","last_peered":"2023-09-23T22:17:16.921923+0000","last_clean":"2023-09-23T22:17:16.921923+0000","last_became_active":"2023-09-23T22:17:00.186019+0000","last_became_peered":"2023-09-23T22:17:00.186019+0000","last_unstale":"2023-09-23T22:17:16.921923+0000","last_undegraded":"2023-09-23T22:17:16.921923+0000","last_fullsized":"2023-09-23T22:17:16.921923+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:14.135203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":129,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718688+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:17:16.718688+0000","last_peered":"2023-09-23T22:17:16.718688+0000","last_clean":"2023-09-23T22:17:16.718688+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:17:16.718688+0000","last_undegraded":"2023-09-23T22:17:16.718688+0000","last_fullsized":"2023-09-23T22:17:16.718688+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"161'6429","reported_seq":8710,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:23.027614+0000","last_change":"2023-09-23T22:16:30.157766+0000","last_active":"2023-09-23T22:17:23.027614+0000","last_peered":"2023-09-23T22:17:23.027614+0000","last_clean":"2023-09-23T22:17:23.027614+0000","last_became_active":"2023-09-23T22:16:30.150277+0000","last_became_peered":"2023-09-23T22:16:30.150277+0000","last_unstale":"2023-09-23T22:17:23.027614+0000","last_undegraded":"2023-09-23T22:17:23.027614+0000","last_fullsized":"2023-09-23T22:17:23.027614+0000","mapping_epoch":140,"log_start":"115'901","ondisk_log_start":"115'901","created":111,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5528,"log_dups_size":0,"ondisk_log_size":5528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:20.415978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346095616,"num_objects":5281,"num_object_clones":0,"num_object_copies":15843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6429,"num_write_kb":374720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1098,"num_bytes_recovered":71958528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":77,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718768+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:17:16.718768+0000","last_peered":"2023-09-23T22:17:16.718768+0000","last_clean":"2023-09-23T22:17:16.718768+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:17:16.718768+0000","last_undegraded":"2023-09-23T22:17:16.718768+0000","last_fullsized":"2023-09-23T22:17:16.718768+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":136,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.720071+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:17:16.720071+0000","last_peered":"2023-09-23T22:17:16.720071+0000","last_clean":"2023-09-23T22:17:16.720071+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:17:16.720071+0000","last_undegraded":"2023-09-23T22:17:16.720071+0000","last_fullsized":"2023-09-23T22:17:16.720071+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"161'6519","reported_seq":9050,"reported_epoch":161,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:17:24.966872+0000","last_change":"2023-09-23T22:17:00.216921+0000","last_active":"2023-09-23T22:17:24.966872+0000","last_peered":"2023-09-23T22:17:24.966872+0000","last_clean":"2023-09-23T22:16:48.124555+0000","last_became_active":"2023-09-23T22:16:50.339530+0000","last_became_peered":"2023-09-23T22:16:50.339530+0000","last_unstale":"2023-09-23T22:17:24.966872+0000","last_undegraded":"2023-09-23T22:17:24.966872+0000","last_fullsized":"2023-09-23T22:16:50.148559+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6519,"log_dups_size":0,"ondisk_log_size":6519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:01.673286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344260608,"num_objects":5253,"num_object_clones":0,"num_object_copies":15759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4647,"num_objects_unfound":0,"num_objects_dirty":5253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6519,"num_write_kb":376704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1230,"num_bytes_recovered":80609280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":606},{"shards":"4,5,6","objects":4647}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":112,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.722429+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:17:16.722429+0000","last_peered":"2023-09-23T22:17:16.722429+0000","last_clean":"2023-09-23T22:17:16.722429+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:17:16.722429+0000","last_undegraded":"2023-09-23T22:17:16.722429+0000","last_fullsized":"2023-09-23T22:17:16.722429+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":129,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.720222+0000","last_change":"2023-09-23T22:16:49.165118+0000","last_active":"2023-09-23T22:17:16.720222+0000","last_peered":"2023-09-23T22:17:16.720222+0000","last_clean":"2023-09-23T22:17:16.720222+0000","last_became_active":"2023-09-23T22:16:49.164802+0000","last_became_peered":"2023-09-23T22:16:49.164802+0000","last_unstale":"2023-09-23T22:17:16.720222+0000","last_undegraded":"2023-09-23T22:17:16.720222+0000","last_fullsized":"2023-09-23T22:17:16.720222+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:23.079142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"161'6479","reported_seq":10302,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:22.385368+0000","last_change":"2023-09-23T22:15:59.633734+0000","last_active":"2023-09-23T22:17:22.385368+0000","last_peered":"2023-09-23T22:17:22.385368+0000","last_clean":"2023-09-23T22:17:22.385368+0000","last_became_active":"2023-09-23T22:15:59.605894+0000","last_became_peered":"2023-09-23T22:15:59.605894+0000","last_unstale":"2023-09-23T22:17:22.385368+0000","last_undegraded":"2023-09-23T22:17:22.385368+0000","last_fullsized":"2023-09-23T22:17:22.385368+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6479,"log_dups_size":0,"ondisk_log_size":6479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:22.683442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357105664,"num_objects":5449,"num_object_clones":0,"num_object_copies":16347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6479,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1867,"num_bytes_recovered":122355712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":126,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.920131+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:17:16.920131+0000","last_peered":"2023-09-23T22:17:16.920131+0000","last_clean":"2023-09-23T22:17:16.920131+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:17:16.920131+0000","last_undegraded":"2023-09-23T22:17:16.920131+0000","last_fullsized":"2023-09-23T22:17:16.920131+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":100,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:19.776953+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:17:19.776953+0000","last_peered":"2023-09-23T22:17:19.776953+0000","last_clean":"2023-09-23T22:17:19.776953+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:17:19.776953+0000","last_undegraded":"2023-09-23T22:17:19.776953+0000","last_fullsized":"2023-09-23T22:17:19.776953+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":707,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715115+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:17:16.715115+0000","last_peered":"2023-09-23T22:17:16.715115+0000","last_clean":"2023-09-23T22:17:16.715115+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:17:16.715115+0000","last_undegraded":"2023-09-23T22:17:16.715115+0000","last_fullsized":"2023-09-23T22:17:16.715115+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":138,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714679+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:17:16.714679+0000","last_peered":"2023-09-23T22:17:16.714679+0000","last_clean":"2023-09-23T22:17:16.714679+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:17:16.714679+0000","last_undegraded":"2023-09-23T22:17:16.714679+0000","last_fullsized":"2023-09-23T22:17:16.714679+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":81,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.713006+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:17:16.713006+0000","last_peered":"2023-09-23T22:17:16.713006+0000","last_clean":"2023-09-23T22:17:16.713006+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:17:16.713006+0000","last_undegraded":"2023-09-23T22:17:16.713006+0000","last_fullsized":"2023-09-23T22:17:16.713006+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"161'6516","reported_seq":11393,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:17:24.576868+0000","last_change":"2023-09-23T22:17:01.449877+0000","last_active":"2023-09-23T22:17:24.576868+0000","last_peered":"2023-09-23T22:17:24.576868+0000","last_clean":"2023-09-23T22:16:48.130011+0000","last_became_active":"2023-09-23T22:17:01.400862+0000","last_became_peered":"2023-09-23T22:17:01.400862+0000","last_unstale":"2023-09-23T22:17:24.576868+0000","last_undegraded":"2023-09-23T22:17:01.381678+0000","last_fullsized":"2023-09-23T22:17:01.178062+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6516,"log_dups_size":0,"ondisk_log_size":6516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:04.734921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342884352,"num_objects":5232,"num_object_clones":0,"num_object_copies":15696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5232,"num_objects_misplaced":5874,"num_objects_unfound":0,"num_objects_dirty":5232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6516,"num_write_kb":375936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2384,"num_bytes_recovered":156237824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,0],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5","objects":-642},{"shards":"0,5,6","objects":5232}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.5","version":"161'6570","reported_seq":10446,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:17:23.000023+0000","last_change":"2023-09-23T22:17:01.389883+0000","last_active":"2023-09-23T22:17:23.000023+0000","last_peered":"2023-09-23T22:17:23.000023+0000","last_clean":"2023-09-23T22:16:50.502857+0000","last_became_active":"2023-09-23T22:17:01.381686+0000","last_became_peered":"2023-09-23T22:17:01.381686+0000","last_unstale":"2023-09-23T22:17:23.000023+0000","last_undegraded":"2023-09-23T22:17:01.202465+0000","last_fullsized":"2023-09-23T22:17:01.175076+0000","mapping_epoch":152,"log_start":"115'1031","ondisk_log_start":"115'1031","created":111,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5539,"log_dups_size":0,"ondisk_log_size":5539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:57.913138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353894400,"num_objects":5400,"num_object_clones":0,"num_object_copies":16200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5985,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6570,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1895,"num_bytes_recovered":124190720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4],"avail_no_missing":["1","0","4","6"],"object_location_counts":[{"shards":"0,1,4","objects":5400},{"shards":"1,4","objects":-585}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":865,"reported_epoch":160,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:17:16.713889+0000","last_change":"2023-09-23T22:17:15.722644+0000","last_active":"2023-09-23T22:17:16.713889+0000","last_peered":"2023-09-23T22:17:16.713889+0000","last_clean":"2023-09-23T22:17:10.678840+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:17:16.713889+0000","last_undegraded":"2023-09-23T22:17:15.714187+0000","last_fullsized":"2023-09-23T22:17:16.713889+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":170,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718645+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:17:16.718645+0000","last_peered":"2023-09-23T22:17:16.718645+0000","last_clean":"2023-09-23T22:17:16.718645+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:17:16.718645+0000","last_undegraded":"2023-09-23T22:17:16.718645+0000","last_fullsized":"2023-09-23T22:17:16.718645+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":135,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714676+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:17:16.714676+0000","last_peered":"2023-09-23T22:17:16.714676+0000","last_clean":"2023-09-23T22:17:16.714676+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:17:16.714676+0000","last_undegraded":"2023-09-23T22:17:16.714676+0000","last_fullsized":"2023-09-23T22:17:16.714676+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.6","version":"161'6279","reported_seq":6338,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:20.860644+0000","last_change":"2023-09-23T22:15:14.823394+0000","last_active":"2023-09-23T22:17:20.860644+0000","last_peered":"2023-09-23T22:17:20.860644+0000","last_clean":"2023-09-23T22:17:20.860644+0000","last_became_active":"2023-09-23T22:15:14.823003+0000","last_became_peered":"2023-09-23T22:15:14.823003+0000","last_unstale":"2023-09-23T22:17:20.860644+0000","last_undegraded":"2023-09-23T22:17:20.860644+0000","last_fullsized":"2023-09-23T22:17:20.860644+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6279,"log_dups_size":0,"ondisk_log_size":6279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:13.496813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353173504,"num_objects":5389,"num_object_clones":0,"num_object_copies":16167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6279,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":501,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.712925+0000","last_change":"2023-09-23T22:15:51.379227+0000","last_active":"2023-09-23T22:17:16.712925+0000","last_peered":"2023-09-23T22:17:16.712925+0000","last_clean":"2023-09-23T22:17:16.712925+0000","last_became_active":"2023-09-23T22:15:43.928977+0000","last_became_peered":"2023-09-23T22:15:43.928977+0000","last_unstale":"2023-09-23T22:17:16.712925+0000","last_undegraded":"2023-09-23T22:17:16.712925+0000","last_fullsized":"2023-09-23T22:17:16.712925+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:04.398533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":109,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714891+0000","last_change":"2023-09-23T22:16:49.251412+0000","last_active":"2023-09-23T22:17:16.714891+0000","last_peered":"2023-09-23T22:17:16.714891+0000","last_clean":"2023-09-23T22:17:16.714891+0000","last_became_active":"2023-09-23T22:16:49.251097+0000","last_became_peered":"2023-09-23T22:16:49.251097+0000","last_unstale":"2023-09-23T22:17:16.714891+0000","last_undegraded":"2023-09-23T22:17:16.714891+0000","last_fullsized":"2023-09-23T22:17:16.714891+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:41.601435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":156,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715797+0000","last_change":"2023-09-23T22:17:00.192930+0000","last_active":"2023-09-23T22:17:16.715797+0000","last_peered":"2023-09-23T22:17:16.715797+0000","last_clean":"2023-09-23T22:17:16.715797+0000","last_became_active":"2023-09-23T22:17:00.192592+0000","last_became_peered":"2023-09-23T22:17:00.192592+0000","last_unstale":"2023-09-23T22:17:16.715797+0000","last_undegraded":"2023-09-23T22:17:16.715797+0000","last_fullsized":"2023-09-23T22:17:16.715797+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:07.279909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":746,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715045+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:17:16.715045+0000","last_peered":"2023-09-23T22:17:16.715045+0000","last_clean":"2023-09-23T22:17:16.715045+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:17:16.715045+0000","last_undegraded":"2023-09-23T22:17:16.715045+0000","last_fullsized":"2023-09-23T22:17:16.715045+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":469,"reported_epoch":160,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:17:16.714501+0000","last_change":"2023-09-23T22:17:01.190810+0000","last_active":"2023-09-23T22:17:16.714501+0000","last_peered":"2023-09-23T22:17:16.714501+0000","last_clean":"2023-09-23T22:08:39.036645+0000","last_became_active":"2023-09-23T22:17:01.190463+0000","last_became_peered":"2023-09-23T22:17:01.190463+0000","last_unstale":"2023-09-23T22:17:16.714501+0000","last_undegraded":"2023-09-23T22:17:01.173656+0000","last_fullsized":"2023-09-23T22:17:01.172963+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:59.602729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"1,6,7","objects":2}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":84,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.922851+0000","last_change":"2023-09-23T22:16:49.155156+0000","last_active":"2023-09-23T22:17:16.922851+0000","last_peered":"2023-09-23T22:17:16.922851+0000","last_clean":"2023-09-23T22:17:16.922851+0000","last_became_active":"2023-09-23T22:16:49.154923+0000","last_became_peered":"2023-09-23T22:16:49.154923+0000","last_unstale":"2023-09-23T22:17:16.922851+0000","last_undegraded":"2023-09-23T22:17:16.922851+0000","last_fullsized":"2023-09-23T22:17:16.922851+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:33.532238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":47,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.720229+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:17:16.720229+0000","last_peered":"2023-09-23T22:17:16.720229+0000","last_clean":"2023-09-23T22:17:16.720229+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:17:16.720229+0000","last_undegraded":"2023-09-23T22:17:16.720229+0000","last_fullsized":"2023-09-23T22:17:16.720229+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"161'6335","reported_seq":6395,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:20.780426+0000","last_change":"2023-09-23T22:15:14.824102+0000","last_active":"2023-09-23T22:17:20.780426+0000","last_peered":"2023-09-23T22:17:20.780426+0000","last_clean":"2023-09-23T22:17:20.780426+0000","last_became_active":"2023-09-23T22:15:14.823790+0000","last_became_peered":"2023-09-23T22:15:14.823790+0000","last_unstale":"2023-09-23T22:17:20.780426+0000","last_undegraded":"2023-09-23T22:17:20.780426+0000","last_fullsized":"2023-09-23T22:17:20.780426+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6335,"log_dups_size":0,"ondisk_log_size":6335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:24.517495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355139584,"num_objects":5419,"num_object_clones":0,"num_object_copies":16257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6335,"num_write_kb":376128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.7","version":"161'6548","reported_seq":11414,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:24.608553+0000","last_change":"2023-09-23T22:17:15.752911+0000","last_active":"2023-09-23T22:17:24.608553+0000","last_peered":"2023-09-23T22:17:24.608553+0000","last_clean":"2023-09-23T22:17:24.608553+0000","last_became_active":"2023-09-23T22:17:15.728788+0000","last_became_peered":"2023-09-23T22:17:15.728788+0000","last_unstale":"2023-09-23T22:17:24.608553+0000","last_undegraded":"2023-09-23T22:17:24.608553+0000","last_fullsized":"2023-09-23T22:17:24.608553+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6548,"log_dups_size":0,"ondisk_log_size":6548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:35.923920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352714752,"num_objects":5382,"num_object_clones":0,"num_object_copies":16146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6548,"num_write_kb":381760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2374,"num_bytes_recovered":155582464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":508,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:18.052165+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:17:18.052165+0000","last_peered":"2023-09-23T22:17:18.052165+0000","last_clean":"2023-09-23T22:17:18.052165+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:17:18.052165+0000","last_undegraded":"2023-09-23T22:17:18.052165+0000","last_fullsized":"2023-09-23T22:17:18.052165+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714227+0000","last_change":"2023-09-23T22:17:15.719993+0000","last_active":"2023-09-23T22:17:16.714227+0000","last_peered":"2023-09-23T22:17:16.714227+0000","last_clean":"2023-09-23T22:17:16.714227+0000","last_became_active":"2023-09-23T22:17:15.719613+0000","last_became_peered":"2023-09-23T22:17:15.719613+0000","last_unstale":"2023-09-23T22:17:16.714227+0000","last_undegraded":"2023-09-23T22:17:16.714227+0000","last_fullsized":"2023-09-23T22:17:16.714227+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:51.213331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":89,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715285+0000","last_change":"2023-09-23T22:17:00.182583+0000","last_active":"2023-09-23T22:17:16.715285+0000","last_peered":"2023-09-23T22:17:16.715285+0000","last_clean":"2023-09-23T22:17:16.715285+0000","last_became_active":"2023-09-23T22:17:00.182250+0000","last_became_peered":"2023-09-23T22:17:00.182250+0000","last_unstale":"2023-09-23T22:17:16.715285+0000","last_undegraded":"2023-09-23T22:17:16.715285+0000","last_fullsized":"2023-09-23T22:17:16.715285+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:53.474239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":579,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.720057+0000","last_change":"2023-09-23T22:17:03.377390+0000","last_active":"2023-09-23T22:17:16.720057+0000","last_peered":"2023-09-23T22:17:16.720057+0000","last_clean":"2023-09-23T22:17:16.720057+0000","last_became_active":"2023-09-23T22:17:00.186427+0000","last_became_peered":"2023-09-23T22:17:00.186427+0000","last_unstale":"2023-09-23T22:17:16.720057+0000","last_undegraded":"2023-09-23T22:17:16.720057+0000","last_fullsized":"2023-09-23T22:17:16.720057+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:46.716124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":100,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715450+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:17:16.715450+0000","last_peered":"2023-09-23T22:17:16.715450+0000","last_clean":"2023-09-23T22:17:16.715450+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:17:16.715450+0000","last_undegraded":"2023-09-23T22:17:16.715450+0000","last_fullsized":"2023-09-23T22:17:16.715450+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714507+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:17:16.714507+0000","last_peered":"2023-09-23T22:17:16.714507+0000","last_clean":"2023-09-23T22:17:16.714507+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:17:16.714507+0000","last_undegraded":"2023-09-23T22:17:16.714507+0000","last_fullsized":"2023-09-23T22:17:16.714507+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"161'6437","reported_seq":8789,"reported_epoch":161,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:17:20.764141+0000","last_change":"2023-09-23T22:17:00.210975+0000","last_active":"2023-09-23T22:17:20.764141+0000","last_peered":"2023-09-23T22:17:20.764141+0000","last_clean":"2023-09-23T22:16:48.102227+0000","last_became_active":"2023-09-23T22:16:50.283490+0000","last_became_peered":"2023-09-23T22:16:50.283490+0000","last_unstale":"2023-09-23T22:17:20.764141+0000","last_undegraded":"2023-09-23T22:17:00.207659+0000","last_fullsized":"2023-09-23T22:16:50.146447+0000","mapping_epoch":147,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6437,"log_dups_size":0,"ondisk_log_size":6437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:48.954402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359858176,"num_objects":5491,"num_object_clones":0,"num_object_copies":16473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4829,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6437,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1127,"num_bytes_recovered":73859072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5","objects":662},{"shards":"0,5,6","objects":4357}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":171,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.922022+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:17:16.922022+0000","last_peered":"2023-09-23T22:17:16.922022+0000","last_clean":"2023-09-23T22:17:16.922022+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:17:16.922022+0000","last_undegraded":"2023-09-23T22:17:16.922022+0000","last_fullsized":"2023-09-23T22:17:16.922022+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":800,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715367+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:17:16.715367+0000","last_peered":"2023-09-23T22:17:16.715367+0000","last_clean":"2023-09-23T22:17:16.715367+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:17:16.715367+0000","last_undegraded":"2023-09-23T22:17:16.715367+0000","last_fullsized":"2023-09-23T22:17:16.715367+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":147,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.920175+0000","last_change":"2023-09-23T22:17:00.187055+0000","last_active":"2023-09-23T22:17:16.920175+0000","last_peered":"2023-09-23T22:17:16.920175+0000","last_clean":"2023-09-23T22:17:16.920175+0000","last_became_active":"2023-09-23T22:17:00.186276+0000","last_became_peered":"2023-09-23T22:17:00.186276+0000","last_unstale":"2023-09-23T22:17:16.920175+0000","last_undegraded":"2023-09-23T22:17:16.920175+0000","last_fullsized":"2023-09-23T22:17:16.920175+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:54.595046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"161'6271","reported_seq":6321,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:22.378133+0000","last_change":"2023-09-23T22:15:14.828787+0000","last_active":"2023-09-23T22:17:22.378133+0000","last_peered":"2023-09-23T22:17:22.378133+0000","last_clean":"2023-09-23T22:17:22.378133+0000","last_became_active":"2023-09-23T22:15:14.828370+0000","last_became_peered":"2023-09-23T22:15:14.828370+0000","last_unstale":"2023-09-23T22:17:22.378133+0000","last_undegraded":"2023-09-23T22:17:22.378133+0000","last_fullsized":"2023-09-23T22:17:22.378133+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6271,"log_dups_size":0,"ondisk_log_size":6271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:35.080268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345047040,"num_objects":5265,"num_object_clones":0,"num_object_copies":15795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6271,"num_write_kb":369152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":224,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.713284+0000","last_change":"2023-09-23T22:16:49.270015+0000","last_active":"2023-09-23T22:17:16.713284+0000","last_peered":"2023-09-23T22:17:16.713284+0000","last_clean":"2023-09-23T22:17:16.713284+0000","last_became_active":"2023-09-23T22:16:49.252726+0000","last_became_peered":"2023-09-23T22:16:49.252726+0000","last_unstale":"2023-09-23T22:17:16.713284+0000","last_undegraded":"2023-09-23T22:17:16.713284+0000","last_fullsized":"2023-09-23T22:17:16.713284+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:18.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":65,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.715520+0000","last_change":"2023-09-23T22:17:00.193883+0000","last_active":"2023-09-23T22:17:16.715520+0000","last_peered":"2023-09-23T22:17:16.715520+0000","last_clean":"2023-09-23T22:17:16.715520+0000","last_became_active":"2023-09-23T22:17:00.182423+0000","last_became_peered":"2023-09-23T22:17:00.182423+0000","last_unstale":"2023-09-23T22:17:16.715520+0000","last_undegraded":"2023-09-23T22:17:16.715520+0000","last_fullsized":"2023-09-23T22:17:16.715520+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:06.249645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":124,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.722489+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:17:16.722489+0000","last_peered":"2023-09-23T22:17:16.722489+0000","last_clean":"2023-09-23T22:17:16.722489+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:17:16.722489+0000","last_undegraded":"2023-09-23T22:17:16.722489+0000","last_fullsized":"2023-09-23T22:17:16.722489+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"161'6396","reported_seq":6456,"reported_epoch":161,"state":"active+clean","last_fresh":"2023-09-23T22:17:23.800467+0000","last_change":"2023-09-23T22:15:14.825150+0000","last_active":"2023-09-23T22:17:23.800467+0000","last_peered":"2023-09-23T22:17:23.800467+0000","last_clean":"2023-09-23T22:17:23.800467+0000","last_became_active":"2023-09-23T22:15:14.824867+0000","last_became_peered":"2023-09-23T22:15:14.824867+0000","last_unstale":"2023-09-23T22:17:23.800467+0000","last_undegraded":"2023-09-23T22:17:23.800467+0000","last_fullsized":"2023-09-23T22:17:23.800467+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6396,"log_dups_size":0,"ondisk_log_size":6396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:30.053753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340393984,"num_objects":5194,"num_object_clones":0,"num_object_copies":15582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6396,"num_write_kb":370880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":50,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718782+0000","last_change":"2023-09-23T22:17:00.186824+0000","last_active":"2023-09-23T22:17:16.718782+0000","last_peered":"2023-09-23T22:17:16.718782+0000","last_clean":"2023-09-23T22:17:16.718782+0000","last_became_active":"2023-09-23T22:17:00.186238+0000","last_became_peered":"2023-09-23T22:17:00.186238+0000","last_unstale":"2023-09-23T22:17:16.718782+0000","last_undegraded":"2023-09-23T22:17:16.718782+0000","last_fullsized":"2023-09-23T22:17:16.718782+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:08.448835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":144,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.713164+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:17:16.713164+0000","last_peered":"2023-09-23T22:17:16.713164+0000","last_clean":"2023-09-23T22:17:16.713164+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:17:16.713164+0000","last_undegraded":"2023-09-23T22:17:16.713164+0000","last_fullsized":"2023-09-23T22:17:16.713164+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":123,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714847+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:17:16.714847+0000","last_peered":"2023-09-23T22:17:16.714847+0000","last_clean":"2023-09-23T22:17:16.714847+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:17:16.714847+0000","last_undegraded":"2023-09-23T22:17:16.714847+0000","last_fullsized":"2023-09-23T22:17:16.714847+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":136,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718863+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:17:16.718863+0000","last_peered":"2023-09-23T22:17:16.718863+0000","last_clean":"2023-09-23T22:17:16.718863+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:17:16.718863+0000","last_undegraded":"2023-09-23T22:17:16.718863+0000","last_fullsized":"2023-09-23T22:17:16.718863+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":120,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.714689+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:17:16.714689+0000","last_peered":"2023-09-23T22:17:16.714689+0000","last_clean":"2023-09-23T22:17:16.714689+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:17:16.714689+0000","last_undegraded":"2023-09-23T22:17:16.714689+0000","last_fullsized":"2023-09-23T22:17:16.714689+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":171,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.922980+0000","last_change":"2023-09-23T22:17:15.719889+0000","last_active":"2023-09-23T22:17:16.922980+0000","last_peered":"2023-09-23T22:17:16.922980+0000","last_clean":"2023-09-23T22:17:16.922980+0000","last_became_active":"2023-09-23T22:17:15.719558+0000","last_became_peered":"2023-09-23T22:17:15.719558+0000","last_unstale":"2023-09-23T22:17:16.922980+0000","last_undegraded":"2023-09-23T22:17:16.922980+0000","last_fullsized":"2023-09-23T22:17:16.922980+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:51.784550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":140,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.921662+0000","last_change":"2023-09-23T22:16:49.145417+0000","last_active":"2023-09-23T22:17:16.921662+0000","last_peered":"2023-09-23T22:17:16.921662+0000","last_clean":"2023-09-23T22:17:16.921662+0000","last_became_active":"2023-09-23T22:16:49.145017+0000","last_became_peered":"2023-09-23T22:16:49.145017+0000","last_unstale":"2023-09-23T22:17:16.921662+0000","last_undegraded":"2023-09-23T22:17:16.921662+0000","last_fullsized":"2023-09-23T22:17:16.921662+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:47.762149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":182,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.920947+0000","last_change":"2023-09-23T22:17:15.721123+0000","last_active":"2023-09-23T22:17:16.920947+0000","last_peered":"2023-09-23T22:17:16.920947+0000","last_clean":"2023-09-23T22:17:16.920947+0000","last_became_active":"2023-09-23T22:17:15.720812+0000","last_became_peered":"2023-09-23T22:17:15.720812+0000","last_unstale":"2023-09-23T22:17:16.920947+0000","last_undegraded":"2023-09-23T22:17:16.920947+0000","last_fullsized":"2023-09-23T22:17:16.920947+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.176278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":72,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.713585+0000","last_change":"2023-09-23T22:17:00.198581+0000","last_active":"2023-09-23T22:17:16.713585+0000","last_peered":"2023-09-23T22:17:16.713585+0000","last_clean":"2023-09-23T22:17:16.713585+0000","last_became_active":"2023-09-23T22:17:00.191620+0000","last_became_peered":"2023-09-23T22:17:00.191620+0000","last_unstale":"2023-09-23T22:17:16.713585+0000","last_undegraded":"2023-09-23T22:17:16.713585+0000","last_fullsized":"2023-09-23T22:17:16.713585+0000","mapping_epoch":151,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:50:13.511541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":129,"reported_epoch":160,"state":"active+clean","last_fresh":"2023-09-23T22:17:16.718482+0000","last_change":"2023-09-23T22:16:49.164638+0000","last_active":"2023-09-23T22:17:16.718482+0000","last_peered":"2023-09-23T22:17:16.718482+0000","last_clean":"2023-09-23T22:17:16.718482+0000","last_became_active":"2023-09-23T22:16:49.164349+0000","last_became_peered":"2023-09-23T22:16:49.164349+0000","last_unstale":"2023-09-23T22:17:16.718482+0000","last_undegraded":"2023-09-23T22:17:16.718482+0000","last_fullsized":"2023-09-23T22:17:16.718482+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:26.536927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":5601230872,"num_objects":85469,"num_object_clones":0,"num_object_copies":256407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16046,"num_objects_misplaced":15708,"num_objects_unfound":0,"num_objects_dirty":85469,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":102143,"num_write_kb":6003521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18221,"num_bytes_recovered":1194131456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17672187904,"data_stored":17672175688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100211,"ondisk_log_size":100211,"up":48,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":5978,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":2,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":113,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234156,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2075128,"kb_used_data":2009456,"kb_used_omap":0,"kb_used_meta":65600,"kb_avail":91670024,"statfs":{"total":95995035648,"available":93870104576,"internally_reserved":0,"allocated":2057682944,"data_stored":2056974041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":67174400},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:16:54 2023","interfaces":[{"interface":"back","average":{"1min":5.9480000000000004,"5min":1.6639999999999999,"15min":0.92200000000000004},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":109.208,"5min":109.208,"15min":109.208},"last":0.26300000000000001},{"interface":"front","average":{"1min":7.2060000000000004,"5min":2.024,"15min":1.0640000000000001},"min":{"1min":0.24299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":109.13800000000001,"5min":109.13800000000001,"15min":109.13800000000001},"last":1.0069999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:17:05 2023","interfaces":[{"interface":"back","average":{"1min":7.1239999999999997,"5min":2.0219999999999998,"15min":1.0509999999999999},"min":{"1min":0.25,"5min":0.246,"15min":0.246},"max":{"1min":123.79300000000001,"5min":123.79300000000001,"15min":123.79300000000001},"last":4.8819999999999997},{"interface":"front","average":{"1min":7.7069999999999999,"5min":2.2400000000000002,"15min":1.1379999999999999},"min":{"1min":0.35199999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":123.66500000000001,"5min":123.66500000000001,"15min":123.66500000000001},"last":1.4910000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:16:32 2023","interfaces":[{"interface":"back","average":{"1min":9.3249999999999993,"5min":2.6520000000000001,"15min":1.556},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":106.63200000000001,"5min":106.63200000000001,"15min":106.63200000000001},"last":8.8490000000000002},{"interface":"front","average":{"1min":9.2430000000000003,"5min":2.3559999999999999,"15min":1.194},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":106.798,"5min":106.798,"15min":106.798},"last":0.93300000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:17:05 2023","interfaces":[{"interface":"back","average":{"1min":7.1360000000000001,"5min":1.948,"15min":1.05},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":113.282,"5min":113.282,"15min":113.282},"last":0.505},{"interface":"front","average":{"1min":6.7800000000000002,"5min":2.0529999999999999,"15min":1.131},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":104.74299999999999,"5min":104.74299999999999,"15min":104.74299999999999},"last":2.5289999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:16:36 2023","interfaces":[{"interface":"back","average":{"1min":2.4380000000000002,"5min":0.92100000000000004,"15min":0.61499999999999999},"min":{"1min":0.29499999999999998,"5min":0.28299999999999997,"15min":0.251},"max":{"1min":33.627000000000002,"5min":33.627000000000002,"15min":33.627000000000002},"last":0.91200000000000003},{"interface":"front","average":{"1min":2.391,"5min":0.89400000000000002,"15min":0.629},"min":{"1min":0.35499999999999998,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":33.113,"5min":33.113,"15min":33.113},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:16:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.60999999999999999,"15min":0.52000000000000002},"min":{"1min":0.33900000000000002,"5min":0.311,"15min":0.28699999999999998},"max":{"1min":2.6589999999999998,"5min":2.6589999999999998,"15min":2.6589999999999998},"last":1.569},{"interface":"front","average":{"1min":2.657,"5min":1.0189999999999999,"15min":0.69999999999999996},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":31.257999999999999,"5min":31.257999999999999,"15min":31.257999999999999},"last":0.95299999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:16:36 2023","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":0.66000000000000003,"15min":0.58299999999999996},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":2.6960000000000002,"5min":2.6960000000000002,"15min":3.0459999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.3919999999999999,"5min":0.80000000000000004,"15min":0.64800000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.308},"max":{"1min":5.1740000000000004,"5min":5.1740000000000004,"15min":5.1740000000000004},"last":0.97999999999999998}]}]},{"osd":6,"up_from":0,"seq":197568495792,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795064,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1821992,"kb_used_data":1722208,"kb_used_omap":7,"kb_used_meta":99768,"kb_avail":91923160,"statfs":{"total":95995035648,"available":94129315840,"internally_reserved":0,"allocated":1763540992,"data_stored":1762671796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":102162804},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":4.1669999999999998,"5min":1.214,"15min":0.64400000000000002},"min":{"1min":0.20499999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":61.890000000000001,"5min":61.890000000000001,"15min":61.890000000000001},"last":1.823},{"interface":"front","average":{"1min":4.0830000000000002,"5min":1.216,"15min":0.67600000000000005},"min":{"1min":0.22700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":62.689999999999998,"5min":62.689999999999998,"15min":62.689999999999998},"last":6.508}]},{"osd":2,"last update":"Sat Sep 23 22:16:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":0.71899999999999997,"15min":0.49199999999999999},"min":{"1min":0.249,"5min":0.216,"15min":0.216},"max":{"1min":17.739999999999998,"5min":17.739999999999998,"15min":17.739999999999998},"last":1.0229999999999999},{"interface":"front","average":{"1min":1.9390000000000001,"5min":0.72799999999999998,"15min":0.51700000000000002},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":17.073,"5min":17.073,"15min":17.073},"last":1.2669999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:17:11 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":0.82399999999999995,"15min":0.54000000000000004},"min":{"1min":0.313,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":22.326000000000001,"5min":22.326000000000001,"15min":22.326000000000001},"last":6.4660000000000002},{"interface":"front","average":{"1min":2.3999999999999999,"5min":0.96999999999999997,"15min":0.59599999999999997},"min":{"1min":0.33800000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":22.259,"5min":22.259,"15min":22.259},"last":4.9669999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:17:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":0.78900000000000003,"15min":0.70799999999999996},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9089999999999998,"5min":6.9089999999999998,"15min":6.9089999999999998},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.71399999999999997,"15min":0.68100000000000005},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":4.4299999999999997,"5min":4.4299999999999997,"15min":4.4299999999999997},"last":1.0349999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.68200000000000005,"15min":0.63700000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.9219999999999997,"5min":4.9219999999999997,"15min":4.9219999999999997},"last":1.0800000000000001},{"interface":"front","average":{"1min":1.3200000000000001,"5min":0.81200000000000006,"15min":0.69399999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.992,"5min":5.992,"15min":5.992},"last":4.125}]},{"osd":6,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":0.81599999999999995,"15min":0.69799999999999995},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":6.0149999999999997,"5min":6.0149999999999997,"15min":6.0149999999999997},"last":3.9249999999999998},{"interface":"front","average":{"1min":1.381,"5min":0.88900000000000001,"15min":0.745},"min":{"1min":0.44500000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.8700000000000001,"5min":6.8700000000000001,"15min":6.8700000000000001},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":0.84799999999999998,"15min":0.71399999999999997},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.1050000000000004,"5min":6.1050000000000004,"15min":6.1050000000000004},"last":1.282},{"interface":"front","average":{"1min":1.1559999999999999,"5min":0.88300000000000001,"15min":0.747},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":6.9359999999999999,"5min":7.0229999999999997,"15min":7.0229999999999997},"last":0.46999999999999997}]}]},{"osd":0,"up_from":93,"seq":399431958598,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2961612,"kb_used_data":2817540,"kb_used_omap":8,"kb_used_meta":144055,"kb_avail":90783540,"statfs":{"total":95995035648,"available":92962344960,"internally_reserved":0,"allocated":2885160960,"data_stored":2884318736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":147513194},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":3.1059999999999999,"5min":1.089,"15min":0.58799999999999997},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":44.776000000000003,"5min":44.776000000000003,"15min":44.776000000000003},"last":4.6920000000000002},{"interface":"front","average":{"1min":4.6959999999999997,"5min":1.3,"15min":0.67600000000000005},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":41.881999999999998,"5min":41.881999999999998,"15min":41.881999999999998},"last":1.1739999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:16:34 2023","interfaces":[{"interface":"back","average":{"1min":1.3480000000000001,"5min":0.59899999999999998,"15min":0.46800000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.0169999999999995,"5min":8.0169999999999995,"15min":8.0169999999999995},"last":4.7249999999999996},{"interface":"front","average":{"1min":2.786,"5min":0.95199999999999996,"15min":0.66500000000000004},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":10.91,"5min":10.91,"15min":10.91},"last":1.3300000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":3.2000000000000002,"5min":1.1639999999999999,"15min":0.68500000000000005},"min":{"1min":0.314,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":35.496000000000002,"5min":35.496000000000002,"15min":35.496000000000002},"last":4.641},{"interface":"front","average":{"1min":4.3220000000000001,"5min":1.3500000000000001,"15min":0.74299999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":35.533000000000001,"5min":35.533000000000001,"15min":35.533000000000001},"last":1.5269999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:16:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":1.0609999999999999,"15min":0.78800000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.5570000000000004,"5min":9.5570000000000004,"15min":9.5570000000000004},"last":1.806},{"interface":"front","average":{"1min":0.92600000000000005,"5min":0.69399999999999995,"15min":0.67200000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":3.762,"5min":3.762,"15min":3.762},"last":4.7850000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:16:52 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.02,"15min":0.79900000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":9.5280000000000005,"5min":9.5280000000000005,"15min":9.5280000000000005},"last":2.1059999999999999},{"interface":"front","average":{"1min":1.2290000000000001,"5min":0.76400000000000001,"15min":0.71099999999999997},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":4.4770000000000003,"5min":4.4770000000000003,"15min":4.4770000000000003},"last":0.34100000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:16:52 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":1.1080000000000001,"15min":0.82099999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":9.4909999999999997,"5min":9.4909999999999997,"15min":9.4909999999999997},"last":1.6899999999999999},{"interface":"front","average":{"1min":1.2090000000000001,"5min":0.75800000000000001,"15min":0.73199999999999998},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.5150000000000001,"5min":3.5150000000000001,"15min":3.5150000000000001},"last":4.7549999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:16:52 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":1.4139999999999999,"15min":0.95799999999999996},"min":{"1min":0.40999999999999998,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":10.641,"5min":23.306000000000001,"15min":23.306000000000001},"last":2.2509999999999999},{"interface":"front","average":{"1min":2.5249999999999999,"5min":1.5700000000000001,"15min":1.0269999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":9.5830000000000002,"5min":23.265999999999998,"15min":23.265999999999998},"last":2.2650000000000001}]}]},{"osd":2,"up_from":103,"seq":442381631537,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2370404,"kb_used_data":2266588,"kb_used_omap":10,"kb_used_meta":103797,"kb_avail":91374748,"statfs":{"total":95995035648,"available":93567741952,"internally_reserved":0,"allocated":2320986112,"data_stored":2320122532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":106288300},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:16:28 2023","interfaces":[{"interface":"back","average":{"1min":2.222,"5min":0.71699999999999997,"15min":0.46200000000000002},"min":{"1min":0.22600000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.3719999999999999,"5min":9.3719999999999999,"15min":9.3719999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":3.121,"5min":0.91800000000000004,"15min":0.54800000000000004},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":14.154,"5min":14.154,"15min":14.154},"last":2.2879999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":0.66700000000000004,"15min":0.504},"min":{"1min":0.29699999999999999,"5min":0.222,"15min":0.222},"max":{"1min":6.8479999999999999,"5min":6.8479999999999999,"15min":6.8479999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":2.1499999999999999,"5min":0.76700000000000002,"15min":0.53500000000000003},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.4240000000000004,"5min":7.4240000000000004,"15min":7.4240000000000004},"last":0.77300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:17:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1040000000000001,"5min":0.91000000000000003,"15min":0.53000000000000003},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.8840000000000003,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.3799999999999999,"5min":0.74199999999999999,"15min":0.50600000000000001},"min":{"1min":0.26800000000000002,"5min":0.246,"15min":0.246},"max":{"1min":5.2519999999999998,"5min":5.2519999999999998,"15min":5.2519999999999998},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 22:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":0.79700000000000004,"15min":0.66400000000000003},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":7.5179999999999998,"5min":7.5179999999999998,"15min":7.5179999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.6990000000000001,"5min":0.83699999999999997,"15min":0.69699999999999995},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.8390000000000004,"5min":6.8390000000000004,"15min":6.8390000000000004},"last":0.63100000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":0.83299999999999996,"15min":0.70099999999999996},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":7.4909999999999997,"5min":7.4909999999999997,"15min":7.4909999999999997},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.6870000000000001,"5min":0.82699999999999996,"15min":0.68000000000000005},"min":{"1min":0.41599999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.7960000000000003,"5min":6.7960000000000003,"15min":6.7960000000000003},"last":4.8940000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":0.86699999999999999,"15min":0.73199999999999998},"min":{"1min":0.38700000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":7.5439999999999996,"5min":7.5439999999999996,"15min":7.5439999999999996},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.5780000000000001,"5min":0.92400000000000004,"15min":0.83199999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":6.8700000000000001,"5min":6.8700000000000001,"15min":6.8700000000000001},"last":0.67000000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:16:28 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":0.86099999999999999,"15min":0.79300000000000004},"min":{"1min":0.42699999999999999,"5min":0.38,"15min":0.38},"max":{"1min":5.2039999999999997,"5min":5.2039999999999997,"15min":5.2039999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.4470000000000001,"5min":0.998,"15min":0.75700000000000001},"min":{"1min":0.44500000000000001,"5min":0.377,"15min":0.377},"max":{"1min":14.199999999999999,"5min":14.199999999999999,"15min":14.199999999999999},"last":0.58799999999999997}]}]},{"osd":3,"up_from":108,"seq":463856468011,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1164112,"kb_used_data":1100552,"kb_used_omap":4,"kb_used_meta":63547,"kb_avail":92581040,"statfs":{"total":95995035648,"available":94802984960,"internally_reserved":0,"allocated":1126965248,"data_stored":1126114952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":65072685},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":2.738,"5min":1.5700000000000001,"15min":0.77500000000000002},"min":{"1min":0.23300000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":27.524000000000001,"5min":77.834999999999994,"15min":77.834999999999994},"last":0.74399999999999999},{"interface":"front","average":{"1min":2.5169999999999999,"5min":1.609,"15min":0.77700000000000002},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":27.600000000000001,"5min":75.558000000000007,"15min":75.558000000000007},"last":1.6419999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":1.5169999999999999,"15min":0.74099999999999999},"min":{"1min":0.23300000000000001,"5min":0.218,"15min":0.218},"max":{"1min":28.091000000000001,"5min":68.284999999999997,"15min":68.284999999999997},"last":3.4060000000000001},{"interface":"front","average":{"1min":2.7999999999999998,"5min":1.581,"15min":0.79900000000000004},"min":{"1min":0.29799999999999999,"5min":0.189,"15min":0.189},"max":{"1min":28.698,"5min":66.082999999999998,"15min":66.082999999999998},"last":1.0820000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":8.3330000000000002,"5min":3.6099999999999999,"15min":1.4650000000000001},"min":{"1min":0.27800000000000002,"5min":0.223,"15min":0.223},"max":{"1min":97.325000000000003,"5min":97.325000000000003,"15min":97.325000000000003},"last":3.5059999999999998},{"interface":"front","average":{"1min":8.9529999999999994,"5min":2.9590000000000001,"15min":1.2689999999999999},"min":{"1min":0.316,"5min":0.254,"15min":0.254},"max":{"1min":97.379999999999995,"5min":97.379999999999995,"15min":97.379999999999995},"last":1.0449999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:17:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.72999999999999998,"15min":0.66000000000000003},"min":{"1min":0.39800000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.7429999999999999,"5min":2.7429999999999999,"15min":2.7429999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.76000000000000001,"15min":0.68999999999999995},"min":{"1min":0.38900000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.0710000000000002,"5min":6.6150000000000002,"15min":6.6150000000000002},"last":2.1019999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.75800000000000001,"15min":0.66900000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.097,"5min":6.5819999999999999,"15min":6.5819999999999999},"last":5.0650000000000004},{"interface":"front","average":{"1min":0.748,"5min":0.72599999999999998,"15min":0.64800000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":2.4620000000000002,"5min":6.2140000000000004,"15min":6.2140000000000004},"last":1.1399999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":0.90600000000000003,"15min":0.73799999999999999},"min":{"1min":0.36299999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.6230000000000002,"5min":6.6440000000000001,"15min":6.6440000000000001},"last":2.2959999999999998},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.83599999999999997,"15min":0.70199999999999996},"min":{"1min":0.38500000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":3.1859999999999999,"5min":6.3799999999999999,"15min":6.3799999999999999},"last":2.2829999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:17:06 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":0.81899999999999995,"15min":0.69299999999999995},"min":{"1min":0.40200000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":6.4930000000000003,"5min":6.4930000000000003,"15min":6.4930000000000003},"last":2.5169999999999999},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.84399999999999997,"15min":0.73699999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":5.569,"5min":5.569,"15min":5.569},"last":0.44800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019062,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2862132,"kb_used_data":2788588,"kb_used_omap":0,"kb_used_meta":73536,"kb_avail":90883020,"statfs":{"total":95995035648,"available":93064212480,"internally_reserved":0,"allocated":2855514112,"data_stored":2854809544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":75300864},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,6,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:16:56 2023","interfaces":[{"interface":"back","average":{"1min":4.4480000000000004,"5min":1.3959999999999999,"15min":0.88400000000000001},"min":{"1min":0.23999999999999999,"5min":0.218,"15min":0.218},"max":{"1min":66.355000000000004,"5min":66.355000000000004,"15min":66.355000000000004},"last":0.35499999999999998},{"interface":"front","average":{"1min":4.3719999999999999,"5min":1.478,"15min":0.92800000000000005},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":66.332999999999998,"5min":66.332999999999998,"15min":66.332999999999998},"last":2.274}]},{"osd":1,"last update":"Sat Sep 23 22:16:56 2023","interfaces":[{"interface":"back","average":{"1min":3.262,"5min":1.109,"15min":0.73299999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":38.878,"5min":38.878,"15min":38.878},"last":0.82499999999999996},{"interface":"front","average":{"1min":4.9480000000000004,"5min":1.5469999999999999,"15min":0.89900000000000002},"min":{"1min":0.32600000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":38.853999999999999,"5min":38.853999999999999,"15min":38.853999999999999},"last":2.3149999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:16:34 2023","interfaces":[{"interface":"back","average":{"1min":3.8130000000000002,"5min":1.2110000000000001,"15min":0.76400000000000001},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":42.445,"5min":42.445,"15min":42.445},"last":1.504},{"interface":"front","average":{"1min":4.8239999999999998,"5min":1.3999999999999999,"15min":0.80800000000000005},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":42.680999999999997,"5min":42.680999999999997,"15min":42.680999999999997},"last":2.3420000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:17:05 2023","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":0.84599999999999997,"15min":0.67800000000000005},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.2350000000000003,"5min":6.2350000000000003,"15min":6.2350000000000003},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.478,"5min":0.88600000000000001,"15min":0.67900000000000005},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.3869999999999996,"5min":6.3869999999999996,"15min":6.3869999999999996},"last":3.0089999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:17:05 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.69899999999999995,"15min":0.54100000000000004},"min":{"1min":0.27800000000000002,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":0.72999999999999998,"5min":8.9499999999999993,"15min":8.9499999999999993},"last":2.6269999999999998},{"interface":"front","average":{"1min":0.51500000000000001,"5min":0.65500000000000003,"15min":0.57699999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.8140000000000001,"5min":4.2779999999999996,"15min":4.2779999999999996},"last":3.0459999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:17:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.67700000000000005,"15min":0.55900000000000005},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":2.7480000000000002,"5min":3.3039999999999998,"15min":3.3039999999999998},"last":0.376},{"interface":"front","average":{"1min":1.0249999999999999,"5min":0.76900000000000002,"15min":0.61899999999999999},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":8.5210000000000008,"5min":8.5210000000000008,"15min":8.5210000000000008},"last":2.3690000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:16:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.80300000000000005,"15min":0.67300000000000004},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":6.319,"5min":8.2159999999999993,"15min":8.2159999999999993},"last":2.3959999999999999},{"interface":"front","average":{"1min":0.90300000000000002,"5min":0.75900000000000001,"15min":0.64200000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.2489999999999997,"5min":5.6280000000000001,"15min":5.6280000000000001},"last":0.52100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757429,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3106536,"kb_used_data":3030496,"kb_used_omap":0,"kb_used_meta":75968,"kb_avail":90638616,"statfs":{"total":95995035648,"available":92813942784,"internally_reserved":0,"allocated":3103227904,"data_stored":3102530857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":77791232},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,5,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:16:47 2023","interfaces":[{"interface":"back","average":{"1min":4.133,"5min":2.7749999999999999,"15min":1.3029999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":64.096000000000004,"5min":133.22499999999999,"15min":133.22499999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":4.8239999999999998,"5min":2.964,"15min":1.419},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":64.108000000000004,"5min":132.78200000000001,"15min":132.78200000000001},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:17:00 2023","interfaces":[{"interface":"back","average":{"1min":6.3319999999999999,"5min":1.7949999999999999,"15min":0.99299999999999999},"min":{"1min":0.32600000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":64.552000000000007,"5min":64.552000000000007,"15min":64.552000000000007},"last":0.39900000000000002},{"interface":"front","average":{"1min":6.4640000000000004,"5min":1.8069999999999999,"15min":0.98299999999999998},"min":{"1min":0.41099999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":65.158000000000001,"5min":65.158000000000001,"15min":65.158000000000001},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:16:33 2023","interfaces":[{"interface":"back","average":{"1min":13.57,"5min":3.1859999999999999,"15min":1.4370000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":88.013000000000005,"5min":88.013000000000005,"15min":88.013000000000005},"last":9.3819999999999997},{"interface":"front","average":{"1min":17.111999999999998,"5min":3.8999999999999999,"15min":1.694},"min":{"1min":0.318,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":87.662999999999997,"5min":87.662999999999997,"15min":87.662999999999997},"last":1.377}]},{"osd":3,"last update":"Sat Sep 23 22:17:02 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.085,"15min":1.081},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":42.052,"5min":88.106999999999999,"15min":88.106999999999999},"last":9.3300000000000001},{"interface":"front","average":{"1min":3.2109999999999999,"5min":2.1829999999999998,"15min":1.1200000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":41.921999999999997,"5min":87.887,"15min":87.887},"last":0.48899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:16:56 2023","interfaces":[{"interface":"back","average":{"1min":4.5300000000000002,"5min":1.3560000000000001,"15min":0.751},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":82.530000000000001,"5min":82.530000000000001,"15min":82.530000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":4.766,"5min":1.474,"15min":0.82099999999999995},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":85.441000000000003,"5min":85.441000000000003,"15min":85.441000000000003},"last":0.32700000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:16:29 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.66200000000000003,"15min":0.55500000000000005},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.0710000000000002,"5min":3.0710000000000002,"15min":3.0710000000000002},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":0.63700000000000001,"15min":0.56000000000000005},"min":{"1min":0.311,"5min":0.311,"15min":0.29699999999999999},"max":{"1min":2.9710000000000001,"5min":2.9710000000000001,"15min":2.9710000000000001},"last":0.70399999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:16:44 2023","interfaces":[{"interface":"back","average":{"1min":0.80600000000000005,"5min":0.66000000000000003,"15min":0.59399999999999997},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32600000000000001},"max":{"1min":2.9319999999999999,"5min":2.9319999999999999,"15min":2.9319999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.76500000000000001,"5min":0.70799999999999996,"15min":0.60199999999999998},"min":{"1min":0.36899999999999999,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":4.3840000000000003,"5min":6.149,"15min":6.149},"last":0.41699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2878275584,"data_stored":2878275584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1748893696,"data_stored":1748893696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2313752576,"data_stored":2313748504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1120141312,"data_stored":1120141312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2850816000,"data_stored":2850816000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3085176832,"data_stored":3085172760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1623195648,"data_stored":1623195648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2051936256,"data_stored":2051932184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:17:27.220 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T22:17:27.220 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-09-23T22:17:27.286 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:27.426 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:27.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:27.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:27.462 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:17:27.537 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:27.596 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:27.643 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:27.806 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:27.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:27.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:27.918 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:27.974 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:17:28.006 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:28.139 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:28.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:28.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:28 smithi026 ceph-mon[215775]: pgmap v339: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 16060/252516 objects degraded (6.360%); 15708/252516 objects misplaced (6.221%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:17:28.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 16060/252516 objects degraded (6.360%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:28.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:28 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4019450635' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:28.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:28 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:28.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:28 smithi026 ceph-mon[223014]: pgmap v339: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 16060/252516 objects degraded (6.360%); 15708/252516 objects misplaced (6.221%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:17:28.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 16060/252516 objects degraded (6.360%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:28.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:28 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4019450635' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:28.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:28 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:28.244 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:28.296 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:28.377 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:28.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:28.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:28.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:28 smithi070 ceph-mon[192652]: pgmap v339: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 16060/252516 objects degraded (6.360%); 15708/252516 objects misplaced (6.221%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:17:28.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 16060/252516 objects degraded (6.360%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:28.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:28 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4019450635' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:28.389 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:28 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:28.489 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:28.535 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:28.636 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:17:28.696 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:28.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:28.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:28.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:28.845 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:28.934 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:28.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:28.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:29.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:29.104 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:29.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:29 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:29.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:29 smithi070 ceph-mon[192652]: failed to encode map e162 with expected crc 2023-09-23T22:17:29.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:29 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:17:29.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:29 smithi070 ceph-mon[192652]: osdmap e162: 8 total, 8 up, 7 in 2023-09-23T22:17:29.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:29 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:29.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:29 smithi026 ceph-mon[215775]: failed to encode map e162 with expected crc 2023-09-23T22:17:29.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:29 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:17:29.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:29 smithi026 ceph-mon[215775]: osdmap e162: 8 total, 8 up, 7 in 2023-09-23T22:17:29.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:29 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:29.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:29 smithi026 ceph-mon[223014]: failed to encode map e162 with expected crc 2023-09-23T22:17:29.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:29 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:17:29.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:29 smithi026 ceph-mon[223014]: osdmap e162: 8 total, 8 up, 7 in 2023-09-23T22:17:29.573 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:29.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:29.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:29.594 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:29.678 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:29.772 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:29.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:17:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:17:29] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-09-23T22:17:30.031 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:30.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:30.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:30.142 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:30.192 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:30.437 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:30.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:30.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:30.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:30.596 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:30.663 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:17:30.704 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:17:30.717 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:30.719 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:30 smithi070 ceph-mon[192652]: pgmap v341: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 16060/249219 objects degraded (6.444%); 15566/249219 objects misplaced (6.246%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:17:30.719 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2865269505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:30.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:30.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:30.722 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:17:30.722 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:17:30.750 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:17:30.787 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:17:30.821 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:30.827 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:17:30.856 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:17:30.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:30 smithi026 ceph-mon[223014]: pgmap v341: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 16060/249219 objects degraded (6.444%); 15566/249219 objects misplaced (6.246%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:17:30.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2865269505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:30.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:30 smithi026 ceph-mon[215775]: pgmap v341: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 16060/249219 objects degraded (6.444%); 15566/249219 objects misplaced (6.246%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:17:30.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2865269505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:30.897 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:17:30.906 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:30.944 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:17:30.963 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:17:30.963 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:17:30.963 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:17:30.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:17:30.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:17:30.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:17:30.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:17:30.964 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:17:31.024 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:31.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:31.132 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:31.196 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:31.281 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:31.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:31.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:31.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:31.486 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:31.970 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:17:32.025 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:32.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:32.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:32.149 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:32.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:17:32.237 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:32.373 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:32.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:32.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:32.489 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:32.608 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:32.737 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:32.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:32.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:32.855 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:32.896 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:32.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:32 smithi026 ceph-mon[215775]: pgmap v342: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 15871/243579 objects degraded (6.516%); 15386/243579 objects misplaced (6.317%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:17:32.952 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:32 smithi026 ceph-mon[223014]: pgmap v342: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 15871/243579 objects degraded (6.516%); 15386/243579 objects misplaced (6.317%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:17:32.961 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:33.100 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:32 smithi070 ceph-mon[192652]: pgmap v342: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 616 op/s; 15871/243579 objects degraded (6.516%); 15386/243579 objects misplaced (6.317%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:17:33.183 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:33.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:33.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:33.289 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:33.374 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:33.710 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:33.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:33.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:33.717 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pg_num to 11 2023-09-23T22:17:33.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15871/243579 objects degraded (6.516%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:33.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:33 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3660649518' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T22:17:33.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15871/243579 objects degraded (6.516%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:33.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:33 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3660649518' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T22:17:33.815 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:33.879 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:33.978 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:33.988 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15871/243579 objects degraded (6.516%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:33.988 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:33 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3660649518' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T22:17:33.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:33.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:34.055 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:17:34.090 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:34.128 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:34.898 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:34.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:34.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:34.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:34 smithi026 ceph-mon[215775]: pgmap v343: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 15871/241743 objects degraded (6.565%); 15223/241743 objects misplaced (6.297%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T22:17:34.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:34 smithi026 ceph-mon[215775]: failed to encode map e163 with expected crc 2023-09-23T22:17:34.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:34 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3660649518' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T22:17:34.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:34 smithi026 ceph-mon[215775]: osdmap e163: 8 total, 8 up, 7 in 2023-09-23T22:17:34.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:34 smithi026 ceph-mon[223014]: pgmap v343: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 15871/241743 objects degraded (6.565%); 15223/241743 objects misplaced (6.297%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T22:17:34.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:34 smithi026 ceph-mon[223014]: failed to encode map e163 with expected crc 2023-09-23T22:17:34.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:34 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3660649518' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T22:17:34.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:34 smithi026 ceph-mon[223014]: osdmap e163: 8 total, 8 up, 7 in 2023-09-23T22:17:35.002 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:35.046 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:34 smithi070 ceph-mon[192652]: pgmap v343: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 15871/241743 objects degraded (6.565%); 15223/241743 objects misplaced (6.297%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T22:17:35.046 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:34 smithi070 ceph-mon[192652]: failed to encode map e163 with expected crc 2023-09-23T22:17:35.046 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:34 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3660649518' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T22:17:35.047 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:34 smithi070 ceph-mon[192652]: osdmap e163: 8 total, 8 up, 7 in 2023-09-23T22:17:35.054 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:35.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:35.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:35.248 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:35.340 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:35.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:35 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T22:17:35.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:35 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T22:17:36.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:36.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:36.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:36.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:35 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T22:17:36.135 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:36.251 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:36.411 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:36.531 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:36.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:36.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:36.643 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:36.694 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:36.781 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:36.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:36.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:36.891 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:36.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[223014]: pgmap v345: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 15677/238131 objects degraded (6.583%); 15057/238131 objects misplaced (6.323%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T22:17:36.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[223014]: failed to encode map e164 with expected crc 2023-09-23T22:17:36.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T22:17:36.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[223014]: osdmap e164: 8 total, 8 up, 7 in 2023-09-23T22:17:36.909 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[223014]: failed to encode map e164 with expected crc 2023-09-23T22:17:36.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[215775]: pgmap v345: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 15677/238131 objects degraded (6.583%); 15057/238131 objects misplaced (6.323%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T22:17:36.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[215775]: failed to encode map e164 with expected crc 2023-09-23T22:17:36.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T22:17:36.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[215775]: osdmap e164: 8 total, 8 up, 7 in 2023-09-23T22:17:36.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:36 smithi026 ceph-mon[215775]: failed to encode map e164 with expected crc 2023-09-23T22:17:36.927 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:111162 5.2 5:43348cf6:::benchmark_data_smithi026_286012_object17354:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:36.946835+0000", 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.035960972000000001, 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.036004580000000001, 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:17:36.984 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 111162 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.946835+0000", 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.946835+0000", 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.946834+0000", 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.946840+0000", 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.297e-06 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.946854+0000", 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4204e-05 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.946864+0000", 2023-09-23T22:17:36.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.149e-06 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.956698+0000", 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0098344639999999994 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.956912+0000", 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021388900000000001 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.957082+0000", 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017023599999999999 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.961015+0000", 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039331039999999998 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:36.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964649+0000", 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036340930000000001 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.980588+0000", 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015939010999999999 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.980650+0000", 2023-09-23T22:17:36.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1797999999999993e-05 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:111163 5.2 5:45b6fa46:::benchmark_data_smithi026_286012_object17355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:36.964785+0000", 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.018011835, 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018164619999999999, 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:17:36.991 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 111163 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964785+0000", 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964785+0000", 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964783+0000", 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:36.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964789+0000", 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.366e-06 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964801+0000", 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2238999999999999e-05 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964814+0000", 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2925000000000001e-05 2023-09-23T22:17:36.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964868+0000", 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3703999999999999e-05 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.965017+0000", 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149574 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.965148+0000", 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130758 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.973449+0000", 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083008219999999994 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:17:36.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.974771+0000", 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013219009999999999 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:111165 5.2 5:430f87ef:::benchmark_data_smithi026_286012_object17357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:36.964830+0000", 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017966407, 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018200796000000002, 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:36.997 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 111165 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964830+0000", 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964830+0000", 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964829+0000", 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:36.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964832+0000", 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.616e-06 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964838+0000", 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0039999999999996e-06 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.964843+0000", 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.322e-06 2023-09-23T22:17:37.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.969025+0000", 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041817249999999999 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.969187+0000", 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000161788 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:17:37.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.969335+0000", 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014806500000000001 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.974778+0000", 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054431009999999997 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:111168 5.7 e165/158 5:e3d60e73:::benchmark_data_smithi026_286012_object17360:head v 165'7033, mlcod=165'7031)", 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:36.968785+0000", 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.014011147, 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01432493, 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.968785+0000", 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:37.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.968785+0000", 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.968784+0000", 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.968793+0000", 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5420000000000005e-06 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.968806+0000", 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2758000000000001e-05 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.968814+0000", 2023-09-23T22:17:37.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4980000000000006e-06 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.973763+0000", 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049490180000000003 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.973786+0000", 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3125999999999998e-05 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.006 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:111162 5.2 e165/147 ondisk, result = 0)", 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:36.976695+0000", 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0061016830000000001, 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064773640000000002, 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:37.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:37.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976695+0000", 2023-09-23T22:17:37.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:37.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:37.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976695+0000", 2023-09-23T22:17:37.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:37.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976693+0000", 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976701+0000", 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3420000000000008e-06 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976722+0000", 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0248e-05 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976731+0000", 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1800000000000002e-06 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:37.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.980569+0000", 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038379880000000001 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.980580+0000", 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1095e-05 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.012 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:111163 5.2 e165/147)", 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:36.976754+0000", 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0060419890000000002, 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064733739999999996, 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976754+0000", 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976754+0000", 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976754+0000", 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:37.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976757+0000", 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0340000000000001e-06 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976763+0000", 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8159999999999999e-06 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.976767+0000", 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3640000000000001e-06 2023-09-23T22:17:37.015 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:111165 5.2 e165/147)", 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:36.982778+0000", 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.8377999999999999e-05, 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049448700000000005, 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:37.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.982778+0000", 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.982778+0000", 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:37.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.982777+0000", 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.982784+0000", 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7560000000000002e-06 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.982795+0000", 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094e-05 2023-09-23T22:17:37.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:36.982802+0000", 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3590000000000002e-06 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 7 2023-09-23T22:17:37.019 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:17:37.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:37.021 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:37.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:37.021 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:36 smithi070 ceph-mon[192652]: pgmap v345: 89 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 599 op/s; 15677/238131 objects degraded (6.583%); 15057/238131 objects misplaced (6.323%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T22:17:37.021 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:36 smithi070 ceph-mon[192652]: failed to encode map e164 with expected crc 2023-09-23T22:17:37.021 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T22:17:37.021 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:36 smithi070 ceph-mon[192652]: osdmap e164: 8 total, 8 up, 7 in 2023-09-23T22:17:37.021 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:36 smithi070 ceph-mon[192652]: failed to encode map e164 with expected crc 2023-09-23T22:17:37.121 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:37.162 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:37.244 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:37.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:37.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:37.351 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:37.422 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:37.745 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:17:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:37 smithi070 ceph-mon[192652]: failed to encode map e165 with expected crc 2023-09-23T22:17:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:37 smithi070 ceph-mon[192652]: osdmap e165: 8 total, 8 up, 7 in 2023-09-23T22:17:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:37 smithi070 ceph-mon[192652]: failed to encode map e165 with expected crc 2023-09-23T22:17:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:37 smithi070 ceph-mon[192652]: failed to encode map e165 with expected crc 2023-09-23T22:17:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:37 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3164869010' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:37 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:38.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:37 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15677/238131 objects degraded (6.583%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:38.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[215775]: failed to encode map e165 with expected crc 2023-09-23T22:17:38.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[215775]: osdmap e165: 8 total, 8 up, 7 in 2023-09-23T22:17:38.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[215775]: failed to encode map e165 with expected crc 2023-09-23T22:17:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[215775]: failed to encode map e165 with expected crc 2023-09-23T22:17:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3164869010' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15677/238131 objects degraded (6.583%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:38.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[223014]: failed to encode map e165 with expected crc 2023-09-23T22:17:38.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[223014]: osdmap e165: 8 total, 8 up, 7 in 2023-09-23T22:17:38.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[223014]: failed to encode map e165 with expected crc 2023-09-23T22:17:38.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[223014]: failed to encode map e165 with expected crc 2023-09-23T22:17:38.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3164869010' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:38.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:17:38.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:37 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15677/238131 objects degraded (6.583%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:38.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[223014]: pgmap v348: 99 pgs: 10 unknown, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 15677/233961 objects degraded (6.701%); 15057/233961 objects misplaced (6.436%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:17:38.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:38.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[223014]: failed to encode map e166 with expected crc 2023-09-23T22:17:38.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:17:38.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[223014]: osdmap e166: 8 total, 8 up, 7 in 2023-09-23T22:17:38.935 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[223014]: failed to encode map e166 with expected crc 2023-09-23T22:17:38.935 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[223014]: failed to encode map e166 with expected crc 2023-09-23T22:17:38.936 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:17:38.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[215775]: pgmap v348: 99 pgs: 10 unknown, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 15677/233961 objects degraded (6.701%); 15057/233961 objects misplaced (6.436%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:17:38.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:38.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[215775]: failed to encode map e166 with expected crc 2023-09-23T22:17:38.936 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:17:38.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[215775]: osdmap e166: 8 total, 8 up, 7 in 2023-09-23T22:17:38.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[215775]: failed to encode map e166 with expected crc 2023-09-23T22:17:38.937 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:38 smithi026 ceph-mon[215775]: failed to encode map e166 with expected crc 2023-09-23T22:17:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:38 smithi070 ceph-mon[192652]: pgmap v348: 99 pgs: 10 unknown, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 15677/233961 objects degraded (6.701%); 15057/233961 objects misplaced (6.436%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:17:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:38 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:38 smithi070 ceph-mon[192652]: failed to encode map e166 with expected crc 2023-09-23T22:17:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:38 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:17:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:38 smithi070 ceph-mon[192652]: osdmap e166: 8 total, 8 up, 7 in 2023-09-23T22:17:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:38 smithi070 ceph-mon[192652]: failed to encode map e166 with expected crc 2023-09-23T22:17:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:38 smithi070 ceph-mon[192652]: failed to encode map e166 with expected crc 2023-09-23T22:17:39.155 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:39.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:39.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:39.273 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:39.298 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:39.365 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:39.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:39.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:39.474 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:39.565 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:39.704 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:39.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:39.727 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:17:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:17:39] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.43.0" 2023-09-23T22:17:39.821 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:39.862 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:39.863 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:39.935 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:39.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:39.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:39 smithi026 ceph-mon[223014]: pgmap v350: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 15677/230382 objects degraded (6.805%); 14857/230382 objects misplaced (6.449%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T22:17:39.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:39.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:39 smithi026 ceph-mon[215775]: pgmap v350: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 15677/230382 objects degraded (6.805%); 14857/230382 objects misplaced (6.449%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T22:17:40.043 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:40.095 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:40.095 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:39 smithi070 ceph-mon[192652]: pgmap v350: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 15677/230382 objects degraded (6.805%); 14857/230382 objects misplaced (6.449%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T22:17:40.098 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:40.182 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:40.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:40.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:40.243 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 7, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:17:40.243 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:17:40.243 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T22:17:40.244 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:17:40.244 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:17:40.244 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T22:17:40.296 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:40.447 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:40.490 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:17:40.555 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:17:40.634 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:17:40.682 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:17:40.725 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:17:40.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:40.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:40.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:40.765 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:17:40.810 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:17:40.840 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:17:40.841 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:17:40.841 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:17:40.841 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:17:40.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:17:40.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:17:40.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:17:40.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:17:40.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:17:40.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:40.942 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:41.041 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:41.050 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/159087446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:41.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:41.152 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:41.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:17:41.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/159087446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:41.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/159087446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:41.196 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:41.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:41 smithi026 ceph-mon[215775]: pgmap v351: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 15468/224295 objects degraded (6.896%); 14693/224295 objects misplaced (6.551%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T22:17:41.932 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:41 smithi026 ceph-mon[223014]: pgmap v351: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 15468/224295 objects degraded (6.896%); 14693/224295 objects misplaced (6.551%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T22:17:42.058 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:42.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:42.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:41 smithi070 ceph-mon[192652]: pgmap v351: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 768 op/s; 15468/224295 objects degraded (6.896%); 14693/224295 objects misplaced (6.551%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T22:17:42.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:17:42.254 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:42.336 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:42.375 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:42.463 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:17:42.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:42.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:17:42.574 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:42.676 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:42.846 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:17:42.846 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:17:42.885 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:42.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:42.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:42.989 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:43.059 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:42 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15468/224295 objects degraded (6.896%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:43.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:42 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15468/224295 objects degraded (6.896%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:43.063 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:43.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:42 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15468/224295 objects degraded (6.896%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:43.978 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:43.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:43.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:44.092 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:44.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:43 smithi070 ceph-mon[192652]: pgmap v352: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 15468/222627 objects degraded (6.948%); 14505/222627 objects misplaced (6.515%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:17:44.185 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:17:44.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:43 smithi026 ceph-mon[215775]: pgmap v352: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 15468/222627 objects degraded (6.948%); 14505/222627 objects misplaced (6.515%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:17:44.215 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:43 smithi026 ceph-mon[223014]: pgmap v352: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 702 op/s; 15468/222627 objects degraded (6.948%); 14505/222627 objects misplaced (6.515%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:17:44.233 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:44.710 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:17:44.978 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:44.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:44.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:45.090 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:45.197 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:45.342 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:45.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:45.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:45.373 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:17:45 smithi070 conmon[172396]: logger=cleanup t=2023-09-23T22:17:45.090862228Z level=info msg="Completed cleanup jobs" duration=148.014643ms 2023-09-23T22:17:45.459 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:45.498 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:45.565 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:45.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:45.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:45.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:45.718 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:45.808 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:45.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:45.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:45.864 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:45.918 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:45.951 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:46.015 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:46.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:46.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:46.039 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:17:46.039 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:17:46.039 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:17:46.125 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:46.176 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:46.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:46 smithi026 ceph-mon[215775]: pgmap v353: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 15245/218904 objects degraded (6.964%); 14343/218904 objects misplaced (6.552%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:17:46.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:46 smithi026 ceph-mon[223014]: pgmap v353: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 15245/218904 objects degraded (6.964%); 14343/218904 objects misplaced (6.552%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:17:46.242 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:46.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:46.308 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:46.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:46.432 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:46.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:46 smithi070 ceph-mon[192652]: pgmap v353: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 601 op/s; 15245/218904 objects degraded (6.964%); 14343/218904 objects misplaced (6.552%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:17:47.146 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:47.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:47.184 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:17:47.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:47.340 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:47.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:47 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/905240136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:47.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:47 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:47.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:47 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/905240136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:47.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:47 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:47.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:47 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/905240136' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:47.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:47 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:17:48.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[215775]: pgmap v354: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 15245/214746 objects degraded (7.099%); 14343/214746 objects misplaced (6.679%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:17:48.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:17:48.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[215775]: failed to encode map e167 with expected crc 2023-09-23T22:17:48.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:17:48.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[215775]: osdmap e167: 8 total, 8 up, 7 in 2023-09-23T22:17:48.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15245/214746 objects degraded (7.099%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:48.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[223014]: pgmap v354: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 15245/214746 objects degraded (7.099%); 14343/214746 objects misplaced (6.679%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:17:48.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:17:48.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[223014]: failed to encode map e167 with expected crc 2023-09-23T22:17:48.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:17:48.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[223014]: osdmap e167: 8 total, 8 up, 7 in 2023-09-23T22:17:48.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15245/214746 objects degraded (7.099%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:48.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:48 smithi070 ceph-mon[192652]: pgmap v354: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.4 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 15245/214746 objects degraded (7.099%); 14343/214746 objects misplaced (6.679%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:17:48.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:48 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:17:48.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:48 smithi070 ceph-mon[192652]: failed to encode map e167 with expected crc 2023-09-23T22:17:48.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:48 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:17:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:48 smithi070 ceph-mon[192652]: osdmap e167: 8 total, 8 up, 7 in 2023-09-23T22:17:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15245/214746 objects degraded (7.099%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:17:48.682 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:48.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:48.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:48.787 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:48.846 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:48.920 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:48.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:48.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:49.026 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:49.057 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:49.125 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:49.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:49.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:49.229 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:49.256 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:49.297 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:49.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:49.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:49.404 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:49.458 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:49.562 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:49.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:49.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:49.674 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:49.723 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:49.794 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:49.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:49.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:49.898 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:49.989 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:17:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:17:49] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.43.0" 2023-09-23T22:17:50.015 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:50.097 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:17:50.098 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:17:50.293 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:50 smithi026 ceph-mon[215775]: pgmap v356: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 15245/211095 objects degraded (7.222%); 14149/211095 objects misplaced (6.703%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:17:50.293 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:50 smithi026 ceph-mon[223014]: pgmap v356: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 15245/211095 objects degraded (7.222%); 14149/211095 objects misplaced (6.703%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:17:50.411 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:50.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:50.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:50.519 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:50.590 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:17:50.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:17:50.604 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:50.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:50 smithi070 ceph-mon[192652]: pgmap v356: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 15245/211095 objects degraded (7.222%); 14149/211095 objects misplaced (6.703%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:17:50.634 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:17:50.676 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:17:50.740 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:17:50.746 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:50.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:50.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:50.779 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:17:50.826 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:17:50.868 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:17:50.873 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:50.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:17:50.916 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:17:50.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:17:50.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:17:50.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:17:50.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:17:50.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:17:50.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:17:50.917 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:17:50.984 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:51.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:51 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1183212647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:51.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:17:51.364 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:51.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:51.472 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:51.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:51 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1183212647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:51.499 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:51 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1183212647' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:17:51.871 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:51.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:51.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:51.981 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:52.017 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:52.098 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:52.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:52.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:52.115 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:52.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:52.223 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:52 smithi026 ceph-mon[215775]: pgmap v357: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 15053/205206 objects degraded (7.336%); 13982/205206 objects misplaced (6.814%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:17:52.224 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:17:52.244 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:52.265 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:52.266 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:52 smithi070 ceph-mon[192652]: pgmap v357: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 15053/205206 objects degraded (7.336%); 13982/205206 objects misplaced (6.814%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:17:52.266 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:17:52.347 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:52.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:52.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:52.457 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:52.465 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 7, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:17:52.465 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:17:52.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:52 smithi026 ceph-mon[223014]: pgmap v357: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 636 op/s; 15053/205206 objects degraded (7.336%); 13982/205206 objects misplaced (6.814%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:17:52.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:17:52.538 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:52.637 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:17:52.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:52.654 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:17:52.755 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:52.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15053/205206 objects degraded (7.336%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:17:53.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15053/205206 objects degraded (7.336%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:17:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15053/205206 objects degraded (7.336%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:17:53.966 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:17:54.355 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:54.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:54.459 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:17:54.550 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:54.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:54 smithi026 ceph-mon[223014]: pgmap v358: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 15053/203430 objects degraded (7.400%); 13787/203430 objects misplaced (6.777%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:17:54.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:17:54.688 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:54.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:54 smithi026 ceph-mon[215775]: pgmap v358: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 15053/203430 objects degraded (7.400%); 13787/203430 objects misplaced (6.777%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:17:54.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:17:54.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:54.688 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:17:54.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:54.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:17:54.717 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:54.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:54.833 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:55.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:54 smithi070 ceph-mon[192652]: pgmap v358: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 15053/203430 objects degraded (7.400%); 13787/203430 objects misplaced (6.777%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:17:55.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:17:55.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:17:55.250 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:55.251 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:55.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:55.361 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:55.395 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:122752 5.9 e167/147 5:98512b4a:::benchmark_data_smithi026_286012_object28944:head v 167'7507, mlcod=121'1772)", 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:55.407860+0000", 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017193646999999999, 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017225997, 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:17:55.426 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.407860+0000", 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.407860+0000", 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.407860+0000", 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.407870+0000", 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0064000000000001e-05 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.407881+0000", 2023-09-23T22:17:55.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1168999999999999e-05 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.407888+0000", 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2849999999999997e-06 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.409876+0000", 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019882469999999998 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.409903+0000", 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7087e-05 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:122759 5.5 5:a48645b2:::benchmark_data_smithi026_286012_object28951:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:55.413489+0000", 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.011565509, 2023-09-23T22:17:55.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011681703999999999, 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 122759 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413489+0000", 2023-09-23T22:17:55.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413489+0000", 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413487+0000", 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413494+0000", 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1470000000000004e-06 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413514+0000", 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0010999999999999e-05 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413523+0000", 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9709999999999993e-06 2023-09-23T22:17:55.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.415429+0000", 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019061460000000001 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.415593+0000", 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000163568 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.415768+0000", 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017503500000000001 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.418799+0000", 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030312749999999999 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.435 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:122756 5.9 e167/147 5:92100056:::benchmark_data_smithi026_286012_object28948:head v 167'7508, mlcod=121'1772)", 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:55.413942+0000", 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.011112196, 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011336347, 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413942+0000", 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413942+0000", 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413941+0000", 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413951+0000", 2023-09-23T22:17:55.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0030999999999999e-05 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413962+0000", 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1260000000000001e-05 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.413969+0000", 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6059999999999999e-06 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.415861+0000", 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018922450000000001 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.415887+0000", 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5816000000000002e-05 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:122754 5.7 e167/158 5:eb088860:::benchmark_data_smithi026_286012_object28946:head v 167'7807, mlcod=167'7805)", 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:55.421881+0000", 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.003172916, 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034654120000000002, 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421881+0000", 2023-09-23T22:17:55.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421881+0000", 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421880+0000", 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421889+0000", 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6470000000000004e-06 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421902+0000", 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2799000000000001e-05 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421912+0000", 2023-09-23T22:17:55.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0618000000000001e-05 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.422131+0000", 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021818299999999999 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.422152+0000", 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1299000000000001e-05 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.443 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:122760 5.7 e167/158)", 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:55.421930+0000", 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.003124291, 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003484516, 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:55.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421930+0000", 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421930+0000", 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421929+0000", 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421934+0000", 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9230000000000001e-06 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421943+0000", 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1629999999999996e-06 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.421948+0000", 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.524e-06 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:122761 5.5 5:aed4d08b:::benchmark_data_smithi026_286012_object28953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e167)", 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:55.422870+0000", 2023-09-23T22:17:55.447 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0021846230000000001, 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026054369999999999, 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 122761 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:55.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.422870+0000", 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.422870+0000", 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.422868+0000", 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.422874+0000", 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8919999999999997e-06 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.422884+0000", 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.022e-05 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:55.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.422892+0000", 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8699999999999992e-06 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.422946+0000", 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4530000000000001e-05 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423141+0000", 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019447299999999999 2023-09-23T22:17:55.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423283+0000", 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000142344 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:122755 5.8 e167/159 5:14534e33:::benchmark_data_smithi026_286012_object28947:head v 167'7515, mlcod=167'7512)", 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:55.423141+0000", 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001912789, 2023-09-23T22:17:55.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024183410000000001, 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423141+0000", 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423141+0000", 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423141+0000", 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423147+0000", 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5889999999999999e-06 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423158+0000", 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.077e-05 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423164+0000", 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4219999999999997e-06 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423229+0000", 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4275000000000004e-05 2023-09-23T22:17:55.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.423249+0000", 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9828999999999999e-05 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:122741 5.8 e167/159 5:1ad291c0:::benchmark_data_smithi026_286012_object28933:head v 167'7514, mlcod=167'7512)", 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:55.405101+0000", 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.019953621000000001, 2023-09-23T22:17:55.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020531984, 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.405101+0000", 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.405101+0000", 2023-09-23T22:17:55.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.405100+0000", 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.405107+0000", 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7770000000000002e-06 2023-09-23T22:17:55.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.405122+0000", 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4792e-05 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.405131+0000", 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6589999999999996e-06 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:17:55.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.405225+0000", 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4065000000000001e-05 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:55.405246+0000", 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1382e-05 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 8 2023-09-23T22:17:55.460 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:17:55.484 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:55.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:55.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:55.587 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:55.702 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:55.941 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:55.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:55.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:56.058 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:17:56.150 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:56.247 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:17:56.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:56.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:17:56.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:56.387 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:56.757 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:17:56.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:56 smithi026 ceph-mon[223014]: pgmap v359: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 14867/200055 objects degraded (7.431%); 13605/200055 objects misplaced (6.801%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:17:56.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2690131867' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:17:56.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:56 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:17:56.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:56 smithi026 ceph-mon[215775]: pgmap v359: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 14867/200055 objects degraded (7.431%); 13605/200055 objects misplaced (6.801%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:17:56.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2690131867' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:17:56.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:56 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:17:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:56 smithi070 ceph-mon[192652]: pgmap v359: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.1 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 14867/200055 objects degraded (7.431%); 13605/200055 objects misplaced (6.801%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:17:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2690131867' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:17:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:56 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:17:57.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:57.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:17:57.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[223014]: osdmap e168: 8 total, 8 up, 7 in 2023-09-23T22:17:57.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[223014]: failed to encode map e168 with expected crc 2023-09-23T22:17:57.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[223014]: failed to encode map e168 with expected crc 2023-09-23T22:17:57.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[223014]: failed to encode map e168 with expected crc 2023-09-23T22:17:57.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[223014]: failed to encode map e168 with expected crc 2023-09-23T22:17:57.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 14867/200055 objects degraded (7.431%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:17:57.891 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:57.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:17:57.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[215775]: osdmap e168: 8 total, 8 up, 7 in 2023-09-23T22:17:57.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[215775]: failed to encode map e168 with expected crc 2023-09-23T22:17:57.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[215775]: failed to encode map e168 with expected crc 2023-09-23T22:17:57.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[215775]: failed to encode map e168 with expected crc 2023-09-23T22:17:57.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[215775]: failed to encode map e168 with expected crc 2023-09-23T22:17:57.892 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:57 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 14867/200055 objects degraded (7.431%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:17:57.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:17:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:57 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:17:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:17:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:57 smithi070 ceph-mon[192652]: osdmap e168: 8 total, 8 up, 7 in 2023-09-23T22:17:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:57 smithi070 ceph-mon[192652]: failed to encode map e168 with expected crc 2023-09-23T22:17:58.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:57 smithi070 ceph-mon[192652]: failed to encode map e168 with expected crc 2023-09-23T22:17:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:57 smithi070 ceph-mon[192652]: failed to encode map e168 with expected crc 2023-09-23T22:17:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:57 smithi070 ceph-mon[192652]: failed to encode map e168 with expected crc 2023-09-23T22:17:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:57 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 14867/200055 objects degraded (7.431%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:17:58.140 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:58.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:58.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:58.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:17:58.386 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:58.626 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:17:58.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:58.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:17:58.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:58 smithi026 ceph-mon[215775]: pgmap v361: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 14867/195924 objects degraded (7.588%); 13605/195924 objects misplaced (6.944%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:17:58.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:17:58.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:17:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:17:58.723 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:58 smithi026 ceph-mon[223014]: pgmap v361: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 14867/195924 objects degraded (7.588%); 13605/195924 objects misplaced (6.944%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:17:58.723 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:17:58.723 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:17:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:17:58.739 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:17:58.806 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:58 smithi070 ceph-mon[192652]: pgmap v361: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 14867/195924 objects degraded (7.588%); 13605/195924 objects misplaced (6.944%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:17:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:17:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:17:59.372 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:17:59.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:59.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:17:59.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:17:59.517 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:59.589 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:17:59.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:59.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:17:59.656 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:17:59.700 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:17:59.731 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:17:59.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:17:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:17:59] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-09-23T22:17:59.794 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:17:59.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:17:59.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:17:59.901 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:17:59.961 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:00.025 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:00.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:00.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:00.132 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:00.213 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:00.259 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:18:00.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:00.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:00.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:00.471 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:18:00.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:00.511 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:18:00.559 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:18:00.612 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:18:00.621 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:00.661 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:18:00.699 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:18:00.727 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:00.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:00.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:00.746 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:18:00.786 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:18:00.796 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:18:00.808 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:18:00.808 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:18:00.808 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:18:00.808 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:18:00.809 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:18:00.809 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:18:00.809 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:18:00.809 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:18:00.835 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:00.887 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:00.983 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:00.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:00.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:00.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:00 smithi026 ceph-mon[215775]: pgmap v362: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 14867/192741 objects degraded (7.713%); 13399/192741 objects misplaced (6.952%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:18:00.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3006210307' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:18:00.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:00 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:18:00.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2145290868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:00.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:00 smithi026 ceph-mon[223014]: pgmap v362: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 14867/192741 objects degraded (7.713%); 13399/192741 objects misplaced (6.952%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:18:00.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3006210307' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:18:00.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:00 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:18:00.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2145290868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:01.090 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:00 smithi070 ceph-mon[192652]: pgmap v362: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 611 op/s; 14867/192741 objects degraded (7.713%); 13399/192741 objects misplaced (6.952%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:18:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3006210307' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:18:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:00 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:18:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2145290868' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:01.146 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:01.213 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:01.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:01.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:01.329 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:01.379 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:01.467 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:01.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:01.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:01.577 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:01.629 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:01.727 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:01.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:01.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:01.800 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[215775]: failed to encode map e169 with expected crc 2023-09-23T22:18:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:18:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[215775]: osdmap e169: 8 total, 8 up, 7 in 2023-09-23T22:18:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[215775]: failed to encode map e169 with expected crc 2023-09-23T22:18:01.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[215775]: pgmap v364: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 14652/186984 objects degraded (7.836%); 13242/186984 objects misplaced (7.082%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:18:01.837 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:01.909 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:02.085 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:02.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:02.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:02.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[223014]: failed to encode map e169 with expected crc 2023-09-23T22:18:02.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:18:02.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[223014]: osdmap e169: 8 total, 8 up, 7 in 2023-09-23T22:18:02.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[223014]: failed to encode map e169 with expected crc 2023-09-23T22:18:02.093 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:01 smithi026 ceph-mon[223014]: pgmap v364: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 14652/186984 objects degraded (7.836%); 13242/186984 objects misplaced (7.082%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:18:02.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:01 smithi070 ceph-mon[192652]: failed to encode map e169 with expected crc 2023-09-23T22:18:02.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:01 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:18:02.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:01 smithi070 ceph-mon[192652]: osdmap e169: 8 total, 8 up, 7 in 2023-09-23T22:18:02.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:01 smithi070 ceph-mon[192652]: failed to encode map e169 with expected crc 2023-09-23T22:18:02.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:01 smithi070 ceph-mon[192652]: pgmap v364: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 14652/186984 objects degraded (7.836%); 13242/186984 objects misplaced (7.082%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:18:02.194 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:02.293 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:02.854 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:02.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:02.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:02.968 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:02.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:02 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 14652/186984 objects degraded (7.836%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:02.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:02 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 14652/186984 objects degraded (7.836%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:03.020 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:03.103 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:03.116 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:02 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 14652/186984 objects degraded (7.836%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:03.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:03.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:03.133 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T22:18:03.217 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:03.266 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:03.337 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:03.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:03.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:03.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:03.553 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:03.608 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:03.719 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:18:03.719 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:18:03.762 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:03.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:03.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:03.867 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:03.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:03 smithi026 ceph-mon[223014]: pgmap v365: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 14652/185316 objects degraded (7.906%); 13040/185316 objects misplaced (7.037%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:18:03.965 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:03 smithi026 ceph-mon[215775]: pgmap v365: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 14652/185316 objects degraded (7.906%); 13040/185316 objects misplaced (7.037%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:18:03.965 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:04.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:03 smithi070 ceph-mon[192652]: pgmap v365: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 14652/185316 objects degraded (7.906%); 13040/185316 objects misplaced (7.037%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:18:04.696 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:18:05.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:18:05.406 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:05.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:05.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:05.518 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:05.558 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:05.636 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:05.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:05.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:05.748 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:05.779 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:05.852 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:05.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:05.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:05.939 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:05.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:06.110 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:06.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:06 smithi026 ceph-mon[215775]: pgmap v366: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 14456/181782 objects degraded (7.952%); 12861/181782 objects misplaced (7.075%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T22:18:06.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:06 smithi026 ceph-mon[223014]: pgmap v366: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 14456/181782 objects degraded (7.952%); 12861/181782 objects misplaced (7.075%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T22:18:06.346 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:06.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:06.457 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:06.542 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:06.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:06 smithi070 ceph-mon[192652]: pgmap v366: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 14456/181782 objects degraded (7.952%); 12861/181782 objects misplaced (7.075%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T22:18:06.702 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:06.709 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:06.823 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:06.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:06.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:06.935 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:06.966 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:07.024 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:07.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:07.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:07.133 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:07.205 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:07.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:07 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:07.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:07 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:07.976 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:07 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:07.976 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:07 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:07.976 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:07 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:07.976 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:07 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:08.074 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:08.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:08.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:08.182 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:08.220 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:08.283 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:08.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:08.390 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:08.425 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:08.505 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:08.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:08.509 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:08.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[223014]: pgmap v367: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 14456/177459 objects degraded (8.146%); 12861/177459 objects misplaced (7.247%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T22:18:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 14456/177459 objects degraded (8.146%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3623940677' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:08.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[215775]: pgmap v367: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 14456/177459 objects degraded (8.146%); 12861/177459 objects misplaced (7.247%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 14456/177459 objects degraded (8.146%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3623940677' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:18:08.668 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:08 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:08.673 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:08.717 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:18:08.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:08 smithi070 ceph-mon[192652]: pgmap v367: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 14456/177459 objects degraded (8.146%); 12861/177459 objects misplaced (7.247%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T22:18:08.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 14456/177459 objects degraded (8.146%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:08.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:08 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3623940677' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:08.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:08 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:08.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:08 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:18:08.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:08 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:18:08.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:08 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:09.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:09.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:09.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:18:09.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[215775]: osdmap e170: 8 total, 8 up, 7 in 2023-09-23T22:18:09.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[215775]: failed to encode map e170 with expected crc 2023-09-23T22:18:09.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[215775]: failed to encode map e170 with expected crc 2023-09-23T22:18:09.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:09.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:09.964 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:18:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:18:09] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-09-23T22:18:09.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:09.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:18:09.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[223014]: osdmap e170: 8 total, 8 up, 7 in 2023-09-23T22:18:09.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[223014]: failed to encode map e170 with expected crc 2023-09-23T22:18:09.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[223014]: failed to encode map e170 with expected crc 2023-09-23T22:18:09.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:09.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:09.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:09.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:10.067 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:10.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:09 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:10.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:09 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:18:10.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:09 smithi070 ceph-mon[192652]: osdmap e170: 8 total, 8 up, 7 in 2023-09-23T22:18:10.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:09 smithi070 ceph-mon[192652]: failed to encode map e170 with expected crc 2023-09-23T22:18:10.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:09 smithi070 ceph-mon[192652]: failed to encode map e170 with expected crc 2023-09-23T22:18:10.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:10.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:10.105 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:10.211 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:10.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:10.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:10.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:10.361 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:10.450 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:10.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:10.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:10.559 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:10.614 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:10.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:10 smithi026 ceph-mon[215775]: pgmap v369: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 14456/173790 objects degraded (8.318%); 12672/173790 objects misplaced (7.292%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:18:10.697 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:10 smithi026 ceph-mon[223014]: pgmap v369: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 14456/173790 objects degraded (8.318%); 12672/173790 objects misplaced (7.292%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:18:10.697 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:10.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:10.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:10.804 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:10.842 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:10.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:10 smithi070 ceph-mon[192652]: pgmap v369: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 14456/173790 objects degraded (8.318%); 12672/173790 objects misplaced (7.292%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:18:10.924 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:10.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:10.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:11.035 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:11.072 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:11.148 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:11.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:11.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:11.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:11.366 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:11.396 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:18:11.446 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:18:11.490 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:18:11.529 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:18:11.574 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:18:11.620 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:11.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:11.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:11.633 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:18:11.697 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:18:11.704 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:18:11.704 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:18:11.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:18:11.704 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19252 5.5 5:ad849924:::benchmark_data_smithi026_286012_object19251:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.619373+0000", 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 163.07589277100001, 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0619795079999998, 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19252 2023-09-23T22:18:11.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619373+0000", 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619373+0000", 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619371+0000", 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619397+0000", 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5233e-05 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.707 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619410+0000", 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3543000000000001e-05 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619419+0000", 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5520000000000005e-06 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.626263+0000", 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068438769999999999 2023-09-23T22:18:11.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.626274+0000", 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1515000000000001e-05 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710603+0000", 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.084329246 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710615+0000", 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1912e-05 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.681047+0000", 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97043182400000005 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.681353+0000", 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030540500000000002 2023-09-23T22:18:11.710 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19260 5.5 5:ac33ab78:::benchmark_data_smithi026_286012_object19259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.619426+0000", 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 163.07584013600001, 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0626961929999998, 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.711 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19260 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619426+0000", 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619426+0000", 2023-09-23T22:18:11.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619425+0000", 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619448+0000", 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.245e-05 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619457+0000", 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.081e-06 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619463+0000", 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5880000000000003e-06 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.626297+0000", 2023-09-23T22:18:11.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068336530000000003 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.626305+0000", 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6240000000000002e-06 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710643+0000", 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0843381729999999 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710652+0000", 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8550000000000008e-06 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.682007+0000", 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97135575100000005 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.682122+0000", 2023-09-23T22:18:11.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114626 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19264 5.5 5:af03d027:::benchmark_data_smithi026_286012_object19263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.622566+0000", 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 163.072699747, 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.068983502, 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19264 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.622566+0000", 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.622566+0000", 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.622565+0000", 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.622590+0000", 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4873000000000001e-05 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.622600+0000", 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0346000000000001e-05 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.622608+0000", 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1699999999999997e-06 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.626326+0000", 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003717999 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.626334+0000", 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6279999999999996e-06 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710679+0000", 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.084345259 2023-09-23T22:18:11.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710687+0000", 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0860000000000002e-06 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.682167+0000", 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97147990500000003 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.682424+0000", 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025670800000000002 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.682632+0000", 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020863699999999999 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.685670+0000", 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030377920000000001 2023-09-23T22:18:11.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.686646+0000", 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00097598599999999998 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.691005+0000", 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043588589999999997 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.691074+0000", 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8573999999999998e-05 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.691550+0000", 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047594400000000001 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.725 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19271 5.5 5:a7ba4133:::benchmark_data_smithi026_286012_object19270:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.637012+0000", 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 163.058253525, 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0551333650000001, 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19271 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.637012+0000", 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.637012+0000", 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.637011+0000", 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:18:11.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.637034+0000", 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3688000000000001e-05 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.637055+0000", 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0545999999999999e-05 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.637063+0000", 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7230000000000004e-06 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.637124+0000", 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0939e-05 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.637137+0000", 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3393e-05 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710730+0000", 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0735927119999999 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710738+0000", 2023-09-23T22:18:11.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5699999999999993e-06 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.683577+0000", 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97283905199999998 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.683734+0000", 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015662 2023-09-23T22:18:11.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.683878+0000", 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014364400000000001 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.685679+0000", 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001801806 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.686731+0000", 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001052054 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.691689+0000", 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049579489999999997 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.691768+0000", 2023-09-23T22:18:11.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8491000000000005e-05 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.692146+0000", 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000377847 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19280 5.5 5:a3b5439d:::benchmark_data_smithi026_286012_object19279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.642585+0000", 2023-09-23T22:18:11.734 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 163.05268122499999, 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0500229050000001, 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19280 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.642585+0000", 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.642585+0000", 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.642583+0000", 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:18:11.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.642601+0000", 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7770000000000001e-05 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.642614+0000", 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2933e-05 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.642620+0000", 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8749999999999997e-06 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.642696+0000", 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5853999999999996e-05 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.642750+0000", 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4394000000000001e-05 2023-09-23T22:18:11.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710764+0000", 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.06801386 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710773+0000", 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.302e-06 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.684635+0000", 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97386125899999998 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.684793+0000", 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015875000000000001 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.684959+0000", 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016521400000000001 2023-09-23T22:18:11.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.686604+0000", 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001645737 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.692192+0000", 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005587669 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.692224+0000", 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2360999999999997e-05 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.692276+0000", 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1317999999999999e-05 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.692608+0000", 2023-09-23T22:18:11.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033189599999999998 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19284 5.5 5:a2be20ca:::benchmark_data_smithi026_286012_object19283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.645737+0000", 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 163.049529167, 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0500752040000001, 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.743 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19284 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.645737+0000", 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.744 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.645737+0000", 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.645735+0000", 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.645758+0000", 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2357e-05 2023-09-23T22:18:11.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.645774+0000", 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5917999999999999e-05 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.645781+0000", 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9970000000000001e-06 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.645843+0000", 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2448999999999998e-05 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.645857+0000", 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3699999999999999e-05 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710799+0000", 2023-09-23T22:18:11.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.064942697 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.710808+0000", 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4570000000000005e-06 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.685703+0000", 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97489538499999995 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.685817+0000", 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113596 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.685982+0000", 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016501299999999999 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.687058+0000", 2023-09-23T22:18:11.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001075672 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.692648+0000", 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055901650000000002 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.695056+0000", 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024084850000000001 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.695123+0000", 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7238999999999998e-05 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.695812+0000", 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00068846900000000004 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:36980 5.5 5:a7a92cf4:::benchmark_data_smithi026_286012_object36979:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:42.905107+0000", 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 148.79015849800001, 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.116179891, 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 36980 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.752 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905107+0000", 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905107+0000", 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905106+0000", 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905131+0000", 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4896999999999999e-05 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905144+0000", 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3421000000000001e-05 2023-09-23T22:18:11.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905151+0000", 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1420000000000004e-06 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905634+0000", 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048328899999999997 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905831+0000", 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019639000000000001 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.906006+0000", 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017515 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.910375+0000", 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043691229999999999 2023-09-23T22:18:11.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.925363+0000", 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014987544 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.925382+0000", 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8910999999999999e-05 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.021183+0000", 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0958009930000001 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.021287+0000", 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104654 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.758 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:36983 5.5 5:a5662841:::benchmark_data_smithi026_286012_object36982:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:42.905157+0000", 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 148.790108465, 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1162371419999999, 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 36983 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905157+0000", 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905157+0000", 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905157+0000", 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:18:11.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905177+0000", 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0346000000000001e-05 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905183+0000", 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1249999999999998e-06 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.905188+0000", 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7890000000000002e-06 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908800+0000", 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036118639999999998 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908922+0000", 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122024 2023-09-23T22:18:11.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.909084+0000", 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016200700000000001 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.912592+0000", 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035080699999999999 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.926061+0000", 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013468968 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.926071+0000", 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6500000000000008e-06 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.021321+0000", 2023-09-23T22:18:11.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0952500629999999 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.021395+0000", 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3671000000000007e-05 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:36988 5.c 5:35dce750:::benchmark_data_smithi026_286012_object36987:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:42.908037+0000", 2023-09-23T22:18:11.765 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 148.787228895, 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1059635379999999, 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 36988 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908037+0000", 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908037+0000", 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908035+0000", 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:18:11.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908061+0000", 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5429e-05 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908079+0000", 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8184000000000001e-05 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908089+0000", 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0254e-05 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908149+0000", 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9966000000000002e-05 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908316+0000", 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016661200000000001 2023-09-23T22:18:11.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.908523+0000", 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000207321 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.927754+0000", 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019230502 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.928827+0000", 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010735129999999999 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.931639+0000", 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028121840000000001 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.931654+0000", 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4494999999999999e-05 2023-09-23T22:18:11.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.013888+0000", 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0822345149999999 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.014001+0000", 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112057 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:37000 5.c 5:310f6fc0:::benchmark_data_smithi026_286012_object36999:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:42.929225+0000", 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 148.76604090999999, 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094877761, 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 37000 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.929225+0000", 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.929225+0000", 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.929223+0000", 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.929249+0000", 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5797000000000001e-05 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.929260+0000", 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1199999999999999e-05 2023-09-23T22:18:11.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.929268+0000", 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6709999999999999e-06 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.932739+0000", 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034710869999999999 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:42.932751+0000", 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1953999999999999e-05 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.014424+0000", 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0816727269999999 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.014521+0000", 2023-09-23T22:18:11.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6866000000000004e-05 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.014661+0000", 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014053899999999999 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.016312+0000", 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016506450000000001 2023-09-23T22:18:11.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.016509+0000", 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019684800000000001 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.023280+0000", 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067716790000000001 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.023335+0000", 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4960000000000002e-05 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:44.024103+0000", 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00076740700000000001 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.780 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79470 5.c 5:3839dd7e:::benchmark_data_smithi026_286012_object79469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.138645+0000", 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.556620861, 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0226527860000001, 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79470 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138645+0000", 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138645+0000", 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138643+0000", 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138676+0000", 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3077000000000002e-05 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138690+0000", 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4126e-05 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138699+0000", 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3089999999999996e-06 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138791+0000", 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2932999999999999e-05 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138804+0000", 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2682e-05 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.154254+0000", 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0154501869999999 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.154453+0000", 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000198934 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.154665+0000", 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000211843 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.157899+0000", 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032335440000000001 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.157942+0000", 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3003999999999997e-05 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161177+0000", 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032349140000000002 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161245+0000", 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8717e-05 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161298+0000", 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2527e-05 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79490 5.c 5:31f4c6c3:::benchmark_data_smithi026_286012_object79489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.166333+0000", 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.52893244099999, 2023-09-23T22:18:11.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.00561488, 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79490 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166333+0000", 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166333+0000", 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166328+0000", 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999948 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166359+0000", 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1356999999999998e-05 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166372+0000", 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.243e-05 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166380+0000", 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9060000000000002e-06 2023-09-23T22:18:11.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166455+0000", 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4969000000000007e-05 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166467+0000", 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2673e-05 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.158005+0000", 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99153777799999998 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.158162+0000", 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015700600000000001 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.158350+0000", 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018749700000000001 2023-09-23T22:18:11.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166611+0000", 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082610119999999999 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166652+0000", 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1117e-05 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.171851+0000", 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051991600000000004 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.171904+0000", 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3298999999999999e-05 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.171948+0000", 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4152e-05 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.795 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79491 5.c 5:39058f9e:::benchmark_data_smithi026_286012_object79490:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.166398+0000", 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.528867373, 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.005660617, 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.796 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79491 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166398+0000", 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166398+0000", 2023-09-23T22:18:11.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166398+0000", 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166429+0000", 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0815000000000002e-05 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166436+0000", 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6059999999999997e-06 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166443+0000", 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3400000000000003e-06 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166497+0000", 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.393e-05 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166505+0000", 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2509999999999994e-06 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161362+0000", 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99485717100000004 2023-09-23T22:18:11.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161527+0000", 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000164599 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161687+0000", 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016049200000000001 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166685+0000", 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004998382 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166760+0000", 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4945999999999997e-05 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.171964+0000", 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005203648 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172020+0000", 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5829999999999999e-05 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172059+0000", 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9208999999999997e-05 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79492 5.c 5:39c85251:::benchmark_data_smithi026_286012_object79491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.166450+0000", 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.528816319, 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0058361490000001, 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79492 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.804 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166450+0000", 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166450+0000", 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166449+0000", 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166471+0000", 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1767999999999999e-05 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166479+0000", 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1340000000000004e-06 2023-09-23T22:18:11.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166484+0000", 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9679999999999997e-06 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166528+0000", 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3801000000000003e-05 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166535+0000", 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3189999999999997e-06 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.164034+0000", 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99749939399999998 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.164170+0000", 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013584400000000001 2023-09-23T22:18:11.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.164260+0000", 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9506999999999995e-05 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172071+0000", 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078108199999999996 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172142+0000", 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1128000000000001e-05 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172198+0000", 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6042999999999999e-05 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172247+0000", 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8841000000000002e-05 2023-09-23T22:18:11.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172286+0000", 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9174999999999997e-05 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79500 5.c 5:3178a3e0:::benchmark_data_smithi026_286012_object79499:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.178188+0000", 2023-09-23T22:18:11.811 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.517077556, 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.994386043, 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79500 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178188+0000", 2023-09-23T22:18:11.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178188+0000", 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178188+0000", 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178210+0000", 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2399e-05 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178223+0000", 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3098999999999999e-05 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178230+0000", 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7700000000000004e-06 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178288+0000", 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8130000000000001e-05 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178298+0000", 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0142999999999999e-05 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166799+0000", 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98850079499999999 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166918+0000", 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011872200000000001 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.167008+0000", 2023-09-23T22:18:11.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0263e-05 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172304+0000", 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052957619999999999 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172371+0000", 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6679000000000003e-05 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172472+0000", 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101407 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172530+0000", 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8408e-05 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172574+0000", 2023-09-23T22:18:11.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4017999999999998e-05 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79525 5.c 5:3c92c974:::benchmark_data_smithi026_286012_object79524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.238208+0000", 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.457057792, 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93452411499999999, 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.819 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79525 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238208+0000", 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238208+0000", 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238206+0000", 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238237+0000", 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1111000000000001e-05 2023-09-23T22:18:11.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238253+0000", 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5276000000000001e-05 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238262+0000", 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5750000000000007e-06 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238329+0000", 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6920000000000003e-05 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238340+0000", 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0890000000000001e-05 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.169053+0000", 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93071300800000001 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.169154+0000", 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100717 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.169255+0000", 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101178 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172312+0000", 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003057073 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172422+0000", 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109694 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172634+0000", 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021268400000000001 2023-09-23T22:18:11.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172681+0000", 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6448e-05 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172732+0000", 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1388000000000003e-05 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.826 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79531 5.c 5:32793f2d:::benchmark_data_smithi026_286012_object79530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.238283+0000", 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.45698250300001, 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95936797200000001, 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79531 2023-09-23T22:18:11.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238283+0000", 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238283+0000", 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238283+0000", 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238306+0000", 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3264999999999998e-05 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238314+0000", 2023-09-23T22:18:11.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0399999999999993e-06 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238320+0000", 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0499999999999997e-06 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238371+0000", 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0946000000000003e-05 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238381+0000", 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4390000000000006e-06 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172794+0000", 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93441326899999999 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172986+0000", 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019185200000000001 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.173147+0000", 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016168399999999999 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.181599+0000", 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084517519999999999 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.186964+0000", 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053651810000000001 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197540+0000", 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010576189999999999 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197597+0000", 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6412000000000001e-05 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197651+0000", 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4431000000000001e-05 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.834 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79556 5.c 5:3b073673:::benchmark_data_smithi026_286012_object79555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.268850+0000", 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.42641553199999, 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.92890088599999998, 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79556 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268850+0000", 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268850+0000", 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268849+0000", 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:18:11.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268877+0000", 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7911000000000001e-05 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268892+0000", 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5311999999999998e-05 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268901+0000", 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5820000000000008e-06 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268966+0000", 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4175000000000001e-05 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268977+0000", 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1469e-05 2023-09-23T22:18:11.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.175810+0000", 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90683305700000005 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.175941+0000", 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013063699999999999 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.176087+0000", 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014580499999999999 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.186996+0000", 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010908951 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197532+0000", 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010536538999999999 2023-09-23T22:18:11.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197660+0000", 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012830099999999999 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197703+0000", 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2586000000000002e-05 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197751+0000", 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8229999999999997e-05 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79578 5.c 5:30fa6cc1:::benchmark_data_smithi026_286012_object79577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.304544+0000", 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.390722071, 2023-09-23T22:18:11.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95892923500000005, 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79578 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304544+0000", 2023-09-23T22:18:11.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304544+0000", 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304542+0000", 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304573+0000", 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1109000000000003e-05 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304595+0000", 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1950000000000002e-05 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304607+0000", 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1562e-05 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304702+0000", 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5725999999999998e-05 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304734+0000", 2023-09-23T22:18:11.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1959999999999999e-05 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.184170+0000", 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87943575799999996 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.184302+0000", 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013180000000000001 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.184454+0000", 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152077 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197767+0000", 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01331267 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.198028+0000", 2023-09-23T22:18:11.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026184100000000002 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263385+0000", 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065356763999999998 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263432+0000", 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7209000000000002e-05 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263473+0000", 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0596999999999998e-05 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79595 5.c 5:344ddd26:::benchmark_data_smithi026_286012_object79594:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.326572+0000", 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 102.36869350000001, 2023-09-23T22:18:11.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.936993611, 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79595 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:11.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326572+0000", 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326572+0000", 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326570+0000", 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326595+0000", 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4989000000000001e-05 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326616+0000", 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1123000000000001e-05 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326625+0000", 2023-09-23T22:18:11.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7860000000000002e-06 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326693+0000", 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8065000000000006e-05 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326704+0000", 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0934000000000001e-05 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.187046+0000", 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.86034200699999996 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.187172+0000", 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125888 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.187317+0000", 2023-09-23T22:18:11.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014485999999999999 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197772+0000", 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01045455 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.198056+0000", 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028478999999999999 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263505+0000", 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065448610000000004 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263538+0000", 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3037000000000002e-05 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263566+0000", 2023-09-23T22:18:11.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7962000000000001e-05 2023-09-23T22:18:11.858 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.858 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.858 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.858 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:11.858 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:11.858 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:18:11.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:11.859 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:18:11.859 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T22:18:11.859 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:18:11.860 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:18:11.860 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:18:11.860 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:18:11.860 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:18:11.860 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:18:11.860 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:18:11.860 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:18:11.913 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:11.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/899983998' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:18:11.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:18:11.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/739809309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:11.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/899983998' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:18:11.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:18:11.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/739809309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:12.009 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:12.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:12.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:12.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/899983998' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:18:12.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:18:12.032 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/739809309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:12.132 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:12.218 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:12.771 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:18:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[223014]: pgmap v370: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 14231/167760 objects degraded (8.483%); 12478/167760 objects misplaced (7.438%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:18:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[223014]: failed to encode map e171 with expected crc 2023-09-23T22:18:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:18:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[223014]: osdmap e171: 8 total, 8 up, 7 in 2023-09-23T22:18:12.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 14231/167760 objects degraded (8.483%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:12.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[215775]: pgmap v370: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 14231/167760 objects degraded (8.483%); 12478/167760 objects misplaced (7.438%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:18:12.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[215775]: failed to encode map e171 with expected crc 2023-09-23T22:18:12.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:18:12.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[215775]: osdmap e171: 8 total, 8 up, 7 in 2023-09-23T22:18:12.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:12 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 14231/167760 objects degraded (8.483%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:12 smithi070 ceph-mon[192652]: pgmap v370: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 640 op/s; 14231/167760 objects degraded (8.483%); 12478/167760 objects misplaced (7.438%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:18:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:12 smithi070 ceph-mon[192652]: failed to encode map e171 with expected crc 2023-09-23T22:18:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:12 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:18:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:12 smithi070 ceph-mon[192652]: osdmap e171: 8 total, 8 up, 7 in 2023-09-23T22:18:13.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:12 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 14231/167760 objects degraded (8.483%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:13.916 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:13.922 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T22:18:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:13.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:14.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:14.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:18:14.131 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:14.265 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:14.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:14.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:14.283 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:14.377 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:14.534 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:14.668 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:14.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:14.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:14.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:14.782 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:14 smithi026 ceph-mon[215775]: pgmap v372: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 14231/165732 objects degraded (8.587%); 12270/165732 objects misplaced (7.404%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:18:14.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:14 smithi026 ceph-mon[223014]: pgmap v372: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 14231/165732 objects degraded (8.587%); 12270/165732 objects misplaced (7.404%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:18:14.824 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:14.915 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:14.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:14.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:15.023 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:15.063 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:14 smithi070 ceph-mon[192652]: pgmap v372: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 668 op/s; 14231/165732 objects degraded (8.587%); 12270/165732 objects misplaced (7.404%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:18:15.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:15.508 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:15.508 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:15.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:15.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:15.683 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:15.914 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:15.955 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:18:15.959 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:15.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:15 smithi026 ceph-mon[215775]: pgmap v373: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 14231/163665 objects degraded (8.695%); 12270/163665 objects misplaced (7.497%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:18:15.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:15 smithi026 ceph-mon[223014]: pgmap v373: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 14231/163665 objects degraded (8.695%); 12270/163665 objects misplaced (7.497%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:18:16.021 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:16.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:16.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:16.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:15 smithi070 ceph-mon[192652]: pgmap v373: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 14231/163665 objects degraded (8.695%); 12270/163665 objects misplaced (7.497%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:18:16.132 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:16.236 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:16.447 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:16.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:16.557 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:16.680 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:18.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:18.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:18.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:18.350 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:18.397 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:18 smithi070 ceph-mon[192652]: pgmap v374: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 14061/158019 objects degraded (8.898%); 12087/158019 objects misplaced (7.649%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:18:18.398 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 14061/158019 objects degraded (8.898%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:18.398 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/74484481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:18:18.398 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:18.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:18 smithi026 ceph-mon[215775]: pgmap v374: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 14061/158019 objects degraded (8.898%); 12087/158019 objects misplaced (7.649%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:18:18.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 14061/158019 objects degraded (8.898%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:18.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/74484481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:18:18.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:18 smithi026 ceph-mon[223014]: pgmap v374: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.2 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 651 op/s; 14061/158019 objects degraded (8.898%); 12087/158019 objects misplaced (7.649%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T22:18:18.406 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 14061/158019 objects degraded (8.898%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:18.406 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/74484481' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:18:18.470 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:18.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:18.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:18.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:18.662 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:18.717 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:18.735 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:18:19.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:19 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:19.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:19 smithi026 ceph-mon[223014]: failed to encode map e172 with expected crc 2023-09-23T22:18:19.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/74484481' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:18:19.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:19 smithi026 ceph-mon[223014]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T22:18:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:19 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:19 smithi026 ceph-mon[215775]: failed to encode map e172 with expected crc 2023-09-23T22:18:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/74484481' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:18:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:19 smithi026 ceph-mon[215775]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T22:18:19.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:19 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:19.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:19 smithi070 ceph-mon[192652]: failed to encode map e172 with expected crc 2023-09-23T22:18:19.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/74484481' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:18:19.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:19 smithi070 ceph-mon[192652]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T22:18:19.890 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:18:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:18:19] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-09-23T22:18:19.987 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:20.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:20.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:20.109 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:20.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:20 smithi026 ceph-mon[215775]: failed to encode map e172 with expected crc 2023-09-23T22:18:20.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:20 smithi026 ceph-mon[215775]: pgmap v376: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 14061/154668 objects degraded (9.091%); 11866/154668 objects misplaced (7.672%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:18:20.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:20 smithi026 ceph-mon[223014]: failed to encode map e172 with expected crc 2023-09-23T22:18:20.154 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:20 smithi026 ceph-mon[223014]: pgmap v376: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 14061/154668 objects degraded (9.091%); 11866/154668 objects misplaced (7.672%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:18:20.206 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:20.250 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:18:20.250 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:18:20.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.250 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:138421 5.a 5:5f1ef40f:::benchmark_data_smithi026_286012_object44613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T22:18:20.250 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:20.234651+0000", 2023-09-23T22:18:20.250 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.013829777999999999, 2023-09-23T22:18:20.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013865100999999999, 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 138421 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234651+0000", 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234651+0000", 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234650+0000", 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234656+0000", 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3520000000000003e-06 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234667+0000", 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0956e-05 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234674+0000", 2023-09-23T22:18:20.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.691e-06 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.239330+0000", 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046562269999999998 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.239454+0000", 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000123946 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4", 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.239570+0000", 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011673 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:138422 5.6 5:6ac092cf:::benchmark_data_smithi026_286012_object44614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:20.234679+0000", 2023-09-23T22:18:20.255 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.01380139, 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013923181999999999, 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 138422 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:20.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234679+0000", 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234679+0000", 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234679+0000", 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:18:20.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234681+0000", 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7599999999999998e-06 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234689+0000", 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2760000000000003e-06 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234693+0000", 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1799999999999998e-06 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234771+0000", 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8215999999999998e-05 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:20.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234919+0000", 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014805499999999999 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.235069+0000", 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015013900000000001 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.260 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:138424 5.4 5:27cb30fa:::benchmark_data_smithi026_286012_object44616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:20.234725+0000", 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.013755642, 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013958238, 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 138424 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:20.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234725+0000", 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234725+0000", 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234724+0000", 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234727+0000", 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7420000000000002e-06 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234746+0000", 2023-09-23T22:18:20.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9097e-05 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234757+0000", 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1178999999999999e-05 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234842+0000", 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4499999999999994e-05 2023-09-23T22:18:20.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.234956+0000", 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114047 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.235069+0000", 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113237 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:20.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.238757+0000", 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036880490000000001 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.238913+0000", 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015608399999999999 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:138422 5.6 e172/111)", 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:20.243355+0000", 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0051255399999999996, 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054163689999999999, 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.243355+0000", 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.243355+0000", 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.243354+0000", 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.243360+0000", 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4790000000000001e-06 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.243374+0000", 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3395e-05 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.243384+0000", 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0342e-05 2023-09-23T22:18:20.269 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:138422 5.6 e172/111)", 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:20.247393+0000", 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001087377, 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001422195, 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:18:20.270 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.247393+0000", 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.247393+0000", 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.247392+0000", 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.247400+0000", 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6449999999999994e-06 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.247418+0000", 2023-09-23T22:18:20.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.717e-05 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.247428+0000", 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0832999999999999e-05 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:138417 5.8 5:1dbe5af0:::benchmark_data_smithi026_286012_object44609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e172)", 2023-09-23T22:18:20.273 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:20.224773+0000", 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.023707669000000001, 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024093125999999999, 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 138417 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.224773+0000", 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.224773+0000", 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.224771+0000", 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:18:20.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.224780+0000", 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6780000000000006e-06 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.224801+0000", 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1631e-05 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.224811+0000", 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3870000000000001e-06 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.225066+0000", 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025489799999999998 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.225218+0000", 2023-09-23T22:18:20.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152232 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.225369+0000", 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015107200000000001 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.231233+0000", 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058642989999999999 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:18:20.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:18:20.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:20.235534+0000", 2023-09-23T22:18:20.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043003199999999998 2023-09-23T22:18:20.279 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.279 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:18:20.279 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.279 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:18:20.279 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:18:20.279 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 6 2023-09-23T22:18:20.279 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:18:20.427 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:20.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:20.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:20.538 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:20.609 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:20.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:20 smithi070 ceph-mon[192652]: failed to encode map e172 with expected crc 2023-09-23T22:18:20.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:20 smithi070 ceph-mon[192652]: pgmap v376: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 14061/154668 objects degraded (9.091%); 11866/154668 objects misplaced (7.672%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:18:20.750 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:20.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:20.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:20.857 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:20.920 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:20.970 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:20.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:20.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:21.075 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:21.162 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:21.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:21 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3302632289' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:18:21.204 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T22:18:21.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:21 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3302632289' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:18:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:21 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3302632289' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:18:21.786 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:18:21.812 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:21.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:21.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:21.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:22.014 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:18:22.027 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:22.108 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:22.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:22.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:22.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[223014]: pgmap v377: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 13875/148818 objects degraded (9.323%); 11678/148818 objects misplaced (7.847%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:18:22.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3302632289' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:18:22.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[223014]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T22:18:22.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[223014]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[223014]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[223014]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[223014]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[215775]: pgmap v377: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 13875/148818 objects degraded (9.323%); 11678/148818 objects misplaced (7.847%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:18:22.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3302632289' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:18:22.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[215775]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T22:18:22.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[215775]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[215775]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[215775]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:22 smithi026 ceph-mon[215775]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:22.590 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:22.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:22.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:22.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:22 smithi070 ceph-mon[192652]: pgmap v377: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 674 op/s; 13875/148818 objects degraded (9.323%); 11678/148818 objects misplaced (7.847%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:18:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3302632289' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:18:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:22 smithi070 ceph-mon[192652]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T22:18:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:22 smithi070 ceph-mon[192652]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:22 smithi070 ceph-mon[192652]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:22 smithi070 ceph-mon[192652]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:22 smithi070 ceph-mon[192652]: failed to encode map e173 with expected crc 2023-09-23T22:18:22.702 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:22.731 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T22:18:22.733 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:22.802 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:22.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:22.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:22.908 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:23.046 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:23.080 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:23.118 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:18:23.143 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:23.187 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:23.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:23.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:23.231 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:18:23.283 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:18:23.298 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:23.324 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:18:23.364 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:18:23.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 13875/148818 objects degraded (9.323%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:23.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1217090564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 13875/148818 objects degraded (9.323%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:23.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1217090564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:23.387 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:23.400 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:18:23.443 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:18:23.490 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:18:23.498 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:18:23.498 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:18:23.498 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:18:23.498 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:18:23.499 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:18:23.499 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:18:23.499 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:18:23.499 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:18:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 13875/148818 objects degraded (9.323%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1217090564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:23.633 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:23.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:23.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:23.741 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:23.833 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:24.282 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:24.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:24.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:24.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:24.419 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:24.714 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:24 smithi026 ceph-mon[215775]: pgmap v379: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 13875/147024 objects degraded (9.437%); 11423/147024 objects misplaced (7.769%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T22:18:24.714 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:24 smithi026 ceph-mon[223014]: pgmap v379: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 13875/147024 objects degraded (9.437%); 11423/147024 objects misplaced (7.769%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T22:18:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:25.106 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:25.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:25.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:24 smithi070 ceph-mon[192652]: pgmap v379: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 13875/147024 objects degraded (9.437%); 11423/147024 objects misplaced (7.769%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T22:18:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:25.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:25.281 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:25.555 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:25.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:25.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:25.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:25.719 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:25.801 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:25.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:25.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:25.914 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:25.987 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:26.300 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:26.347 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:26.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:26.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:26.458 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:26.491 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:26.570 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:26.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:26.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:26.675 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:26.761 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:18:26.762 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:18:26.818 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:26 smithi070 ceph-mon[192652]: pgmap v380: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 13875/145317 objects degraded (9.548%); 11423/145317 objects misplaced (7.861%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T22:18:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:26 smithi026 ceph-mon[215775]: pgmap v380: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 13875/145317 objects degraded (9.548%); 11423/145317 objects misplaced (7.861%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T22:18:26.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:26 smithi026 ceph-mon[223014]: pgmap v380: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 3.0 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 13875/145317 objects degraded (9.548%); 11423/145317 objects misplaced (7.861%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T22:18:27.410 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:27.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:27.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:27.517 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:27.596 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:27.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:27 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 13875/145317 objects degraded (9.548%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:27.913 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1692631142' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:27.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:27 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:27.914 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:27 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 13875/145317 objects degraded (9.548%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:27.914 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1692631142' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:27.914 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:27 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:27.940 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:18:28.070 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:28.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:28.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:28.091 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:18:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:27 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 13875/145317 objects degraded (9.548%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1692631142' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:28.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:27 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:28.181 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:28.224 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:28.285 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:28.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:28.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:28.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:28.513 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:28.715 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:18:28.767 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:28.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:28.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:28.885 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:28.920 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:28.998 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:29.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:29.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:29.109 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:28 smithi070 ceph-mon[192652]: pgmap v381: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.8 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 13681/139377 objects degraded (9.816%); 11244/139377 objects misplaced (8.067%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:18:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:28 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:18:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:28 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:18:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:28 smithi070 ceph-mon[192652]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T22:18:29.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:28 smithi026 ceph-mon[223014]: pgmap v381: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.8 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 13681/139377 objects degraded (9.816%); 11244/139377 objects misplaced (8.067%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:18:29.189 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:28 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:18:29.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:28 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:18:29.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:28 smithi026 ceph-mon[223014]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T22:18:29.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:28 smithi026 ceph-mon[215775]: pgmap v381: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.8 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 629 op/s; 13681/139377 objects degraded (9.816%); 11244/139377 objects misplaced (8.067%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T22:18:29.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:28 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:18:29.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:28 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:18:29.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:28 smithi026 ceph-mon[215775]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T22:18:29.201 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:29.401 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:29.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:29.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:29.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:29.542 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:29.866 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:29.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:29.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:29.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:29 smithi026 ceph-mon[215775]: failed to encode map e174 with expected crc 2023-09-23T22:18:29.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:29 smithi026 ceph-mon[215775]: failed to encode map e174 with expected crc 2023-09-23T22:18:29.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:29 smithi026 ceph-mon[215775]: pgmap v383: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.8 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 13681/135741 objects degraded (10.079%); 11015/135741 objects misplaced (8.115%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T22:18:29.875 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:18:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:18:29] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.43.0" 2023-09-23T22:18:29.875 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:29 smithi026 ceph-mon[223014]: failed to encode map e174 with expected crc 2023-09-23T22:18:29.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:29 smithi026 ceph-mon[223014]: failed to encode map e174 with expected crc 2023-09-23T22:18:29.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:29 smithi026 ceph-mon[223014]: pgmap v383: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.8 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 13681/135741 objects degraded (10.079%); 11015/135741 objects misplaced (8.115%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T22:18:29.975 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:30.089 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:29 smithi070 ceph-mon[192652]: failed to encode map e174 with expected crc 2023-09-23T22:18:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:29 smithi070 ceph-mon[192652]: failed to encode map e174 with expected crc 2023-09-23T22:18:30.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:29 smithi070 ceph-mon[192652]: pgmap v383: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.8 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 13681/135741 objects degraded (10.079%); 11015/135741 objects misplaced (8.115%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T22:18:30.148 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:30.268 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:30.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:30.374 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:30.410 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:30.482 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:30.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:30.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:30.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:30.692 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:30.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1651203524' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:18:30.789 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:30 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:18:30.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1651203524' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:18:30.790 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:30 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:18:30.799 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T22:18:30.838 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:31.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1651203524' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:18:31.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:30 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:18:31.415 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:31.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:31.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:31.522 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:31.577 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:31.682 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:31.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:31.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:31.794 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:31.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:31 smithi070 ceph-mon[192652]: failed to encode map e175 with expected crc 2023-09-23T22:18:31.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:31 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:18:31.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:31 smithi070 ceph-mon[192652]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T22:18:31.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:31 smithi070 ceph-mon[192652]: failed to encode map e175 with expected crc 2023-09-23T22:18:31.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:31 smithi070 ceph-mon[192652]: pgmap v385: 99 pgs: 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 13470/129918 objects degraded (10.368%); 10789/129918 objects misplaced (8.304%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:18:31.844 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:31 smithi070 ceph-mon[192652]: osdmap e176: 8 total, 8 up, 7 in 2023-09-23T22:18:31.861 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:31.900 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:31.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:31.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:32.006 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:32.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:18:32.046 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:32.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[215775]: failed to encode map e175 with expected crc 2023-09-23T22:18:32.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:18:32.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[215775]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T22:18:32.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[215775]: failed to encode map e175 with expected crc 2023-09-23T22:18:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[215775]: pgmap v385: 99 pgs: 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 13470/129918 objects degraded (10.368%); 10789/129918 objects misplaced (8.304%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:18:32.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[215775]: osdmap e176: 8 total, 8 up, 7 in 2023-09-23T22:18:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[223014]: failed to encode map e175 with expected crc 2023-09-23T22:18:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:18:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[223014]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T22:18:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[223014]: failed to encode map e175 with expected crc 2023-09-23T22:18:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[223014]: pgmap v385: 99 pgs: 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 13470/129918 objects degraded (10.368%); 10789/129918 objects misplaced (8.304%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:18:32.132 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:31 smithi026 ceph-mon[223014]: osdmap e176: 8 total, 8 up, 7 in 2023-09-23T22:18:32.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:18:32.510 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:32.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:32.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:32.614 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:32.745 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:32.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:32 smithi026 ceph-mon[223014]: failed to encode map e176 with expected crc 2023-09-23T22:18:32.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:32 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 13470/129918 objects degraded (10.368%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:32.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:32 smithi026 ceph-mon[215775]: failed to encode map e176 with expected crc 2023-09-23T22:18:32.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:32 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 13470/129918 objects degraded (10.368%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:32 smithi070 ceph-mon[192652]: failed to encode map e176 with expected crc 2023-09-23T22:18:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:32 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 13470/129918 objects degraded (10.368%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:33.240 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:33.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:33.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:33.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:33.372 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:33.415 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:33.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:33.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:33.521 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:33.596 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:33.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:33 smithi026 ceph-mon[215775]: failed to encode map e176 with expected crc 2023-09-23T22:18:33.929 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:33.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:33.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:34.064 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:34.083 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:18:34.083 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:18:34.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:33 smithi026 ceph-mon[223014]: failed to encode map e176 with expected crc 2023-09-23T22:18:34.113 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:33 smithi026 ceph-mon[223014]: pgmap v387: 99 pgs: 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 13470/128085 objects degraded (10.516%); 10536/128085 objects misplaced (8.226%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T22:18:34.114 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:33 smithi026 ceph-mon[215775]: pgmap v387: 99 pgs: 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 13470/128085 objects degraded (10.516%); 10536/128085 objects misplaced (8.226%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T22:18:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:33 smithi070 ceph-mon[192652]: failed to encode map e176 with expected crc 2023-09-23T22:18:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:33 smithi070 ceph-mon[192652]: pgmap v387: 99 pgs: 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 13470/128085 objects degraded (10.516%); 10536/128085 objects misplaced (8.226%); 7.4 MiB/s, 117 objects/s recovering 2023-09-23T22:18:34.125 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:34.174 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:34.281 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:34.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:34.386 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:34.450 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:35.065 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:35.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:35.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:35.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:18:35.294 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:35.615 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:35.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:35.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:18:35.729 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:35.754 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:18:35.772 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:35.804 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:18:35.834 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:18:35.842 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:35.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:35.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:35.890 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:18:35.926 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:18:35.953 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:35.985 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:36.025 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:36.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:36.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:36.067 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:18:36.067 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:18:36.067 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:18:36.067 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:18:36.067 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:18:36.067 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:18:36.067 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:18:36.068 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:18:36.068 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:18:36.068 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:18:36.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:36.188 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:36.265 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:36.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:36.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:36.371 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:36 smithi070 ceph-mon[192652]: pgmap v388: 99 pgs: 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 13470/126243 objects degraded (10.670%); 10536/126243 objects misplaced (8.346%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:18:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:36 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3271647242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:36.441 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:36.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:36 smithi026 ceph-mon[215775]: pgmap v388: 99 pgs: 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 13470/126243 objects degraded (10.670%); 10536/126243 objects misplaced (8.346%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:18:36.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:36 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3271647242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:36.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:36 smithi026 ceph-mon[223014]: pgmap v388: 99 pgs: 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.6 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 13470/126243 objects degraded (10.670%); 10536/126243 objects misplaced (8.346%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T22:18:36.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:36 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3271647242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:36.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:36.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:36.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:36.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:36.648 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:37.495 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:37.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:37.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:37.603 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:37.644 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:37.716 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:37.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:37.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:37.832 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:37.877 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:37.944 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:37.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:37.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:38.050 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:38.123 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:38.151 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:38.357 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:38.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:38.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:38 smithi070 ceph-mon[192652]: pgmap v389: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.5 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 13282/120399 objects degraded (11.032%); 10307/120399 objects misplaced (8.561%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:18:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 13282/120399 objects degraded (11.032%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:38.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:38 smithi026 ceph-mon[223014]: pgmap v389: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.5 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 13282/120399 objects degraded (11.032%); 10307/120399 objects misplaced (8.561%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:18:38.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 13282/120399 objects degraded (11.032%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:38.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:38 smithi026 ceph-mon[215775]: pgmap v389: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.5 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 13282/120399 objects degraded (11.032%); 10307/120399 objects misplaced (8.561%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:18:38.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 13282/120399 objects degraded (11.032%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:38.474 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:38.525 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:38.596 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:38.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:38.709 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:38.797 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:39.041 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:39.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:39.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:39.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:39.211 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:18:39.219 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:39.231 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:18:39.334 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:39.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:39 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/327765035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:18:39.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:39.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:39.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:39 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1831419678' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:18:39.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:39 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/327765035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:18:39.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:39.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:39.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:39 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1831419678' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:18:39.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:39.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:39.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:39.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:39 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/327765035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:18:39.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:39.481 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:39.481 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:39 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1831419678' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:18:39.483 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:39.545 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:39.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:39.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:39.650 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:39.679 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:39.799 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:18:39.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:18:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:18:39] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.43.0" 2023-09-23T22:18:40.179 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:40.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:40.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:40.283 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:40.324 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:40.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: pgmap v390: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 13282/116868 objects degraded (11.365%); 10067/116868 objects misplaced (8.614%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:18:40.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/327765035' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:18:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1831419678' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:18:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: osdmap e177: 8 total, 8 up, 7 in 2023-09-23T22:18:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[215775]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: pgmap v390: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 13282/116868 objects degraded (11.365%); 10067/116868 objects misplaced (8.614%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:18:40.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:40.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/327765035' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:18:40.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1831419678' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:18:40.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: osdmap e177: 8 total, 8 up, 7 in 2023-09-23T22:18:40.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:40 smithi026 ceph-mon[223014]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: pgmap v390: 99 pgs: 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.4 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 13282/116868 objects degraded (11.365%); 10067/116868 objects misplaced (8.614%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:18:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/327765035' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:18:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1831419678' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:18:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: osdmap e177: 8 total, 8 up, 7 in 2023-09-23T22:18:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:40 smithi070 ceph-mon[192652]: failed to encode map e177 with expected crc 2023-09-23T22:18:40.767 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T22:18:41.000 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:41.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:41.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:41.110 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:41.136 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:41.183 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:41.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:41.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:41.199 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:41.286 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:41.314 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:41.354 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:18:41.354 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:18:41.355 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:18:41.355 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:18:41.361 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:41.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:41.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:41.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[223014]: osdmap e178: 8 total, 8 up, 7 in 2023-09-23T22:18:41.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[223014]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[223014]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[223014]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.382 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[223014]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[223014]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[223014]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[215775]: osdmap e178: 8 total, 8 up, 7 in 2023-09-23T22:18:41.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[215775]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[215775]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[215775]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.383 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[215775]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[215775]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:41 smithi026 ceph-mon[215775]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.385 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:41.467 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:41.517 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:41.628 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:41 smithi070 ceph-mon[192652]: osdmap e178: 8 total, 8 up, 7 in 2023-09-23T22:18:41.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:41 smithi070 ceph-mon[192652]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:41 smithi070 ceph-mon[192652]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:41 smithi070 ceph-mon[192652]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:41 smithi070 ceph-mon[192652]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:41 smithi070 ceph-mon[192652]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:41 smithi070 ceph-mon[192652]: failed to encode map e178 with expected crc 2023-09-23T22:18:41.735 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:41.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:41.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:41.841 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:41.913 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:42.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[215775]: pgmap v393: 99 pgs: 1 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 10778/112326 objects degraded (9.595%); 12059/112326 objects misplaced (10.736%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:18:42.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[223014]: pgmap v393: 99 pgs: 1 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 10778/112326 objects degraded (9.595%); 12059/112326 objects misplaced (10.736%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:18:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[223014]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[223014]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[223014]: osdmap e179: 8 total, 8 up, 7 in 2023-09-23T22:18:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[223014]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[223014]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[215775]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[215775]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[215775]: osdmap e179: 8 total, 8 up, 7 in 2023-09-23T22:18:42.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[215775]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:42 smithi026 ceph-mon[215775]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:42 smithi070 ceph-mon[192652]: pgmap v393: 99 pgs: 1 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 654 op/s; 10778/112326 objects degraded (9.595%); 12059/112326 objects misplaced (10.736%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:18:42.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:42 smithi070 ceph-mon[192652]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:42 smithi070 ceph-mon[192652]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:42 smithi070 ceph-mon[192652]: osdmap e179: 8 total, 8 up, 7 in 2023-09-23T22:18:42.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:42 smithi070 ceph-mon[192652]: failed to encode map e179 with expected crc 2023-09-23T22:18:42.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:42 smithi070 ceph-mon[192652]: failed to encode map e179 with expected crc 2023-09-23T22:18:43.029 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:18:43.470 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:43.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10778/112326 objects degraded (9.595%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:43.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:43.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:43.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10778/112326 objects degraded (9.595%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:43.579 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10778/112326 objects degraded (9.595%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:43.647 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:43.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:18:43.903 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:43.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:44.019 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:44.169 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:44.286 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:44.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:44.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:44.405 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:44.448 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:44.529 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:44.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:44.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:44.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:44.696 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:44.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:44 smithi026 ceph-mon[215775]: pgmap v395: 99 pgs: 1 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.3 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 10778/111297 objects degraded (9.684%); 11598/111297 objects misplaced (10.421%); 11 MiB/s, 173 objects/s recovering 2023-09-23T22:18:44.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:44 smithi026 ceph-mon[223014]: pgmap v395: 99 pgs: 1 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.3 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 10778/111297 objects degraded (9.684%); 11598/111297 objects misplaced (10.421%); 11 MiB/s, 173 objects/s recovering 2023-09-23T22:18:45.024 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:18:45.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:45.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:18:45.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:44 smithi070 ceph-mon[192652]: pgmap v395: 99 pgs: 1 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.3 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 10778/111297 objects degraded (9.684%); 11598/111297 objects misplaced (10.421%); 11 MiB/s, 173 objects/s recovering 2023-09-23T22:18:45.131 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:45.219 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:45.331 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:45.473 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:45.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:45.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:45.579 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:45.639 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:18:45.689 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:45.696 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:18:45.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1261037127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:45.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1261037127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:45.782 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:18:45.823 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:18:45.864 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:18:45.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:18:45.944 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:45.956 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:18:46.004 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:18:46.045 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:18:46.045 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:18:46.045 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:18:46.046 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:18:46.046 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:18:46.046 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:18:46.046 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:18:46.046 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:18:46.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1261037127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:46 smithi026 ceph-mon[215775]: pgmap v396: 99 pgs: 1 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 10778/110166 objects degraded (9.783%); 11598/110166 objects misplaced (10.528%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:18:46.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:46 smithi026 ceph-mon[223014]: pgmap v396: 99 pgs: 1 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 10778/110166 objects degraded (9.783%); 11598/110166 objects misplaced (10.528%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:18:47.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:46 smithi070 ceph-mon[192652]: pgmap v396: 99 pgs: 1 remapped+peering, 1 peering, 3 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 2.2 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 369 op/s; 10778/110166 objects degraded (9.783%); 11598/110166 objects misplaced (10.528%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:18:47.680 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:47.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:47.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:47.836 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:47.888 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:47.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:47.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:47.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:47 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10778/110166 objects degraded (9.783%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:47.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:47 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10778/110166 objects degraded (9.783%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:47.994 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:48.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:48.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:47 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10778/110166 objects degraded (9.783%), 6 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:48.334 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:48.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:48.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:48.441 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:48.478 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:48.556 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:48.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:48.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:48.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:48.693 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:48.800 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:48.808 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T22:18:48.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:48.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:48.918 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:48.944 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:48.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:48 smithi026 ceph-mon[215775]: pgmap v397: 99 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 18193/105123 objects degraded (17.306%); 9655/105123 objects misplaced (9.184%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:18:48.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1372807808' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:18:48.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:48 smithi026 ceph-mon[223014]: pgmap v397: 99 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 18193/105123 objects degraded (17.306%); 9655/105123 objects misplaced (9.184%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:18:48.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1372807808' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:18:48.989 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:48.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:48.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:49.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:49.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:48 smithi070 ceph-mon[192652]: pgmap v397: 99 pgs: 3 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.1 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 495 op/s; 18193/105123 objects degraded (17.306%); 9655/105123 objects misplaced (9.184%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:18:49.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1372807808' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:18:49.172 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:49.738 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:18:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:18:49] "GET /metrics HTTP/1.1" 200 36395 "" "Prometheus/2.43.0" 2023-09-23T22:18:49.825 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:18:49.900 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:18:50.026 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:50.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:50.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:50.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[215775]: failed to encode map e180 with expected crc 2023-09-23T22:18:50.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1372807808' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:18:50.031 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[215775]: osdmap e180: 8 total, 8 up, 7 in 2023-09-23T22:18:50.032 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[215775]: failed to encode map e180 with expected crc 2023-09-23T22:18:50.032 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4018277458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:50.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[223014]: failed to encode map e180 with expected crc 2023-09-23T22:18:50.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1372807808' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:18:50.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[223014]: osdmap e180: 8 total, 8 up, 7 in 2023-09-23T22:18:50.032 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[223014]: failed to encode map e180 with expected crc 2023-09-23T22:18:50.033 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:49 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4018277458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:50.073 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T22:18:50.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:49 smithi070 ceph-mon[192652]: failed to encode map e180 with expected crc 2023-09-23T22:18:50.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:49 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1372807808' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:18:50.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:49 smithi070 ceph-mon[192652]: osdmap e180: 8 total, 8 up, 7 in 2023-09-23T22:18:50.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:49 smithi070 ceph-mon[192652]: failed to encode map e180 with expected crc 2023-09-23T22:18:50.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:49 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4018277458' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:18:50.131 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:50.246 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:50.877 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:50.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:50.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:50.986 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:51.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:50 smithi070 ceph-mon[192652]: pgmap v399: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 18193/101670 objects degraded (17.894%); 9355/101670 objects misplaced (9.201%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T22:18:51.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:50 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:51.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:50 smithi070 ceph-mon[192652]: failed to encode map e181 with expected crc 2023-09-23T22:18:51.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:50 smithi070 ceph-mon[192652]: failed to encode map e181 with expected crc 2023-09-23T22:18:51.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4018277458' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:18:51.018 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:50 smithi070 ceph-mon[192652]: osdmap e181: 8 total, 8 up, 7 in 2023-09-23T22:18:51.019 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:51.028 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[223014]: pgmap v399: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 18193/101670 objects degraded (17.894%); 9355/101670 objects misplaced (9.201%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T22:18:51.028 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:51.028 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[223014]: failed to encode map e181 with expected crc 2023-09-23T22:18:51.028 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[223014]: failed to encode map e181 with expected crc 2023-09-23T22:18:51.028 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4018277458' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:18:51.028 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[223014]: osdmap e181: 8 total, 8 up, 7 in 2023-09-23T22:18:51.028 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[215775]: pgmap v399: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 18193/101670 objects degraded (17.894%); 9355/101670 objects misplaced (9.201%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T22:18:51.028 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:18:51.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[215775]: failed to encode map e181 with expected crc 2023-09-23T22:18:51.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[215775]: failed to encode map e181 with expected crc 2023-09-23T22:18:51.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4018277458' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:18:51.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:50 smithi026 ceph-mon[215775]: osdmap e181: 8 total, 8 up, 7 in 2023-09-23T22:18:51.120 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:51.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:51.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:51.225 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:18:51.274 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:51.337 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:18:51.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:51.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:18:51.442 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:51.474 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:18:51.474 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:18:51.547 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:51.562 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:51.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:51.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:51.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:51.863 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:18:51.915 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:52.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:51 smithi070 ceph-mon[192652]: pgmap v401: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.0 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 18048/99471 objects degraded (18.144%); 9210/99471 objects misplaced (9.259%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:18:52.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:51 smithi026 ceph-mon[223014]: pgmap v401: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.0 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 18048/99471 objects degraded (18.144%); 9210/99471 objects misplaced (9.259%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:18:52.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:51 smithi026 ceph-mon[215775]: pgmap v401: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 2.0 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 18048/99471 objects degraded (18.144%); 9210/99471 objects misplaced (9.259%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:18:52.486 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:18:52.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:52.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:18:52.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:52.662 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:52.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:18:52.689 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:52.842 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:52.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:52.949 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:52.990 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:53.017 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:52 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 18048/99471 objects degraded (18.144%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:53.017 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:52 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 18048/99471 objects degraded (18.144%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:53.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:18:53.060 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:53.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:53.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:53.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:52 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 18048/99471 objects degraded (18.144%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:53.166 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:53.203 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:53.271 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:53.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:53.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:53.379 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:53.465 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:54.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:53 smithi070 ceph-mon[192652]: pgmap v402: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.9 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 18048/94413 objects degraded (19.116%); 8858/94413 objects misplaced (9.382%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:18:54.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:53 smithi026 ceph-mon[215775]: pgmap v402: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.9 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 18048/94413 objects degraded (19.116%); 8858/94413 objects misplaced (9.382%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:18:54.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:53 smithi026 ceph-mon[223014]: pgmap v402: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.9 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 658 op/s; 18048/94413 objects degraded (19.116%); 8858/94413 objects misplaced (9.382%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:18:54.313 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:54.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:54.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:54.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:54.498 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:54.981 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:54.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:54.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:55.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:55.050 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:55.086 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:55.118 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:55.177 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:55.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:55.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:55.285 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:55.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:55.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:55.328 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:55.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:18:55.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:18:55.383 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:55.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:55.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:55.488 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:18:55.533 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:55.601 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:18:55.609 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:55.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:18:55.746 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:18:55.759 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:18:55.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:55.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:18:55.826 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:18:55.874 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:18:55.876 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:55.908 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:55.924 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:18:55.963 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:18:55.964 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:55.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:55.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:55.998 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:18:56.032 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:18:56.074 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:56.076 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:18:56.076 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:18:56.076 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:18:56.076 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:18:56.076 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:18:56.076 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:18:56.076 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:18:56.076 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:18:56.106 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:56.168 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:56.174 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:56 smithi070 ceph-mon[192652]: pgmap v403: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.9 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 18048/92877 objects degraded (19.432%); 8858/92877 objects misplaced (9.537%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T22:18:56.174 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3361444000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:56.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:56.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:56 smithi026 ceph-mon[223014]: pgmap v403: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.9 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 18048/92877 objects degraded (19.432%); 8858/92877 objects misplaced (9.537%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T22:18:56.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3361444000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:56.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:56 smithi026 ceph-mon[215775]: pgmap v403: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.9 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 509 op/s; 18048/92877 objects degraded (19.432%); 8858/92877 objects misplaced (9.537%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T22:18:56.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3361444000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:18:56.276 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:18:56.365 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:57.091 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T22:18:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1956761496' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:18:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:18:57.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1956761496' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:18:57.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:18:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1956761496' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:18:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:18:57.691 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:18:57.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:57.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:18:57.795 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:18:57.821 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:57.864 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:18:57.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:57.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:18:57.969 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:18:57.996 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:58.041 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:18:58.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:58.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:18:58.098 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:18:58.098 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:58 smithi070 ceph-mon[192652]: osdmap e182: 8 total, 8 up, 7 in 2023-09-23T22:18:58.098 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:58 smithi070 ceph-mon[192652]: failed to encode map e182 with expected crc 2023-09-23T22:18:58.098 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:58 smithi070 ceph-mon[192652]: failed to encode map e182 with expected crc 2023-09-23T22:18:58.098 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:58 smithi070 ceph-mon[192652]: pgmap v405: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 17961/87213 objects degraded (20.594%); 8675/87213 objects misplaced (9.947%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:18:58.098 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 18048/92877 objects degraded (19.432%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:58.098 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:58 smithi070 ceph-mon[192652]: osdmap e183: 8 total, 8 up, 7 in 2023-09-23T22:18:58.098 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:58 smithi070 ceph-mon[192652]: failed to encode map e183 with expected crc 2023-09-23T22:18:58.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:18:58.178 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:58.222 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:18:58.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:18:58.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:18:58.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:18:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:18:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[223014]: osdmap e182: 8 total, 8 up, 7 in 2023-09-23T22:18:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[223014]: failed to encode map e182 with expected crc 2023-09-23T22:18:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[223014]: failed to encode map e182 with expected crc 2023-09-23T22:18:58.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[223014]: pgmap v405: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 17961/87213 objects degraded (20.594%); 8675/87213 objects misplaced (9.947%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 18048/92877 objects degraded (19.432%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[223014]: osdmap e183: 8 total, 8 up, 7 in 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[223014]: failed to encode map e183 with expected crc 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[215775]: osdmap e182: 8 total, 8 up, 7 in 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[215775]: failed to encode map e182 with expected crc 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[215775]: failed to encode map e182 with expected crc 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[215775]: pgmap v405: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 1.8 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 602 op/s; 17961/87213 objects degraded (20.594%); 8675/87213 objects misplaced (9.947%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 18048/92877 objects degraded (19.432%), 8 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:18:58.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[215775]: osdmap e183: 8 total, 8 up, 7 in 2023-09-23T22:18:58.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:58 smithi026 ceph-mon[215775]: failed to encode map e183 with expected crc 2023-09-23T22:18:58.363 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:18:59.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:59 smithi070 ceph-mon[192652]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:59 smithi070 ceph-mon[192652]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:59 smithi070 ceph-mon[192652]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:59 smithi070 ceph-mon[192652]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:59 smithi070 ceph-mon[192652]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:59 smithi070 ceph-mon[192652]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:59 smithi070 ceph-mon[192652]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:18:59 smithi070 ceph-mon[192652]: osdmap e184: 8 total, 8 up, 7 in 2023-09-23T22:18:59.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[223014]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[223014]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[223014]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[223014]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[223014]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[223014]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[223014]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[223014]: osdmap e184: 8 total, 8 up, 7 in 2023-09-23T22:18:59.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[215775]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[215775]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[215775]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[215775]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[215775]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[215775]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[215775]: failed to encode map e183 with expected crc 2023-09-23T22:18:59.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:18:59 smithi026 ceph-mon[215775]: osdmap e184: 8 total, 8 up, 7 in 2023-09-23T22:18:59.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:18:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:18:59] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2023-09-23T22:19:00.020 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:00.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:00.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:00.124 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:00.201 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:00.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:00 smithi070 ceph-mon[192652]: failed to encode map e184 with expected crc 2023-09-23T22:19:00.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:00 smithi070 ceph-mon[192652]: 5.3 continuing backfill to osd.3 from (126'2448,181'9995] MIN to 181'9995 2023-09-23T22:19:00.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:00 smithi070 ceph-mon[192652]: failed to encode map e184 with expected crc 2023-09-23T22:19:00.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:00 smithi070 ceph-mon[192652]: 4.1f continuing backfill to osd.0 from (0'0,0'0] MIN to 0'0 2023-09-23T22:19:00.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:00 smithi070 ceph-mon[192652]: pgmap v408: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.7 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 16219/84180 objects degraded (19.267%); 8312/84180 objects misplaced (9.874%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:19:00.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[215775]: failed to encode map e184 with expected crc 2023-09-23T22:19:00.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[215775]: 5.3 continuing backfill to osd.3 from (126'2448,181'9995] MIN to 181'9995 2023-09-23T22:19:00.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[215775]: failed to encode map e184 with expected crc 2023-09-23T22:19:00.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[215775]: 4.1f continuing backfill to osd.0 from (0'0,0'0] MIN to 0'0 2023-09-23T22:19:00.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[215775]: pgmap v408: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.7 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 16219/84180 objects degraded (19.267%); 8312/84180 objects misplaced (9.874%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:19:00.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[223014]: failed to encode map e184 with expected crc 2023-09-23T22:19:00.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[223014]: 5.3 continuing backfill to osd.3 from (126'2448,181'9995] MIN to 181'9995 2023-09-23T22:19:00.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[223014]: failed to encode map e184 with expected crc 2023-09-23T22:19:00.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[223014]: 4.1f continuing backfill to osd.0 from (0'0,0'0] MIN to 0'0 2023-09-23T22:19:00.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:00 smithi026 ceph-mon[223014]: pgmap v408: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 1.7 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 568 op/s; 16219/84180 objects degraded (19.267%); 8312/84180 objects misplaced (9.874%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:19:00.619 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:19:00.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:19:00.637 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:00.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:00.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:00.748 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:00.852 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:01.156 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:01.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:01.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:01.270 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:01.382 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:01.496 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:01.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:01.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:01.509 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:19:01.509 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:19:01.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:19:01.509 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:19:01.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.509 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:39434 5.0 5:00e131c4:::benchmark_data_smithi026_286012_object39433:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T22:19:01.509 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:46.461706+0000", 2023-09-23T22:19:01.509 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 195.03542008400001, 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12994860999999999, 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 39434 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.461706+0000", 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.461706+0000", 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.461704+0000", 2023-09-23T22:19:01.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.461742+0000", 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7793999999999998e-05 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.461757+0000", 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4647999999999999e-05 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.461765+0000", 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8910000000000001e-06 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.461819+0000", 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3681000000000003e-05 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.461955+0000", 2023-09-23T22:19:01.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013545300000000001 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.462114+0000", 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159421 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591500+0000", 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12938644499999999 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591512+0000", 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1821000000000001e-05 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591568+0000", 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.52e-05 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591616+0000", 2023-09-23T22:19:01.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8424999999999998e-05 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591654+0000", 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8501999999999998e-05 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:39439 5.0 5:0f8939be:::benchmark_data_smithi026_286012_object39438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e126)", 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:46.463923+0000", 2023-09-23T22:19:01.516 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 195.03320267500001, 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13097114200000001, 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 39439 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.463923+0000", 2023-09-23T22:19:01.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.463923+0000", 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.463922+0000", 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.463970+0000", 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8068999999999997e-05 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.463993+0000", 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3184999999999999e-05 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464002+0000", 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5469999999999997e-06 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464061+0000", 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9414999999999999e-05 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464177+0000", 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011558899999999999 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464325+0000", 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014819799999999999 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.594728+0000", 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13040294099999999 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.594775+0000", 2023-09-23T22:19:01.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7417999999999999e-05 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.594796+0000", 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1131000000000001e-05 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.594849+0000", 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.304e-05 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.594894+0000", 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5126000000000001e-05 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:39429 5.1 e126/111 5:8ca29b3b:::benchmark_data_smithi026_286012_object39428:head v 126'2380, mlcod=126'2375)", 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:46.464034+0000", 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 195.03309162299999, 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12611878900000001, 2023-09-23T22:19:01.523 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464034+0000", 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464034+0000", 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464034+0000", 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464066+0000", 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1866000000000003e-05 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464075+0000", 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0820000000000005e-06 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464081+0000", 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2910000000000004e-06 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464147+0000", 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5969999999999993e-05 2023-09-23T22:19:01.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.464168+0000", 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0687e-05 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.589814+0000", 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.125646014 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.590153+0000", 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033952899999999998 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:39434 5.0 e126/111 ondisk, result = 0)", 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:46.466741+0000", 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 195.030384435, 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12492296, 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:19:01.528 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.466741+0000", 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.466741+0000", 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.466741+0000", 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.466747+0000", 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1419999999999998e-06 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.589808+0000", 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.123061029 2023-09-23T22:19:01.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.589821+0000", 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2814e-05 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591555+0000", 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017344159999999999 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591562+0000", 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2089999999999999e-06 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591664+0000", 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000102277 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.532 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.34674.0:39434 5.0 e126/111 ondisk, result = 0)", 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:46.469805+0000", 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 195.02732054699999, 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.121707836, 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.469805+0000", 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.469805+0000", 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.469803+0000", 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.469813+0000", 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0043e-05 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.589797+0000", 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.119983458 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.589813+0000", 2023-09-23T22:19:01.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6541999999999998e-05 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591486+0000", 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001672483 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591494+0000", 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9889999999999992e-06 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591513+0000", 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9244000000000001e-05 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:39458 5.a e126/111 5:5d97bcdd:::benchmark_data_smithi026_286012_object39457:head v 126'2415, mlcod=126'2413)", 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:46.473346+0000", 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 195.02377948700001, 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.118602915, 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.537 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.473346+0000", 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.473346+0000", 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.473345+0000", 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.473372+0000", 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7070999999999999e-05 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.589958+0000", 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.116585868 2023-09-23T22:19:01.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.589966+0000", 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9650000000000008e-06 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.590011+0000", 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5018999999999997e-05 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.590047+0000", 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.612e-05 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591920+0000", 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001873336 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:46.591949+0000", 2023-09-23T22:19:01.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8986999999999999e-05 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:44005 5.0 5:0d99eca6:::benchmark_data_smithi026_286012_object44004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:50.163920+0000", 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 191.33320617999999, 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10662313900000001, 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.542 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 44005 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.163920+0000", 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.163920+0000", 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.163917+0000", 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.163944+0000", 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6452999999999999e-05 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.163962+0000", 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8094999999999999e-05 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.163972+0000", 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.893e-06 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.164040+0000", 2023-09-23T22:19:01.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8244999999999995e-05 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.164174+0000", 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013446500000000001 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.164303+0000", 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012848300000000001 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.165881+0000", 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001577976 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.167384+0000", 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001503023 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270441+0000", 2023-09-23T22:19:01.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.103057234 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270495+0000", 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3711999999999997e-05 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270543+0000", 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8090000000000002e-05 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.548 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:44008 5.a e127/111 5:52a3b827:::benchmark_data_smithi026_286012_object44007:head v 127'2688, mlcod=127'2683)", 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:50.170412+0000", 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 191.32671423100001, 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.109874844, 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.170412+0000", 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.170412+0000", 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.170411+0000", 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.170435+0000", 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3862999999999998e-05 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270393+0000", 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.099958063999999999 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270409+0000", 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6169000000000001e-05 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270462+0000", 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2964999999999999e-05 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270485+0000", 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3129000000000002e-05 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.280243+0000", 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097578140000000001 2023-09-23T22:19:01.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.280287+0000", 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3665000000000003e-05 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:44015 5.0 5:0dd9b0b3:::benchmark_data_smithi026_286012_object44014:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e127)", 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:50.171859+0000", 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 191.32526734300001, 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10593005599999999, 2023-09-23T22:19:01.553 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 44015 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.171859+0000", 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.171859+0000", 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.171856+0000", 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.171881+0000", 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4564000000000001e-05 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270393+0000", 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.098511766000000001 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270410+0000", 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6789000000000001e-05 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270571+0000", 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001617 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270681+0000", 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010968600000000001 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.270843+0000", 2023-09-23T22:19:01.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016217299999999999 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.277530+0000", 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066869049999999999 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.277590+0000", 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9490000000000001e-05 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.277684+0000", 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4278999999999996e-05 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.277749+0000", 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5139000000000001e-05 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:50.277789+0000", 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9635000000000003e-05 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:78904 5.0 5:0504264f:::benchmark_data_smithi026_286012_object78903:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:28.377753+0000", 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 153.119373319, 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.14455474200000001, 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.560 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 78904 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.377753+0000", 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.377753+0000", 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.377751+0000", 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.377778+0000", 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7188000000000001e-05 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.377794+0000", 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6300999999999999e-05 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.377804+0000", 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6439999999999995e-06 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.377853+0000", 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8986999999999997e-05 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.378003+0000", 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014971799999999999 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.378146+0000", 2023-09-23T22:19:01.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014362899999999999 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.380673+0000", 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025269139999999999 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.380843+0000", 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016985000000000001 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.522201+0000", 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.14135811300000001 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.522262+0000", 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0968999999999999e-05 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.522307+0000", 2023-09-23T22:19:01.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5259999999999997e-05 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:78915 5.0 5:06854e7d:::benchmark_data_smithi026_286012_object78914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:28.380976+0000", 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 153.116149517, 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.14385921300000001, 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.567 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 78915 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.380976+0000", 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.380976+0000", 2023-09-23T22:19:01.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.380975+0000", 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.380995+0000", 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.984e-05 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.381004+0000", 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8990000000000007e-06 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.381009+0000", 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1599999999999997e-06 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.381053+0000", 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4236999999999997e-05 2023-09-23T22:19:01.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.381177+0000", 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000123306 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.381314+0000", 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013715999999999999 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.383893+0000", 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025789020000000001 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.383992+0000", 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8889999999999997e-05 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.524737+0000", 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.140745124 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.524798+0000", 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0968999999999999e-05 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:28.524836+0000", 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7849999999999998e-05 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.573 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:93663 5.a e145/111 5:59c82de1:::benchmark_data_smithi026_286012_object93662:head v 145'5832, mlcod=145'5830)", 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.135798+0000", 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 133.36132817800001, 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.131569192, 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.135798+0000", 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.135798+0000", 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.135797+0000", 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.135826+0000", 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9328000000000001e-05 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.135840+0000", 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4225e-05 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.135848+0000", 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8760000000000005e-06 2023-09-23T22:19:01.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.135917+0000", 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8718000000000002e-05 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.135938+0000", 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1242000000000001e-05 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.267337+0000", 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13139943000000001 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.267367+0000", 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9487000000000001e-05 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:93665 5.a e145/111 5:5c35879a:::benchmark_data_smithi026_286012_object93664:head v 145'5833, mlcod=145'5830)", 2023-09-23T22:19:01.578 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.137090+0000", 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 133.360036213, 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13030172500000001, 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137090+0000", 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137090+0000", 2023-09-23T22:19:01.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137089+0000", 2023-09-23T22:19:01.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:19:01.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137117+0000", 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7695000000000002e-05 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137128+0000", 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0998999999999999e-05 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137135+0000", 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6179999999999997e-06 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.138782+0000", 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001646925 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.138802+0000", 2023-09-23T22:19:01.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9850000000000001e-05 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.267371+0000", 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12856879500000001 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.267391+0000", 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0611999999999998e-05 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.584 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.34674.0:93671 5.a e145/111 5:52600061:::benchmark_data_smithi026_286012_object93670:head v 145'5834, mlcod=145'5830)", 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.137777+0000", 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 133.35934886999999, 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13773982000000001, 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137777+0000", 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137777+0000", 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137776+0000", 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:19:01.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137797+0000", 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0772999999999999e-05 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137806+0000", 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0100000000000001e-06 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.137812+0000", 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1650000000000003e-06 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.267786+0000", 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.129973428 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.267858+0000", 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2336999999999995e-05 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.275464+0000", 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076061460000000003 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.275517+0000", 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2510999999999999e-05 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.589 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93677 5.0 5:05c0540c:::benchmark_data_smithi026_286012_object93676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.141597+0000", 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 133.355529225, 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.14839171200000001, 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.590 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93677 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.141597+0000", 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.141597+0000", 2023-09-23T22:19:01.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.141595+0000", 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.141632+0000", 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7406000000000001e-05 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.263669+0000", 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12203647099999999 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.263680+0000", 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.079e-05 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.264292+0000", 2023-09-23T22:19:01.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00061220399999999998 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.264425+0000", 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013275400000000001 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.264577+0000", 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015197799999999999 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.272675+0000", 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080980400000000008 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.274069+0000", 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001393959 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.289897+0000", 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015828689 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.289951+0000", 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3677000000000001e-05 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.289988+0000", 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7351000000000002e-05 2023-09-23T22:19:01.596 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:98884 5.0 5:04d5db28:::benchmark_data_smithi026_286012_object5076:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:17.424382+0000", 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 104.07274417399999, 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16158092399999999, 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.597 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 98884 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.424382+0000", 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.424382+0000", 2023-09-23T22:19:01.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.424380+0000", 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.424387+0000", 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8839999999999997e-06 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.424404+0000", 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7087000000000001e-05 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.424414+0000", 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0344e-05 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.424545+0000", 2023-09-23T22:19:01.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130109 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.424683+0000", 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013819799999999999 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.424814+0000", 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013090899999999999 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.427532+0000", 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027187660000000001 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.430916+0000", 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033833959999999999 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.585874+0000", 2023-09-23T22:19:01.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.15495842200000001 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.585922+0000", 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.812e-05 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:17.585963+0000", 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0293000000000002e-05 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.603 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:101985 5.0 5:0407da77:::benchmark_data_smithi026_286012_object8177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:22.049023+0000", 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 99.448103176000004, 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.112404491, 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.604 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 101985 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.049023+0000", 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.049023+0000", 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.049020+0000", 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.049031+0000", 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0148e-05 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.049050+0000", 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9187000000000001e-05 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.049061+0000", 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1419e-05 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.049117+0000", 2023-09-23T22:19:01.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6229e-05 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.049255+0000", 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013719000000000001 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.049330+0000", 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4900000000000005e-05 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.051809+0000", 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024791169999999999 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.053837+0000", 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020280400000000001 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.161326+0000", 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.107489456 2023-09-23T22:19:01.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.161381+0000", 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5213999999999997e-05 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.161427+0000", 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5868000000000002e-05 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.610 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:101993 5.0 5:06026ba7:::benchmark_data_smithi026_286012_object8185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:22.055182+0000", 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 99.441943541000001, 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.106348026, 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.611 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 101993 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.055182+0000", 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.055182+0000", 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.055181+0000", 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.055187+0000", 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3820000000000003e-06 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.055198+0000", 2023-09-23T22:19:01.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1505000000000001e-05 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.055205+0000", 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1949999999999997e-06 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.055252+0000", 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6414e-05 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.055371+0000", 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118941 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.055472+0000", 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101068 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.058381+0000", 2023-09-23T22:19:01.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029092100000000002 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.059674+0000", 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012932830000000001 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.161466+0000", 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10179199 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.161500+0000", 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3970999999999997e-05 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:22.161530+0000", 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0199e-05 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:107802 5.0 5:0033011f:::benchmark_data_smithi026_286012_object13994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:31.538764+0000", 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 89.958361838000002, 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.142925147, 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107802 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.538764+0000", 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.538764+0000", 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.538761+0000", 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.538772+0000", 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0176999999999999e-05 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.538791+0000", 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8923000000000002e-05 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.538801+0000", 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0239e-05 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.538861+0000", 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0139000000000001e-05 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.538977+0000", 2023-09-23T22:19:01.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011603399999999999 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.539078+0000", 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100832 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.542594+0000", 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035160410000000001 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.547603+0000", 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050087270000000001 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.681599+0000", 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.133996799 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.681651+0000", 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1230999999999998e-05 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.681689+0000", 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8618999999999997e-05 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.624 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:107805 5.0 5:08ce62cc:::benchmark_data_smithi026_286012_object13997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e162)", 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:17:31.544696+0000", 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 89.952430121000006, 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.140899204, 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 107805 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:19:01.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.544696+0000", 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.544696+0000", 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:19:01.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.544694+0000", 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.544702+0000", 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6820000000000003e-06 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.544743+0000", 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1109999999999998e-05 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.544751+0000", 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4190000000000002e-06 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.544818+0000", 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6249999999999998e-05 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:19:01.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.544939+0000", 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012155300000000001 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.545032+0000", 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2641999999999998e-05 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.547692+0000", 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026596509999999999 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.547777+0000", 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5020999999999998e-05 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.685501+0000", 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13772474300000001 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:19:01.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.685553+0000", 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1221999999999998e-05 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:17:31.685595+0000", 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2457999999999999e-05 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:19:01.631 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:19:01.632 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:01.632 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:01.674 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:01.747 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:01.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:01.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:01.854 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:01.881 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:01.927 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:01.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:01.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:02.032 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:02.059 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:02.105 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:02.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:02.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:02.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[223014]: pgmap v409: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 89 active+clean; 1.7 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 17837/82701 objects degraded (21.568%); 626/82701 objects misplaced (0.757%); 6.3 MiB/s, 98 objects/s recovering 2023-09-23T22:19:02.130 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:19:02.214 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:02.247 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:02 smithi070 ceph-mon[192652]: pgmap v409: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 89 active+clean; 1.7 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 17837/82701 objects degraded (21.568%); 626/82701 objects misplaced (0.757%); 6.3 MiB/s, 98 objects/s recovering 2023-09-23T22:19:02.248 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:02 smithi070 ceph-mon[192652]: osdmap e185: 8 total, 8 up, 7 in 2023-09-23T22:19:02.248 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:02 smithi070 ceph-mon[192652]: failed to encode map e185 with expected crc 2023-09-23T22:19:02.248 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:02 smithi070 ceph-mon[192652]: failed to encode map e185 with expected crc 2023-09-23T22:19:02.248 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1612983546' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:19:02.248 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:02 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:19:02.250 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:02.322 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:02.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:02.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:02.430 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[215775]: pgmap v409: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 89 active+clean; 1.7 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 17837/82701 objects degraded (21.568%); 626/82701 objects misplaced (0.757%); 6.3 MiB/s, 98 objects/s recovering 2023-09-23T22:19:02.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[215775]: osdmap e185: 8 total, 8 up, 7 in 2023-09-23T22:19:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[215775]: failed to encode map e185 with expected crc 2023-09-23T22:19:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[215775]: failed to encode map e185 with expected crc 2023-09-23T22:19:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1612983546' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:19:02.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:19:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[223014]: osdmap e185: 8 total, 8 up, 7 in 2023-09-23T22:19:02.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[223014]: failed to encode map e185 with expected crc 2023-09-23T22:19:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[223014]: failed to encode map e185 with expected crc 2023-09-23T22:19:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1612983546' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:19:02.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:02 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:19:02.491 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:19:02.515 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:02.858 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:02.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:02.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:02.973 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:02.998 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:03.039 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:03.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:03.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:03.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:03.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[215775]: failed to encode map e186 with expected crc 2023-09-23T22:19:03.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:19:03.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[215775]: osdmap e186: 8 total, 8 up, 7 in 2023-09-23T22:19:03.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[215775]: failed to encode map e186 with expected crc 2023-09-23T22:19:03.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17837/82701 objects degraded (21.568%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:03.145 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:03.180 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:03.243 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:03.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:03.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:03 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:03.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:03 smithi070 ceph-mon[192652]: failed to encode map e186 with expected crc 2023-09-23T22:19:03.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:03 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:19:03.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:03 smithi070 ceph-mon[192652]: osdmap e186: 8 total, 8 up, 7 in 2023-09-23T22:19:03.253 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:03 smithi070 ceph-mon[192652]: failed to encode map e186 with expected crc 2023-09-23T22:19:03.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17837/82701 objects degraded (21.568%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:03.351 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:03.416 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:03.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:03.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[223014]: failed to encode map e186 with expected crc 2023-09-23T22:19:03.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:19:03.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[223014]: osdmap e186: 8 total, 8 up, 7 in 2023-09-23T22:19:03.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[223014]: failed to encode map e186 with expected crc 2023-09-23T22:19:03.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17837/82701 objects degraded (21.568%), 7 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:03.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:19:03.640 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:03.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:03.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:03.752 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:03.832 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:04.010 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:04.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:04.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:04.050 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:04.118 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:04.148 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:04.192 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:04.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:04.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:04.295 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:04.367 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:04.600 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:04.601 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:04.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:04.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:04.721 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:04.825 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:04.940 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:04 smithi026 ceph-mon[215775]: pgmap v412: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 91 active+clean; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 352 op/s; 17701/79803 objects degraded (22.181%); 10 MiB/s, 161 objects/s recovering 2023-09-23T22:19:04.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:04 smithi026 ceph-mon[223014]: pgmap v412: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 91 active+clean; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 352 op/s; 17701/79803 objects degraded (22.181%); 10 MiB/s, 161 objects/s recovering 2023-09-23T22:19:05.111 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:05.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:05.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:05.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:04 smithi070 ceph-mon[192652]: pgmap v412: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 91 active+clean; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 352 op/s; 17701/79803 objects degraded (22.181%); 10 MiB/s, 161 objects/s recovering 2023-09-23T22:19:05.217 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:05.303 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:05.487 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:05.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:05.563 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:19:05.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:05.647 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:05.775 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:19:05.777 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:19:06.002 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:06.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:06.105 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:06.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:06.211 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:06.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:06.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:06.316 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:06.354 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:06.417 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:06.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:06.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:06.607 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:06.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:06 smithi026 ceph-mon[215775]: pgmap v413: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 91 active+clean; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 17701/78702 objects degraded (22.491%); 8.7 MiB/s, 136 objects/s recovering 2023-09-23T22:19:06.970 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:06 smithi026 ceph-mon[215775]: from='client.44858 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:06.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:06 smithi026 ceph-mon[223014]: pgmap v413: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 91 active+clean; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 17701/78702 objects degraded (22.491%); 8.7 MiB/s, 136 objects/s recovering 2023-09-23T22:19:06.971 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:06 smithi026 ceph-mon[223014]: from='client.44858 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:07.006 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":413,"stamp":"2023-09-23T22:19:05.117371+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1722161099,"num_objects":26234,"num_object_clones":0,"num_object_copies":78702,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":17701,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26234,"num_whiteouts":0,"num_read":6469,"num_read_kb":13097,"num_write":162552,"num_write_kb":6018662,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27084,"num_bytes_recovered":1772427663,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109154,"ondisk_log_size":109154,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":326,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6357736,"kb_used_data":5519472,"kb_used_omap":31,"kb_used_meta":837984,"kb_avail":649858328,"statfs":{"total":671965249536,"available":665454927872,"internally_reserved":0,"allocated":5651939328,"data_stored":5645544188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":858096567},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,0,0,4,0,0,10,7,10,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-128319488,"num_objects":-1958,"num_object_clones":0,"num_object_copies":-5874,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1482,"num_objects_misplaced":-8312,"num_objects_unfound":0,"num_objects_dirty":-1958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1958,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":876,"num_bytes_recovered":58325954,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.417795"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127656+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:19:01.127656+0000","last_peered":"2023-09-23T22:19:01.127656+0000","last_clean":"2023-09-23T22:19:01.127656+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:19:01.127656+0000","last_undegraded":"2023-09-23T22:19:01.127656+0000","last_fullsized":"2023-09-23T22:19:01.127656+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":165,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130866+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:19:01.130866+0000","last_peered":"2023-09-23T22:19:01.130866+0000","last_clean":"2023-09-23T22:19:01.130866+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:19:01.130866+0000","last_undegraded":"2023-09-23T22:19:01.130866+0000","last_fullsized":"2023-09-23T22:19:01.130866+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":125,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.126779+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:19:01.126779+0000","last_peered":"2023-09-23T22:19:01.126779+0000","last_clean":"2023-09-23T22:19:01.126779+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:19:01.126779+0000","last_undegraded":"2023-09-23T22:19:01.126779+0000","last_fullsized":"2023-09-23T22:19:01.126779+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701122+0000","last_change":"2023-09-23T22:18:57.724501+0000","last_active":"2023-09-23T22:18:58.701122+0000","last_peered":"2023-09-23T22:18:58.701122+0000","last_clean":"2023-09-23T22:18:58.701122+0000","last_became_active":"2023-09-23T22:18:57.724203+0000","last_became_peered":"2023-09-23T22:18:57.724203+0000","last_unstale":"2023-09-23T22:18:58.701122+0000","last_undegraded":"2023-09-23T22:18:58.701122+0000","last_fullsized":"2023-09-23T22:18:58.701122+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:52.168267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":205,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130355+0000","last_change":"2023-09-23T22:18:57.722574+0000","last_active":"2023-09-23T22:19:01.130355+0000","last_peered":"2023-09-23T22:19:01.130355+0000","last_clean":"2023-09-23T22:19:01.130355+0000","last_became_active":"2023-09-23T22:18:57.721800+0000","last_became_peered":"2023-09-23T22:18:57.721800+0000","last_unstale":"2023-09-23T22:19:01.130355+0000","last_undegraded":"2023-09-23T22:19:01.130355+0000","last_fullsized":"2023-09-23T22:19:01.130355+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:09.303838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":120,"reported_epoch":184,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:18:58.729966+0000","last_change":"2023-09-23T22:18:58.730170+0000","last_active":"2023-09-23T22:18:58.729966+0000","last_peered":"2023-09-23T22:18:58.729966+0000","last_clean":"2023-09-23T22:18:49.795932+0000","last_became_active":"2023-09-23T22:18:58.729860+0000","last_became_peered":"2023-09-23T22:18:58.729860+0000","last_unstale":"2023-09-23T22:18:58.729966+0000","last_undegraded":"2023-09-23T22:18:58.729966+0000","last_fullsized":"2023-09-23T22:18:58.729966+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:30.948774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,3],"avail_no_missing":["5","0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":50,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701587+0000","last_change":"2023-09-23T22:18:57.729038+0000","last_active":"2023-09-23T22:18:58.701587+0000","last_peered":"2023-09-23T22:18:58.701587+0000","last_clean":"2023-09-23T22:18:58.701587+0000","last_became_active":"2023-09-23T22:18:57.728650+0000","last_became_peered":"2023-09-23T22:18:57.728650+0000","last_unstale":"2023-09-23T22:18:58.701587+0000","last_undegraded":"2023-09-23T22:18:58.701587+0000","last_fullsized":"2023-09-23T22:18:58.701587+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:11.461883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":160,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130424+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:19:01.130424+0000","last_peered":"2023-09-23T22:19:01.130424+0000","last_clean":"2023-09-23T22:19:01.130424+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:19:01.130424+0000","last_undegraded":"2023-09-23T22:19:01.130424+0000","last_fullsized":"2023-09-23T22:19:01.130424+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":144,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.715227+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:18:57.715227+0000","last_peered":"2023-09-23T22:18:57.715227+0000","last_clean":"2023-09-23T22:18:57.715227+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:18:57.715227+0000","last_undegraded":"2023-09-23T22:18:57.715227+0000","last_fullsized":"2023-09-23T22:18:57.715227+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":110,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131940+0000","last_change":"2023-09-23T22:16:49.154341+0000","last_active":"2023-09-23T22:19:01.131940+0000","last_peered":"2023-09-23T22:19:01.131940+0000","last_clean":"2023-09-23T22:19:01.131940+0000","last_became_active":"2023-09-23T22:16:49.154041+0000","last_became_peered":"2023-09-23T22:16:49.154041+0000","last_unstale":"2023-09-23T22:19:01.131940+0000","last_undegraded":"2023-09-23T22:19:01.131940+0000","last_fullsized":"2023-09-23T22:19:01.131940+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:05.733262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":97,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130792+0000","last_change":"2023-09-23T22:15:14.829731+0000","last_active":"2023-09-23T22:19:01.130792+0000","last_peered":"2023-09-23T22:19:01.130792+0000","last_clean":"2023-09-23T22:19:01.130792+0000","last_became_active":"2023-09-23T22:15:14.829437+0000","last_became_peered":"2023-09-23T22:15:14.829437+0000","last_unstale":"2023-09-23T22:19:01.130792+0000","last_undegraded":"2023-09-23T22:19:01.130792+0000","last_fullsized":"2023-09-23T22:19:01.130792+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:45.456558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.124844+0000","last_change":"2023-09-23T22:18:57.723165+0000","last_active":"2023-09-23T22:19:01.124844+0000","last_peered":"2023-09-23T22:19:01.124844+0000","last_clean":"2023-09-23T22:19:01.124844+0000","last_became_active":"2023-09-23T22:18:57.722863+0000","last_became_peered":"2023-09-23T22:18:57.722863+0000","last_unstale":"2023-09-23T22:19:01.124844+0000","last_undegraded":"2023-09-23T22:19:01.124844+0000","last_fullsized":"2023-09-23T22:19:01.124844+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:19.645029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":170,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127699+0000","last_change":"2023-09-23T22:16:49.265991+0000","last_active":"2023-09-23T22:19:01.127699+0000","last_peered":"2023-09-23T22:19:01.127699+0000","last_clean":"2023-09-23T22:19:01.127699+0000","last_became_active":"2023-09-23T22:16:49.251169+0000","last_became_peered":"2023-09-23T22:16:49.251169+0000","last_unstale":"2023-09-23T22:19:01.127699+0000","last_undegraded":"2023-09-23T22:19:01.127699+0000","last_fullsized":"2023-09-23T22:19:01.127699+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.502218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128377+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:19:01.128377+0000","last_peered":"2023-09-23T22:19:01.128377+0000","last_clean":"2023-09-23T22:19:01.128377+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:19:01.128377+0000","last_undegraded":"2023-09-23T22:19:01.128377+0000","last_fullsized":"2023-09-23T22:19:01.128377+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":97,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128422+0000","last_change":"2023-09-23T22:15:14.826143+0000","last_active":"2023-09-23T22:19:01.128422+0000","last_peered":"2023-09-23T22:19:01.128422+0000","last_clean":"2023-09-23T22:19:01.128422+0000","last_became_active":"2023-09-23T22:15:14.825780+0000","last_became_peered":"2023-09-23T22:15:14.825780+0000","last_unstale":"2023-09-23T22:19:01.128422+0000","last_undegraded":"2023-09-23T22:19:01.128422+0000","last_fullsized":"2023-09-23T22:19:01.128422+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:22.608676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130553+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:19:01.130553+0000","last_peered":"2023-09-23T22:19:01.130553+0000","last_clean":"2023-09-23T22:19:01.130553+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:19:01.130553+0000","last_undegraded":"2023-09-23T22:19:01.130553+0000","last_fullsized":"2023-09-23T22:19:01.130553+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":12,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.703281+0000","last_change":"2023-09-23T22:18:57.726112+0000","last_active":"2023-09-23T22:18:58.703281+0000","last_peered":"2023-09-23T22:18:58.703281+0000","last_clean":"2023-09-23T22:18:58.703281+0000","last_became_active":"2023-09-23T22:18:57.725558+0000","last_became_peered":"2023-09-23T22:18:57.725558+0000","last_unstale":"2023-09-23T22:18:58.703281+0000","last_undegraded":"2023-09-23T22:18:58.703281+0000","last_fullsized":"2023-09-23T22:18:58.703281+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:38.315664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":143,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.715360+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:18:57.715360+0000","last_peered":"2023-09-23T22:18:57.715360+0000","last_clean":"2023-09-23T22:18:57.715360+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:18:57.715360+0000","last_undegraded":"2023-09-23T22:18:57.715360+0000","last_fullsized":"2023-09-23T22:18:57.715360+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130107+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:19:01.130107+0000","last_peered":"2023-09-23T22:19:01.130107+0000","last_clean":"2023-09-23T22:19:01.130107+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:19:01.130107+0000","last_undegraded":"2023-09-23T22:19:01.130107+0000","last_fullsized":"2023-09-23T22:19:01.130107+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701454+0000","last_change":"2023-09-23T22:18:57.726691+0000","last_active":"2023-09-23T22:18:58.701454+0000","last_peered":"2023-09-23T22:18:58.701454+0000","last_clean":"2023-09-23T22:18:58.701454+0000","last_became_active":"2023-09-23T22:18:57.726386+0000","last_became_peered":"2023-09-23T22:18:57.726386+0000","last_unstale":"2023-09-23T22:18:58.701454+0000","last_undegraded":"2023-09-23T22:18:58.701454+0000","last_fullsized":"2023-09-23T22:18:58.701454+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:34.829149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":152,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.730336+0000","last_change":"2023-09-23T22:18:57.728767+0000","last_active":"2023-09-23T22:18:57.730336+0000","last_peered":"2023-09-23T22:18:57.730336+0000","last_clean":"2023-09-23T22:18:57.730336+0000","last_became_active":"2023-09-23T22:18:57.728377+0000","last_became_peered":"2023-09-23T22:18:57.728377+0000","last_unstale":"2023-09-23T22:18:57.730336+0000","last_undegraded":"2023-09-23T22:18:57.730336+0000","last_fullsized":"2023-09-23T22:18:57.730336+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:01:01.370904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131068+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:19:01.131068+0000","last_peered":"2023-09-23T22:19:01.131068+0000","last_clean":"2023-09-23T22:19:01.131068+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:19:01.131068+0000","last_undegraded":"2023-09-23T22:19:01.131068+0000","last_fullsized":"2023-09-23T22:19:01.131068+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"186'9926","reported_seq":21556,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.393456+0000","last_change":"2023-09-23T22:19:02.159483+0000","last_active":"2023-09-23T22:19:02.393456+0000","last_peered":"2023-09-23T22:19:02.393456+0000","last_clean":"2023-09-23T22:19:02.393456+0000","last_became_active":"2023-09-23T22:19:02.132959+0000","last_became_peered":"2023-09-23T22:19:02.132959+0000","last_unstale":"2023-09-23T22:19:02.393456+0000","last_undegraded":"2023-09-23T22:19:02.393456+0000","last_fullsized":"2023-09-23T22:19:02.393456+0000","mapping_epoch":185,"log_start":"133'3365","ondisk_log_start":"133'3365","created":111,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:20.126557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106168320,"num_objects":1620,"num_object_clones":0,"num_object_copies":4860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9926,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5765,"num_bytes_recovered":377815040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128061+0000","last_change":"2023-09-23T22:19:00.682889+0000","last_active":"2023-09-23T22:19:01.128061+0000","last_peered":"2023-09-23T22:19:01.128061+0000","last_clean":"2023-09-23T22:19:01.128061+0000","last_became_active":"2023-09-23T22:18:57.718002+0000","last_became_peered":"2023-09-23T22:18:57.718002+0000","last_unstale":"2023-09-23T22:19:01.128061+0000","last_undegraded":"2023-09-23T22:19:01.128061+0000","last_fullsized":"2023-09-23T22:19:01.128061+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:22.517277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.714957+0000","last_change":"2023-09-23T22:16:49.158580+0000","last_active":"2023-09-23T22:18:57.714957+0000","last_peered":"2023-09-23T22:18:57.714957+0000","last_clean":"2023-09-23T22:18:57.714957+0000","last_became_active":"2023-09-23T22:16:49.158266+0000","last_became_peered":"2023-09-23T22:16:49.158266+0000","last_unstale":"2023-09-23T22:18:57.714957+0000","last_undegraded":"2023-09-23T22:18:57.714957+0000","last_fullsized":"2023-09-23T22:18:57.714957+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:03.791658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"184'9855","reported_seq":15186,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:19:00.934183+0000","last_change":"2023-09-23T22:17:16.903782+0000","last_active":"2023-09-23T22:19:00.934183+0000","last_peered":"2023-09-23T22:19:00.934183+0000","last_clean":"2023-09-23T22:19:00.934183+0000","last_became_active":"2023-09-23T22:17:16.895165+0000","last_became_peered":"2023-09-23T22:17:16.895165+0000","last_unstale":"2023-09-23T22:19:00.934183+0000","last_undegraded":"2023-09-23T22:19:00.934183+0000","last_fullsized":"2023-09-23T22:19:00.934183+0000","mapping_epoch":159,"log_start":"127'2700","ondisk_log_start":"127'2700","created":111,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7155,"log_dups_size":0,"ondisk_log_size":7155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:31.886804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109903872,"num_objects":1677,"num_object_clones":0,"num_object_copies":5031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9855,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2600,"num_bytes_recovered":170393600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":97,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127512+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:19:01.127512+0000","last_peered":"2023-09-23T22:19:01.127512+0000","last_clean":"2023-09-23T22:19:01.127512+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:19:01.127512+0000","last_undegraded":"2023-09-23T22:19:01.127512+0000","last_fullsized":"2023-09-23T22:19:01.127512+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130055+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:19:01.130055+0000","last_peered":"2023-09-23T22:19:01.130055+0000","last_clean":"2023-09-23T22:19:01.130055+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:19:01.130055+0000","last_undegraded":"2023-09-23T22:19:01.130055+0000","last_fullsized":"2023-09-23T22:19:01.130055+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"184'9880","reported_seq":16421,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:01.075591+0000","last_change":"2023-09-23T22:19:00.690164+0000","last_active":"2023-09-23T22:19:01.075591+0000","last_peered":"2023-09-23T22:19:01.075591+0000","last_clean":"2023-09-23T22:18:39.199855+0000","last_became_active":"2023-09-23T22:18:41.409775+0000","last_became_peered":"2023-09-23T22:18:41.409775+0000","last_unstale":"2023-09-23T22:19:01.075591+0000","last_undegraded":"2023-09-23T22:18:41.249089+0000","last_fullsized":"2023-09-23T22:18:41.215237+0000","mapping_epoch":178,"log_start":"127'2700","ondisk_log_start":"127'2700","created":111,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7180,"log_dups_size":0,"ondisk_log_size":7180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:30.052669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107085824,"num_objects":1634,"num_object_clones":0,"num_object_copies":4902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5621,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9880,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3204,"num_bytes_recovered":209977344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-575},{"shards":"0,2,7","objects":1569}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":109,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.715290+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:18:57.715290+0000","last_peered":"2023-09-23T22:18:57.715290+0000","last_clean":"2023-09-23T22:18:57.715290+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:18:57.715290+0000","last_undegraded":"2023-09-23T22:18:57.715290+0000","last_fullsized":"2023-09-23T22:18:57.715290+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.130612+0000","last_change":"2023-09-23T22:16:49.161676+0000","last_active":"2023-09-23T22:19:02.130612+0000","last_peered":"2023-09-23T22:19:02.130612+0000","last_clean":"2023-09-23T22:19:02.130612+0000","last_became_active":"2023-09-23T22:16:49.161247+0000","last_became_peered":"2023-09-23T22:16:49.161247+0000","last_unstale":"2023-09-23T22:19:02.130612+0000","last_undegraded":"2023-09-23T22:19:02.130612+0000","last_fullsized":"2023-09-23T22:19:02.130612+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:30.708371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"184'10045","reported_seq":10127,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.114269+0000","last_change":"2023-09-23T22:15:14.826723+0000","last_active":"2023-09-23T22:19:01.114269+0000","last_peered":"2023-09-23T22:19:01.114269+0000","last_clean":"2023-09-23T22:19:01.114269+0000","last_became_active":"2023-09-23T22:15:14.826454+0000","last_became_peered":"2023-09-23T22:15:14.826454+0000","last_unstale":"2023-09-23T22:19:01.114269+0000","last_undegraded":"2023-09-23T22:19:01.114269+0000","last_fullsized":"2023-09-23T22:19:01.114269+0000","mapping_epoch":111,"log_start":"129'2900","ondisk_log_start":"129'2900","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7145,"log_dups_size":0,"ondisk_log_size":7145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:11.166077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108068864,"num_objects":1649,"num_object_clones":0,"num_object_copies":4947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10045,"num_write_kb":374208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":154,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.715278+0000","last_change":"2023-09-23T22:16:49.164833+0000","last_active":"2023-09-23T22:18:57.715278+0000","last_peered":"2023-09-23T22:18:57.715278+0000","last_clean":"2023-09-23T22:18:57.715278+0000","last_became_active":"2023-09-23T22:16:49.163833+0000","last_became_peered":"2023-09-23T22:16:49.163833+0000","last_unstale":"2023-09-23T22:18:57.715278+0000","last_undegraded":"2023-09-23T22:18:57.715278+0000","last_fullsized":"2023-09-23T22:18:57.715278+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.891395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":36,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.704326+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:18:57.704326+0000","last_peered":"2023-09-23T22:18:57.704326+0000","last_clean":"2023-09-23T22:18:57.704326+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:18:57.704326+0000","last_undegraded":"2023-09-23T22:18:57.704326+0000","last_fullsized":"2023-09-23T22:18:57.704326+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":448,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.700769+0000","last_change":"2023-09-23T22:18:57.721431+0000","last_active":"2023-09-23T22:18:58.700769+0000","last_peered":"2023-09-23T22:18:58.700769+0000","last_clean":"2023-09-23T22:18:58.700769+0000","last_became_active":"2023-09-23T22:18:57.721043+0000","last_became_peered":"2023-09-23T22:18:57.721043+0000","last_unstale":"2023-09-23T22:18:58.700769+0000","last_undegraded":"2023-09-23T22:18:58.700769+0000","last_fullsized":"2023-09-23T22:18:58.700769+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"184'10143","reported_seq":10227,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.030966+0000","last_change":"2023-09-23T22:15:14.824965+0000","last_active":"2023-09-23T22:19:01.030966+0000","last_peered":"2023-09-23T22:19:01.030966+0000","last_clean":"2023-09-23T22:19:01.030966+0000","last_became_active":"2023-09-23T22:15:14.824626+0000","last_became_peered":"2023-09-23T22:15:14.824626+0000","last_unstale":"2023-09-23T22:19:01.030966+0000","last_undegraded":"2023-09-23T22:19:01.030966+0000","last_fullsized":"2023-09-23T22:19:01.030966+0000","mapping_epoch":111,"log_start":"129'2930","ondisk_log_start":"129'2930","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7213,"log_dups_size":0,"ondisk_log_size":7213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.613455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108003352,"num_objects":1649,"num_object_clones":0,"num_object_copies":4947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1649,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10143,"num_write_kb":377281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":197,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131661+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:19:01.131661+0000","last_peered":"2023-09-23T22:19:01.131661+0000","last_clean":"2023-09-23T22:19:01.131661+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:19:01.131661+0000","last_undegraded":"2023-09-23T22:19:01.131661+0000","last_fullsized":"2023-09-23T22:19:01.131661+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127531+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:19:01.127531+0000","last_peered":"2023-09-23T22:19:01.127531+0000","last_clean":"2023-09-23T22:19:01.127531+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:19:01.127531+0000","last_undegraded":"2023-09-23T22:19:01.127531+0000","last_fullsized":"2023-09-23T22:19:01.127531+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":446,"reported_epoch":184,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:18:58.700794+0000","last_change":"2023-09-23T22:18:57.743507+0000","last_active":"2023-09-23T22:18:58.700794+0000","last_peered":"2023-09-23T22:18:58.700794+0000","last_clean":"2023-09-23T22:08:39.036645+0000","last_became_active":"2023-09-23T22:18:57.742130+0000","last_became_peered":"2023-09-23T22:18:57.742130+0000","last_unstale":"2023-09-23T22:18:58.700794+0000","last_undegraded":"2023-09-23T22:18:57.716066+0000","last_fullsized":"2023-09-23T22:18:58.700794+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"0,1,7","objects":-1},{"shards":"1,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.c","version":"186'10164","reported_seq":12471,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:03.252416+0000","last_change":"2023-09-23T22:16:30.157766+0000","last_active":"2023-09-23T22:19:03.252416+0000","last_peered":"2023-09-23T22:19:03.252416+0000","last_clean":"2023-09-23T22:19:03.252416+0000","last_became_active":"2023-09-23T22:16:30.150277+0000","last_became_peered":"2023-09-23T22:16:30.150277+0000","last_unstale":"2023-09-23T22:19:03.252416+0000","last_undegraded":"2023-09-23T22:19:03.252416+0000","last_fullsized":"2023-09-23T22:19:03.252416+0000","mapping_epoch":140,"log_start":"142'5325","ondisk_log_start":"142'5325","created":111,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":4839,"log_dups_size":0,"ondisk_log_size":4839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:20.415978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101318656,"num_objects":1546,"num_object_clones":0,"num_object_copies":4638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10164,"num_write_kb":374720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1098,"num_bytes_recovered":71958528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":102,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127788+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:19:01.127788+0000","last_peered":"2023-09-23T22:19:01.127788+0000","last_clean":"2023-09-23T22:19:01.127788+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:19:01.127788+0000","last_undegraded":"2023-09-23T22:19:01.127788+0000","last_fullsized":"2023-09-23T22:19:01.127788+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":161,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127907+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:19:01.127907+0000","last_peered":"2023-09-23T22:19:01.127907+0000","last_clean":"2023-09-23T22:19:01.127907+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:19:01.127907+0000","last_undegraded":"2023-09-23T22:19:01.127907+0000","last_fullsized":"2023-09-23T22:19:01.127907+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"184'10069","reported_seq":18342,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:18:59.834081+0000","last_change":"2023-09-23T22:18:40.214871+0000","last_active":"2023-09-23T22:18:59.834081+0000","last_peered":"2023-09-23T22:18:59.834081+0000","last_clean":"2023-09-23T22:16:48.124555+0000","last_became_active":"2023-09-23T22:16:50.339530+0000","last_became_peered":"2023-09-23T22:16:50.339530+0000","last_unstale":"2023-09-23T22:18:59.834081+0000","last_undegraded":"2023-09-23T22:18:40.214039+0000","last_fullsized":"2023-09-23T22:16:50.148559+0000","mapping_epoch":147,"log_start":"126'2524","ondisk_log_start":"126'2524","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7545,"log_dups_size":0,"ondisk_log_size":7545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:01.673286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111607808,"num_objects":1703,"num_object_clones":0,"num_object_copies":5109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1789,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10069,"num_write_kb":376704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4088,"num_bytes_recovered":267911168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":-86},{"shards":"4,5,6","objects":1694}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":137,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.129896+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:19:01.129896+0000","last_peered":"2023-09-23T22:19:01.129896+0000","last_clean":"2023-09-23T22:19:01.129896+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:19:01.129896+0000","last_undegraded":"2023-09-23T22:19:01.129896+0000","last_fullsized":"2023-09-23T22:19:01.129896+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127734+0000","last_change":"2023-09-23T22:16:49.165118+0000","last_active":"2023-09-23T22:19:01.127734+0000","last_peered":"2023-09-23T22:19:01.127734+0000","last_clean":"2023-09-23T22:19:01.127734+0000","last_became_active":"2023-09-23T22:16:49.164802+0000","last_became_peered":"2023-09-23T22:16:49.164802+0000","last_unstale":"2023-09-23T22:19:01.127734+0000","last_undegraded":"2023-09-23T22:19:01.127734+0000","last_fullsized":"2023-09-23T22:19:01.127734+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:23.079142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":448,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.703391+0000","last_change":"2023-09-23T22:18:57.726339+0000","last_active":"2023-09-23T22:18:58.703391+0000","last_peered":"2023-09-23T22:18:58.703391+0000","last_clean":"2023-09-23T22:18:58.703391+0000","last_became_active":"2023-09-23T22:18:57.725773+0000","last_became_peered":"2023-09-23T22:18:57.725773+0000","last_unstale":"2023-09-23T22:18:58.703391+0000","last_undegraded":"2023-09-23T22:18:58.703391+0000","last_fullsized":"2023-09-23T22:18:58.703391+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"186'10226","reported_seq":14074,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.416530+0000","last_change":"2023-09-23T22:15:59.633734+0000","last_active":"2023-09-23T22:19:02.416530+0000","last_peered":"2023-09-23T22:19:02.416530+0000","last_clean":"2023-09-23T22:19:02.416530+0000","last_became_active":"2023-09-23T22:15:59.605894+0000","last_became_peered":"2023-09-23T22:15:59.605894+0000","last_unstale":"2023-09-23T22:19:02.416530+0000","last_undegraded":"2023-09-23T22:19:02.416530+0000","last_fullsized":"2023-09-23T22:19:02.416530+0000","mapping_epoch":132,"log_start":"133'3647","ondisk_log_start":"133'3647","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:22.683442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111542272,"num_objects":1702,"num_object_clones":0,"num_object_copies":5106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10226,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1867,"num_bytes_recovered":122355712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130206+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:19:01.130206+0000","last_peered":"2023-09-23T22:19:01.130206+0000","last_clean":"2023-09-23T22:19:01.130206+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:19:01.130206+0000","last_undegraded":"2023-09-23T22:19:01.130206+0000","last_fullsized":"2023-09-23T22:19:01.130206+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":125,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.125455+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:19:01.125455+0000","last_peered":"2023-09-23T22:19:01.125455+0000","last_clean":"2023-09-23T22:19:01.125455+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:19:01.125455+0000","last_undegraded":"2023-09-23T22:19:01.125455+0000","last_fullsized":"2023-09-23T22:19:01.125455+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":448,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701102+0000","last_change":"2023-09-23T22:18:57.725616+0000","last_active":"2023-09-23T22:18:58.701102+0000","last_peered":"2023-09-23T22:18:58.701102+0000","last_clean":"2023-09-23T22:18:58.701102+0000","last_became_active":"2023-09-23T22:18:57.725224+0000","last_became_peered":"2023-09-23T22:18:57.725224+0000","last_unstale":"2023-09-23T22:18:58.701102+0000","last_undegraded":"2023-09-23T22:18:58.701102+0000","last_fullsized":"2023-09-23T22:18:58.701102+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":732,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.122316+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:19:01.122316+0000","last_peered":"2023-09-23T22:19:01.122316+0000","last_clean":"2023-09-23T22:19:01.122316+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:19:01.122316+0000","last_undegraded":"2023-09-23T22:19:01.122316+0000","last_fullsized":"2023-09-23T22:19:01.122316+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":161,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.715415+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:18:57.715415+0000","last_peered":"2023-09-23T22:18:57.715415+0000","last_clean":"2023-09-23T22:18:57.715415+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:18:57.715415+0000","last_undegraded":"2023-09-23T22:18:57.715415+0000","last_fullsized":"2023-09-23T22:18:57.715415+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.707968+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:18:57.707968+0000","last_peered":"2023-09-23T22:18:57.707968+0000","last_clean":"2023-09-23T22:18:57.707968+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:18:57.707968+0000","last_undegraded":"2023-09-23T22:18:57.707968+0000","last_fullsized":"2023-09-23T22:18:57.707968+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"184'10059","reported_seq":15096,"reported_epoch":184,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:19:01.003753+0000","last_change":"2023-09-23T22:18:59.033329+0000","last_active":"2023-09-23T22:19:01.003753+0000","last_peered":"2023-09-23T22:19:01.003753+0000","last_clean":"2023-09-23T22:16:48.130011+0000","last_became_active":"2023-09-23T22:18:58.953635+0000","last_became_peered":"2023-09-23T22:18:58.953635+0000","last_unstale":"2023-09-23T22:19:01.003753+0000","last_undegraded":"2023-09-23T22:18:58.948229+0000","last_fullsized":"2023-09-23T22:19:01.003753+0000","mapping_epoch":183,"log_start":"136'4113","ondisk_log_start":"136'4113","created":111,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5946,"log_dups_size":0,"ondisk_log_size":5946,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:59.274843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110690304,"num_objects":1689,"num_object_clones":0,"num_object_copies":5067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1689,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10059,"num_write_kb":375936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2449,"num_bytes_recovered":160432128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,1],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":1689}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":448,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701764+0000","last_change":"2023-09-23T22:18:57.742555+0000","last_active":"2023-09-23T22:18:58.701764+0000","last_peered":"2023-09-23T22:18:58.701764+0000","last_clean":"2023-09-23T22:18:58.701764+0000","last_became_active":"2023-09-23T22:18:57.742205+0000","last_became_peered":"2023-09-23T22:18:57.742205+0000","last_unstale":"2023-09-23T22:18:58.701764+0000","last_undegraded":"2023-09-23T22:18:58.701764+0000","last_fullsized":"2023-09-23T22:18:58.701764+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"186'10366","reported_seq":14276,"reported_epoch":186,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:03.245501+0000","last_change":"2023-09-23T22:18:57.918095+0000","last_active":"2023-09-23T22:19:03.245501+0000","last_peered":"2023-09-23T22:19:03.245501+0000","last_clean":"2023-09-23T22:16:50.502857+0000","last_became_active":"2023-09-23T22:18:57.873881+0000","last_became_peered":"2023-09-23T22:18:57.873881+0000","last_unstale":"2023-09-23T22:19:03.245501+0000","last_undegraded":"2023-09-23T22:18:57.734113+0000","last_fullsized":"2023-09-23T22:18:57.704742+0000","mapping_epoch":182,"log_start":"142'5540","ondisk_log_start":"142'5540","created":111,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":4826,"log_dups_size":0,"ondisk_log_size":4826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:58.431178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105119744,"num_objects":1604,"num_object_clones":0,"num_object_copies":4812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10366,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1895,"num_bytes_recovered":124190720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[1,4],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4","objects":1604}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":888,"reported_epoch":183,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:18:57.715155+0000","last_change":"2023-09-23T22:17:15.722644+0000","last_active":"2023-09-23T22:18:57.715155+0000","last_peered":"2023-09-23T22:18:57.715155+0000","last_clean":"2023-09-23T22:17:10.678840+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:18:57.715155+0000","last_undegraded":"2023-09-23T22:17:15.714187+0000","last_fullsized":"2023-09-23T22:18:57.715155+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":154,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,3,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":195,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128335+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:19:01.128335+0000","last_peered":"2023-09-23T22:19:01.128335+0000","last_clean":"2023-09-23T22:19:01.128335+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:19:01.128335+0000","last_undegraded":"2023-09-23T22:19:01.128335+0000","last_fullsized":"2023-09-23T22:19:01.128335+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":158,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.715048+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:18:57.715048+0000","last_peered":"2023-09-23T22:18:57.715048+0000","last_clean":"2023-09-23T22:18:57.715048+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:18:57.715048+0000","last_undegraded":"2023-09-23T22:18:57.715048+0000","last_fullsized":"2023-09-23T22:18:57.715048+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":448,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.703139+0000","last_change":"2023-09-23T22:18:57.725923+0000","last_active":"2023-09-23T22:18:58.703139+0000","last_peered":"2023-09-23T22:18:58.703139+0000","last_clean":"2023-09-23T22:18:58.703139+0000","last_became_active":"2023-09-23T22:18:57.725260+0000","last_became_peered":"2023-09-23T22:18:57.725260+0000","last_unstale":"2023-09-23T22:18:58.703139+0000","last_undegraded":"2023-09-23T22:18:58.703139+0000","last_fullsized":"2023-09-23T22:18:58.703139+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"184'10023","reported_seq":10106,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:19:00.938480+0000","last_change":"2023-09-23T22:15:14.823394+0000","last_active":"2023-09-23T22:19:00.938480+0000","last_peered":"2023-09-23T22:19:00.938480+0000","last_clean":"2023-09-23T22:19:00.938480+0000","last_became_active":"2023-09-23T22:15:14.823003+0000","last_became_peered":"2023-09-23T22:15:14.823003+0000","last_unstale":"2023-09-23T22:19:00.938480+0000","last_undegraded":"2023-09-23T22:19:00.938480+0000","last_fullsized":"2023-09-23T22:19:00.938480+0000","mapping_epoch":111,"log_start":"127'2800","ondisk_log_start":"127'2800","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7223,"log_dups_size":0,"ondisk_log_size":7223,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:13.496813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107806720,"num_objects":1645,"num_object_clones":0,"num_object_copies":4935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10023,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":525,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.706190+0000","last_change":"2023-09-23T22:15:51.379227+0000","last_active":"2023-09-23T22:18:57.706190+0000","last_peered":"2023-09-23T22:18:57.706190+0000","last_clean":"2023-09-23T22:18:57.706190+0000","last_became_active":"2023-09-23T22:15:43.928977+0000","last_became_peered":"2023-09-23T22:15:43.928977+0000","last_unstale":"2023-09-23T22:18:57.706190+0000","last_undegraded":"2023-09-23T22:18:57.706190+0000","last_fullsized":"2023-09-23T22:18:57.706190+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:04.398533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":135,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.120981+0000","last_change":"2023-09-23T22:16:49.251412+0000","last_active":"2023-09-23T22:19:01.120981+0000","last_peered":"2023-09-23T22:19:01.120981+0000","last_clean":"2023-09-23T22:19:01.120981+0000","last_became_active":"2023-09-23T22:16:49.251097+0000","last_became_peered":"2023-09-23T22:16:49.251097+0000","last_unstale":"2023-09-23T22:19:01.120981+0000","last_undegraded":"2023-09-23T22:19:01.120981+0000","last_fullsized":"2023-09-23T22:19:01.120981+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:41.601435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":190,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.125498+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:19:01.125498+0000","last_peered":"2023-09-23T22:19:01.125498+0000","last_clean":"2023-09-23T22:19:01.125498+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:19:01.125498+0000","last_undegraded":"2023-09-23T22:19:01.125498+0000","last_fullsized":"2023-09-23T22:19:01.125498+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":769,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.715810+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:18:57.715810+0000","last_peered":"2023-09-23T22:18:57.715810+0000","last_clean":"2023-09-23T22:18:57.715810+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:18:57.715810+0000","last_undegraded":"2023-09-23T22:18:57.715810+0000","last_fullsized":"2023-09-23T22:18:57.715810+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":448,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.700899+0000","last_change":"2023-09-23T22:18:57.743602+0000","last_active":"2023-09-23T22:18:58.700899+0000","last_peered":"2023-09-23T22:18:58.700899+0000","last_clean":"2023-09-23T22:18:58.700899+0000","last_became_active":"2023-09-23T22:18:57.743004+0000","last_became_peered":"2023-09-23T22:18:57.743004+0000","last_unstale":"2023-09-23T22:18:58.700899+0000","last_undegraded":"2023-09-23T22:18:58.700899+0000","last_fullsized":"2023-09-23T22:18:58.700899+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":110,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.135190+0000","last_change":"2023-09-23T22:16:49.155156+0000","last_active":"2023-09-23T22:19:01.135190+0000","last_peered":"2023-09-23T22:19:01.135190+0000","last_clean":"2023-09-23T22:19:01.135190+0000","last_became_active":"2023-09-23T22:16:49.154923+0000","last_became_peered":"2023-09-23T22:16:49.154923+0000","last_unstale":"2023-09-23T22:19:01.135190+0000","last_undegraded":"2023-09-23T22:19:01.135190+0000","last_fullsized":"2023-09-23T22:19:01.135190+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:33.532238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":72,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128134+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:19:01.128134+0000","last_peered":"2023-09-23T22:19:01.128134+0000","last_clean":"2023-09-23T22:19:01.128134+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:19:01.128134+0000","last_undegraded":"2023-09-23T22:19:01.128134+0000","last_fullsized":"2023-09-23T22:19:01.128134+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"184'10144","reported_seq":10227,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.016327+0000","last_change":"2023-09-23T22:15:14.824102+0000","last_active":"2023-09-23T22:19:01.016327+0000","last_peered":"2023-09-23T22:19:01.016327+0000","last_clean":"2023-09-23T22:19:01.016327+0000","last_became_active":"2023-09-23T22:15:14.823790+0000","last_became_peered":"2023-09-23T22:15:14.823790+0000","last_unstale":"2023-09-23T22:19:01.016327+0000","last_undegraded":"2023-09-23T22:19:01.016327+0000","last_fullsized":"2023-09-23T22:19:01.016327+0000","mapping_epoch":111,"log_start":"129'3000","ondisk_log_start":"129'3000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7144,"log_dups_size":0,"ondisk_log_size":7144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:24.517495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105512960,"num_objects":1610,"num_object_clones":0,"num_object_copies":4830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10144,"num_write_kb":376128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":448,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.702112+0000","last_change":"2023-09-23T22:18:57.720920+0000","last_active":"2023-09-23T22:18:58.702112+0000","last_peered":"2023-09-23T22:18:58.702112+0000","last_clean":"2023-09-23T22:18:58.702112+0000","last_became_active":"2023-09-23T22:18:57.720584+0000","last_became_peered":"2023-09-23T22:18:57.720584+0000","last_unstale":"2023-09-23T22:18:58.702112+0000","last_undegraded":"2023-09-23T22:18:58.702112+0000","last_fullsized":"2023-09-23T22:18:58.702112+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"184'10272","reported_seq":15161,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:19:00.123880+0000","last_change":"2023-09-23T22:17:15.752911+0000","last_active":"2023-09-23T22:19:00.123880+0000","last_peered":"2023-09-23T22:19:00.123880+0000","last_clean":"2023-09-23T22:19:00.123880+0000","last_became_active":"2023-09-23T22:17:15.728788+0000","last_became_peered":"2023-09-23T22:17:15.728788+0000","last_unstale":"2023-09-23T22:19:00.123880+0000","last_undegraded":"2023-09-23T22:19:00.123880+0000","last_fullsized":"2023-09-23T22:19:00.123880+0000","mapping_epoch":158,"log_start":"127'2763","ondisk_log_start":"127'2763","created":111,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7509,"log_dups_size":0,"ondisk_log_size":7509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:35.923920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108658688,"num_objects":1658,"num_object_clones":0,"num_object_copies":4974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10272,"num_write_kb":381760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2374,"num_bytes_recovered":155582464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":533,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.121116+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:19:01.121116+0000","last_peered":"2023-09-23T22:19:01.121116+0000","last_clean":"2023-09-23T22:19:01.121116+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:19:01.121116+0000","last_undegraded":"2023-09-23T22:19:01.121116+0000","last_fullsized":"2023-09-23T22:19:01.121116+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":45,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.714071+0000","last_change":"2023-09-23T22:18:31.795915+0000","last_active":"2023-09-23T22:18:57.714071+0000","last_peered":"2023-09-23T22:18:57.714071+0000","last_clean":"2023-09-23T22:18:57.714071+0000","last_became_active":"2023-09-23T22:18:31.795608+0000","last_became_peered":"2023-09-23T22:18:31.795608+0000","last_unstale":"2023-09-23T22:18:57.714071+0000","last_undegraded":"2023-09-23T22:18:57.714071+0000","last_fullsized":"2023-09-23T22:18:57.714071+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:49.789927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701044+0000","last_change":"2023-09-23T22:18:57.721257+0000","last_active":"2023-09-23T22:18:58.701044+0000","last_peered":"2023-09-23T22:18:58.701044+0000","last_clean":"2023-09-23T22:18:58.701044+0000","last_became_active":"2023-09-23T22:18:57.720548+0000","last_became_peered":"2023-09-23T22:18:57.720548+0000","last_unstale":"2023-09-23T22:18:58.701044+0000","last_undegraded":"2023-09-23T22:18:58.701044+0000","last_fullsized":"2023-09-23T22:18:58.701044+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:10.709024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":448,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701378+0000","last_change":"2023-09-23T22:18:57.727276+0000","last_active":"2023-09-23T22:18:58.701378+0000","last_peered":"2023-09-23T22:18:58.701378+0000","last_clean":"2023-09-23T22:18:58.701378+0000","last_became_active":"2023-09-23T22:18:57.726938+0000","last_became_peered":"2023-09-23T22:18:57.726938+0000","last_unstale":"2023-09-23T22:18:58.701378+0000","last_undegraded":"2023-09-23T22:18:58.701378+0000","last_fullsized":"2023-09-23T22:18:58.701378+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":73,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701366+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:18:58.701366+0000","last_peered":"2023-09-23T22:18:58.701366+0000","last_clean":"2023-09-23T22:18:58.701366+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:18:58.701366+0000","last_undegraded":"2023-09-23T22:18:58.701366+0000","last_fullsized":"2023-09-23T22:18:58.701366+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":125,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.122639+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:19:01.122639+0000","last_peered":"2023-09-23T22:19:01.122639+0000","last_clean":"2023-09-23T22:19:01.122639+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:19:01.122639+0000","last_undegraded":"2023-09-23T22:19:01.122639+0000","last_fullsized":"2023-09-23T22:19:01.122639+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":36,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.704244+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:18:57.704244+0000","last_peered":"2023-09-23T22:18:57.704244+0000","last_clean":"2023-09-23T22:18:57.704244+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:18:57.704244+0000","last_undegraded":"2023-09-23T22:18:57.704244+0000","last_fullsized":"2023-09-23T22:18:57.704244+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"184'10234","reported_seq":13577,"reported_epoch":184,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:01.032154+0000","last_change":"2023-09-23T22:18:57.739452+0000","last_active":"2023-09-23T22:19:01.032154+0000","last_peered":"2023-09-23T22:19:01.032154+0000","last_clean":"2023-09-23T22:16:48.102227+0000","last_became_active":"2023-09-23T22:16:50.283490+0000","last_became_peered":"2023-09-23T22:16:50.283490+0000","last_unstale":"2023-09-23T22:19:01.032154+0000","last_undegraded":"2023-09-23T22:17:00.207659+0000","last_fullsized":"2023-09-23T22:16:50.146447+0000","mapping_epoch":147,"log_start":"136'4277","ondisk_log_start":"136'4277","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5957,"log_dups_size":0,"ondisk_log_size":5957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:48.954402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111017984,"num_objects":1694,"num_object_clones":0,"num_object_copies":5082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4346,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10234,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1610,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5","objects":-2652},{"shards":"0,5,6","objects":1685}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":448,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701170+0000","last_change":"2023-09-23T22:18:57.742449+0000","last_active":"2023-09-23T22:18:58.701170+0000","last_peered":"2023-09-23T22:18:58.701170+0000","last_clean":"2023-09-23T22:18:58.701170+0000","last_became_active":"2023-09-23T22:18:57.742146+0000","last_became_peered":"2023-09-23T22:18:57.742146+0000","last_unstale":"2023-09-23T22:18:58.701170+0000","last_undegraded":"2023-09-23T22:18:58.701170+0000","last_fullsized":"2023-09-23T22:18:58.701170+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":197,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131150+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:19:01.131150+0000","last_peered":"2023-09-23T22:19:01.131150+0000","last_clean":"2023-09-23T22:19:01.131150+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:19:01.131150+0000","last_undegraded":"2023-09-23T22:19:01.131150+0000","last_fullsized":"2023-09-23T22:19:01.131150+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":825,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.121323+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:19:01.121323+0000","last_peered":"2023-09-23T22:19:01.121323+0000","last_clean":"2023-09-23T22:19:01.121323+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:19:01.121323+0000","last_undegraded":"2023-09-23T22:19:01.121323+0000","last_fullsized":"2023-09-23T22:19:01.121323+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":182,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130280+0000","last_change":"2023-09-23T22:18:57.722771+0000","last_active":"2023-09-23T22:19:01.130280+0000","last_peered":"2023-09-23T22:19:01.130280+0000","last_clean":"2023-09-23T22:19:01.130280+0000","last_became_active":"2023-09-23T22:18:57.722258+0000","last_became_peered":"2023-09-23T22:18:57.722258+0000","last_unstale":"2023-09-23T22:19:01.130280+0000","last_undegraded":"2023-09-23T22:19:01.130280+0000","last_fullsized":"2023-09-23T22:19:01.130280+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:18.267091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"186'9942","reported_seq":10017,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.367285+0000","last_change":"2023-09-23T22:15:14.828787+0000","last_active":"2023-09-23T22:19:02.367285+0000","last_peered":"2023-09-23T22:19:02.367285+0000","last_clean":"2023-09-23T22:19:02.367285+0000","last_became_active":"2023-09-23T22:15:14.828370+0000","last_became_peered":"2023-09-23T22:15:14.828370+0000","last_unstale":"2023-09-23T22:19:02.367285+0000","last_undegraded":"2023-09-23T22:19:02.367285+0000","last_fullsized":"2023-09-23T22:19:02.367285+0000","mapping_epoch":111,"log_start":"133'3370","ondisk_log_start":"133'3370","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6572,"log_dups_size":0,"ondisk_log_size":6572,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:35.080268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104464384,"num_objects":1594,"num_object_clones":0,"num_object_copies":4782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9942,"num_write_kb":369152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":452,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:19:00.918752+0000","last_change":"2023-09-23T22:19:00.918399+0000","last_active":"2023-09-23T22:19:00.918752+0000","last_peered":"2023-09-23T22:19:00.918752+0000","last_clean":"2023-09-23T22:19:00.918752+0000","last_became_active":"2023-09-23T22:18:57.720614+0000","last_became_peered":"2023-09-23T22:18:57.720614+0000","last_unstale":"2023-09-23T22:19:00.918752+0000","last_undegraded":"2023-09-23T22:19:00.918752+0000","last_fullsized":"2023-09-23T22:19:00.918752+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":248,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.708440+0000","last_change":"2023-09-23T22:16:49.270015+0000","last_active":"2023-09-23T22:18:58.708440+0000","last_peered":"2023-09-23T22:18:58.708440+0000","last_clean":"2023-09-23T22:18:58.708440+0000","last_became_active":"2023-09-23T22:16:49.252726+0000","last_became_peered":"2023-09-23T22:16:49.252726+0000","last_unstale":"2023-09-23T22:18:58.708440+0000","last_undegraded":"2023-09-23T22:18:58.708440+0000","last_fullsized":"2023-09-23T22:18:58.708440+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:18.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":16,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:19:00.704626+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:19:00.704626+0000","last_peered":"2023-09-23T22:19:00.704626+0000","last_clean":"2023-09-23T22:19:00.704626+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:19:00.704626+0000","last_undegraded":"2023-09-23T22:19:00.704626+0000","last_fullsized":"2023-09-23T22:19:00.704626+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":149,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.129975+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:19:01.129975+0000","last_peered":"2023-09-23T22:19:01.129975+0000","last_clean":"2023-09-23T22:19:01.129975+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:19:01.129975+0000","last_undegraded":"2023-09-23T22:19:01.129975+0000","last_fullsized":"2023-09-23T22:19:01.129975+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"186'10058","reported_seq":10143,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:03.744564+0000","last_change":"2023-09-23T22:15:14.825150+0000","last_active":"2023-09-23T22:19:03.744564+0000","last_peered":"2023-09-23T22:19:03.744564+0000","last_clean":"2023-09-23T22:19:03.744564+0000","last_became_active":"2023-09-23T22:15:14.824867+0000","last_became_peered":"2023-09-23T22:15:14.824867+0000","last_unstale":"2023-09-23T22:19:03.744564+0000","last_undegraded":"2023-09-23T22:19:03.744564+0000","last_fullsized":"2023-09-23T22:19:03.744564+0000","mapping_epoch":111,"log_start":"129'3100","ondisk_log_start":"129'3100","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6958,"log_dups_size":0,"ondisk_log_size":6958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:30.053753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100401152,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10058,"num_write_kb":370880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":12,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.702259+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:18:58.702259+0000","last_peered":"2023-09-23T22:18:58.702259+0000","last_clean":"2023-09-23T22:18:58.702259+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:18:58.702259+0000","last_undegraded":"2023-09-23T22:18:58.702259+0000","last_fullsized":"2023-09-23T22:18:58.702259+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":168,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.707620+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:18:57.707620+0000","last_peered":"2023-09-23T22:18:57.707620+0000","last_clean":"2023-09-23T22:18:57.707620+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:18:57.707620+0000","last_undegraded":"2023-09-23T22:18:57.707620+0000","last_fullsized":"2023-09-23T22:18:57.707620+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":146,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.715366+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:18:57.715366+0000","last_peered":"2023-09-23T22:18:57.715366+0000","last_clean":"2023-09-23T22:18:57.715366+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:18:57.715366+0000","last_undegraded":"2023-09-23T22:18:57.715366+0000","last_fullsized":"2023-09-23T22:18:57.715366+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":161,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128461+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:19:01.128461+0000","last_peered":"2023-09-23T22:19:01.128461+0000","last_clean":"2023-09-23T22:19:01.128461+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:19:01.128461+0000","last_undegraded":"2023-09-23T22:19:01.128461+0000","last_fullsized":"2023-09-23T22:19:01.128461+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":143,"reported_epoch":183,"state":"active+clean","last_fresh":"2023-09-23T22:18:57.715430+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:18:57.715430+0000","last_peered":"2023-09-23T22:18:57.715430+0000","last_clean":"2023-09-23T22:18:57.715430+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:18:57.715430+0000","last_undegraded":"2023-09-23T22:18:57.715430+0000","last_fullsized":"2023-09-23T22:18:57.715430+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":20,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127722+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:19:01.127722+0000","last_peered":"2023-09-23T22:19:01.127722+0000","last_clean":"2023-09-23T22:19:01.127722+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:19:01.127722+0000","last_undegraded":"2023-09-23T22:19:01.127722+0000","last_fullsized":"2023-09-23T22:19:01.127722+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":166,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133160+0000","last_change":"2023-09-23T22:16:49.145417+0000","last_active":"2023-09-23T22:19:01.133160+0000","last_peered":"2023-09-23T22:19:01.133160+0000","last_clean":"2023-09-23T22:19:01.133160+0000","last_became_active":"2023-09-23T22:16:49.145017+0000","last_became_peered":"2023-09-23T22:16:49.145017+0000","last_unstale":"2023-09-23T22:19:01.133160+0000","last_undegraded":"2023-09-23T22:19:01.133160+0000","last_fullsized":"2023-09-23T22:19:01.133160+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:47.762149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":208,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131817+0000","last_change":"2023-09-23T22:17:15.721123+0000","last_active":"2023-09-23T22:19:01.131817+0000","last_peered":"2023-09-23T22:19:01.131817+0000","last_clean":"2023-09-23T22:19:01.131817+0000","last_became_active":"2023-09-23T22:17:15.720812+0000","last_became_peered":"2023-09-23T22:17:15.720812+0000","last_unstale":"2023-09-23T22:19:01.131817+0000","last_undegraded":"2023-09-23T22:19:01.131817+0000","last_fullsized":"2023-09-23T22:19:01.131817+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.176278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":56,"reported_epoch":184,"state":"active+clean","last_fresh":"2023-09-23T22:18:58.701503+0000","last_change":"2023-09-23T22:18:57.729455+0000","last_active":"2023-09-23T22:18:58.701503+0000","last_peered":"2023-09-23T22:18:58.701503+0000","last_clean":"2023-09-23T22:18:58.701503+0000","last_became_active":"2023-09-23T22:18:57.729088+0000","last_became_peered":"2023-09-23T22:18:57.729088+0000","last_unstale":"2023-09-23T22:18:58.701503+0000","last_undegraded":"2023-09-23T22:18:58.701503+0000","last_fullsized":"2023-09-23T22:18:58.701503+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:17.861334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127295+0000","last_change":"2023-09-23T22:16:49.164638+0000","last_active":"2023-09-23T22:19:01.127295+0000","last_peered":"2023-09-23T22:19:01.127295+0000","last_clean":"2023-09-23T22:19:01.127295+0000","last_became_active":"2023-09-23T22:16:49.164349+0000","last_became_peered":"2023-09-23T22:16:49.164349+0000","last_unstale":"2023-09-23T22:19:01.127295+0000","last_undegraded":"2023-09-23T22:19:01.127295+0000","last_fullsized":"2023-09-23T22:19:01.127295+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:26.536927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1717370904,"num_objects":26206,"num_object_clones":0,"num_object_copies":78618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17688,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26206,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":161406,"num_write_kb":6003521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26950,"num_bytes_recovered":1766129664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7201173504,"data_stored":7201161288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106352,"ondisk_log_size":106352,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4892,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2098784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234176,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":727652,"kb_used_data":649372,"kb_used_omap":0,"kb_used_meta":78272,"kb_avail":93017500,"statfs":{"total":95995035648,"available":95249920000,"internally_reserved":0,"allocated":664956928,"data_stored":664147446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":80150528},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":2.2080000000000002,"15min":1.105},"min":{"1min":0.23100000000000001,"5min":0.223,"15min":0.223},"max":{"1min":21.977,"5min":109.208,"15min":109.208},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.548,"5min":2.536,"15min":1.2370000000000001},"min":{"1min":0.28199999999999997,"5min":0.217,"15min":0.217},"max":{"1min":11.698,"5min":109.13800000000001,"15min":109.13800000000001},"last":2.774}]},{"osd":1,"last update":"Sat Sep 23 22:18:05 2023","interfaces":[{"interface":"back","average":{"1min":7.6849999999999996,"5min":3.4460000000000002,"15min":1.526},"min":{"1min":0.33500000000000002,"5min":0.246,"15min":0.246},"max":{"1min":122.461,"5min":123.79300000000001,"15min":123.79300000000001},"last":6.8319999999999999},{"interface":"front","average":{"1min":8.4710000000000001,"5min":3.8170000000000002,"15min":1.663},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":122.268,"5min":123.66500000000001,"15min":123.66500000000001},"last":5.766}]},{"osd":2,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":6.2619999999999996,"5min":6.1609999999999996,"15min":2.7250000000000001},"min":{"1min":0.78600000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":37.698,"5min":125.101,"15min":125.101},"last":3.7559999999999998},{"interface":"front","average":{"1min":5.4589999999999996,"5min":5.8220000000000001,"15min":2.3500000000000001},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":37.679000000000002,"5min":126.021,"15min":126.021},"last":3.6520000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:18:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":2.23,"15min":1.1439999999999999},"min":{"1min":0.35099999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.4950000000000001,"5min":113.282,"15min":113.282},"last":3.2810000000000001},{"interface":"front","average":{"1min":2.1909999999999998,"5min":2.3570000000000002,"15min":1.2330000000000001},"min":{"1min":0.37,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.8300000000000001,"5min":104.74299999999999,"15min":104.74299999999999},"last":7.1799999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.3009999999999999,"15min":0.749},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.251},"max":{"1min":10.305999999999999,"5min":33.627000000000002,"15min":33.627000000000002},"last":3.6760000000000002},{"interface":"front","average":{"1min":1.335,"5min":1.111,"15min":0.70299999999999996},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.1300000000000008,"5min":33.113,"15min":33.113},"last":0.67500000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.0109999999999999,"15min":0.65700000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.9219999999999997,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":3.5129999999999999},{"interface":"front","average":{"1min":1.946,"5min":1.5369999999999999,"15min":0.88200000000000001},"min":{"1min":0.35499999999999998,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":10.423999999999999,"5min":31.257999999999999,"15min":31.257999999999999},"last":3.722}]},{"osd":6,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":0.999,"15min":0.68899999999999995},"min":{"1min":0.33700000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":7.9459999999999997,"5min":9.0139999999999993,"15min":9.0139999999999993},"last":0.438},{"interface":"front","average":{"1min":2.2930000000000001,"5min":1.3500000000000001,"15min":0.84199999999999997},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.308},"max":{"1min":10.347,"5min":13.362,"15min":13.362},"last":3.7869999999999999}]}]},{"osd":6,"up_from":0,"seq":197568495812,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795084,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":784260,"kb_used_data":622012,"kb_used_omap":7,"kb_used_meta":162104,"kb_avail":92960892,"statfs":{"total":95995035648,"available":95191953408,"internally_reserved":0,"allocated":636940288,"data_stored":635926010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":165994868},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":1.472,"15min":0.72999999999999998},"min":{"1min":0.249,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":6.6859999999999999,"5min":61.890000000000001,"15min":61.890000000000001},"last":0.20399999999999999},{"interface":"front","average":{"1min":3.1110000000000002,"5min":1.7569999999999999,"15min":0.85599999999999998},"min":{"1min":0.27300000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":13.243,"5min":62.689999999999998,"15min":62.689999999999998},"last":0.27300000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":5.3970000000000002,"5min":2.1629999999999998,"15min":0.97299999999999998},"min":{"1min":0.29099999999999998,"5min":0.216,"15min":0.216},"max":{"1min":33.902000000000001,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":7.593,"5min":2.302,"15min":1.042},"min":{"1min":0.48399999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":34.781999999999996,"5min":34.781999999999996,"15min":34.781999999999996},"last":0.309}]},{"osd":3,"last update":"Sat Sep 23 22:18:13 2023","interfaces":[{"interface":"back","average":{"1min":3.492,"5min":1.4430000000000001,"15min":0.746},"min":{"1min":0.34499999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":13.198,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":2.8759999999999999,"5min":1.4630000000000001,"15min":0.76100000000000001},"min":{"1min":0.28299999999999997,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.3599999999999994,"5min":22.259,"15min":22.259},"last":0.25900000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":0.876,"15min":0.73699999999999999},"min":{"1min":0.28299999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.7530000000000001,"5min":6.9089999999999998,"15min":6.9089999999999998},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.6960000000000002,"5min":1.121,"15min":0.81699999999999995},"min":{"1min":0.39600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":9.1780000000000008,"5min":9.1780000000000008,"15min":9.1780000000000008},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5259999999999998,"5min":1.0640000000000001,"15min":0.76400000000000001},"min":{"1min":0.42999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":8.1539999999999999,"5min":8.1539999999999999,"15min":8.1539999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.1320000000000001,"5min":1.111,"15min":0.79400000000000004},"min":{"1min":0.32700000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6130000000000004,"5min":6.6130000000000004,"15min":6.6130000000000004},"last":0.29599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":1.1950000000000001,"15min":0.82499999999999996},"min":{"1min":0.38900000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.989000000000001,"5min":10.989000000000001,"15min":10.989000000000001},"last":0.372},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.048,"15min":0.79800000000000004},"min":{"1min":0.38,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.7969999999999997,"5min":6.8700000000000001,"15min":6.8700000000000001},"last":0.35699999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":2.831,"5min":1.2849999999999999,"15min":0.85899999999999999},"min":{"1min":0.41199999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":10.948,"5min":10.948,"15min":10.948},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.153,"5min":1.1779999999999999,"15min":0.84599999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":7.8970000000000002,"5min":7.8970000000000002,"15min":7.8970000000000002},"last":0.45300000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958618,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1292956,"kb_used_data":1134868,"kb_used_omap":8,"kb_used_meta":158007,"kb_avail":92452196,"statfs":{"total":95995035648,"available":94671048704,"internally_reserved":0,"allocated":1162104832,"data_stored":1161113235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":161800042},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,0,0,0,8,3],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:18:09 2023","interfaces":[{"interface":"back","average":{"1min":4.4109999999999996,"5min":1.903,"15min":0.85899999999999999},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":30.010999999999999,"5min":44.776000000000003,"15min":44.776000000000003},"last":1.159},{"interface":"front","average":{"1min":5.0750000000000002,"5min":2.242,"15min":0.98999999999999999},"min":{"1min":0.27100000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":48.838000000000001,"5min":48.838000000000001,"15min":48.838000000000001},"last":2.6509999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:18:40 2023","interfaces":[{"interface":"back","average":{"1min":3.9319999999999999,"5min":1.502,"15min":0.76900000000000002},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":25.893000000000001,"5min":25.893000000000001,"15min":25.893000000000001},"last":8.8300000000000001},{"interface":"front","average":{"1min":7.742,"5min":2.7810000000000001,"15min":1.2749999999999999},"min":{"1min":0.28999999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":66.665000000000006,"5min":66.665000000000006,"15min":66.665000000000006},"last":7.726}]},{"osd":3,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":3.8399999999999999,"5min":1.843,"15min":0.91100000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":20.358000000000001,"5min":35.496000000000002,"15min":35.496000000000002},"last":1.1200000000000001},{"interface":"front","average":{"1min":3.2749999999999999,"5min":1.917,"15min":0.93200000000000005},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":20.422000000000001,"5min":35.533000000000001,"15min":35.533000000000001},"last":0.77800000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.2880000000000003,"5min":2.2050000000000001,"15min":1.1679999999999999},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.217000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":1.29},{"interface":"front","average":{"1min":2.9860000000000002,"5min":1.2569999999999999,"15min":0.85799999999999998},"min":{"1min":0.32700000000000001,"5min":0.308,"15min":0.308},"max":{"1min":22.209,"5min":22.209,"15min":22.209},"last":1.018}]},{"osd":5,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.6959999999999997,"5min":2.173,"15min":1.1799999999999999},"min":{"1min":0.36099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.372,"5min":12.372,"15min":12.372},"last":1.3140000000000001},{"interface":"front","average":{"1min":2.6699999999999999,"5min":1.286,"15min":0.88200000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":20.390999999999998,"5min":20.390999999999998,"15min":20.390999999999998},"last":1.0549999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.5609999999999999,"5min":2.2149999999999999,"15min":1.1879999999999999},"min":{"1min":0.46400000000000002,"5min":0.379,"15min":0.379},"max":{"1min":10.275,"5min":10.275,"15min":10.275},"last":1.228},{"interface":"front","average":{"1min":2.867,"5min":1.3080000000000001,"15min":0.91000000000000003},"min":{"1min":0.38600000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":22.263000000000002,"5min":22.263000000000002,"15min":22.263000000000002},"last":1.0840000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.1180000000000003,"5min":2.3079999999999998,"15min":1.252},"min":{"1min":0.44500000000000001,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":20.491,"5min":23.306000000000001,"15min":23.306000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":4.5170000000000003,"5min":2.6419999999999999,"15min":1.379},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.257999999999999,"5min":23.265999999999998,"15min":23.265999999999998},"last":1.264}]}]},{"osd":2,"up_from":103,"seq":442381631557,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":792092,"kb_used_data":643988,"kb_used_omap":10,"kb_used_meta":148085,"kb_avail":92953060,"statfs":{"total":95995035648,"available":95183933440,"internally_reserved":0,"allocated":659443712,"data_stored":658453938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":151639212},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":4.7610000000000001,"5min":1.9370000000000001,"15min":0.86899999999999999},"min":{"1min":0.308,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":15.26,"5min":15.26,"15min":15.26},"last":0.67000000000000004},{"interface":"front","average":{"1min":4.7240000000000002,"5min":2.2749999999999999,"15min":1},"min":{"1min":0.21199999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":19.149999999999999,"5min":19.149999999999999,"15min":19.149999999999999},"last":6.3600000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":4.5199999999999996,"5min":3.9489999999999998,"15min":1.5980000000000001},"min":{"1min":0.32600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":19.869,"5min":157.636,"15min":157.636},"last":6.7149999999999999},{"interface":"front","average":{"1min":4.1710000000000003,"5min":4.0369999999999999,"15min":1.625},"min":{"1min":0.28100000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":17.768999999999998,"5min":155.63300000000001,"15min":155.63300000000001},"last":6.3289999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:18:05 2023","interfaces":[{"interface":"back","average":{"1min":3.4769999999999999,"5min":1.5369999999999999,"15min":0.73899999999999999},"min":{"1min":0.24199999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":13.192,"5min":13.192,"15min":13.192},"last":1.1419999999999999},{"interface":"front","average":{"1min":4.6390000000000002,"5min":1.5920000000000001,"15min":0.78900000000000003},"min":{"1min":0.32400000000000001,"5min":0.246,"15min":0.246},"max":{"1min":14.259,"5min":14.259,"15min":14.259},"last":20.581}]},{"osd":4,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":1.5289999999999999,"15min":0.90800000000000003},"min":{"1min":0.26800000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.452,"5min":11.244,"15min":11.244},"last":0.81799999999999995},{"interface":"front","average":{"1min":2.7029999999999998,"5min":1.544,"15min":0.93200000000000005},"min":{"1min":0.372,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.2959999999999994,"5min":9.2799999999999994,"15min":9.2799999999999994},"last":0.73899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":1.671,"15min":0.97999999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.5009999999999994,"5min":11.291,"15min":11.291},"last":0.84999999999999998},{"interface":"front","average":{"1min":2.911,"5min":1.617,"15min":0.94299999999999995},"min":{"1min":0.36799999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.327,"5min":9.4499999999999993,"15min":9.4499999999999993},"last":0.78000000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":3.589,"5min":1.7130000000000001,"15min":1.014},"min":{"1min":0.38400000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":17.640999999999998,"5min":17.640999999999998,"15min":17.640999999999998},"last":0.88500000000000001},{"interface":"front","average":{"1min":2.8839999999999999,"5min":1.5169999999999999,"15min":1.0289999999999999},"min":{"1min":0.42299999999999999,"5min":0.375,"15min":0.375},"max":{"1min":11.875,"5min":11.875,"15min":11.875},"last":0.90500000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":3.8969999999999998,"5min":1.714,"15min":1.0780000000000001},"min":{"1min":0.38500000000000001,"5min":0.38,"15min":0.38},"max":{"1min":10.135,"5min":10.135,"15min":10.135},"last":6.7640000000000002},{"interface":"front","average":{"1min":3.46,"5min":1.9279999999999999,"15min":1.0669999999999999},"min":{"1min":0.51000000000000001,"5min":0.377,"15min":0.377},"max":{"1min":17.738,"5min":17.738,"15min":17.738},"last":0.86799999999999999}]}]},{"osd":3,"up_from":108,"seq":463856468031,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462064,"kb_used_data":394408,"kb_used_omap":4,"kb_used_meta":67643,"kb_avail":93283088,"statfs":{"total":95995035648,"available":95521882112,"internally_reserved":0,"allocated":403873792,"data_stored":402875006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":69266989},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:06 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.946,"15min":0.90000000000000002},"min":{"1min":0.24399999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.135,"5min":77.834999999999994,"15min":77.834999999999994},"last":6.7329999999999997},{"interface":"front","average":{"1min":3.3140000000000001,"5min":2.2000000000000002,"15min":0.97399999999999998},"min":{"1min":0.27400000000000002,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.983000000000001,"5min":75.558000000000007,"15min":75.558000000000007},"last":6.5810000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:18:06 2023","interfaces":[{"interface":"back","average":{"1min":3.3530000000000002,"5min":2.117,"15min":0.94099999999999995},"min":{"1min":0.29099999999999998,"5min":0.218,"15min":0.218},"max":{"1min":14.125999999999999,"5min":68.284999999999997,"15min":68.284999999999997},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.468,"5min":1.792,"15min":0.87},"min":{"1min":0.29799999999999999,"5min":0.189,"15min":0.189},"max":{"1min":6.3600000000000003,"5min":66.082999999999998,"15min":66.082999999999998},"last":15.554}]},{"osd":2,"last update":"Sat Sep 23 22:18:06 2023","interfaces":[{"interface":"back","average":{"1min":3.597,"5min":4.25,"15min":1.6779999999999999},"min":{"1min":0.33000000000000002,"5min":0.223,"15min":0.223},"max":{"1min":11.901,"5min":97.325000000000003,"15min":97.325000000000003},"last":17.931000000000001},{"interface":"front","average":{"1min":3.3130000000000002,"5min":3.5369999999999999,"15min":1.462},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":12.101000000000001,"5min":97.379999999999995,"15min":97.379999999999995},"last":9.6319999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:18:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":0.93999999999999995,"15min":0.72999999999999998},"min":{"1min":0.316,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.2329999999999997,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":6.7610000000000001},{"interface":"front","average":{"1min":2.7989999999999999,"5min":1.1879999999999999,"15min":0.83299999999999996},"min":{"1min":0.44800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.036,"5min":12.036,"15min":12.036},"last":6.6200000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:18:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4649999999999999,"5min":1.1259999999999999,"15min":0.79200000000000004},"min":{"1min":0.36799999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":14.292,"5min":14.292,"15min":14.292},"last":6.6529999999999996},{"interface":"front","average":{"1min":2.4929999999999999,"5min":1.103,"15min":0.77400000000000002},"min":{"1min":0.35399999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":11.56,"5min":11.56,"15min":11.56},"last":0.44700000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:18:06 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":1.2170000000000001,"15min":0.84099999999999997},"min":{"1min":0.436,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":12.068,"5min":12.068,"15min":12.068},"last":6.694},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.228,"15min":0.83199999999999996},"min":{"1min":0.38,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":11.586,"5min":11.586,"15min":11.586},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:18:06 2023","interfaces":[{"interface":"back","average":{"1min":3.048,"5min":1.302,"15min":0.85399999999999998},"min":{"1min":0.51100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":11.624000000000001,"5min":11.624000000000001,"15min":11.624000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.0409999999999999,"15min":0.80300000000000005},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":12.382,"5min":12.382,"15min":12.382},"last":6.7880000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019082,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":995100,"kb_used_data":887892,"kb_used_omap":0,"kb_used_meta":107200,"kb_avail":92750052,"statfs":{"total":95995035648,"available":94976053248,"internally_reserved":0,"allocated":909201408,"data_stored":908399803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":109772800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2,3,4],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.9970000000000001,"15min":1.0840000000000001},"min":{"1min":0.23000000000000001,"5min":0.218,"15min":0.218},"max":{"1min":7.5789999999999997,"5min":66.355000000000004,"15min":66.355000000000004},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.4299999999999999,"5min":2.2200000000000002,"15min":1.1699999999999999},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.5949999999999998,"5min":66.332999999999998,"15min":66.332999999999998},"last":0.52200000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":1.748,"15min":0.94599999999999995},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.625,"5min":38.878,"15min":38.878},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.4220000000000002,"15min":1.1899999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":10.236000000000001,"5min":38.853999999999999,"15min":38.853999999999999},"last":0.504}]},{"osd":2,"last update":"Sat Sep 23 22:18:40 2023","interfaces":[{"interface":"back","average":{"1min":8.1180000000000003,"5min":2.8780000000000001,"15min":1.3200000000000001},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.255},"max":{"1min":133.22900000000001,"5min":133.22900000000001,"15min":133.22900000000001},"last":1.925},{"interface":"front","average":{"1min":8.4529999999999994,"5min":3.4510000000000001,"15min":1.4910000000000001},"min":{"1min":0.33000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":133.60300000000001,"5min":133.60300000000001,"15min":133.60300000000001},"last":0.36299999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:18:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":1.1739999999999999,"15min":0.78700000000000003},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.7110000000000003,"5min":7.7110000000000003,"15min":7.7110000000000003},"last":6.6900000000000004},{"interface":"front","average":{"1min":2.5880000000000001,"5min":1.288,"15min":0.81299999999999994},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":9.8109999999999999,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":6.5670000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:18:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":0.86399999999999999,"15min":0.60199999999999998},"min":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":8.6050000000000004,"5min":8.9499999999999993,"15min":8.9499999999999993},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.252,"5min":0.79400000000000004,"15min":0.623},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.8789999999999996,"5min":7.8789999999999996,"15min":7.8789999999999996},"last":0.376}]},{"osd":6,"last update":"Sat Sep 23 22:18:24 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.75,"15min":0.58599999999999997},"min":{"1min":0.309,"5min":0.251,"15min":0.251},"max":{"1min":3.6699999999999999,"5min":3.6699999999999999,"15min":3.6699999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.84599999999999997,"15min":0.64600000000000002},"min":{"1min":0.32100000000000001,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":3.052,"5min":8.5210000000000008,"15min":8.5210000000000008},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.1180000000000001,"15min":0.76000000000000001},"min":{"1min":0.33100000000000002,"5min":0.307,"15min":0.307},"max":{"1min":5.3440000000000003,"5min":8.577,"15min":8.577},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.90400000000000003,"15min":0.69299999999999995},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.7799999999999998,"5min":5.6280000000000001,"15min":5.6280000000000001},"last":0.39100000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757448,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1303612,"kb_used_data":1186932,"kb_used_omap":0,"kb_used_meta":116672,"kb_avail":92441540,"statfs":{"total":95995035648,"available":94660136960,"internally_reserved":0,"allocated":1215418368,"data_stored":1214628750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":119472128},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,1,6,1],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":3.113,"15min":1.4139999999999999},"min":{"1min":0.26800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.6849999999999996,"5min":133.22499999999999,"15min":133.22499999999999},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.732,"5min":3.387,"15min":1.554},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.444,"5min":132.78200000000001,"15min":132.78200000000001},"last":0.438}]},{"osd":1,"last update":"Sat Sep 23 22:18:04 2023","interfaces":[{"interface":"back","average":{"1min":7.3369999999999997,"5min":3.1440000000000001,"15min":1.4419999999999999},"min":{"1min":0.39900000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":78.597999999999999,"5min":78.597999999999999,"15min":78.597999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":7.2839999999999998,"5min":3.1499999999999999,"15min":1.4299999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":78.609999999999999,"5min":78.609999999999999,"15min":78.609999999999999},"last":0.44700000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:18:39 2023","interfaces":[{"interface":"back","average":{"1min":12.324999999999999,"5min":6.0250000000000004,"15min":2.383},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":104.254,"5min":104.254,"15min":104.254},"last":0.39400000000000002},{"interface":"front","average":{"1min":12.984,"5min":6.6600000000000001,"15min":2.6139999999999999},"min":{"1min":0.34699999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":104.217,"5min":104.217,"15min":104.217},"last":0.46400000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:18:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3149999999999999,"5min":2.4319999999999999,"15min":1.196},"min":{"1min":0.34100000000000003,"5min":0.246,"15min":0.246},"max":{"1min":9.3300000000000001,"5min":88.106999999999999,"15min":88.106999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.1779999999999999,"5min":2.5009999999999999,"15min":1.226},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.1509999999999998,"5min":87.887,"15min":87.887},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:17:58 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.5429999999999999,"15min":0.81899999999999995},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26800000000000002},"max":{"1min":7.7880000000000003,"5min":82.530000000000001,"15min":82.530000000000001},"last":0.375},{"interface":"front","average":{"1min":1.2689999999999999,"5min":1.6180000000000001,"15min":0.877},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":5.6079999999999997,"5min":85.441000000000003,"15min":85.441000000000003},"last":0.26100000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:18:30 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.117,"15min":1.0509999999999999},"min":{"1min":0.32800000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.6519999999999992,"5min":114.83499999999999,"15min":114.83499999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.9119999999999999,"15min":0.995},"min":{"1min":0.31,"5min":0.31,"15min":0.29699999999999999},"max":{"1min":7.4640000000000004,"5min":110.905,"15min":110.905},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:18:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.1850000000000001,"15min":0.77200000000000002},"min":{"1min":0.35899999999999999,"5min":0.33500000000000002,"15min":0.32600000000000001},"max":{"1min":8.2159999999999993,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.1950000000000001,"5min":0.97599999999999998,"15min":0.69699999999999995},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.33200000000000002},"max":{"1min":5.2460000000000004,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":0.41799999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1153171456,"data_stored":1153171456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":620560384,"data_stored":620560384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":647761920,"data_stored":647757848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":400424960,"data_stored":400424960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":900333568,"data_stored":900333568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1208422400,"data_stored":1208418328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1617362944,"data_stored":1617362944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":653135872,"data_stored":653131800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:19:07.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:19:07.105 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:07.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:07.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:06 smithi070 ceph-mon[192652]: pgmap v413: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 91 active+clean; 1.6 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 17701/78702 objects degraded (22.491%); 8.7 MiB/s, 136 objects/s recovering 2023-09-23T22:19:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:06 smithi070 ceph-mon[192652]: from='client.44858 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:07.219 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:07.314 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:07.402 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:07.509 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:19:07.539 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:07.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:07.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:07.551 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:19:07.598 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:19:07.637 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:19:07.652 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:07.695 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:19:07.740 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:19:07.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:07.786 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:19:07.804 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:19:07.823 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:19:07.823 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:19:07.823 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:19:07.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:19:07.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:19:07.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:19:07.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:19:07.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:19:07.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2368729695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:07.960 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:07 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17701/78702 objects degraded (22.491%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:07.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2368729695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:07.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:07 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17701/78702 objects degraded (22.491%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:07.998 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:08.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:08.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:08.104 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2368729695' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:07 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17701/78702 objects degraded (22.491%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:08.188 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:08.962 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:08.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:09.067 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:09.092 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:08 smithi070 ceph-mon[192652]: pgmap v414: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.5 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 427 op/s; 15837/73911 objects degraded (21.427%); 1402/73911 objects misplaced (1.897%); 11 MiB/s, 167 objects/s recovering 2023-09-23T22:19:09.092 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:08 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:19:09.095 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:09.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:08 smithi026 ceph-mon[215775]: pgmap v414: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.5 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 427 op/s; 15837/73911 objects degraded (21.427%); 1402/73911 objects misplaced (1.897%); 11 MiB/s, 167 objects/s recovering 2023-09-23T22:19:09.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:08 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:19:09.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:08 smithi026 ceph-mon[223014]: pgmap v414: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.5 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 427 op/s; 15837/73911 objects degraded (21.427%); 1402/73911 objects misplaced (1.897%); 11 MiB/s, 167 objects/s recovering 2023-09-23T22:19:09.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:08 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:19:09.568 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:09.615 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:19:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:19:09] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2023-09-23T22:19:09.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:09.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:10.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:10.965 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:10.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:10.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:11.060 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:10 smithi070 ceph-mon[192652]: pgmap v415: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.4 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 15837/70824 objects degraded (22.361%); 1402/70824 objects misplaced (1.980%); 10 MiB/s, 153 objects/s recovering 2023-09-23T22:19:11.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:11.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:10 smithi026 ceph-mon[223014]: pgmap v415: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.4 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 15837/70824 objects degraded (22.361%); 1402/70824 objects misplaced (1.980%); 10 MiB/s, 153 objects/s recovering 2023-09-23T22:19:11.143 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:10 smithi026 ceph-mon[215775]: pgmap v415: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.4 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 15837/70824 objects degraded (22.361%); 1402/70824 objects misplaced (1.980%); 10 MiB/s, 153 objects/s recovering 2023-09-23T22:19:11.156 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:11.298 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:19:11.428 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:11.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:11.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:11.549 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:11.596 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:11.708 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:19:11.709 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:19:11.737 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:11.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:11.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:11.824 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:19:11.842 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:11.900 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:12.049 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1635033122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:12.050 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1635033122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:12.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1635033122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:12.213 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:12.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:12.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:12.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:12.418 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:12.554 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:12.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:12.660 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:12.696 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:12.709 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:19:12.722 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:19:12.988 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:12 smithi070 ceph-mon[192652]: pgmap v416: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.4 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 15638/68559 objects degraded (22.810%); 1109/68559 objects misplaced (1.618%); 11 MiB/s, 171 objects/s recovering 2023-09-23T22:19:12.988 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:12 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:19:12.988 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1635033122' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:19:12.988 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:12 smithi070 ceph-mon[192652]: osdmap e187: 8 total, 8 up, 7 in 2023-09-23T22:19:12.988 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:12 smithi070 ceph-mon[192652]: failed to encode map e187 with expected crc 2023-09-23T22:19:12.988 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:12 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15638/68559 objects degraded (22.810%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:12.999 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:13.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:13.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:13.012 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:19:13.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[223014]: pgmap v416: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.4 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 15638/68559 objects degraded (22.810%); 1109/68559 objects misplaced (1.618%); 11 MiB/s, 171 objects/s recovering 2023-09-23T22:19:13.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:19:13.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1635033122' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:19:13.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[223014]: osdmap e187: 8 total, 8 up, 7 in 2023-09-23T22:19:13.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[223014]: failed to encode map e187 with expected crc 2023-09-23T22:19:13.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15638/68559 objects degraded (22.810%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:13.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[215775]: pgmap v416: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.4 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 15638/68559 objects degraded (22.810%); 1109/68559 objects misplaced (1.618%); 11 MiB/s, 171 objects/s recovering 2023-09-23T22:19:13.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:19:13.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1635033122' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:19:13.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[215775]: osdmap e187: 8 total, 8 up, 7 in 2023-09-23T22:19:13.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[215775]: failed to encode map e187 with expected crc 2023-09-23T22:19:13.090 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:12 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15638/68559 objects degraded (22.810%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:13.107 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:13.157 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:13.227 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:13.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:13.334 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:13.396 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:13.590 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:13.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:13.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:13.654 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:13.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:13.732 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:13.761 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":417,"stamp":"2023-09-23T22:19:11.786874+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1426266059,"num_objects":21719,"num_object_clones":0,"num_object_copies":65157,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":15638,"num_objects_misplaced":1109,"num_objects_unfound":0,"num_objects_dirty":21719,"num_whiteouts":0,"num_read":6469,"num_read_kb":13097,"num_write":167067,"num_write_kb":6018662,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28039,"num_bytes_recovered":1838422434,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110169,"ondisk_log_size":110169,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":324,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5642100,"kb_used_data":4781116,"kb_used_omap":31,"kb_used_meta":860640,"kb_avail":650573964,"statfs":{"total":671965249536,"available":666187739136,"internally_reserved":0,"allocated":4895862784,"data_stored":4889439884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":881296311},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,4,2,2,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-339804160,"num_objects":-5185,"num_object_clones":0,"num_object_copies":-15555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2063,"num_objects_misplaced":483,"num_objects_unfound":0,"num_objects_dirty":-5185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5185,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1582,"num_bytes_recovered":107020329,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.663132"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127656+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:19:01.127656+0000","last_peered":"2023-09-23T22:19:01.127656+0000","last_clean":"2023-09-23T22:19:01.127656+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:19:01.127656+0000","last_undegraded":"2023-09-23T22:19:01.127656+0000","last_fullsized":"2023-09-23T22:19:01.127656+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":165,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130866+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:19:01.130866+0000","last_peered":"2023-09-23T22:19:01.130866+0000","last_clean":"2023-09-23T22:19:01.130866+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:19:01.130866+0000","last_undegraded":"2023-09-23T22:19:01.130866+0000","last_fullsized":"2023-09-23T22:19:01.130866+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":125,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.126779+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:19:01.126779+0000","last_peered":"2023-09-23T22:19:01.126779+0000","last_clean":"2023-09-23T22:19:01.126779+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:19:01.126779+0000","last_undegraded":"2023-09-23T22:19:01.126779+0000","last_fullsized":"2023-09-23T22:19:01.126779+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.166307+0000","last_change":"2023-09-23T22:18:57.724501+0000","last_active":"2023-09-23T22:19:01.166307+0000","last_peered":"2023-09-23T22:19:01.166307+0000","last_clean":"2023-09-23T22:19:01.166307+0000","last_became_active":"2023-09-23T22:18:57.724203+0000","last_became_peered":"2023-09-23T22:18:57.724203+0000","last_unstale":"2023-09-23T22:19:01.166307+0000","last_undegraded":"2023-09-23T22:19:01.166307+0000","last_fullsized":"2023-09-23T22:19:01.166307+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:52.168267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":205,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130355+0000","last_change":"2023-09-23T22:18:57.722574+0000","last_active":"2023-09-23T22:19:01.130355+0000","last_peered":"2023-09-23T22:19:01.130355+0000","last_clean":"2023-09-23T22:19:01.130355+0000","last_became_active":"2023-09-23T22:18:57.721800+0000","last_became_peered":"2023-09-23T22:18:57.721800+0000","last_unstale":"2023-09-23T22:19:01.130355+0000","last_undegraded":"2023-09-23T22:19:01.130355+0000","last_fullsized":"2023-09-23T22:19:01.130355+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:09.303838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":185,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:19:01.134161+0000","last_change":"2023-09-23T22:18:58.730170+0000","last_active":"2023-09-23T22:19:01.134161+0000","last_peered":"2023-09-23T22:19:01.134161+0000","last_clean":"2023-09-23T22:18:49.795932+0000","last_became_active":"2023-09-23T22:18:58.729860+0000","last_became_peered":"2023-09-23T22:18:58.729860+0000","last_unstale":"2023-09-23T22:19:01.134161+0000","last_undegraded":"2023-09-23T22:19:01.134161+0000","last_fullsized":"2023-09-23T22:19:01.134161+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:30.948774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,3],"avail_no_missing":["5","0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.164590+0000","last_change":"2023-09-23T22:18:57.729038+0000","last_active":"2023-09-23T22:19:01.164590+0000","last_peered":"2023-09-23T22:19:01.164590+0000","last_clean":"2023-09-23T22:19:01.164590+0000","last_became_active":"2023-09-23T22:18:57.728650+0000","last_became_peered":"2023-09-23T22:18:57.728650+0000","last_unstale":"2023-09-23T22:19:01.164590+0000","last_undegraded":"2023-09-23T22:19:01.164590+0000","last_fullsized":"2023-09-23T22:19:01.164590+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:11.461883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":160,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130424+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:19:01.130424+0000","last_peered":"2023-09-23T22:19:01.130424+0000","last_clean":"2023-09-23T22:19:01.130424+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:19:01.130424+0000","last_undegraded":"2023-09-23T22:19:01.130424+0000","last_fullsized":"2023-09-23T22:19:01.130424+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":146,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133876+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:19:01.133876+0000","last_peered":"2023-09-23T22:19:01.133876+0000","last_clean":"2023-09-23T22:19:01.133876+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:19:01.133876+0000","last_undegraded":"2023-09-23T22:19:01.133876+0000","last_fullsized":"2023-09-23T22:19:01.133876+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":110,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131940+0000","last_change":"2023-09-23T22:16:49.154341+0000","last_active":"2023-09-23T22:19:01.131940+0000","last_peered":"2023-09-23T22:19:01.131940+0000","last_clean":"2023-09-23T22:19:01.131940+0000","last_became_active":"2023-09-23T22:16:49.154041+0000","last_became_peered":"2023-09-23T22:16:49.154041+0000","last_unstale":"2023-09-23T22:19:01.131940+0000","last_undegraded":"2023-09-23T22:19:01.131940+0000","last_fullsized":"2023-09-23T22:19:01.131940+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:05.733262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":97,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130792+0000","last_change":"2023-09-23T22:15:14.829731+0000","last_active":"2023-09-23T22:19:01.130792+0000","last_peered":"2023-09-23T22:19:01.130792+0000","last_clean":"2023-09-23T22:19:01.130792+0000","last_became_active":"2023-09-23T22:15:14.829437+0000","last_became_peered":"2023-09-23T22:15:14.829437+0000","last_unstale":"2023-09-23T22:19:01.130792+0000","last_undegraded":"2023-09-23T22:19:01.130792+0000","last_fullsized":"2023-09-23T22:19:01.130792+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:45.456558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.124844+0000","last_change":"2023-09-23T22:18:57.723165+0000","last_active":"2023-09-23T22:19:01.124844+0000","last_peered":"2023-09-23T22:19:01.124844+0000","last_clean":"2023-09-23T22:19:01.124844+0000","last_became_active":"2023-09-23T22:18:57.722863+0000","last_became_peered":"2023-09-23T22:18:57.722863+0000","last_unstale":"2023-09-23T22:19:01.124844+0000","last_undegraded":"2023-09-23T22:19:01.124844+0000","last_fullsized":"2023-09-23T22:19:01.124844+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:19.645029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":170,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127699+0000","last_change":"2023-09-23T22:16:49.265991+0000","last_active":"2023-09-23T22:19:01.127699+0000","last_peered":"2023-09-23T22:19:01.127699+0000","last_clean":"2023-09-23T22:19:01.127699+0000","last_became_active":"2023-09-23T22:16:49.251169+0000","last_became_peered":"2023-09-23T22:16:49.251169+0000","last_unstale":"2023-09-23T22:19:01.127699+0000","last_undegraded":"2023-09-23T22:19:01.127699+0000","last_fullsized":"2023-09-23T22:19:01.127699+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.502218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128377+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:19:01.128377+0000","last_peered":"2023-09-23T22:19:01.128377+0000","last_clean":"2023-09-23T22:19:01.128377+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:19:01.128377+0000","last_undegraded":"2023-09-23T22:19:01.128377+0000","last_fullsized":"2023-09-23T22:19:01.128377+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":97,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128422+0000","last_change":"2023-09-23T22:15:14.826143+0000","last_active":"2023-09-23T22:19:01.128422+0000","last_peered":"2023-09-23T22:19:01.128422+0000","last_clean":"2023-09-23T22:19:01.128422+0000","last_became_active":"2023-09-23T22:15:14.825780+0000","last_became_peered":"2023-09-23T22:15:14.825780+0000","last_unstale":"2023-09-23T22:19:01.128422+0000","last_undegraded":"2023-09-23T22:19:01.128422+0000","last_fullsized":"2023-09-23T22:19:01.128422+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:22.608676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130553+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:19:01.130553+0000","last_peered":"2023-09-23T22:19:01.130553+0000","last_clean":"2023-09-23T22:19:01.130553+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:19:01.130553+0000","last_undegraded":"2023-09-23T22:19:01.130553+0000","last_fullsized":"2023-09-23T22:19:01.130553+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.157666+0000","last_change":"2023-09-23T22:18:57.726112+0000","last_active":"2023-09-23T22:19:01.157666+0000","last_peered":"2023-09-23T22:19:01.157666+0000","last_clean":"2023-09-23T22:19:01.157666+0000","last_became_active":"2023-09-23T22:18:57.725558+0000","last_became_peered":"2023-09-23T22:18:57.725558+0000","last_unstale":"2023-09-23T22:19:01.157666+0000","last_undegraded":"2023-09-23T22:19:01.157666+0000","last_fullsized":"2023-09-23T22:19:01.157666+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:38.315664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":145,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133787+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:19:01.133787+0000","last_peered":"2023-09-23T22:19:01.133787+0000","last_clean":"2023-09-23T22:19:01.133787+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:19:01.133787+0000","last_undegraded":"2023-09-23T22:19:01.133787+0000","last_fullsized":"2023-09-23T22:19:01.133787+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130107+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:19:01.130107+0000","last_peered":"2023-09-23T22:19:01.130107+0000","last_clean":"2023-09-23T22:19:01.130107+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:19:01.130107+0000","last_undegraded":"2023-09-23T22:19:01.130107+0000","last_fullsized":"2023-09-23T22:19:01.130107+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.166496+0000","last_change":"2023-09-23T22:18:57.726691+0000","last_active":"2023-09-23T22:19:01.166496+0000","last_peered":"2023-09-23T22:19:01.166496+0000","last_clean":"2023-09-23T22:19:01.166496+0000","last_became_active":"2023-09-23T22:18:57.726386+0000","last_became_peered":"2023-09-23T22:18:57.726386+0000","last_unstale":"2023-09-23T22:19:01.166496+0000","last_undegraded":"2023-09-23T22:19:01.166496+0000","last_fullsized":"2023-09-23T22:19:01.166496+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:34.829149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161551+0000","last_change":"2023-09-23T22:18:57.728767+0000","last_active":"2023-09-23T22:19:02.161551+0000","last_peered":"2023-09-23T22:19:02.161551+0000","last_clean":"2023-09-23T22:19:02.161551+0000","last_became_active":"2023-09-23T22:18:57.728377+0000","last_became_peered":"2023-09-23T22:18:57.728377+0000","last_unstale":"2023-09-23T22:19:02.161551+0000","last_undegraded":"2023-09-23T22:19:02.161551+0000","last_fullsized":"2023-09-23T22:19:02.161551+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:01:01.370904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":151,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131068+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:19:01.131068+0000","last_peered":"2023-09-23T22:19:01.131068+0000","last_clean":"2023-09-23T22:19:01.131068+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:19:01.131068+0000","last_undegraded":"2023-09-23T22:19:01.131068+0000","last_fullsized":"2023-09-23T22:19:01.131068+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"186'10084","reported_seq":21717,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:07.401696+0000","last_change":"2023-09-23T22:19:02.159483+0000","last_active":"2023-09-23T22:19:07.401696+0000","last_peered":"2023-09-23T22:19:07.401696+0000","last_clean":"2023-09-23T22:19:07.401696+0000","last_became_active":"2023-09-23T22:19:02.132959+0000","last_became_peered":"2023-09-23T22:19:02.132959+0000","last_unstale":"2023-09-23T22:19:07.401696+0000","last_undegraded":"2023-09-23T22:19:07.401696+0000","last_fullsized":"2023-09-23T22:19:07.401696+0000","mapping_epoch":185,"log_start":"133'3365","ondisk_log_start":"133'3365","created":111,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6719,"log_dups_size":0,"ondisk_log_size":6719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:20.126557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95813632,"num_objects":1462,"num_object_clones":0,"num_object_copies":4386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1462,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10084,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5765,"num_bytes_recovered":377815040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128061+0000","last_change":"2023-09-23T22:19:00.682889+0000","last_active":"2023-09-23T22:19:01.128061+0000","last_peered":"2023-09-23T22:19:01.128061+0000","last_clean":"2023-09-23T22:19:01.128061+0000","last_became_active":"2023-09-23T22:18:57.718002+0000","last_became_peered":"2023-09-23T22:18:57.718002+0000","last_unstale":"2023-09-23T22:19:01.128061+0000","last_undegraded":"2023-09-23T22:19:01.128061+0000","last_fullsized":"2023-09-23T22:19:01.128061+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:22.517277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":138,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133018+0000","last_change":"2023-09-23T22:16:49.158580+0000","last_active":"2023-09-23T22:19:01.133018+0000","last_peered":"2023-09-23T22:19:01.133018+0000","last_clean":"2023-09-23T22:19:01.133018+0000","last_became_active":"2023-09-23T22:16:49.158266+0000","last_became_peered":"2023-09-23T22:16:49.158266+0000","last_unstale":"2023-09-23T22:19:01.133018+0000","last_undegraded":"2023-09-23T22:19:01.133018+0000","last_fullsized":"2023-09-23T22:19:01.133018+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:03.791658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"186'10220","reported_seq":15553,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.132410+0000","last_change":"2023-09-23T22:17:16.903782+0000","last_active":"2023-09-23T22:19:11.132410+0000","last_peered":"2023-09-23T22:19:11.132410+0000","last_clean":"2023-09-23T22:19:11.132410+0000","last_became_active":"2023-09-23T22:17:16.895165+0000","last_became_peered":"2023-09-23T22:17:16.895165+0000","last_unstale":"2023-09-23T22:19:11.132410+0000","last_undegraded":"2023-09-23T22:19:11.132410+0000","last_fullsized":"2023-09-23T22:19:11.132410+0000","mapping_epoch":159,"log_start":"129'3000","ondisk_log_start":"129'3000","created":111,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7220,"log_dups_size":0,"ondisk_log_size":7220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:31.886804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85983232,"num_objects":1312,"num_object_clones":0,"num_object_copies":3936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10220,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2600,"num_bytes_recovered":170393600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":97,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127512+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:19:01.127512+0000","last_peered":"2023-09-23T22:19:01.127512+0000","last_clean":"2023-09-23T22:19:01.127512+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:19:01.127512+0000","last_undegraded":"2023-09-23T22:19:01.127512+0000","last_fullsized":"2023-09-23T22:19:01.127512+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130055+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:19:01.130055+0000","last_peered":"2023-09-23T22:19:01.130055+0000","last_clean":"2023-09-23T22:19:01.130055+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:19:01.130055+0000","last_undegraded":"2023-09-23T22:19:01.130055+0000","last_fullsized":"2023-09-23T22:19:01.130055+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"186'10206","reported_seq":16748,"reported_epoch":186,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:11.139064+0000","last_change":"2023-09-23T22:19:00.690164+0000","last_active":"2023-09-23T22:19:11.139064+0000","last_peered":"2023-09-23T22:19:11.139064+0000","last_clean":"2023-09-23T22:18:39.199855+0000","last_became_active":"2023-09-23T22:18:41.409775+0000","last_became_peered":"2023-09-23T22:18:41.409775+0000","last_unstale":"2023-09-23T22:19:11.139064+0000","last_undegraded":"2023-09-23T22:18:41.249089+0000","last_fullsized":"2023-09-23T22:18:41.215237+0000","mapping_epoch":178,"log_start":"129'3000","ondisk_log_start":"129'3000","created":111,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7206,"log_dups_size":0,"ondisk_log_size":7206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:30.052669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85721088,"num_objects":1308,"num_object_clones":0,"num_object_copies":3924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5621,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10206,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3204,"num_bytes_recovered":209977344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-901},{"shards":"0,2,7","objects":1308}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":111,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133747+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:19:01.133747+0000","last_peered":"2023-09-23T22:19:01.133747+0000","last_clean":"2023-09-23T22:19:01.133747+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:19:01.133747+0000","last_undegraded":"2023-09-23T22:19:01.133747+0000","last_fullsized":"2023-09-23T22:19:01.133747+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.130612+0000","last_change":"2023-09-23T22:16:49.161676+0000","last_active":"2023-09-23T22:19:02.130612+0000","last_peered":"2023-09-23T22:19:02.130612+0000","last_clean":"2023-09-23T22:19:02.130612+0000","last_became_active":"2023-09-23T22:16:49.161247+0000","last_became_peered":"2023-09-23T22:16:49.161247+0000","last_unstale":"2023-09-23T22:19:02.130612+0000","last_undegraded":"2023-09-23T22:19:02.130612+0000","last_fullsized":"2023-09-23T22:19:02.130612+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:30.708371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"186'10370","reported_seq":10455,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.058930+0000","last_change":"2023-09-23T22:15:14.826723+0000","last_active":"2023-09-23T22:19:11.058930+0000","last_peered":"2023-09-23T22:19:11.058930+0000","last_clean":"2023-09-23T22:19:11.058930+0000","last_became_active":"2023-09-23T22:15:14.826454+0000","last_became_peered":"2023-09-23T22:15:14.826454+0000","last_unstale":"2023-09-23T22:19:11.058930+0000","last_undegraded":"2023-09-23T22:19:11.058930+0000","last_fullsized":"2023-09-23T22:19:11.058930+0000","mapping_epoch":111,"log_start":"133'3200","ondisk_log_start":"133'3200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7170,"log_dups_size":0,"ondisk_log_size":7170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:11.166077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86769664,"num_objects":1324,"num_object_clones":0,"num_object_copies":3972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10370,"num_write_kb":374208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":156,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133627+0000","last_change":"2023-09-23T22:16:49.164833+0000","last_active":"2023-09-23T22:19:01.133627+0000","last_peered":"2023-09-23T22:19:01.133627+0000","last_clean":"2023-09-23T22:19:01.133627+0000","last_became_active":"2023-09-23T22:16:49.163833+0000","last_became_peered":"2023-09-23T22:16:49.163833+0000","last_unstale":"2023-09-23T22:19:01.133627+0000","last_undegraded":"2023-09-23T22:19:01.133627+0000","last_fullsized":"2023-09-23T22:19:01.133627+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.891395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":38,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.159524+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:19:01.159524+0000","last_peered":"2023-09-23T22:19:01.159524+0000","last_clean":"2023-09-23T22:19:01.159524+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:19:01.159524+0000","last_undegraded":"2023-09-23T22:19:01.159524+0000","last_fullsized":"2023-09-23T22:19:01.159524+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":450,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.143626+0000","last_change":"2023-09-23T22:18:57.721431+0000","last_active":"2023-09-23T22:19:01.143626+0000","last_peered":"2023-09-23T22:19:01.143626+0000","last_clean":"2023-09-23T22:19:01.143626+0000","last_became_active":"2023-09-23T22:18:57.721043+0000","last_became_peered":"2023-09-23T22:18:57.721043+0000","last_unstale":"2023-09-23T22:19:01.143626+0000","last_undegraded":"2023-09-23T22:19:01.143626+0000","last_fullsized":"2023-09-23T22:19:01.143626+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"186'10483","reported_seq":10569,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.132448+0000","last_change":"2023-09-23T22:15:14.824965+0000","last_active":"2023-09-23T22:19:11.132448+0000","last_peered":"2023-09-23T22:19:11.132448+0000","last_clean":"2023-09-23T22:19:11.132448+0000","last_became_active":"2023-09-23T22:15:14.824626+0000","last_became_peered":"2023-09-23T22:15:14.824626+0000","last_unstale":"2023-09-23T22:19:11.132448+0000","last_undegraded":"2023-09-23T22:19:11.132448+0000","last_fullsized":"2023-09-23T22:19:11.132448+0000","mapping_epoch":111,"log_start":"133'3330","ondisk_log_start":"133'3330","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7153,"log_dups_size":0,"ondisk_log_size":7153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.613455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85721112,"num_objects":1309,"num_object_clones":0,"num_object_copies":3927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1309,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10483,"num_write_kb":377281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":197,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131661+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:19:01.131661+0000","last_peered":"2023-09-23T22:19:01.131661+0000","last_clean":"2023-09-23T22:19:01.131661+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:19:01.131661+0000","last_undegraded":"2023-09-23T22:19:01.131661+0000","last_fullsized":"2023-09-23T22:19:01.131661+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127531+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:19:01.127531+0000","last_peered":"2023-09-23T22:19:01.127531+0000","last_clean":"2023-09-23T22:19:01.127531+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:19:01.127531+0000","last_undegraded":"2023-09-23T22:19:01.127531+0000","last_fullsized":"2023-09-23T22:19:01.127531+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":448,"reported_epoch":185,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:19:01.144231+0000","last_change":"2023-09-23T22:18:57.743507+0000","last_active":"2023-09-23T22:19:01.144231+0000","last_peered":"2023-09-23T22:19:01.144231+0000","last_clean":"2023-09-23T22:08:39.036645+0000","last_became_active":"2023-09-23T22:18:57.742130+0000","last_became_peered":"2023-09-23T22:18:57.742130+0000","last_unstale":"2023-09-23T22:19:01.144231+0000","last_undegraded":"2023-09-23T22:18:57.716066+0000","last_fullsized":"2023-09-23T22:19:01.144231+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"0,1,7","objects":-1},{"shards":"1,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.c","version":"186'10334","reported_seq":12641,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:08.221666+0000","last_change":"2023-09-23T22:16:30.157766+0000","last_active":"2023-09-23T22:19:08.221666+0000","last_peered":"2023-09-23T22:19:08.221666+0000","last_clean":"2023-09-23T22:19:08.221666+0000","last_became_active":"2023-09-23T22:16:30.150277+0000","last_became_peered":"2023-09-23T22:16:30.150277+0000","last_unstale":"2023-09-23T22:19:08.221666+0000","last_undegraded":"2023-09-23T22:19:08.221666+0000","last_fullsized":"2023-09-23T22:19:08.221666+0000","mapping_epoch":140,"log_start":"142'5325","ondisk_log_start":"142'5325","created":111,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5009,"log_dups_size":0,"ondisk_log_size":5009,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:20.415978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90177536,"num_objects":1376,"num_object_clones":0,"num_object_copies":4128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10334,"num_write_kb":374720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1098,"num_bytes_recovered":71958528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":102,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127788+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:19:01.127788+0000","last_peered":"2023-09-23T22:19:01.127788+0000","last_clean":"2023-09-23T22:19:01.127788+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:19:01.127788+0000","last_undegraded":"2023-09-23T22:19:01.127788+0000","last_fullsized":"2023-09-23T22:19:01.127788+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":161,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127907+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:19:01.127907+0000","last_peered":"2023-09-23T22:19:01.127907+0000","last_clean":"2023-09-23T22:19:01.127907+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:19:01.127907+0000","last_undegraded":"2023-09-23T22:19:01.127907+0000","last_fullsized":"2023-09-23T22:19:01.127907+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"186'10423","reported_seq":19827,"reported_epoch":186,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:10.124695+0000","last_change":"2023-09-23T22:19:00.894300+0000","last_active":"2023-09-23T22:19:10.124695+0000","last_peered":"2023-09-23T22:19:10.124695+0000","last_clean":"2023-09-23T22:16:48.124555+0000","last_became_active":"2023-09-23T22:16:50.339530+0000","last_became_peered":"2023-09-23T22:16:50.339530+0000","last_unstale":"2023-09-23T22:19:10.124695+0000","last_undegraded":"2023-09-23T22:18:40.214039+0000","last_fullsized":"2023-09-23T22:16:50.148559+0000","mapping_epoch":147,"log_start":"127'2824","ondisk_log_start":"127'2824","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7599,"log_dups_size":0,"ondisk_log_size":7599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:01.673286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88408064,"num_objects":1349,"num_object_clones":0,"num_object_copies":4047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":116,"num_objects_misplaced":1109,"num_objects_unfound":0,"num_objects_dirty":1349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10423,"num_write_kb":376704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4652,"num_bytes_recovered":304873472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":124},{"shards":"4,5,6","objects":1109}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":137,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.129896+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:19:01.129896+0000","last_peered":"2023-09-23T22:19:01.129896+0000","last_clean":"2023-09-23T22:19:01.129896+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:19:01.129896+0000","last_undegraded":"2023-09-23T22:19:01.129896+0000","last_fullsized":"2023-09-23T22:19:01.129896+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127734+0000","last_change":"2023-09-23T22:16:49.165118+0000","last_active":"2023-09-23T22:19:01.127734+0000","last_peered":"2023-09-23T22:19:01.127734+0000","last_clean":"2023-09-23T22:19:01.127734+0000","last_became_active":"2023-09-23T22:16:49.164802+0000","last_became_peered":"2023-09-23T22:16:49.164802+0000","last_unstale":"2023-09-23T22:19:01.127734+0000","last_undegraded":"2023-09-23T22:19:01.127734+0000","last_fullsized":"2023-09-23T22:19:01.127734+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:23.079142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":450,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.157757+0000","last_change":"2023-09-23T22:18:57.726339+0000","last_active":"2023-09-23T22:19:01.157757+0000","last_peered":"2023-09-23T22:19:01.157757+0000","last_clean":"2023-09-23T22:19:01.157757+0000","last_became_active":"2023-09-23T22:18:57.725773+0000","last_became_peered":"2023-09-23T22:18:57.725773+0000","last_unstale":"2023-09-23T22:19:01.157757+0000","last_undegraded":"2023-09-23T22:19:01.157757+0000","last_fullsized":"2023-09-23T22:19:01.157757+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"186'10405","reported_seq":14253,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:07.426034+0000","last_change":"2023-09-23T22:15:59.633734+0000","last_active":"2023-09-23T22:19:07.426034+0000","last_peered":"2023-09-23T22:19:07.426034+0000","last_clean":"2023-09-23T22:19:07.426034+0000","last_became_active":"2023-09-23T22:15:59.605894+0000","last_became_peered":"2023-09-23T22:15:59.605894+0000","last_unstale":"2023-09-23T22:19:07.426034+0000","last_undegraded":"2023-09-23T22:19:07.426034+0000","last_fullsized":"2023-09-23T22:19:07.426034+0000","mapping_epoch":132,"log_start":"133'3647","ondisk_log_start":"133'3647","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6758,"log_dups_size":0,"ondisk_log_size":6758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:22.683442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99811328,"num_objects":1523,"num_object_clones":0,"num_object_copies":4569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10405,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1867,"num_bytes_recovered":122355712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":152,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130206+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:19:01.130206+0000","last_peered":"2023-09-23T22:19:01.130206+0000","last_clean":"2023-09-23T22:19:01.130206+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:19:01.130206+0000","last_undegraded":"2023-09-23T22:19:01.130206+0000","last_fullsized":"2023-09-23T22:19:01.130206+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":125,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.125455+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:19:01.125455+0000","last_peered":"2023-09-23T22:19:01.125455+0000","last_clean":"2023-09-23T22:19:01.125455+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:19:01.125455+0000","last_undegraded":"2023-09-23T22:19:01.125455+0000","last_fullsized":"2023-09-23T22:19:01.125455+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":450,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.164266+0000","last_change":"2023-09-23T22:18:57.725616+0000","last_active":"2023-09-23T22:19:01.164266+0000","last_peered":"2023-09-23T22:19:01.164266+0000","last_clean":"2023-09-23T22:19:01.164266+0000","last_became_active":"2023-09-23T22:18:57.725224+0000","last_became_peered":"2023-09-23T22:18:57.725224+0000","last_unstale":"2023-09-23T22:19:01.164266+0000","last_undegraded":"2023-09-23T22:19:01.164266+0000","last_fullsized":"2023-09-23T22:19:01.164266+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":733,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.130627+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:19:02.130627+0000","last_peered":"2023-09-23T22:19:02.130627+0000","last_clean":"2023-09-23T22:19:02.130627+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:19:02.130627+0000","last_undegraded":"2023-09-23T22:19:02.130627+0000","last_fullsized":"2023-09-23T22:19:02.130627+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":163,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133863+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:19:01.133863+0000","last_peered":"2023-09-23T22:19:01.133863+0000","last_clean":"2023-09-23T22:19:01.133863+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:19:01.133863+0000","last_undegraded":"2023-09-23T22:19:01.133863+0000","last_fullsized":"2023-09-23T22:19:01.133863+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.145186+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:19:01.145186+0000","last_peered":"2023-09-23T22:19:01.145186+0000","last_clean":"2023-09-23T22:19:01.145186+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:19:01.145186+0000","last_undegraded":"2023-09-23T22:19:01.145186+0000","last_fullsized":"2023-09-23T22:19:01.145186+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"186'10437","reported_seq":16234,"reported_epoch":186,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:19:11.091158+0000","last_change":"2023-09-23T22:18:59.033329+0000","last_active":"2023-09-23T22:19:11.091158+0000","last_peered":"2023-09-23T22:19:11.091158+0000","last_clean":"2023-09-23T22:16:48.130011+0000","last_became_active":"2023-09-23T22:18:58.953635+0000","last_became_peered":"2023-09-23T22:18:58.953635+0000","last_unstale":"2023-09-23T22:19:11.091158+0000","last_undegraded":"2023-09-23T22:18:58.948229+0000","last_fullsized":"2023-09-23T22:19:11.091158+0000","mapping_epoch":183,"log_start":"139'4513","ondisk_log_start":"139'4513","created":111,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5924,"log_dups_size":0,"ondisk_log_size":5924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:59.274843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85917696,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10437,"num_write_kb":375936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2828,"num_bytes_recovered":185270272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,1],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,1,5","objects":1},{"shards":"0,5,6","objects":1310}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":450,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.144010+0000","last_change":"2023-09-23T22:18:57.742555+0000","last_active":"2023-09-23T22:19:01.144010+0000","last_peered":"2023-09-23T22:19:01.144010+0000","last_clean":"2023-09-23T22:19:01.144010+0000","last_became_active":"2023-09-23T22:18:57.742205+0000","last_became_peered":"2023-09-23T22:18:57.742205+0000","last_unstale":"2023-09-23T22:19:01.144010+0000","last_undegraded":"2023-09-23T22:19:01.144010+0000","last_fullsized":"2023-09-23T22:19:01.144010+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"186'10551","reported_seq":14461,"reported_epoch":186,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:08.261104+0000","last_change":"2023-09-23T22:18:57.918095+0000","last_active":"2023-09-23T22:19:08.261104+0000","last_peered":"2023-09-23T22:19:08.261104+0000","last_clean":"2023-09-23T22:16:50.502857+0000","last_became_active":"2023-09-23T22:18:57.873881+0000","last_became_peered":"2023-09-23T22:18:57.873881+0000","last_unstale":"2023-09-23T22:19:08.261104+0000","last_undegraded":"2023-09-23T22:18:57.734113+0000","last_fullsized":"2023-09-23T22:18:57.704742+0000","mapping_epoch":182,"log_start":"142'5540","ondisk_log_start":"142'5540","created":111,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5011,"log_dups_size":0,"ondisk_log_size":5011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:58.431178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92995584,"num_objects":1419,"num_object_clones":0,"num_object_copies":4257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10551,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1895,"num_bytes_recovered":124190720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[1,4],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4","objects":1419}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":918,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133269+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:19:01.133269+0000","last_peered":"2023-09-23T22:19:01.133269+0000","last_clean":"2023-09-23T22:19:01.133269+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:19:01.133269+0000","last_undegraded":"2023-09-23T22:19:01.133269+0000","last_fullsized":"2023-09-23T22:19:01.133269+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":195,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128335+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:19:01.128335+0000","last_peered":"2023-09-23T22:19:01.128335+0000","last_clean":"2023-09-23T22:19:01.128335+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:19:01.128335+0000","last_undegraded":"2023-09-23T22:19:01.128335+0000","last_fullsized":"2023-09-23T22:19:01.128335+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":160,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133461+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:19:01.133461+0000","last_peered":"2023-09-23T22:19:01.133461+0000","last_clean":"2023-09-23T22:19:01.133461+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:19:01.133461+0000","last_undegraded":"2023-09-23T22:19:01.133461+0000","last_fullsized":"2023-09-23T22:19:01.133461+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":450,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.157503+0000","last_change":"2023-09-23T22:18:57.725923+0000","last_active":"2023-09-23T22:19:01.157503+0000","last_peered":"2023-09-23T22:19:01.157503+0000","last_clean":"2023-09-23T22:19:01.157503+0000","last_became_active":"2023-09-23T22:18:57.725260+0000","last_became_peered":"2023-09-23T22:18:57.725260+0000","last_unstale":"2023-09-23T22:19:01.157503+0000","last_undegraded":"2023-09-23T22:19:01.157503+0000","last_fullsized":"2023-09-23T22:19:01.157503+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"186'10387","reported_seq":10472,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.127967+0000","last_change":"2023-09-23T22:15:14.823394+0000","last_active":"2023-09-23T22:19:11.127967+0000","last_peered":"2023-09-23T22:19:11.127967+0000","last_clean":"2023-09-23T22:19:11.127967+0000","last_became_active":"2023-09-23T22:15:14.823003+0000","last_became_peered":"2023-09-23T22:15:14.823003+0000","last_unstale":"2023-09-23T22:19:11.127967+0000","last_undegraded":"2023-09-23T22:19:11.127967+0000","last_fullsized":"2023-09-23T22:19:11.127967+0000","mapping_epoch":111,"log_start":"132'3200","ondisk_log_start":"132'3200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7187,"log_dups_size":0,"ondisk_log_size":7187,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:13.496813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83951616,"num_objects":1281,"num_object_clones":0,"num_object_copies":3843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10387,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":527,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.137780+0000","last_change":"2023-09-23T22:15:51.379227+0000","last_active":"2023-09-23T22:19:01.137780+0000","last_peered":"2023-09-23T22:19:01.137780+0000","last_clean":"2023-09-23T22:19:01.137780+0000","last_became_active":"2023-09-23T22:15:43.928977+0000","last_became_peered":"2023-09-23T22:15:43.928977+0000","last_unstale":"2023-09-23T22:19:01.137780+0000","last_undegraded":"2023-09-23T22:19:01.137780+0000","last_fullsized":"2023-09-23T22:19:01.137780+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:04.398533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":135,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.120981+0000","last_change":"2023-09-23T22:16:49.251412+0000","last_active":"2023-09-23T22:19:01.120981+0000","last_peered":"2023-09-23T22:19:01.120981+0000","last_clean":"2023-09-23T22:19:01.120981+0000","last_became_active":"2023-09-23T22:16:49.251097+0000","last_became_peered":"2023-09-23T22:16:49.251097+0000","last_unstale":"2023-09-23T22:19:01.120981+0000","last_undegraded":"2023-09-23T22:19:01.120981+0000","last_fullsized":"2023-09-23T22:19:01.120981+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:41.601435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":190,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.125498+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:19:01.125498+0000","last_peered":"2023-09-23T22:19:01.125498+0000","last_clean":"2023-09-23T22:19:01.125498+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:19:01.125498+0000","last_undegraded":"2023-09-23T22:19:01.125498+0000","last_fullsized":"2023-09-23T22:19:01.125498+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":771,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133387+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:19:01.133387+0000","last_peered":"2023-09-23T22:19:01.133387+0000","last_clean":"2023-09-23T22:19:01.133387+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:19:01.133387+0000","last_undegraded":"2023-09-23T22:19:01.133387+0000","last_fullsized":"2023-09-23T22:19:01.133387+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":450,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.144495+0000","last_change":"2023-09-23T22:18:57.743602+0000","last_active":"2023-09-23T22:19:01.144495+0000","last_peered":"2023-09-23T22:19:01.144495+0000","last_clean":"2023-09-23T22:19:01.144495+0000","last_became_active":"2023-09-23T22:18:57.743004+0000","last_became_peered":"2023-09-23T22:18:57.743004+0000","last_unstale":"2023-09-23T22:19:01.144495+0000","last_undegraded":"2023-09-23T22:19:01.144495+0000","last_fullsized":"2023-09-23T22:19:01.144495+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":110,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.135190+0000","last_change":"2023-09-23T22:16:49.155156+0000","last_active":"2023-09-23T22:19:01.135190+0000","last_peered":"2023-09-23T22:19:01.135190+0000","last_clean":"2023-09-23T22:19:01.135190+0000","last_became_active":"2023-09-23T22:16:49.154923+0000","last_became_peered":"2023-09-23T22:16:49.154923+0000","last_unstale":"2023-09-23T22:19:01.135190+0000","last_undegraded":"2023-09-23T22:19:01.135190+0000","last_fullsized":"2023-09-23T22:19:01.135190+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:33.532238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":72,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128134+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:19:01.128134+0000","last_peered":"2023-09-23T22:19:01.128134+0000","last_clean":"2023-09-23T22:19:01.128134+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:19:01.128134+0000","last_undegraded":"2023-09-23T22:19:01.128134+0000","last_fullsized":"2023-09-23T22:19:01.128134+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"186'10495","reported_seq":10580,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.139491+0000","last_change":"2023-09-23T22:15:14.824102+0000","last_active":"2023-09-23T22:19:11.139491+0000","last_peered":"2023-09-23T22:19:11.139491+0000","last_clean":"2023-09-23T22:19:11.139491+0000","last_became_active":"2023-09-23T22:15:14.823790+0000","last_became_peered":"2023-09-23T22:15:14.823790+0000","last_unstale":"2023-09-23T22:19:11.139491+0000","last_undegraded":"2023-09-23T22:19:11.139491+0000","last_fullsized":"2023-09-23T22:19:11.139491+0000","mapping_epoch":111,"log_start":"133'3300","ondisk_log_start":"133'3300","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7195,"log_dups_size":0,"ondisk_log_size":7195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:24.517495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82509824,"num_objects":1259,"num_object_clones":0,"num_object_copies":3777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10495,"num_write_kb":376128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":450,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.159198+0000","last_change":"2023-09-23T22:18:57.720920+0000","last_active":"2023-09-23T22:19:01.159198+0000","last_peered":"2023-09-23T22:19:01.159198+0000","last_clean":"2023-09-23T22:19:01.159198+0000","last_became_active":"2023-09-23T22:18:57.720584+0000","last_became_peered":"2023-09-23T22:18:57.720584+0000","last_unstale":"2023-09-23T22:19:01.159198+0000","last_undegraded":"2023-09-23T22:19:01.159198+0000","last_fullsized":"2023-09-23T22:19:01.159198+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"186'10604","reported_seq":15495,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:10.096435+0000","last_change":"2023-09-23T22:17:15.752911+0000","last_active":"2023-09-23T22:19:10.096435+0000","last_peered":"2023-09-23T22:19:10.096435+0000","last_clean":"2023-09-23T22:19:10.096435+0000","last_became_active":"2023-09-23T22:17:15.728788+0000","last_became_peered":"2023-09-23T22:17:15.728788+0000","last_unstale":"2023-09-23T22:19:10.096435+0000","last_undegraded":"2023-09-23T22:19:10.096435+0000","last_fullsized":"2023-09-23T22:19:10.096435+0000","mapping_epoch":158,"log_start":"129'3063","ondisk_log_start":"129'3063","created":111,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:35.923920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86900736,"num_objects":1326,"num_object_clones":0,"num_object_copies":3978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10604,"num_write_kb":381760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2374,"num_bytes_recovered":155582464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":533,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.121116+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:19:01.121116+0000","last_peered":"2023-09-23T22:19:01.121116+0000","last_clean":"2023-09-23T22:19:01.121116+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:19:01.121116+0000","last_undegraded":"2023-09-23T22:19:01.121116+0000","last_fullsized":"2023-09-23T22:19:01.121116+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":47,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.143887+0000","last_change":"2023-09-23T22:18:31.795915+0000","last_active":"2023-09-23T22:19:01.143887+0000","last_peered":"2023-09-23T22:19:01.143887+0000","last_clean":"2023-09-23T22:19:01.143887+0000","last_became_active":"2023-09-23T22:18:31.795608+0000","last_became_peered":"2023-09-23T22:18:31.795608+0000","last_unstale":"2023-09-23T22:19:01.143887+0000","last_undegraded":"2023-09-23T22:19:01.143887+0000","last_fullsized":"2023-09-23T22:19:01.143887+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:49.789927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.166257+0000","last_change":"2023-09-23T22:18:57.721257+0000","last_active":"2023-09-23T22:19:01.166257+0000","last_peered":"2023-09-23T22:19:01.166257+0000","last_clean":"2023-09-23T22:19:01.166257+0000","last_became_active":"2023-09-23T22:18:57.720548+0000","last_became_peered":"2023-09-23T22:18:57.720548+0000","last_unstale":"2023-09-23T22:19:01.166257+0000","last_undegraded":"2023-09-23T22:19:01.166257+0000","last_fullsized":"2023-09-23T22:19:01.166257+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:10.709024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":450,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.166437+0000","last_change":"2023-09-23T22:18:57.727276+0000","last_active":"2023-09-23T22:19:01.166437+0000","last_peered":"2023-09-23T22:19:01.166437+0000","last_clean":"2023-09-23T22:19:01.166437+0000","last_became_active":"2023-09-23T22:18:57.726938+0000","last_became_peered":"2023-09-23T22:18:57.726938+0000","last_unstale":"2023-09-23T22:19:01.166437+0000","last_undegraded":"2023-09-23T22:19:01.166437+0000","last_fullsized":"2023-09-23T22:19:01.166437+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":75,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.183245+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:19:01.183245+0000","last_peered":"2023-09-23T22:19:01.183245+0000","last_clean":"2023-09-23T22:19:01.183245+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:19:01.183245+0000","last_undegraded":"2023-09-23T22:19:01.183245+0000","last_fullsized":"2023-09-23T22:19:01.183245+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":126,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.130797+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:19:02.130797+0000","last_peered":"2023-09-23T22:19:02.130797+0000","last_clean":"2023-09-23T22:19:02.130797+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:19:02.130797+0000","last_undegraded":"2023-09-23T22:19:02.130797+0000","last_fullsized":"2023-09-23T22:19:02.130797+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.159459+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:19:01.159459+0000","last_peered":"2023-09-23T22:19:01.159459+0000","last_clean":"2023-09-23T22:19:01.159459+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:19:01.159459+0000","last_undegraded":"2023-09-23T22:19:01.159459+0000","last_fullsized":"2023-09-23T22:19:01.159459+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"186'10585","reported_seq":13932,"reported_epoch":186,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:11.024500+0000","last_change":"2023-09-23T22:18:57.739452+0000","last_active":"2023-09-23T22:19:11.024500+0000","last_peered":"2023-09-23T22:19:11.024500+0000","last_clean":"2023-09-23T22:16:48.102227+0000","last_became_active":"2023-09-23T22:16:50.283490+0000","last_became_peered":"2023-09-23T22:16:50.283490+0000","last_unstale":"2023-09-23T22:19:11.024500+0000","last_undegraded":"2023-09-23T22:17:00.207659+0000","last_fullsized":"2023-09-23T22:16:50.146447+0000","mapping_epoch":147,"log_start":"139'4677","ondisk_log_start":"139'4677","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:48.954402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88014848,"num_objects":1343,"num_object_clones":0,"num_object_copies":4029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4346,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10585,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1610,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5","objects":-3003},{"shards":"0,5,6","objects":1334}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":450,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.167196+0000","last_change":"2023-09-23T22:18:57.742449+0000","last_active":"2023-09-23T22:19:01.167196+0000","last_peered":"2023-09-23T22:19:01.167196+0000","last_clean":"2023-09-23T22:19:01.167196+0000","last_became_active":"2023-09-23T22:18:57.742146+0000","last_became_peered":"2023-09-23T22:18:57.742146+0000","last_unstale":"2023-09-23T22:19:01.167196+0000","last_undegraded":"2023-09-23T22:19:01.167196+0000","last_fullsized":"2023-09-23T22:19:01.167196+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":197,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131150+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:19:01.131150+0000","last_peered":"2023-09-23T22:19:01.131150+0000","last_clean":"2023-09-23T22:19:01.131150+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:19:01.131150+0000","last_undegraded":"2023-09-23T22:19:01.131150+0000","last_fullsized":"2023-09-23T22:19:01.131150+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":825,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.121323+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:19:01.121323+0000","last_peered":"2023-09-23T22:19:01.121323+0000","last_clean":"2023-09-23T22:19:01.121323+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:19:01.121323+0000","last_undegraded":"2023-09-23T22:19:01.121323+0000","last_fullsized":"2023-09-23T22:19:01.121323+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":182,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.130280+0000","last_change":"2023-09-23T22:18:57.722771+0000","last_active":"2023-09-23T22:19:01.130280+0000","last_peered":"2023-09-23T22:19:01.130280+0000","last_clean":"2023-09-23T22:19:01.130280+0000","last_became_active":"2023-09-23T22:18:57.722258+0000","last_became_peered":"2023-09-23T22:18:57.722258+0000","last_unstale":"2023-09-23T22:19:01.130280+0000","last_undegraded":"2023-09-23T22:19:01.130280+0000","last_fullsized":"2023-09-23T22:19:01.130280+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:18.267091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"186'10111","reported_seq":10186,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:07.285139+0000","last_change":"2023-09-23T22:15:14.828787+0000","last_active":"2023-09-23T22:19:07.285139+0000","last_peered":"2023-09-23T22:19:07.285139+0000","last_clean":"2023-09-23T22:19:07.285139+0000","last_became_active":"2023-09-23T22:15:14.828370+0000","last_became_peered":"2023-09-23T22:15:14.828370+0000","last_unstale":"2023-09-23T22:19:07.285139+0000","last_undegraded":"2023-09-23T22:19:07.285139+0000","last_fullsized":"2023-09-23T22:19:07.285139+0000","mapping_epoch":111,"log_start":"133'3370","ondisk_log_start":"133'3370","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6741,"log_dups_size":0,"ondisk_log_size":6741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:35.080268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93388800,"num_objects":1425,"num_object_clones":0,"num_object_copies":4275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10111,"num_write_kb":369152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":454,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.163826+0000","last_change":"2023-09-23T22:19:00.918399+0000","last_active":"2023-09-23T22:19:01.163826+0000","last_peered":"2023-09-23T22:19:01.163826+0000","last_clean":"2023-09-23T22:19:01.163826+0000","last_became_active":"2023-09-23T22:18:57.720614+0000","last_became_peered":"2023-09-23T22:18:57.720614+0000","last_unstale":"2023-09-23T22:19:01.163826+0000","last_undegraded":"2023-09-23T22:19:01.163826+0000","last_fullsized":"2023-09-23T22:19:01.163826+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":250,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.156297+0000","last_change":"2023-09-23T22:16:49.270015+0000","last_active":"2023-09-23T22:19:01.156297+0000","last_peered":"2023-09-23T22:19:01.156297+0000","last_clean":"2023-09-23T22:19:01.156297+0000","last_became_active":"2023-09-23T22:16:49.252726+0000","last_became_peered":"2023-09-23T22:16:49.252726+0000","last_unstale":"2023-09-23T22:19:01.156297+0000","last_undegraded":"2023-09-23T22:19:01.156297+0000","last_fullsized":"2023-09-23T22:19:01.156297+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:18.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":18,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.166378+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:19:01.166378+0000","last_peered":"2023-09-23T22:19:01.166378+0000","last_clean":"2023-09-23T22:19:01.166378+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:19:01.166378+0000","last_undegraded":"2023-09-23T22:19:01.166378+0000","last_fullsized":"2023-09-23T22:19:01.166378+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":149,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.129975+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:19:01.129975+0000","last_peered":"2023-09-23T22:19:01.129975+0000","last_clean":"2023-09-23T22:19:01.129975+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:19:01.129975+0000","last_undegraded":"2023-09-23T22:19:01.129975+0000","last_fullsized":"2023-09-23T22:19:01.129975+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"186'10226","reported_seq":10310,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:08.857062+0000","last_change":"2023-09-23T22:15:14.825150+0000","last_active":"2023-09-23T22:19:08.857062+0000","last_peered":"2023-09-23T22:19:08.857062+0000","last_clean":"2023-09-23T22:19:08.857062+0000","last_became_active":"2023-09-23T22:15:14.824867+0000","last_became_peered":"2023-09-23T22:15:14.824867+0000","last_unstale":"2023-09-23T22:19:08.857062+0000","last_undegraded":"2023-09-23T22:19:08.857062+0000","last_fullsized":"2023-09-23T22:19:08.857062+0000","mapping_epoch":111,"log_start":"132'3200","ondisk_log_start":"132'3200","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:30.053753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89391104,"num_objects":1364,"num_object_clones":0,"num_object_copies":4092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10226,"num_write_kb":370880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":14,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.159385+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:19:01.159385+0000","last_peered":"2023-09-23T22:19:01.159385+0000","last_clean":"2023-09-23T22:19:01.159385+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:19:01.159385+0000","last_undegraded":"2023-09-23T22:19:01.159385+0000","last_fullsized":"2023-09-23T22:19:01.159385+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":170,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.146231+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:19:01.146231+0000","last_peered":"2023-09-23T22:19:01.146231+0000","last_clean":"2023-09-23T22:19:01.146231+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:19:01.146231+0000","last_undegraded":"2023-09-23T22:19:01.146231+0000","last_fullsized":"2023-09-23T22:19:01.146231+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":148,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133679+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:19:01.133679+0000","last_peered":"2023-09-23T22:19:01.133679+0000","last_clean":"2023-09-23T22:19:01.133679+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:19:01.133679+0000","last_undegraded":"2023-09-23T22:19:01.133679+0000","last_fullsized":"2023-09-23T22:19:01.133679+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":161,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.128461+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:19:01.128461+0000","last_peered":"2023-09-23T22:19:01.128461+0000","last_clean":"2023-09-23T22:19:01.128461+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:19:01.128461+0000","last_undegraded":"2023-09-23T22:19:01.128461+0000","last_fullsized":"2023-09-23T22:19:01.128461+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":145,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133822+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:19:01.133822+0000","last_peered":"2023-09-23T22:19:01.133822+0000","last_clean":"2023-09-23T22:19:01.133822+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:19:01.133822+0000","last_undegraded":"2023-09-23T22:19:01.133822+0000","last_fullsized":"2023-09-23T22:19:01.133822+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":20,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127722+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:19:01.127722+0000","last_peered":"2023-09-23T22:19:01.127722+0000","last_clean":"2023-09-23T22:19:01.127722+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:19:01.127722+0000","last_undegraded":"2023-09-23T22:19:01.127722+0000","last_fullsized":"2023-09-23T22:19:01.127722+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":166,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.133160+0000","last_change":"2023-09-23T22:16:49.145417+0000","last_active":"2023-09-23T22:19:01.133160+0000","last_peered":"2023-09-23T22:19:01.133160+0000","last_clean":"2023-09-23T22:19:01.133160+0000","last_became_active":"2023-09-23T22:16:49.145017+0000","last_became_peered":"2023-09-23T22:16:49.145017+0000","last_unstale":"2023-09-23T22:19:01.133160+0000","last_undegraded":"2023-09-23T22:19:01.133160+0000","last_fullsized":"2023-09-23T22:19:01.133160+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:47.762149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":208,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.131817+0000","last_change":"2023-09-23T22:17:15.721123+0000","last_active":"2023-09-23T22:19:01.131817+0000","last_peered":"2023-09-23T22:19:01.131817+0000","last_clean":"2023-09-23T22:19:01.131817+0000","last_became_active":"2023-09-23T22:17:15.720812+0000","last_became_peered":"2023-09-23T22:17:15.720812+0000","last_unstale":"2023-09-23T22:19:01.131817+0000","last_undegraded":"2023-09-23T22:19:01.131817+0000","last_fullsized":"2023-09-23T22:19:01.131817+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.176278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":58,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.164518+0000","last_change":"2023-09-23T22:18:57.729455+0000","last_active":"2023-09-23T22:19:01.164518+0000","last_peered":"2023-09-23T22:19:01.164518+0000","last_clean":"2023-09-23T22:19:01.164518+0000","last_became_active":"2023-09-23T22:18:57.729088+0000","last_became_peered":"2023-09-23T22:18:57.729088+0000","last_unstale":"2023-09-23T22:19:01.164518+0000","last_undegraded":"2023-09-23T22:19:01.164518+0000","last_fullsized":"2023-09-23T22:19:01.164518+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:17.861334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":154,"reported_epoch":185,"state":"active+clean","last_fresh":"2023-09-23T22:19:01.127295+0000","last_change":"2023-09-23T22:16:49.164638+0000","last_active":"2023-09-23T22:19:01.127295+0000","last_peered":"2023-09-23T22:19:01.127295+0000","last_clean":"2023-09-23T22:19:01.127295+0000","last_became_active":"2023-09-23T22:16:49.164349+0000","last_became_peered":"2023-09-23T22:16:49.164349+0000","last_unstale":"2023-09-23T22:19:01.127295+0000","last_undegraded":"2023-09-23T22:19:01.127295+0000","last_fullsized":"2023-09-23T22:19:01.127295+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:26.536927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1421475864,"num_objects":21691,"num_object_clones":0,"num_object_copies":65073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15637,"num_objects_misplaced":1109,"num_objects_unfound":0,"num_objects_dirty":21691,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165921,"num_write_kb":6003521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27893,"num_bytes_recovered":1827930112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6435188736,"data_stored":6435176520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107367,"ondisk_log_size":107367,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4892,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2098784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234177,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":673124,"kb_used_data":590172,"kb_used_omap":0,"kb_used_meta":82944,"kb_avail":93072028,"statfs":{"total":95995035648,"available":95305756672,"internally_reserved":0,"allocated":604336128,"data_stored":603526646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":84934656},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":2.2080000000000002,"15min":1.105},"min":{"1min":0.23100000000000001,"5min":0.223,"15min":0.223},"max":{"1min":21.977,"5min":109.208,"15min":109.208},"last":0.309},{"interface":"front","average":{"1min":2.548,"5min":2.536,"15min":1.2370000000000001},"min":{"1min":0.28199999999999997,"5min":0.217,"15min":0.217},"max":{"1min":11.698,"5min":109.13800000000001,"15min":109.13800000000001},"last":7.4489999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":4.4370000000000003,"5min":4.2199999999999998,"15min":1.784},"min":{"1min":0.254,"5min":0.246,"15min":0.246},"max":{"1min":27.800000000000001,"5min":123.79300000000001,"15min":123.79300000000001},"last":5.5270000000000001},{"interface":"front","average":{"1min":4.3490000000000002,"5min":4.569,"15min":1.9139999999999999},"min":{"1min":0.316,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":27.785,"5min":123.66500000000001,"15min":123.66500000000001},"last":5.266}]},{"osd":2,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":6.2619999999999996,"5min":6.1609999999999996,"15min":2.7250000000000001},"min":{"1min":0.78600000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":37.698,"5min":125.101,"15min":125.101},"last":10.598000000000001},{"interface":"front","average":{"1min":5.4589999999999996,"5min":5.8220000000000001,"15min":2.3500000000000001},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":37.679000000000002,"5min":126.021,"15min":126.021},"last":4.2389999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":2.4710000000000001,"15min":1.2250000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.399,"5min":113.282,"15min":113.282},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.9769999999999999,"5min":2.8180000000000001,"15min":1.3859999999999999},"min":{"1min":0.41299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.2809999999999997,"5min":104.74299999999999,"15min":104.74299999999999},"last":7.2809999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.3009999999999999,"15min":0.749},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.251},"max":{"1min":10.305999999999999,"5min":33.627000000000002,"15min":33.627000000000002},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.335,"5min":1.111,"15min":0.70299999999999996},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.1300000000000008,"5min":33.113,"15min":33.113},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.0109999999999999,"15min":0.65700000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.9219999999999997,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.946,"5min":1.5369999999999999,"15min":0.88200000000000001},"min":{"1min":0.35499999999999998,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":10.423999999999999,"5min":31.257999999999999,"15min":31.257999999999999},"last":0.379}]},{"osd":6,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":0.999,"15min":0.68899999999999995},"min":{"1min":0.33700000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":7.9459999999999997,"5min":9.0139999999999993,"15min":9.0139999999999993},"last":0.66600000000000004},{"interface":"front","average":{"1min":2.2930000000000001,"5min":1.3500000000000001,"15min":0.84199999999999997},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.308},"max":{"1min":10.347,"5min":13.362,"15min":13.362},"last":0.53600000000000003}]}]},{"osd":6,"up_from":0,"seq":197568495813,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795085,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":721924,"kb_used_data":555644,"kb_used_omap":7,"kb_used_meta":166200,"kb_avail":93023228,"statfs":{"total":95995035648,"available":95255785472,"internally_reserved":0,"allocated":568979456,"data_stored":567965178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":170189172},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":1.472,"15min":0.72999999999999998},"min":{"1min":0.249,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":6.6859999999999999,"5min":61.890000000000001,"15min":61.890000000000001},"last":0.224},{"interface":"front","average":{"1min":3.1110000000000002,"5min":1.7569999999999999,"15min":0.85599999999999998},"min":{"1min":0.27300000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":13.243,"5min":62.689999999999998,"15min":62.689999999999998},"last":4.1719999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":5.3970000000000002,"5min":2.1629999999999998,"15min":0.97299999999999998},"min":{"1min":0.29099999999999998,"5min":0.216,"15min":0.216},"max":{"1min":33.902000000000001,"5min":33.902000000000001,"15min":33.902000000000001},"last":4.3300000000000001},{"interface":"front","average":{"1min":7.593,"5min":2.302,"15min":1.042},"min":{"1min":0.48399999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":34.781999999999996,"5min":34.781999999999996,"15min":34.781999999999996},"last":10.192}]},{"osd":3,"last update":"Sat Sep 23 22:18:13 2023","interfaces":[{"interface":"back","average":{"1min":3.492,"5min":1.4430000000000001,"15min":0.746},"min":{"1min":0.34499999999999997,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":13.198,"5min":22.326000000000001,"15min":22.326000000000001},"last":4.5590000000000002},{"interface":"front","average":{"1min":2.8759999999999999,"5min":1.4630000000000001,"15min":0.76100000000000001},"min":{"1min":0.28299999999999997,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":9.3599999999999994,"5min":22.259,"15min":22.259},"last":4.5800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":0.876,"15min":0.73699999999999999},"min":{"1min":0.28299999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.7530000000000001,"5min":6.9089999999999998,"15min":6.9089999999999998},"last":0.28899999999999998},{"interface":"front","average":{"1min":2.6960000000000002,"5min":1.121,"15min":0.81699999999999995},"min":{"1min":0.39600000000000002,"5min":0.318,"15min":0.318},"max":{"1min":9.1780000000000008,"5min":9.1780000000000008,"15min":9.1780000000000008},"last":4.3579999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5259999999999998,"5min":1.0640000000000001,"15min":0.76400000000000001},"min":{"1min":0.42999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":8.1539999999999999,"5min":8.1539999999999999,"15min":8.1539999999999999},"last":4.4080000000000004},{"interface":"front","average":{"1min":2.1320000000000001,"5min":1.111,"15min":0.79400000000000004},"min":{"1min":0.32700000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6130000000000004,"5min":6.6130000000000004,"15min":6.6130000000000004},"last":4.2850000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":1.1950000000000001,"15min":0.82499999999999996},"min":{"1min":0.38900000000000001,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.989000000000001,"5min":10.989000000000001,"15min":10.989000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.048,"15min":0.79800000000000004},"min":{"1min":0.38,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":5.7969999999999997,"5min":6.8700000000000001,"15min":6.8700000000000001},"last":4.0990000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":2.831,"5min":1.2849999999999999,"15min":0.85899999999999999},"min":{"1min":0.41199999999999998,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":10.948,"5min":10.948,"15min":10.948},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.153,"5min":1.1779999999999999,"15min":0.84599999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":7.8970000000000002,"5min":7.8970000000000002,"15min":7.8970000000000002},"last":4.1420000000000003}]}]},{"osd":0,"up_from":93,"seq":399431958620,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1122552,"kb_used_data":956336,"kb_used_omap":8,"kb_used_meta":166199,"kb_avail":92622600,"statfs":{"total":95995035648,"available":94845542400,"internally_reserved":0,"allocated":979288064,"data_stored":978284723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":170188650},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:18:09 2023","interfaces":[{"interface":"back","average":{"1min":4.4109999999999996,"5min":1.903,"15min":0.85899999999999999},"min":{"1min":0.21199999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":30.010999999999999,"5min":44.776000000000003,"15min":44.776000000000003},"last":1.1779999999999999},{"interface":"front","average":{"1min":5.0750000000000002,"5min":2.242,"15min":0.98999999999999999},"min":{"1min":0.27100000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":48.838000000000001,"5min":48.838000000000001,"15min":48.838000000000001},"last":1.1899999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:18:40 2023","interfaces":[{"interface":"back","average":{"1min":3.9319999999999999,"5min":1.502,"15min":0.76900000000000002},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":25.893000000000001,"5min":25.893000000000001,"15min":25.893000000000001},"last":3.3410000000000002},{"interface":"front","average":{"1min":7.742,"5min":2.7810000000000001,"15min":1.2749999999999999},"min":{"1min":0.28999999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":66.665000000000006,"5min":66.665000000000006,"15min":66.665000000000006},"last":1.532}]},{"osd":3,"last update":"Sat Sep 23 22:18:08 2023","interfaces":[{"interface":"back","average":{"1min":3.8399999999999999,"5min":1.843,"15min":0.91100000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":20.358000000000001,"5min":35.496000000000002,"15min":35.496000000000002},"last":4.1970000000000001},{"interface":"front","average":{"1min":3.2749999999999999,"5min":1.917,"15min":0.93200000000000005},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":20.422000000000001,"5min":35.533000000000001,"15min":35.533000000000001},"last":0.82399999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.2880000000000003,"5min":2.2050000000000001,"15min":1.1679999999999999},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.217000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":9.5830000000000002},{"interface":"front","average":{"1min":2.9860000000000002,"5min":1.2569999999999999,"15min":0.85799999999999998},"min":{"1min":0.32700000000000001,"5min":0.308,"15min":0.308},"max":{"1min":22.209,"5min":22.209,"15min":22.209},"last":0.41999999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.6959999999999997,"5min":2.173,"15min":1.1799999999999999},"min":{"1min":0.36099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.372,"5min":12.372,"15min":12.372},"last":9.5},{"interface":"front","average":{"1min":2.6699999999999999,"5min":1.286,"15min":0.88200000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":20.390999999999998,"5min":20.390999999999998,"15min":20.390999999999998},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.5609999999999999,"5min":2.2149999999999999,"15min":1.1879999999999999},"min":{"1min":0.46400000000000002,"5min":0.379,"15min":0.379},"max":{"1min":10.275,"5min":10.275,"15min":10.275},"last":9.4239999999999995},{"interface":"front","average":{"1min":2.867,"5min":1.3080000000000001,"15min":0.91000000000000003},"min":{"1min":0.38600000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":22.263000000000002,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.1180000000000003,"5min":2.3079999999999998,"15min":1.252},"min":{"1min":0.44500000000000001,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":20.491,"5min":23.306000000000001,"15min":23.306000000000001},"last":1.1639999999999999},{"interface":"front","average":{"1min":4.5170000000000003,"5min":2.6419999999999999,"15min":1.379},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.257999999999999,"5min":23.265999999999998,"15min":23.265999999999998},"last":9.6799999999999997}]}]},{"osd":2,"up_from":103,"seq":442381631559,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":685300,"kb_used_data":527148,"kb_used_omap":10,"kb_used_meta":157941,"kb_avail":93059852,"statfs":{"total":95995035648,"available":95293288448,"internally_reserved":0,"allocated":539799552,"data_stored":538801770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":161731756},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":4.7610000000000001,"5min":1.9370000000000001,"15min":0.86899999999999999},"min":{"1min":0.308,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":15.26,"5min":15.26,"15min":15.26},"last":0.80600000000000005},{"interface":"front","average":{"1min":4.7240000000000002,"5min":2.2749999999999999,"15min":1},"min":{"1min":0.21199999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":19.149999999999999,"5min":19.149999999999999,"15min":19.149999999999999},"last":3.2349999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":4.5199999999999996,"5min":3.9489999999999998,"15min":1.5980000000000001},"min":{"1min":0.32600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":19.869,"5min":157.636,"15min":157.636},"last":3.7000000000000002},{"interface":"front","average":{"1min":4.1710000000000003,"5min":4.0369999999999999,"15min":1.625},"min":{"1min":0.28100000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":17.768999999999998,"5min":155.63300000000001,"15min":155.63300000000001},"last":3.8220000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:19:05 2023","interfaces":[{"interface":"back","average":{"1min":4.1289999999999996,"5min":2.2949999999999999,"15min":0.99199999999999999},"min":{"1min":0.25,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":19.852,"5min":19.852,"15min":19.852},"last":0.78200000000000003},{"interface":"front","average":{"1min":4.4340000000000002,"5min":2.4009999999999998,"15min":1.0589999999999999},"min":{"1min":0.27900000000000003,"5min":0.246,"15min":0.246},"max":{"1min":20.581,"5min":20.581,"15min":20.581},"last":1.0760000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":1.5289999999999999,"15min":0.90800000000000003},"min":{"1min":0.26800000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.452,"5min":11.244,"15min":11.244},"last":3.1880000000000002},{"interface":"front","average":{"1min":2.7029999999999998,"5min":1.544,"15min":0.93200000000000005},"min":{"1min":0.372,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.2959999999999994,"5min":9.2799999999999994,"15min":9.2799999999999994},"last":0.69199999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":1.671,"15min":0.97999999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.5009999999999994,"5min":11.291,"15min":11.291},"last":3.2639999999999998},{"interface":"front","average":{"1min":2.911,"5min":1.617,"15min":0.94299999999999995},"min":{"1min":0.36799999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.327,"5min":9.4499999999999993,"15min":9.4499999999999993},"last":0.72699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":3.589,"5min":1.7130000000000001,"15min":1.014},"min":{"1min":0.38400000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":17.640999999999998,"5min":17.640999999999998,"15min":17.640999999999998},"last":3.2909999999999999},{"interface":"front","average":{"1min":2.8839999999999999,"5min":1.5169999999999999,"15min":1.0289999999999999},"min":{"1min":0.42299999999999999,"5min":0.375,"15min":0.375},"max":{"1min":11.875,"5min":11.875,"15min":11.875},"last":0.754}]},{"osd":7,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":3.8969999999999998,"5min":1.714,"15min":1.0780000000000001},"min":{"1min":0.38500000000000001,"5min":0.38,"15min":0.38},"max":{"1min":10.135,"5min":10.135,"15min":10.135},"last":2.2730000000000001},{"interface":"front","average":{"1min":3.46,"5min":1.9279999999999999,"15min":1.0669999999999999},"min":{"1min":0.51000000000000001,"5min":0.377,"15min":0.377},"max":{"1min":17.738,"5min":17.738,"15min":17.738},"last":5.649}]}]},{"osd":3,"up_from":108,"seq":463856468032,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":432944,"kb_used_data":361192,"kb_used_omap":4,"kb_used_meta":71739,"kb_avail":93312208,"statfs":{"total":95995035648,"available":95551700992,"internally_reserved":0,"allocated":369860608,"data_stored":368861822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":73461293},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":4.3410000000000002,"5min":2.738,"15min":1.1639999999999999},"min":{"1min":0.23300000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":32.731999999999999,"5min":77.834999999999994,"15min":77.834999999999994},"last":0.495},{"interface":"front","average":{"1min":4.8730000000000002,"5min":3.1019999999999999,"15min":1.2749999999999999},"min":{"1min":0.26500000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":32.786999999999999,"5min":75.558000000000007,"15min":75.558000000000007},"last":4.3810000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":7.4589999999999996,"5min":3.5379999999999998,"15min":1.4139999999999999},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":82.951999999999998,"5min":82.951999999999998,"15min":82.951999999999998},"last":3.859},{"interface":"front","average":{"1min":9.4529999999999994,"5min":3.601,"15min":1.4730000000000001},"min":{"1min":0.22600000000000001,"5min":0.189,"15min":0.189},"max":{"1min":86.001000000000005,"5min":86.001000000000005,"15min":86.001000000000005},"last":0.54800000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":17.736000000000001,"5min":7.7190000000000003,"15min":2.835},"min":{"1min":0.28000000000000003,"5min":0.223,"15min":0.223},"max":{"1min":99.962999999999994,"5min":99.962999999999994,"15min":99.962999999999994},"last":0.53100000000000003},{"interface":"front","average":{"1min":18.577000000000002,"5min":7.1669999999999998,"15min":2.6720000000000002},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":100.429,"5min":100.429,"15min":100.429},"last":0.52400000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:18:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":0.93999999999999995,"15min":0.72999999999999998},"min":{"1min":0.316,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.2329999999999997,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.7989999999999999,"5min":1.1879999999999999,"15min":0.83299999999999996},"min":{"1min":0.44800000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":12.036,"5min":12.036,"15min":12.036},"last":4.1929999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.397,"15min":0.88200000000000001},"min":{"1min":0.32800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.6529999999999996,"5min":14.292,"15min":14.292},"last":4.2229999999999999},{"interface":"front","average":{"1min":2.726,"5min":1.526,"15min":0.91500000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.484,"5min":11.56,"15min":11.56},"last":0.443}]},{"osd":6,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":2.6190000000000002,"5min":1.6100000000000001,"15min":0.97199999999999998},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.948,"5min":12.068,"15min":12.068},"last":4.1449999999999996},{"interface":"front","average":{"1min":2.0939999999999999,"5min":1.52,"15min":0.93000000000000005},"min":{"1min":0.35599999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.0660000000000007,"5min":11.586,"15min":11.586},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":1.607,"15min":0.95599999999999996},"min":{"1min":0.39600000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":9.1630000000000003,"5min":11.624000000000001,"15min":11.624000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.504,"5min":1.405,"15min":0.92400000000000004},"min":{"1min":0.40400000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.7880000000000003,"5min":12.382,"15min":12.382},"last":3.2970000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019083,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":886108,"kb_used_data":786004,"kb_used_omap":0,"kb_used_meta":100096,"kb_avail":92859044,"statfs":{"total":95995035648,"available":95087661056,"internally_reserved":0,"allocated":804868096,"data_stored":804066491,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":102498304},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.9970000000000001,"15min":1.0840000000000001},"min":{"1min":0.23000000000000001,"5min":0.218,"15min":0.218},"max":{"1min":7.5789999999999997,"5min":66.355000000000004,"15min":66.355000000000004},"last":0.316},{"interface":"front","average":{"1min":1.4299999999999999,"5min":2.2200000000000002,"15min":1.1699999999999999},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.5949999999999998,"5min":66.332999999999998,"15min":66.332999999999998},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":1.748,"15min":0.94599999999999995},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.625,"5min":38.878,"15min":38.878},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.4220000000000002,"15min":1.1899999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":10.236000000000001,"5min":38.853999999999999,"15min":38.853999999999999},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:18:40 2023","interfaces":[{"interface":"back","average":{"1min":8.1180000000000003,"5min":2.8780000000000001,"15min":1.3200000000000001},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.255},"max":{"1min":133.22900000000001,"5min":133.22900000000001,"15min":133.22900000000001},"last":0.76500000000000001},{"interface":"front","average":{"1min":8.4529999999999994,"5min":3.4510000000000001,"15min":1.4910000000000001},"min":{"1min":0.33000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":133.60300000000001,"5min":133.60300000000001,"15min":133.60300000000001},"last":1.085}]},{"osd":3,"last update":"Sat Sep 23 22:18:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2330000000000001,"5min":1.1739999999999999,"15min":0.78700000000000003},"min":{"1min":0.32600000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.7110000000000003,"5min":7.7110000000000003,"15min":7.7110000000000003},"last":1.591},{"interface":"front","average":{"1min":2.5880000000000001,"5min":1.288,"15min":0.81299999999999994},"min":{"1min":0.38100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":9.8109999999999999,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":0.40300000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:18:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":0.86399999999999999,"15min":0.60199999999999998},"min":{"1min":0.29399999999999998,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":8.6050000000000004,"5min":8.9499999999999993,"15min":8.9499999999999993},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.252,"5min":0.79400000000000004,"15min":0.623},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.8789999999999996,"5min":7.8789999999999996,"15min":7.8789999999999996},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:18:24 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.75,"15min":0.58599999999999997},"min":{"1min":0.309,"5min":0.251,"15min":0.251},"max":{"1min":3.6699999999999999,"5min":3.6699999999999999,"15min":3.6699999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.84599999999999997,"15min":0.64600000000000002},"min":{"1min":0.32100000000000001,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":3.052,"5min":8.5210000000000008,"15min":8.5210000000000008},"last":0.50600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.1180000000000001,"15min":0.76000000000000001},"min":{"1min":0.33100000000000002,"5min":0.307,"15min":0.307},"max":{"1min":5.3440000000000003,"5min":8.577,"15min":8.577},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.90400000000000003,"15min":0.69299999999999995},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.7799999999999998,"5min":5.6280000000000001,"15min":5.6280000000000001},"last":0.56100000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757450,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1120148,"kb_used_data":1004620,"kb_used_omap":0,"kb_used_meta":115520,"kb_avail":92625004,"statfs":{"total":95995035648,"available":94848004096,"internally_reserved":0,"allocated":1028730880,"data_stored":1027933254,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":118292480},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":3.113,"15min":1.4139999999999999},"min":{"1min":0.26800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.6849999999999996,"5min":133.22499999999999,"15min":133.22499999999999},"last":4},{"interface":"front","average":{"1min":1.732,"5min":3.387,"15min":1.554},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.444,"5min":132.78200000000001,"15min":132.78200000000001},"last":0.28999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":7.7859999999999996,"5min":4.5830000000000002,"15min":1.9219999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":83.759,"5min":83.759,"15min":83.759},"last":1.423},{"interface":"front","average":{"1min":7.8920000000000003,"5min":4.6139999999999999,"15min":1.9179999999999999},"min":{"1min":0.30399999999999999,"5min":0.252,"15min":0.252},"max":{"1min":83.741,"5min":83.741,"15min":83.741},"last":1.4370000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:18:39 2023","interfaces":[{"interface":"back","average":{"1min":12.324999999999999,"5min":6.0250000000000004,"15min":2.383},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":104.254,"5min":104.254,"15min":104.254},"last":0.40600000000000003},{"interface":"front","average":{"1min":12.984,"5min":6.6600000000000001,"15min":2.6139999999999999},"min":{"1min":0.34699999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":104.217,"5min":104.217,"15min":104.217},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.615,"5min":2.6389999999999998,"15min":1.2649999999999999},"min":{"1min":0.31900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":4.7549999999999999,"5min":88.106999999999999,"15min":88.106999999999999},"last":1.7589999999999999},{"interface":"front","average":{"1min":2.3929999999999998,"5min":2.8610000000000002,"15min":1.3460000000000001},"min":{"1min":0.38800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1369999999999996,"5min":87.887,"15min":87.887},"last":2.6299999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:19:01 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.6559999999999999,"15min":0.86399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.4880000000000004,"5min":82.530000000000001,"15min":82.530000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.7070000000000001,"15min":0.91800000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.577,"5min":85.441000000000003,"15min":85.441000000000003},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:18:30 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.117,"15min":1.0509999999999999},"min":{"1min":0.32800000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.6519999999999992,"5min":114.83499999999999,"15min":114.83499999999999},"last":0.439},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.9119999999999999,"15min":0.995},"min":{"1min":0.31,"5min":0.31,"15min":0.29699999999999999},"max":{"1min":7.4640000000000004,"5min":110.905,"15min":110.905},"last":0.46100000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:18:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.1850000000000001,"15min":0.77200000000000002},"min":{"1min":0.35899999999999999,"5min":0.33500000000000002,"15min":0.32600000000000001},"max":{"1min":8.2159999999999993,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":1.615},{"interface":"front","average":{"1min":1.1950000000000001,"5min":0.97599999999999998,"15min":0.69699999999999995},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.33200000000000002},"max":{"1min":5.2460000000000004,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":0.57399999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":970522624,"data_stored":970522624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":551288832,"data_stored":551288832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":528027648,"data_stored":528023576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":365428736,"data_stored":365428736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":798687232,"data_stored":798687232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1021841408,"data_stored":1021837336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1607532544,"data_stored":1607532544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":591859712,"data_stored":591855640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:19:13.768 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:19:13.809 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:13.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:13.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:13.860 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:13.915 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:13.955 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:13.980 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:13 smithi070 ceph-mon[192652]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:13 smithi026 ceph-mon[215775]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:13 smithi026 ceph-mon[223014]: from='client.54724 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:14.022 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:14.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:14.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:14.131 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:14.229 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:14.356 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:14.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:14.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:14.465 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:14.535 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:14.777 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:14.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:14.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:14.881 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:14.957 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:15.122 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:14 smithi070 ceph-mon[192652]: pgmap v418: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.3 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 15638/63474 objects degraded (24.637%); 1109/63474 objects misplaced (1.747%); 6.3 MiB/s, 95 objects/s recovering 2023-09-23T22:19:15.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:14 smithi026 ceph-mon[223014]: pgmap v418: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.3 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 15638/63474 objects degraded (24.637%); 1109/63474 objects misplaced (1.747%); 6.3 MiB/s, 95 objects/s recovering 2023-09-23T22:19:15.236 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:15.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:14 smithi026 ceph-mon[215775]: pgmap v418: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.3 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 15638/63474 objects degraded (24.637%); 1109/63474 objects misplaced (1.747%); 6.3 MiB/s, 95 objects/s recovering 2023-09-23T22:19:15.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:15.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:15.349 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:15.427 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:15.536 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:15.594 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:15.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:15.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:15.704 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:19:15.726 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:15.768 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:15.851 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:15.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:15.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:15 smithi070 ceph-mon[192652]: pgmap v419: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.3 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 15638/61872 objects degraded (25.275%); 1109/61872 objects misplaced (1.792%); 6.3 MiB/s, 95 objects/s recovering 2023-09-23T22:19:15.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:15.877 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:19:15.877 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:19:15.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:16.075 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:15 smithi026 ceph-mon[215775]: pgmap v419: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.3 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 15638/61872 objects degraded (25.275%); 1109/61872 objects misplaced (1.792%); 6.3 MiB/s, 95 objects/s recovering 2023-09-23T22:19:16.075 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:15 smithi026 ceph-mon[223014]: pgmap v419: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.3 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 15638/61872 objects degraded (25.275%); 1109/61872 objects misplaced (1.792%); 6.3 MiB/s, 95 objects/s recovering 2023-09-23T22:19:16.109 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:16.590 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:16.591 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:19:16.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:16.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:16.637 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:19:16.670 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:19:16.701 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:16.711 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:19:16.755 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:19:16.760 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:16.809 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:19:16.856 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:19:16.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:19:16.925 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:16.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:16.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:16.941 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:19:16.941 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:19:16.941 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:19:16.941 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:19:16.941 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:19:16.941 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:19:16.941 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:19:16.941 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:19:17.032 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:17.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1211412621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:17.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1211412621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:17.117 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:17.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1211412621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:17.156 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:19:17.725 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:17.836 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:17.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:17.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:17.950 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:18.012 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:18.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:17 smithi070 ceph-mon[192652]: pgmap v420: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.2 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 15456/56679 objects degraded (27.269%); 812/56679 objects misplaced (1.433%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:19:18.134 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:17 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15638/61872 objects degraded (25.275%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:18.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:17 smithi026 ceph-mon[223014]: pgmap v420: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.2 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 15456/56679 objects degraded (27.269%); 812/56679 objects misplaced (1.433%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:19:18.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:17 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15638/61872 objects degraded (25.275%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:18.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:17 smithi026 ceph-mon[215775]: pgmap v420: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.2 GiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 15456/56679 objects degraded (27.269%); 812/56679 objects misplaced (1.433%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:19:18.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:17 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15638/61872 objects degraded (25.275%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:18.247 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:18.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:18.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:18.358 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:18.399 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:18.512 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:18.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:18.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:18.616 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:18.722 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:18.861 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:19.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:19.051 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:19.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:19.152 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:19.198 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:19.321 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:19.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:19.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:19.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:19 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:19.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:19 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:19.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:19 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:19.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:19 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:19.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:19 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:19 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:19.431 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:19.461 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:19.580 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:19.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:19.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:19.622 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:19:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:19:19] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2023-09-23T22:19:19.626 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:19:19.626 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15456/56679 objects degraded (27.269%), 6 pgs degraded, 2 pgs undersized","count":8},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":543,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":187,"num_osds":8,"num_up_osds":8,"osd_up_since":1695507239,"num_in_osds":7,"osd_in_since":1695507408,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":92},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":99,"num_pools":5,"num_objects":18893,"data_bytes":1241061323,"bytes_used":5273432064,"bytes_avail":666691817472,"bytes_total":671965249536,"degraded_objects":15456,"degraded_total":56679,"degraded_ratio":0.27269359021859946,"misplaced_objects":812,"misplaced_total":56679,"misplaced_ratio":0.014326293689020626,"recovering_objects_per_sec":97,"recovering_bytes_per_sec":6361540,"recovering_keys_per_sec":0,"num_objects_recovered":971,"num_bytes_recovered":63635456,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":574},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T22:14:59.012277+0000","services":{}},"progress_events":{"2fcb4d05-59c9-4a0a-bf68-bf36661520e1":{"message":"Global Recovery Event (3m)\n [==========================..] (remaining: 17s)","progress":0.92929291725158691,"add_to_ceph_s":true},"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:19:19.685 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:19.739 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:19.800 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:19.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:19.840 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:19:19.840 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:19:19.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19249 5.e 5:741020f8:::benchmark_data_smithi026_286012_object19248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.612049+0000", 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 231.22575492199999, 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0748528140000002, 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.841 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19249 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612049+0000", 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612049+0000", 2023-09-23T22:19:19.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612048+0000", 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612100+0000", 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2673999999999997e-05 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612113+0000", 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2835e-05 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612118+0000", 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9479999999999999e-06 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612163+0000", 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4447999999999999e-05 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612311+0000", 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000148877 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612464+0000", 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000152452 2023-09-23T22:19:19.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.613528+0000", 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010644700000000001 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.613796+0000", 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026775600000000002 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.625193+0000", 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011396692999999999 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.625207+0000", 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4555e-05 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.622896+0000", 2023-09-23T22:19:19.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.997689041 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.622909+0000", 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2702e-05 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.686798+0000", 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0638887100000001 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.686902+0000", 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104128 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37003 5.e 5:74658dc9:::benchmark_data_smithi026_286012_object37002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.939383+0000", 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 216.89842053500001, 2023-09-23T22:19:19.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0753498450000001, 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37003 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939383+0000", 2023-09-23T22:19:19.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939383+0000", 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939382+0000", 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939413+0000", 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0855000000000002e-05 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939426+0000", 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3127e-05 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939431+0000", 2023-09-23T22:19:19.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0690000000000001e-06 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939491+0000", 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0344999999999999e-05 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.939504+0000", 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2612999999999999e-05 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.006978+0000", 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0674743799999999 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.007104+0000", 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012563799999999999 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:19.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.007235+0000", 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000130636 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.009090+0000", 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018550039999999999 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.009127+0000", 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7441999999999998e-05 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.013897+0000", 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047696400000000003 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.013966+0000", 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9115000000000005e-05 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.014733+0000", 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00076754000000000004 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:100913 5.9 5:991cdcd6:::benchmark_data_smithi026_286012_object7105:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.306330+0000", 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.531474172, 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.209402436, 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.857 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100913 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.306330+0000", 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.306330+0000", 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.306329+0000", 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.306334+0000", 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6789999999999996e-06 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.306342+0000", 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2260000000000005e-06 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.306346+0000", 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7409999999999998e-06 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.309159+0000", 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028131200000000001 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.309293+0000", 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000133774 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.309386+0000", 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3979000000000003e-05 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.315350+0000", 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059636489999999997 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.315357+0000", 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1980000000000004e-06 2023-09-23T22:19:19.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.513585+0000", 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19822747499999999 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.513636+0000", 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0973999999999997e-05 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.515732+0000", 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020964809999999999 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:100917 5.9 5:9a1e71ed:::benchmark_data_smithi026_286012_object7109:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e161)", 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.313929+0000", 2023-09-23T22:19:19.864 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.523874761, 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.207139079, 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 100917 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.313929+0000", 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.313929+0000", 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.313928+0000", 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.313935+0000", 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.703e-06 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.313957+0000", 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2195999999999999e-05 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.313964+0000", 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8639999999999998e-06 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.315381+0000", 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014172189999999999 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.315486+0000", 2023-09-23T22:19:19.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010512400000000001 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.315625+0000", 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013886200000000001 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.322587+0000", 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069619870000000002 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.322627+0000", 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.032e-05 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.519190+0000", 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19656256799999999 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.519248+0000", 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8483000000000002e-05 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.521068+0000", 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001820185 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.871 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161176 5.9 5:945d6bac:::benchmark_data_smithi026_286012_object67368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.166084+0000", 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.671719947, 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99363560299999998, 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161176 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.872 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166084+0000", 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166084+0000", 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166082+0000", 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166091+0000", 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7220000000000005e-06 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166109+0000", 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8105e-05 2023-09-23T22:19:19.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166116+0000", 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4440000000000002e-06 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166180+0000", 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3662000000000001e-05 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166194+0000", 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4032000000000001e-05 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.156511+0000", 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99031726200000003 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.156629+0000", 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011803600000000001 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.156707+0000", 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7717000000000001e-05 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159591+0000", 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002884148 2023-09-23T22:19:19.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159602+0000", 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1090999999999999e-05 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159659+0000", 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7086000000000002e-05 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159698+0000", 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8359000000000003e-05 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159720+0000", 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1844e-05 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.879 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161183 5.9 5:9d2505ba:::benchmark_data_smithi026_286012_object67375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.174941+0000", 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.662863154, 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99311035800000003, 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161183 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174941+0000", 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174941+0000", 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174940+0000", 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174956+0000", 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6464e-05 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174973+0000", 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6818000000000001e-05 2023-09-23T22:19:19.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174978+0000", 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5899999999999998e-06 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.175022+0000", 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4005999999999998e-05 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.175030+0000", 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1430000000000001e-06 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159758+0000", 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98472881700000003 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159842+0000", 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3813999999999994e-05 2023-09-23T22:19:19.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159906+0000", 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3689000000000006e-05 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.167923+0000", 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0080168219999999998 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.167986+0000", 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2785000000000003e-05 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.167992+0000", 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7510000000000002e-06 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168025+0000", 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2543999999999999e-05 2023-09-23T22:19:19.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168051+0000", 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6344000000000001e-05 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161184 5.9 5:9ee6e8a5:::benchmark_data_smithi026_286012_object67376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.184105+0000", 2023-09-23T22:19:19.887 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.653699027999998, 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98725831399999997, 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161184 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184105+0000", 2023-09-23T22:19:19.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184105+0000", 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184104+0000", 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184109+0000", 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2270000000000001e-06 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184123+0000", 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3202e-05 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184127+0000", 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7500000000000003e-06 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184175+0000", 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7769000000000003e-05 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184184+0000", 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9900000000000003e-06 2023-09-23T22:19:19.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.162842+0000", 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97865803299999998 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.162967+0000", 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012465299999999999 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.163088+0000", 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012078699999999999 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168063+0000", 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049753080000000003 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168098+0000", 2023-09-23T22:19:19.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4718000000000002e-05 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171274+0000", 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031765180000000001 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171324+0000", 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9665e-05 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171363+0000", 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9535e-05 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161224 5.9 5:9cc03ca7:::benchmark_data_smithi026_286012_object67416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.255475+0000", 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.582328911000001, 2023-09-23T22:19:19.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91600351700000004, 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161224 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255475+0000", 2023-09-23T22:19:19.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255475+0000", 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255474+0000", 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255482+0000", 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8900000000000007e-06 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255495+0000", 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3318e-05 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255501+0000", 2023-09-23T22:19:19.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9490000000000001e-06 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255544+0000", 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3299e-05 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255551+0000", 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7850000000000003e-06 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.165528+0000", 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90997731400000004 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.165626+0000", 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7516000000000006e-05 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.165692+0000", 2023-09-23T22:19:19.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5684000000000003e-05 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171210+0000", 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055180790000000004 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171245+0000", 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5228999999999998e-05 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171403+0000", 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015777799999999999 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171445+0000", 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2277000000000002e-05 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171479+0000", 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3555e-05 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161261 5.9 5:9621e124:::benchmark_data_smithi026_286012_object67453:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.357700+0000", 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.480103516, 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81973434199999995, 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.903 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161261 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.357700+0000", 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.357700+0000", 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.357699+0000", 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.357706+0000", 2023-09-23T22:19:19.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3790000000000003e-06 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.357719+0000", 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3013999999999999e-05 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.357723+0000", 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8760000000000004e-06 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.357773+0000", 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9872e-05 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.357780+0000", 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5690000000000001e-06 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168121+0000", 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81034072599999996 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168205+0000", 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4839000000000005e-05 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168297+0000", 2023-09-23T22:19:19.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1331999999999999e-05 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.177294+0000", 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0089970540000000009 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.177306+0000", 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1732999999999999e-05 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.177344+0000", 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8936999999999997e-05 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.177395+0000", 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0157000000000001e-05 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.177435+0000", 2023-09-23T22:19:19.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0207000000000003e-05 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161264 5.9 5:9a8c542e:::benchmark_data_smithi026_286012_object67456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.364135+0000", 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.473668543999999, 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.81912996400000004, 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.911 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161264 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.364135+0000", 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.364135+0000", 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.364134+0000", 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.364144+0000", 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5540000000000003e-06 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.364157+0000", 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.359e-05 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.364162+0000", 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8849999999999998e-06 2023-09-23T22:19:19.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.364214+0000", 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1198e-05 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.364225+0000", 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1447000000000001e-05 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171501+0000", 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.80727624399999998 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171604+0000", 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010249 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171709+0000", 2023-09-23T22:19:19.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105688 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.180293+0000", 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0085836739999999995 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183156+0000", 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028628389999999998 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183189+0000", 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3027000000000001e-05 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183224+0000", 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4656000000000002e-05 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183265+0000", 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1817999999999998e-05 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161291 5.9 5:9a4a0f1c:::benchmark_data_smithi026_286012_object67483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.415674+0000", 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.422129907999999, 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.76777367100000005, 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.919 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161291 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.415674+0000", 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.415674+0000", 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.415673+0000", 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.415680+0000", 2023-09-23T22:19:19.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.559e-06 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.415696+0000", 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5786000000000001e-05 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.415702+0000", 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1550000000000004e-06 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.415759+0000", 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7302000000000001e-05 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.415771+0000", 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1681e-05 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.174489+0000", 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75871752800000003 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.174585+0000", 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6544000000000003e-05 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.174648+0000", 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2614999999999996e-05 2023-09-23T22:19:19.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183164+0000", 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0085158860000000003 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183331+0000", 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016722899999999999 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183373+0000", 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2602999999999999e-05 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183413+0000", 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9530999999999998e-05 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183448+0000", 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4786000000000002e-05 2023-09-23T22:19:19.926 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161303 5.9 5:9534c909:::benchmark_data_smithi026_286012_object67495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.438163+0000", 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.399641348999999, 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.754254011, 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.927 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161303 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.438163+0000", 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.438163+0000", 2023-09-23T22:19:19.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.438161+0000", 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.438169+0000", 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7170000000000008e-06 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.438187+0000", 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8391000000000001e-05 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.438197+0000", 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9569999999999997e-06 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.438252+0000", 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4627999999999999e-05 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.438260+0000", 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.541e-06 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.177482+0000", 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73922193800000002 2023-09-23T22:19:19.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.177603+0000", 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121144 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.177712+0000", 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010833 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183339+0000", 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056276269999999996 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183490+0000", 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015095899999999999 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192341+0000", 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0088510370000000008 2023-09-23T22:19:19.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192380+0000", 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8574e-05 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192417+0000", 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6773000000000002e-05 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.934 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161318 5.9 5:986bb684:::benchmark_data_smithi026_286012_object67510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.480230+0000", 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.35757371, 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.71237420399999996, 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.935 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161318 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.480230+0000", 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.480230+0000", 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.480230+0000", 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.480232+0000", 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.261e-06 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.480238+0000", 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5520000000000004e-06 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.480240+0000", 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.537e-06 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.480301+0000", 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0692000000000001e-05 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.480311+0000", 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8959999999999998e-06 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.180341+0000", 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.70003020999999999 2023-09-23T22:19:19.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.180459+0000", 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118168 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.180575+0000", 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011590399999999999 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183523+0000", 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029480520000000001 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192260+0000", 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0087367320000000005 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192533+0000", 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000273033 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192577+0000", 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4357000000000002e-05 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192604+0000", 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7121e-05 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161364 5.9 5:9ec735ef:::benchmark_data_smithi026_286012_object67556:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.605126+0000", 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.232677855999999, 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.58761849099999997, 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161364 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.943 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.605126+0000", 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.605126+0000", 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.605125+0000", 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.605131+0000", 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7169999999999996e-06 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.605143+0000", 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2417e-05 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.605148+0000", 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0860000000000001e-06 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.605195+0000", 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7057e-05 2023-09-23T22:19:19.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.605204+0000", 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2479999999999996e-06 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183545+0000", 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57834091600000004 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183647+0000", 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102477 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.183749+0000", 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102225 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192296+0000", 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0085465300000000001 2023-09-23T22:19:19.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192302+0000", 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0800000000000002e-06 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192659+0000", 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035761799999999998 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192708+0000", 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8466e-05 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192745+0000", 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6733999999999998e-05 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.950 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161369 5.9 5:9725bd27:::benchmark_data_smithi026_286012_object67561:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.616643+0000", 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.221160981000001, 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57624556699999996, 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161369 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.616643+0000", 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.616643+0000", 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.616642+0000", 2023-09-23T22:19:19.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.616649+0000", 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0829999999999998e-06 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.616664+0000", 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.504e-05 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.616670+0000", 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6069999999999996e-06 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.616746+0000", 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5668999999999997e-05 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.616753+0000", 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8229999999999997e-06 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.186437+0000", 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.56968372499999997 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.186534+0000", 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7323000000000003e-05 2023-09-23T22:19:19.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.186596+0000", 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1817999999999997e-05 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192306+0000", 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057104640000000002 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192475+0000", 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016826800000000001 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192807+0000", 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000332121 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192851+0000", 2023-09-23T22:19:19.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4443000000000001e-05 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192889+0000", 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7511000000000001e-05 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161373 5.9 5:9c679044:::benchmark_data_smithi026_286012_object67565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.958 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.626696+0000", 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.211108478, 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.56979785900000002, 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161373 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.959 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.626696+0000", 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.626696+0000", 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.626694+0000", 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.626701+0000", 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2450000000000001e-06 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.626717+0000", 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5435e-05 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.626723+0000", 2023-09-23T22:19:19.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4999999999999996e-06 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.626788+0000", 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4948000000000002e-05 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.626802+0000", 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4107999999999999e-05 2023-09-23T22:19:19.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.189513+0000", 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.56271090400000001 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.189630+0000", 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011644100000000001 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.189741+0000", 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110803 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.196313+0000", 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0065719910000000001 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.196405+0000", 2023-09-23T22:19:19.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2546999999999993e-05 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.196412+0000", 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9140000000000002e-06 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.196451+0000", 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9434999999999998e-05 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.196493+0000", 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1906999999999997e-05 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161407 5.9 5:98681d1d:::benchmark_data_smithi026_286012_object67599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.700472+0000", 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.137332174000001, 2023-09-23T22:19:19.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.49929712300000001, 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161407 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.700472+0000", 2023-09-23T22:19:19.967 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.700472+0000", 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.700471+0000", 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.700476+0000", 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7360000000000001e-06 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.700487+0000", 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1188e-05 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.700491+0000", 2023-09-23T22:19:19.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5950000000000001e-06 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.700534+0000", 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2970999999999999e-05 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.700540+0000", 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9699999999999996e-06 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.192931+0000", 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.49239071600000001 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.193045+0000", 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011417999999999999 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.193156+0000", 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110432 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.196357+0000", 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032014439999999999 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.199643+0000", 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032856209999999999 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.199675+0000", 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2394999999999997e-05 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.199725+0000", 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9897e-05 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.199769+0000", 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3884999999999997e-05 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161432 5.9 5:94d46212:::benchmark_data_smithi026_286012_object67624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.753392+0000", 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.084411985999999, 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.46373786900000002, 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.974 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161432 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.753392+0000", 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.753392+0000", 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.753391+0000", 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.753398+0000", 2023-09-23T22:19:19.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8979999999999999e-06 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.753411+0000", 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3733999999999999e-05 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.753434+0000", 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2575999999999999e-05 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.753497+0000", 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3351999999999999e-05 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.753508+0000", 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0825000000000001e-05 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.196528+0000", 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.44301942300000002 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.196642+0000", 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114434 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.196745+0000", 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102773 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.206730+0000", 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0099855240000000008 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.206798+0000", 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7749999999999993e-05 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.217045+0000", 2023-09-23T22:19:19.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010247300000000001 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.217093+0000", 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7777e-05 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.217130+0000", 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6708000000000002e-05 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.981 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161436 5.9 5:9c01565e:::benchmark_data_smithi026_286012_object67628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.760790+0000", 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.077014393999999, 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.45652717500000001, 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.982 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161436 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.760790+0000", 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.760790+0000", 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.760789+0000", 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.760796+0000", 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3549999999999999e-06 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.760809+0000", 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2516e-05 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.760815+0000", 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9070000000000004e-06 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.760861+0000", 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6623999999999999e-05 2023-09-23T22:19:19.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.760869+0000", 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0749999999999998e-06 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.199860+0000", 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.43899058899999999 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.199989+0000", 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000129469 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.200071+0000", 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1667000000000003e-05 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.206791+0000", 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067203080000000004 2023-09-23T22:19:19.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.206803+0000", 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1636e-05 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.217232+0000", 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010429081999999999 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.217285+0000", 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3173000000000001e-05 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.217317+0000", 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1557999999999998e-05 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.989 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:171055 5.e 5:7e0f0b19:::benchmark_data_smithi026_286012_object77247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:18.713082+0000", 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.124721992, 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.213489019, 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 171055 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:19.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.713082+0000", 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.713082+0000", 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:19.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.713081+0000", 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.713088+0000", 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8080000000000005e-06 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.885924+0000", 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.172835508 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.885974+0000", 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9672999999999997e-05 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.886030+0000", 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6589000000000003e-05 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:19.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.886162+0000", 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131748 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.886289+0000", 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012667900000000001 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.896137+0000", 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0098482670000000008 2023-09-23T22:19:19.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.896186+0000", 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8760000000000001e-05 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.926468+0000", 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030282839999999998 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:19.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.926526+0000", 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7469000000000002e-05 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:18.926571+0000", 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5130999999999999e-05 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:19.996 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:19:19.997 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:20.039 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:20.117 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:20.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:20.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:20.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:20.246 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:20.288 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:20.290 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:20 smithi070 ceph-mon[192652]: pgmap v421: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.1 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 15456/53436 objects degraded (28.924%); 812/53436 objects misplaced (1.520%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:19:20.290 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1334454296' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:19:20.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:20.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:20.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:20.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:20 smithi026 ceph-mon[215775]: pgmap v421: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.1 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 15456/53436 objects degraded (28.924%); 812/53436 objects misplaced (1.520%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:19:20.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1334454296' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:19:20.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:20 smithi026 ceph-mon[223014]: pgmap v421: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.1 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 15456/53436 objects degraded (28.924%); 812/53436 objects misplaced (1.520%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:19:20.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1334454296' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:19:20.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:19:20.484 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:20.603 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:20.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:20.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:20.707 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:20.759 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:19:20.779 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:21.192 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:21.456 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:21.458 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:21.459 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:21.459 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:19:21.459 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:19:21.459 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:21.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:21.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:21.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:21.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:19:21.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:19:21.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:21.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:21.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:21.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:19:21.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:19:21.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:19:21.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:21.483 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:19:21.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:19:21.583 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:21.667 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:21.915 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:21.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:21.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:22.023 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:22.051 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:22.162 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:22.359 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:22.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:22 smithi026 ceph-mon[215775]: pgmap v422: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 15456/52374 objects degraded (29.511%); 516/52374 objects misplaced (0.985%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:19:22.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:22 smithi026 ceph-mon[223014]: pgmap v422: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 15456/52374 objects degraded (29.511%); 516/52374 objects misplaced (0.985%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:19:22.477 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:22.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:22.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:22.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:22 smithi070 ceph-mon[192652]: pgmap v422: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 1.1 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 539 op/s; 15456/52374 objects degraded (29.511%); 516/52374 objects misplaced (0.985%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:19:22.583 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:22.659 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:22.891 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:22.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:22.996 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:23.076 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:23.172 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:19:23.172 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:173509 5.6 e187/111 5:65de4d18:::benchmark_data_smithi026_286012_object79701:head v 187'10787, mlcod=187'10785)", 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:23.164420+0000", 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0060080730000000001, 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060307190000000004, 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.173 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:23.174 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164420+0000", 2023-09-23T22:19:23.174 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:23.174 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.174 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.174 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:23.174 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164420+0000", 2023-09-23T22:19:23.174 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:23.174 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.174 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164419+0000", 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164427+0000", 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2230000000000007e-06 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164443+0000", 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5574999999999999e-05 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164449+0000", 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8090000000000004e-06 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164487+0000", 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7602999999999999e-05 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164505+0000", 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8145999999999999e-05 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:23.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.178 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.178 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:173510 5.6 e187/111 5:6a8097a1:::benchmark_data_smithi026_286012_object79702:head v 187'10788, mlcod=187'10785)", 2023-09-23T22:19:23.178 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:23.164777+0000", 2023-09-23T22:19:23.178 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0056507190000000002, 2023-09-23T22:19:23.178 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005714093, 2023-09-23T22:19:23.178 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:23.178 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:19:23.178 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:23.178 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164777+0000", 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164777+0000", 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164776+0000", 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164783+0000", 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2559999999999999e-06 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164796+0000", 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3616e-05 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.164803+0000", 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.64e-06 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.167310+0000", 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025066849999999998 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:23.167334+0000", 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3983999999999999e-05 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:23.182 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:23.183 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:23.183 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T22:19:23.183 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 2 2023-09-23T22:19:23.183 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:19:23.189 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:23.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:23.192 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:23.216 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:19:23.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:23.323 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:23.366 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:23.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:23.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:23.469 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:23.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1693142711' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:19:23.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15456/52374 objects degraded (29.511%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:23.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1693142711' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:19:23.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15456/52374 objects degraded (29.511%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:23.554 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1693142711' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:19:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15456/52374 objects degraded (29.511%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:24.030 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:19:24.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:24 smithi070 ceph-mon[192652]: pgmap v423: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 975 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 15311/46653 objects degraded (32.819%); 516/46653 objects misplaced (1.106%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T22:19:24.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:24 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:24.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:24 smithi070 ceph-mon[192652]: failed to encode map e188 with expected crc 2023-09-23T22:19:24.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:24 smithi070 ceph-mon[192652]: failed to encode map e188 with expected crc 2023-09-23T22:19:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1693142711' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:19:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:24 smithi070 ceph-mon[192652]: osdmap e188: 8 total, 8 up, 7 in 2023-09-23T22:19:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:24.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[223014]: pgmap v423: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 975 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 15311/46653 objects degraded (32.819%); 516/46653 objects misplaced (1.106%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T22:19:24.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:24.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[223014]: failed to encode map e188 with expected crc 2023-09-23T22:19:24.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[223014]: failed to encode map e188 with expected crc 2023-09-23T22:19:24.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1693142711' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:19:24.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[223014]: osdmap e188: 8 total, 8 up, 7 in 2023-09-23T22:19:24.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:24.465 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[215775]: pgmap v423: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 975 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 15311/46653 objects degraded (32.819%); 516/46653 objects misplaced (1.106%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T22:19:24.465 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:24.465 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[215775]: failed to encode map e188 with expected crc 2023-09-23T22:19:24.465 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[215775]: failed to encode map e188 with expected crc 2023-09-23T22:19:24.465 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1693142711' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:19:24.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[215775]: osdmap e188: 8 total, 8 up, 7 in 2023-09-23T22:19:24.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:24.470 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:24.806 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:24.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:24.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:24.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:25.036 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:25.197 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:25.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:25.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:25.309 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:25.337 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:25.379 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:25.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:25.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:25.482 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:25.512 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:25.518 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:19:25.558 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:19:25.623 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:19:25.664 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:19:25.700 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:19:25.745 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:19:25.771 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:25.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:25.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:25.794 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:19:25.846 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:19:25.865 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:19:25.865 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:19:25.865 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:19:25.865 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:19:25.865 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:19:25.866 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:19:25.866 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:19:25.866 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:19:25.878 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:25.986 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:26.206 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:26.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:26.228 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:19:26.246 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:19:26.283 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:19:26.283 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:19:26.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:26 smithi070 ceph-mon[192652]: pgmap v425: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 941 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 15311/45045 objects degraded (33.990%); 516/45045 objects misplaced (1.146%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:19:26.307 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1322548060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:26.314 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:26.343 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:26.387 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:26.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:26.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:26.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:26 smithi026 ceph-mon[215775]: pgmap v425: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 941 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 15311/45045 objects degraded (33.990%); 516/45045 objects misplaced (1.146%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:19:26.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1322548060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:26.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:26 smithi026 ceph-mon[223014]: pgmap v425: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 941 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 15311/45045 objects degraded (33.990%); 516/45045 objects misplaced (1.146%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:19:26.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1322548060' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:26.491 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:26.558 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:26.645 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:26.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:26.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:26.749 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:26.799 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:27.055 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:19:27.100 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:19:27.117 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:27.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:27.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:27.238 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:27.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:27 smithi026 ceph-mon[223014]: from='client.34965 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:27.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:27 smithi026 ceph-mon[215775]: from='client.34965 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:27.370 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:27.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:27 smithi070 ceph-mon[192652]: from='client.34965 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:27.619 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:27.645 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:27.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:27.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:27.674 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":425,"stamp":"2023-09-23T22:19:25.123917+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":986912715,"num_objects":15015,"num_object_clones":0,"num_object_copies":45045,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":15311,"num_objects_misplaced":516,"num_objects_unfound":0,"num_objects_dirty":15015,"num_whiteouts":0,"num_read":6469,"num_read_kb":13097,"num_write":173771,"num_write_kb":6018662,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28960,"num_bytes_recovered":1898781090,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111073,"ondisk_log_size":111073,"up":297,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":324,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4353764,"kb_used_data":3521644,"kb_used_omap":31,"kb_used_meta":831712,"kb_avail":651862300,"statfs":{"total":671965249536,"available":667506995200,"internally_reserved":0,"allocated":3606163456,"data_stored":3599700755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":851674039},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,9,9,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-367591424,"num_objects":-5609,"num_object_clones":0,"num_object_copies":-16827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-327,"num_objects_misplaced":-593,"num_objects_unfound":0,"num_objects_dirty":-5609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5609,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":921,"num_bytes_recovered":60358656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003221"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":164,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129134+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:19:02.129134+0000","last_peered":"2023-09-23T22:19:02.129134+0000","last_clean":"2023-09-23T22:19:02.129134+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:19:02.129134+0000","last_undegraded":"2023-09-23T22:19:02.129134+0000","last_fullsized":"2023-09-23T22:19:02.129134+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":167,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.790821+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:19:11.790821+0000","last_peered":"2023-09-23T22:19:11.790821+0000","last_clean":"2023-09-23T22:19:11.790821+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:19:11.790821+0000","last_undegraded":"2023-09-23T22:19:11.790821+0000","last_fullsized":"2023-09-23T22:19:11.790821+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":127,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.789148+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:19:11.789148+0000","last_peered":"2023-09-23T22:19:11.789148+0000","last_clean":"2023-09-23T22:19:11.789148+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:19:11.789148+0000","last_undegraded":"2023-09-23T22:19:11.789148+0000","last_fullsized":"2023-09-23T22:19:11.789148+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":15,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161834+0000","last_change":"2023-09-23T22:18:57.724501+0000","last_active":"2023-09-23T22:19:02.161834+0000","last_peered":"2023-09-23T22:19:02.161834+0000","last_clean":"2023-09-23T22:19:02.161834+0000","last_became_active":"2023-09-23T22:18:57.724203+0000","last_became_peered":"2023-09-23T22:18:57.724203+0000","last_unstale":"2023-09-23T22:19:02.161834+0000","last_undegraded":"2023-09-23T22:19:02.161834+0000","last_fullsized":"2023-09-23T22:19:02.161834+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:52.168267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":207,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.790037+0000","last_change":"2023-09-23T22:18:57.722574+0000","last_active":"2023-09-23T22:19:11.790037+0000","last_peered":"2023-09-23T22:19:11.790037+0000","last_clean":"2023-09-23T22:19:11.790037+0000","last_became_active":"2023-09-23T22:18:57.721800+0000","last_became_peered":"2023-09-23T22:18:57.721800+0000","last_unstale":"2023-09-23T22:19:11.790037+0000","last_undegraded":"2023-09-23T22:19:11.790037+0000","last_fullsized":"2023-09-23T22:19:11.790037+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:09.303838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":123,"reported_epoch":186,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:19:02.172373+0000","last_change":"2023-09-23T22:18:58.730170+0000","last_active":"2023-09-23T22:19:02.172373+0000","last_peered":"2023-09-23T22:19:02.172373+0000","last_clean":"2023-09-23T22:18:49.795932+0000","last_became_active":"2023-09-23T22:18:58.729860+0000","last_became_peered":"2023-09-23T22:18:58.729860+0000","last_unstale":"2023-09-23T22:19:02.172373+0000","last_undegraded":"2023-09-23T22:19:02.172373+0000","last_fullsized":"2023-09-23T22:19:02.172373+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:30.948774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,3],"avail_no_missing":["5","0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.165084+0000","last_change":"2023-09-23T22:18:57.729038+0000","last_active":"2023-09-23T22:19:02.165084+0000","last_peered":"2023-09-23T22:19:02.165084+0000","last_clean":"2023-09-23T22:19:02.165084+0000","last_became_active":"2023-09-23T22:18:57.728650+0000","last_became_peered":"2023-09-23T22:18:57.728650+0000","last_unstale":"2023-09-23T22:19:02.165084+0000","last_undegraded":"2023-09-23T22:19:02.165084+0000","last_fullsized":"2023-09-23T22:19:02.165084+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:11.461883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":162,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.790103+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:19:11.790103+0000","last_peered":"2023-09-23T22:19:11.790103+0000","last_clean":"2023-09-23T22:19:11.790103+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:19:11.790103+0000","last_undegraded":"2023-09-23T22:19:11.790103+0000","last_fullsized":"2023-09-23T22:19:11.790103+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":147,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.172748+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:19:02.172748+0000","last_peered":"2023-09-23T22:19:02.172748+0000","last_clean":"2023-09-23T22:19:02.172748+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:19:02.172748+0000","last_undegraded":"2023-09-23T22:19:02.172748+0000","last_fullsized":"2023-09-23T22:19:02.172748+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":112,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.790086+0000","last_change":"2023-09-23T22:16:49.154341+0000","last_active":"2023-09-23T22:19:11.790086+0000","last_peered":"2023-09-23T22:19:11.790086+0000","last_clean":"2023-09-23T22:19:11.790086+0000","last_became_active":"2023-09-23T22:16:49.154041+0000","last_became_peered":"2023-09-23T22:16:49.154041+0000","last_unstale":"2023-09-23T22:19:11.790086+0000","last_undegraded":"2023-09-23T22:19:11.790086+0000","last_fullsized":"2023-09-23T22:19:11.790086+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:05.733262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":99,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.790737+0000","last_change":"2023-09-23T22:15:14.829731+0000","last_active":"2023-09-23T22:19:11.790737+0000","last_peered":"2023-09-23T22:19:11.790737+0000","last_clean":"2023-09-23T22:19:11.790737+0000","last_became_active":"2023-09-23T22:15:14.829437+0000","last_became_peered":"2023-09-23T22:15:14.829437+0000","last_unstale":"2023-09-23T22:19:11.790737+0000","last_undegraded":"2023-09-23T22:19:11.790737+0000","last_fullsized":"2023-09-23T22:19:11.790737+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:45.456558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":154,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.789290+0000","last_change":"2023-09-23T22:18:57.723165+0000","last_active":"2023-09-23T22:19:11.789290+0000","last_peered":"2023-09-23T22:19:11.789290+0000","last_clean":"2023-09-23T22:19:11.789290+0000","last_became_active":"2023-09-23T22:18:57.722863+0000","last_became_peered":"2023-09-23T22:18:57.722863+0000","last_unstale":"2023-09-23T22:19:11.789290+0000","last_undegraded":"2023-09-23T22:19:11.789290+0000","last_fullsized":"2023-09-23T22:19:11.789290+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:19.645029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":171,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129231+0000","last_change":"2023-09-23T22:16:49.265991+0000","last_active":"2023-09-23T22:19:02.129231+0000","last_peered":"2023-09-23T22:19:02.129231+0000","last_clean":"2023-09-23T22:19:02.129231+0000","last_became_active":"2023-09-23T22:16:49.251169+0000","last_became_peered":"2023-09-23T22:16:49.251169+0000","last_unstale":"2023-09-23T22:19:02.129231+0000","last_undegraded":"2023-09-23T22:19:02.129231+0000","last_fullsized":"2023-09-23T22:19:02.129231+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.502218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129064+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:19:02.129064+0000","last_peered":"2023-09-23T22:19:02.129064+0000","last_clean":"2023-09-23T22:19:02.129064+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:19:02.129064+0000","last_undegraded":"2023-09-23T22:19:02.129064+0000","last_fullsized":"2023-09-23T22:19:02.129064+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":98,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129258+0000","last_change":"2023-09-23T22:15:14.826143+0000","last_active":"2023-09-23T22:19:02.129258+0000","last_peered":"2023-09-23T22:19:02.129258+0000","last_clean":"2023-09-23T22:19:02.129258+0000","last_became_active":"2023-09-23T22:15:14.825780+0000","last_became_peered":"2023-09-23T22:15:14.825780+0000","last_unstale":"2023-09-23T22:19:02.129258+0000","last_undegraded":"2023-09-23T22:19:02.129258+0000","last_fullsized":"2023-09-23T22:19:02.129258+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:22.608676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":153,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.790748+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:19:11.790748+0000","last_peered":"2023-09-23T22:19:11.790748+0000","last_clean":"2023-09-23T22:19:11.790748+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:19:11.790748+0000","last_undegraded":"2023-09-23T22:19:11.790748+0000","last_fullsized":"2023-09-23T22:19:11.790748+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.162921+0000","last_change":"2023-09-23T22:18:57.726112+0000","last_active":"2023-09-23T22:19:02.162921+0000","last_peered":"2023-09-23T22:19:02.162921+0000","last_clean":"2023-09-23T22:19:02.162921+0000","last_became_active":"2023-09-23T22:18:57.725558+0000","last_became_peered":"2023-09-23T22:18:57.725558+0000","last_unstale":"2023-09-23T22:19:02.162921+0000","last_undegraded":"2023-09-23T22:19:02.162921+0000","last_fullsized":"2023-09-23T22:19:02.162921+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:38.315664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":146,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.172108+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:19:02.172108+0000","last_peered":"2023-09-23T22:19:02.172108+0000","last_clean":"2023-09-23T22:19:02.172108+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:19:02.172108+0000","last_undegraded":"2023-09-23T22:19:02.172108+0000","last_fullsized":"2023-09-23T22:19:02.172108+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":110,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129827+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:19:02.129827+0000","last_peered":"2023-09-23T22:19:02.129827+0000","last_clean":"2023-09-23T22:19:02.129827+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:19:02.129827+0000","last_undegraded":"2023-09-23T22:19:02.129827+0000","last_fullsized":"2023-09-23T22:19:02.129827+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":15,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.164640+0000","last_change":"2023-09-23T22:18:57.726691+0000","last_active":"2023-09-23T22:19:02.164640+0000","last_peered":"2023-09-23T22:19:02.164640+0000","last_clean":"2023-09-23T22:19:02.164640+0000","last_became_active":"2023-09-23T22:18:57.726386+0000","last_became_peered":"2023-09-23T22:18:57.726386+0000","last_unstale":"2023-09-23T22:19:02.164640+0000","last_undegraded":"2023-09-23T22:19:02.164640+0000","last_fullsized":"2023-09-23T22:19:02.164640+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:34.829149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":156,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.093551+0000","last_change":"2023-09-23T22:18:57.728767+0000","last_active":"2023-09-23T22:19:11.093551+0000","last_peered":"2023-09-23T22:19:11.093551+0000","last_clean":"2023-09-23T22:19:11.093551+0000","last_became_active":"2023-09-23T22:18:57.728377+0000","last_became_peered":"2023-09-23T22:18:57.728377+0000","last_unstale":"2023-09-23T22:19:11.093551+0000","last_undegraded":"2023-09-23T22:19:11.093551+0000","last_fullsized":"2023-09-23T22:19:11.093551+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:01:01.370904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":153,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.791093+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:19:11.791093+0000","last_peered":"2023-09-23T22:19:11.791093+0000","last_clean":"2023-09-23T22:19:11.791093+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:19:11.791093+0000","last_undegraded":"2023-09-23T22:19:11.791093+0000","last_fullsized":"2023-09-23T22:19:11.791093+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"187'10618","reported_seq":22252,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:22.424826+0000","last_change":"2023-09-23T22:19:02.159483+0000","last_active":"2023-09-23T22:19:22.424826+0000","last_peered":"2023-09-23T22:19:22.424826+0000","last_clean":"2023-09-23T22:19:22.424826+0000","last_became_active":"2023-09-23T22:19:02.132959+0000","last_became_peered":"2023-09-23T22:19:02.132959+0000","last_unstale":"2023-09-23T22:19:22.424826+0000","last_undegraded":"2023-09-23T22:19:22.424826+0000","last_fullsized":"2023-09-23T22:19:22.424826+0000","mapping_epoch":185,"log_start":"136'3865","ondisk_log_start":"136'3865","created":111,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:20.126557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60817408,"num_objects":928,"num_object_clones":0,"num_object_copies":2784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10618,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5765,"num_bytes_recovered":377815040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":154,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129648+0000","last_change":"2023-09-23T22:19:00.682889+0000","last_active":"2023-09-23T22:19:02.129648+0000","last_peered":"2023-09-23T22:19:02.129648+0000","last_clean":"2023-09-23T22:19:02.129648+0000","last_became_active":"2023-09-23T22:18:57.718002+0000","last_became_peered":"2023-09-23T22:18:57.718002+0000","last_unstale":"2023-09-23T22:19:02.129648+0000","last_undegraded":"2023-09-23T22:19:02.129648+0000","last_fullsized":"2023-09-23T22:19:02.129648+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:22.517277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":139,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.170556+0000","last_change":"2023-09-23T22:16:49.158580+0000","last_active":"2023-09-23T22:19:02.170556+0000","last_peered":"2023-09-23T22:19:02.170556+0000","last_clean":"2023-09-23T22:19:02.170556+0000","last_became_active":"2023-09-23T22:16:49.158266+0000","last_became_peered":"2023-09-23T22:16:49.158266+0000","last_unstale":"2023-09-23T22:19:02.170556+0000","last_undegraded":"2023-09-23T22:19:02.170556+0000","last_fullsized":"2023-09-23T22:19:02.170556+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:03.791658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"187'10601","reported_seq":15935,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:21.125343+0000","last_change":"2023-09-23T22:17:16.903782+0000","last_active":"2023-09-23T22:19:21.125343+0000","last_peered":"2023-09-23T22:19:21.125343+0000","last_clean":"2023-09-23T22:19:21.125343+0000","last_became_active":"2023-09-23T22:17:16.895165+0000","last_became_peered":"2023-09-23T22:17:16.895165+0000","last_unstale":"2023-09-23T22:19:21.125343+0000","last_undegraded":"2023-09-23T22:19:21.125343+0000","last_fullsized":"2023-09-23T22:19:21.125343+0000","mapping_epoch":159,"log_start":"133'3400","ondisk_log_start":"133'3400","created":111,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7201,"log_dups_size":0,"ondisk_log_size":7201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:31.886804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61014016,"num_objects":931,"num_object_clones":0,"num_object_copies":2793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10601,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2600,"num_bytes_recovered":170393600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":98,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129180+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:19:02.129180+0000","last_peered":"2023-09-23T22:19:02.129180+0000","last_clean":"2023-09-23T22:19:02.129180+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:19:02.129180+0000","last_undegraded":"2023-09-23T22:19:02.129180+0000","last_fullsized":"2023-09-23T22:19:02.129180+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":154,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129221+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:19:02.129221+0000","last_peered":"2023-09-23T22:19:02.129221+0000","last_clean":"2023-09-23T22:19:02.129221+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:19:02.129221+0000","last_undegraded":"2023-09-23T22:19:02.129221+0000","last_fullsized":"2023-09-23T22:19:02.129221+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"187'10544","reported_seq":17088,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:21.156578+0000","last_change":"2023-09-23T22:19:00.690164+0000","last_active":"2023-09-23T22:19:21.156578+0000","last_peered":"2023-09-23T22:19:21.156578+0000","last_clean":"2023-09-23T22:18:39.199855+0000","last_became_active":"2023-09-23T22:18:41.409775+0000","last_became_peered":"2023-09-23T22:18:41.409775+0000","last_unstale":"2023-09-23T22:19:21.156578+0000","last_undegraded":"2023-09-23T22:18:41.249089+0000","last_fullsized":"2023-09-23T22:18:41.215237+0000","mapping_epoch":178,"log_start":"133'3400","ondisk_log_start":"133'3400","created":111,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7144,"log_dups_size":0,"ondisk_log_size":7144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:30.052669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63569920,"num_objects":970,"num_object_clones":0,"num_object_copies":2910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5621,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10544,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3204,"num_bytes_recovered":209977344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-1239},{"shards":"0,2,7","objects":970}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":112,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.172040+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:19:02.172040+0000","last_peered":"2023-09-23T22:19:02.172040+0000","last_clean":"2023-09-23T22:19:02.172040+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:19:02.172040+0000","last_undegraded":"2023-09-23T22:19:02.172040+0000","last_fullsized":"2023-09-23T22:19:02.172040+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":103,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.789382+0000","last_change":"2023-09-23T22:16:49.161676+0000","last_active":"2023-09-23T22:19:11.789382+0000","last_peered":"2023-09-23T22:19:11.789382+0000","last_clean":"2023-09-23T22:19:11.789382+0000","last_became_active":"2023-09-23T22:16:49.161247+0000","last_became_peered":"2023-09-23T22:16:49.161247+0000","last_unstale":"2023-09-23T22:19:11.789382+0000","last_undegraded":"2023-09-23T22:19:11.789382+0000","last_fullsized":"2023-09-23T22:19:11.789382+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:30.708371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"187'10714","reported_seq":10800,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:21.184642+0000","last_change":"2023-09-23T22:15:14.826723+0000","last_active":"2023-09-23T22:19:21.184642+0000","last_peered":"2023-09-23T22:19:21.184642+0000","last_clean":"2023-09-23T22:19:21.184642+0000","last_became_active":"2023-09-23T22:15:14.826454+0000","last_became_peered":"2023-09-23T22:15:14.826454+0000","last_unstale":"2023-09-23T22:19:21.184642+0000","last_undegraded":"2023-09-23T22:19:21.184642+0000","last_fullsized":"2023-09-23T22:19:21.184642+0000","mapping_epoch":111,"log_start":"133'3500","ondisk_log_start":"133'3500","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7214,"log_dups_size":0,"ondisk_log_size":7214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:11.166077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64225280,"num_objects":980,"num_object_clones":0,"num_object_copies":2940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10714,"num_write_kb":374208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":157,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.172548+0000","last_change":"2023-09-23T22:16:49.164833+0000","last_active":"2023-09-23T22:19:02.172548+0000","last_peered":"2023-09-23T22:19:02.172548+0000","last_clean":"2023-09-23T22:19:02.172548+0000","last_became_active":"2023-09-23T22:16:49.163833+0000","last_became_peered":"2023-09-23T22:16:49.163833+0000","last_unstale":"2023-09-23T22:19:02.172548+0000","last_undegraded":"2023-09-23T22:19:02.172548+0000","last_fullsized":"2023-09-23T22:19:02.172548+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.891395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":39,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161961+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:19:02.161961+0000","last_peered":"2023-09-23T22:19:02.161961+0000","last_clean":"2023-09-23T22:19:02.161961+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:19:02.161961+0000","last_undegraded":"2023-09-23T22:19:02.161961+0000","last_fullsized":"2023-09-23T22:19:02.161961+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":451,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161358+0000","last_change":"2023-09-23T22:18:57.721431+0000","last_active":"2023-09-23T22:19:02.161358+0000","last_peered":"2023-09-23T22:19:02.161358+0000","last_clean":"2023-09-23T22:19:02.161358+0000","last_became_active":"2023-09-23T22:18:57.721043+0000","last_became_peered":"2023-09-23T22:18:57.721043+0000","last_unstale":"2023-09-23T22:19:02.161358+0000","last_undegraded":"2023-09-23T22:19:02.161358+0000","last_fullsized":"2023-09-23T22:19:02.161358+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"187'10847","reported_seq":10934,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:21.147478+0000","last_change":"2023-09-23T22:15:14.824965+0000","last_active":"2023-09-23T22:19:21.147478+0000","last_peered":"2023-09-23T22:19:21.147478+0000","last_clean":"2023-09-23T22:19:21.147478+0000","last_became_active":"2023-09-23T22:15:14.824626+0000","last_became_peered":"2023-09-23T22:15:14.824626+0000","last_unstale":"2023-09-23T22:19:21.147478+0000","last_undegraded":"2023-09-23T22:19:21.147478+0000","last_fullsized":"2023-09-23T22:19:21.147478+0000","mapping_epoch":111,"log_start":"133'3630","ondisk_log_start":"133'3630","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7217,"log_dups_size":0,"ondisk_log_size":7217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.613455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61866008,"num_objects":945,"num_object_clones":0,"num_object_copies":2835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":945,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10847,"num_write_kb":377281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":199,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.791309+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:19:11.791309+0000","last_peered":"2023-09-23T22:19:11.791309+0000","last_clean":"2023-09-23T22:19:11.791309+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:19:11.791309+0000","last_undegraded":"2023-09-23T22:19:11.791309+0000","last_fullsized":"2023-09-23T22:19:11.791309+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129086+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:19:02.129086+0000","last_peered":"2023-09-23T22:19:02.129086+0000","last_clean":"2023-09-23T22:19:02.129086+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:19:02.129086+0000","last_undegraded":"2023-09-23T22:19:02.129086+0000","last_fullsized":"2023-09-23T22:19:02.129086+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":449,"reported_epoch":186,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:19:02.161378+0000","last_change":"2023-09-23T22:18:57.743507+0000","last_active":"2023-09-23T22:19:02.161378+0000","last_peered":"2023-09-23T22:19:02.161378+0000","last_clean":"2023-09-23T22:08:39.036645+0000","last_became_active":"2023-09-23T22:18:57.742130+0000","last_became_peered":"2023-09-23T22:18:57.742130+0000","last_unstale":"2023-09-23T22:19:02.161378+0000","last_undegraded":"2023-09-23T22:18:57.716066+0000","last_fullsized":"2023-09-23T22:19:02.161378+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"0,1,7","objects":-1},{"shards":"1,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.c","version":"187'10826","reported_seq":13134,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:23.095486+0000","last_change":"2023-09-23T22:16:30.157766+0000","last_active":"2023-09-23T22:19:23.095486+0000","last_peered":"2023-09-23T22:19:23.095486+0000","last_clean":"2023-09-23T22:19:23.095486+0000","last_became_active":"2023-09-23T22:16:30.150277+0000","last_became_peered":"2023-09-23T22:16:30.150277+0000","last_unstale":"2023-09-23T22:19:23.095486+0000","last_undegraded":"2023-09-23T22:19:23.095486+0000","last_fullsized":"2023-09-23T22:19:23.095486+0000","mapping_epoch":140,"log_start":"144'5425","ondisk_log_start":"144'5425","created":111,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:20.415978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57933824,"num_objects":884,"num_object_clones":0,"num_object_copies":2652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10826,"num_write_kb":374720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1098,"num_bytes_recovered":71958528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":103,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129317+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:19:02.129317+0000","last_peered":"2023-09-23T22:19:02.129317+0000","last_clean":"2023-09-23T22:19:02.129317+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:19:02.129317+0000","last_undegraded":"2023-09-23T22:19:02.129317+0000","last_fullsized":"2023-09-23T22:19:02.129317+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":162,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129475+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:19:02.129475+0000","last_peered":"2023-09-23T22:19:02.129475+0000","last_clean":"2023-09-23T22:19:02.129475+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:19:02.129475+0000","last_undegraded":"2023-09-23T22:19:02.129475+0000","last_fullsized":"2023-09-23T22:19:02.129475+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"187'10789","reported_seq":21380,"reported_epoch":187,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:20.116280+0000","last_change":"2023-09-23T22:19:00.894300+0000","last_active":"2023-09-23T22:19:20.116280+0000","last_peered":"2023-09-23T22:19:20.116280+0000","last_clean":"2023-09-23T22:16:48.124555+0000","last_became_active":"2023-09-23T22:16:50.339530+0000","last_became_peered":"2023-09-23T22:16:50.339530+0000","last_unstale":"2023-09-23T22:19:20.116280+0000","last_undegraded":"2023-09-23T22:18:40.214039+0000","last_fullsized":"2023-09-23T22:16:50.148559+0000","mapping_epoch":147,"log_start":"133'3224","ondisk_log_start":"133'3224","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:01.673286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64421888,"num_objects":983,"num_object_clones":0,"num_object_copies":2949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":116,"num_objects_misplaced":516,"num_objects_unfound":0,"num_objects_dirty":983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10789,"num_write_kb":376704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5245,"num_bytes_recovered":343736320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4],"avail_no_missing":["5","4","6"],"object_location_counts":[{"shards":"4,5","objects":351},{"shards":"4,5,6","objects":516}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":138,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.128936+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:19:02.128936+0000","last_peered":"2023-09-23T22:19:02.128936+0000","last_clean":"2023-09-23T22:19:02.128936+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:19:02.128936+0000","last_undegraded":"2023-09-23T22:19:02.128936+0000","last_fullsized":"2023-09-23T22:19:02.128936+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129398+0000","last_change":"2023-09-23T22:16:49.165118+0000","last_active":"2023-09-23T22:19:02.129398+0000","last_peered":"2023-09-23T22:19:02.129398+0000","last_clean":"2023-09-23T22:19:02.129398+0000","last_became_active":"2023-09-23T22:16:49.164802+0000","last_became_peered":"2023-09-23T22:16:49.164802+0000","last_unstale":"2023-09-23T22:19:02.129398+0000","last_undegraded":"2023-09-23T22:19:02.129398+0000","last_fullsized":"2023-09-23T22:19:02.129398+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:23.079142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":451,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.162997+0000","last_change":"2023-09-23T22:18:57.726339+0000","last_active":"2023-09-23T22:19:02.162997+0000","last_peered":"2023-09-23T22:19:02.162997+0000","last_clean":"2023-09-23T22:19:02.162997+0000","last_became_active":"2023-09-23T22:18:57.725773+0000","last_became_peered":"2023-09-23T22:18:57.725773+0000","last_unstale":"2023-09-23T22:19:02.162997+0000","last_undegraded":"2023-09-23T22:19:02.162997+0000","last_fullsized":"2023-09-23T22:19:02.162997+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"187'10982","reported_seq":14831,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:22.423423+0000","last_change":"2023-09-23T22:15:59.633734+0000","last_active":"2023-09-23T22:19:22.423423+0000","last_peered":"2023-09-23T22:19:22.423423+0000","last_clean":"2023-09-23T22:19:22.423423+0000","last_became_active":"2023-09-23T22:15:59.605894+0000","last_became_peered":"2023-09-23T22:15:59.605894+0000","last_unstale":"2023-09-23T22:19:22.423423+0000","last_undegraded":"2023-09-23T22:19:22.423423+0000","last_fullsized":"2023-09-23T22:19:22.423423+0000","mapping_epoch":132,"log_start":"136'4247","ondisk_log_start":"136'4247","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:22.683442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61997056,"num_objects":946,"num_object_clones":0,"num_object_copies":2838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10982,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1867,"num_bytes_recovered":122355712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":154,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.789892+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:19:11.789892+0000","last_peered":"2023-09-23T22:19:11.789892+0000","last_clean":"2023-09-23T22:19:11.789892+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:19:11.789892+0000","last_undegraded":"2023-09-23T22:19:11.789892+0000","last_fullsized":"2023-09-23T22:19:11.789892+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":127,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.789891+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:19:11.789891+0000","last_peered":"2023-09-23T22:19:11.789891+0000","last_clean":"2023-09-23T22:19:11.789891+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:19:11.789891+0000","last_undegraded":"2023-09-23T22:19:11.789891+0000","last_fullsized":"2023-09-23T22:19:11.789891+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":451,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.164778+0000","last_change":"2023-09-23T22:18:57.725616+0000","last_active":"2023-09-23T22:19:02.164778+0000","last_peered":"2023-09-23T22:19:02.164778+0000","last_clean":"2023-09-23T22:19:02.164778+0000","last_became_active":"2023-09-23T22:18:57.725224+0000","last_became_peered":"2023-09-23T22:18:57.725224+0000","last_unstale":"2023-09-23T22:19:02.164778+0000","last_undegraded":"2023-09-23T22:19:02.164778+0000","last_fullsized":"2023-09-23T22:19:02.164778+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":735,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.790103+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:19:11.790103+0000","last_peered":"2023-09-23T22:19:11.790103+0000","last_clean":"2023-09-23T22:19:11.790103+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:19:11.790103+0000","last_undegraded":"2023-09-23T22:19:11.790103+0000","last_fullsized":"2023-09-23T22:19:11.790103+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":164,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.172267+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:19:02.172267+0000","last_peered":"2023-09-23T22:19:02.172267+0000","last_clean":"2023-09-23T22:19:02.172267+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:19:02.172267+0000","last_undegraded":"2023-09-23T22:19:02.172267+0000","last_fullsized":"2023-09-23T22:19:02.172267+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":108,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.173565+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:19:02.173565+0000","last_peered":"2023-09-23T22:19:02.173565+0000","last_clean":"2023-09-23T22:19:02.173565+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:19:02.173565+0000","last_undegraded":"2023-09-23T22:19:02.173565+0000","last_fullsized":"2023-09-23T22:19:02.173565+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"187'10764","reported_seq":17218,"reported_epoch":187,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:19:21.113734+0000","last_change":"2023-09-23T22:18:59.033329+0000","last_active":"2023-09-23T22:19:21.113734+0000","last_peered":"2023-09-23T22:19:21.113734+0000","last_clean":"2023-09-23T22:16:48.130011+0000","last_became_active":"2023-09-23T22:18:58.953635+0000","last_became_peered":"2023-09-23T22:18:58.953635+0000","last_unstale":"2023-09-23T22:19:21.113734+0000","last_undegraded":"2023-09-23T22:18:58.948229+0000","last_fullsized":"2023-09-23T22:19:21.113734+0000","mapping_epoch":183,"log_start":"139'4813","ondisk_log_start":"139'4813","created":111,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5951,"log_dups_size":0,"ondisk_log_size":5951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:59.274843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64487424,"num_objects":984,"num_object_clones":0,"num_object_copies":2952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":984,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10764,"num_write_kb":375936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3156,"num_bytes_recovered":206766080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,1],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,1,5","objects":2},{"shards":"0,5,6","objects":982}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":451,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161745+0000","last_change":"2023-09-23T22:18:57.742555+0000","last_active":"2023-09-23T22:19:02.161745+0000","last_peered":"2023-09-23T22:19:02.161745+0000","last_clean":"2023-09-23T22:19:02.161745+0000","last_became_active":"2023-09-23T22:18:57.742205+0000","last_became_peered":"2023-09-23T22:18:57.742205+0000","last_unstale":"2023-09-23T22:19:02.161745+0000","last_undegraded":"2023-09-23T22:19:02.161745+0000","last_fullsized":"2023-09-23T22:19:02.161745+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"187'11117","reported_seq":15027,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:23.150522+0000","last_change":"2023-09-23T22:18:57.918095+0000","last_active":"2023-09-23T22:19:23.150522+0000","last_peered":"2023-09-23T22:19:23.150522+0000","last_clean":"2023-09-23T22:16:50.502857+0000","last_became_active":"2023-09-23T22:18:57.873881+0000","last_became_peered":"2023-09-23T22:18:57.873881+0000","last_unstale":"2023-09-23T22:19:23.150522+0000","last_undegraded":"2023-09-23T22:18:57.734113+0000","last_fullsized":"2023-09-23T22:18:57.704742+0000","mapping_epoch":182,"log_start":"145'5740","ondisk_log_start":"145'5740","created":111,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5377,"log_dups_size":0,"ondisk_log_size":5377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:58.431178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55902208,"num_objects":853,"num_object_clones":0,"num_object_copies":2559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11117,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1895,"num_bytes_recovered":124190720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[1,4],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4","objects":853}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":919,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.171248+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:19:02.171248+0000","last_peered":"2023-09-23T22:19:02.171248+0000","last_clean":"2023-09-23T22:19:02.171248+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:19:02.171248+0000","last_undegraded":"2023-09-23T22:19:02.171248+0000","last_fullsized":"2023-09-23T22:19:02.171248+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":196,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.128973+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:19:02.128973+0000","last_peered":"2023-09-23T22:19:02.128973+0000","last_clean":"2023-09-23T22:19:02.128973+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:19:02.128973+0000","last_undegraded":"2023-09-23T22:19:02.128973+0000","last_fullsized":"2023-09-23T22:19:02.128973+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":161,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.172362+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:19:02.172362+0000","last_peered":"2023-09-23T22:19:02.172362+0000","last_clean":"2023-09-23T22:19:02.172362+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:19:02.172362+0000","last_undegraded":"2023-09-23T22:19:02.172362+0000","last_fullsized":"2023-09-23T22:19:02.172362+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":451,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.162785+0000","last_change":"2023-09-23T22:18:57.725923+0000","last_active":"2023-09-23T22:19:02.162785+0000","last_peered":"2023-09-23T22:19:02.162785+0000","last_clean":"2023-09-23T22:19:02.162785+0000","last_became_active":"2023-09-23T22:18:57.725260+0000","last_became_peered":"2023-09-23T22:18:57.725260+0000","last_unstale":"2023-09-23T22:19:02.162785+0000","last_undegraded":"2023-09-23T22:19:02.162785+0000","last_fullsized":"2023-09-23T22:19:02.162785+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"187'10704","reported_seq":10790,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:21.003915+0000","last_change":"2023-09-23T22:15:14.823394+0000","last_active":"2023-09-23T22:19:21.003915+0000","last_peered":"2023-09-23T22:19:21.003915+0000","last_clean":"2023-09-23T22:19:21.003915+0000","last_became_active":"2023-09-23T22:15:14.823003+0000","last_became_peered":"2023-09-23T22:15:14.823003+0000","last_unstale":"2023-09-23T22:19:21.003915+0000","last_undegraded":"2023-09-23T22:19:21.003915+0000","last_fullsized":"2023-09-23T22:19:21.003915+0000","mapping_epoch":111,"log_start":"133'3500","ondisk_log_start":"133'3500","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7204,"log_dups_size":0,"ondisk_log_size":7204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:13.496813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63176704,"num_objects":964,"num_object_clones":0,"num_object_copies":2892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10704,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":528,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.173398+0000","last_change":"2023-09-23T22:15:51.379227+0000","last_active":"2023-09-23T22:19:02.173398+0000","last_peered":"2023-09-23T22:19:02.173398+0000","last_clean":"2023-09-23T22:19:02.173398+0000","last_became_active":"2023-09-23T22:15:43.928977+0000","last_became_peered":"2023-09-23T22:15:43.928977+0000","last_unstale":"2023-09-23T22:19:02.173398+0000","last_undegraded":"2023-09-23T22:19:02.173398+0000","last_fullsized":"2023-09-23T22:19:02.173398+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:04.398533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":137,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.795786+0000","last_change":"2023-09-23T22:16:49.251412+0000","last_active":"2023-09-23T22:19:11.795786+0000","last_peered":"2023-09-23T22:19:11.795786+0000","last_clean":"2023-09-23T22:19:11.795786+0000","last_became_active":"2023-09-23T22:16:49.251097+0000","last_became_peered":"2023-09-23T22:16:49.251097+0000","last_unstale":"2023-09-23T22:19:11.795786+0000","last_undegraded":"2023-09-23T22:19:11.795786+0000","last_fullsized":"2023-09-23T22:19:11.795786+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:41.601435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":193,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-09-23T22:19:23.404006+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:19:23.404006+0000","last_peered":"2023-09-23T22:19:23.404006+0000","last_clean":"2023-09-23T22:19:23.404006+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:19:23.404006+0000","last_undegraded":"2023-09-23T22:19:23.404006+0000","last_fullsized":"2023-09-23T22:19:23.404006+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":772,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.171680+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:19:02.171680+0000","last_peered":"2023-09-23T22:19:02.171680+0000","last_clean":"2023-09-23T22:19:02.171680+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:19:02.171680+0000","last_undegraded":"2023-09-23T22:19:02.171680+0000","last_fullsized":"2023-09-23T22:19:02.171680+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":451,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161451+0000","last_change":"2023-09-23T22:18:57.743602+0000","last_active":"2023-09-23T22:19:02.161451+0000","last_peered":"2023-09-23T22:19:02.161451+0000","last_clean":"2023-09-23T22:19:02.161451+0000","last_became_active":"2023-09-23T22:18:57.743004+0000","last_became_peered":"2023-09-23T22:18:57.743004+0000","last_unstale":"2023-09-23T22:19:02.161451+0000","last_undegraded":"2023-09-23T22:19:02.161451+0000","last_fullsized":"2023-09-23T22:19:02.161451+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":112,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.790941+0000","last_change":"2023-09-23T22:16:49.155156+0000","last_active":"2023-09-23T22:19:11.790941+0000","last_peered":"2023-09-23T22:19:11.790941+0000","last_clean":"2023-09-23T22:19:11.790941+0000","last_became_active":"2023-09-23T22:16:49.154923+0000","last_became_peered":"2023-09-23T22:16:49.154923+0000","last_unstale":"2023-09-23T22:19:11.790941+0000","last_undegraded":"2023-09-23T22:19:11.790941+0000","last_fullsized":"2023-09-23T22:19:11.790941+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:33.532238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":73,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129714+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:19:02.129714+0000","last_peered":"2023-09-23T22:19:02.129714+0000","last_clean":"2023-09-23T22:19:02.129714+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:19:02.129714+0000","last_undegraded":"2023-09-23T22:19:02.129714+0000","last_fullsized":"2023-09-23T22:19:02.129714+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"187'10815","reported_seq":10901,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:21.169674+0000","last_change":"2023-09-23T22:15:14.824102+0000","last_active":"2023-09-23T22:19:21.169674+0000","last_peered":"2023-09-23T22:19:21.169674+0000","last_clean":"2023-09-23T22:19:21.169674+0000","last_became_active":"2023-09-23T22:15:14.823790+0000","last_became_peered":"2023-09-23T22:15:14.823790+0000","last_unstale":"2023-09-23T22:19:21.169674+0000","last_undegraded":"2023-09-23T22:19:21.169674+0000","last_fullsized":"2023-09-23T22:19:21.169674+0000","mapping_epoch":111,"log_start":"133'3600","ondisk_log_start":"133'3600","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7215,"log_dups_size":0,"ondisk_log_size":7215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:24.517495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10815,"num_write_kb":376128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":451,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161661+0000","last_change":"2023-09-23T22:18:57.720920+0000","last_active":"2023-09-23T22:19:02.161661+0000","last_peered":"2023-09-23T22:19:02.161661+0000","last_clean":"2023-09-23T22:19:02.161661+0000","last_became_active":"2023-09-23T22:18:57.720584+0000","last_became_peered":"2023-09-23T22:18:57.720584+0000","last_unstale":"2023-09-23T22:19:02.161661+0000","last_undegraded":"2023-09-23T22:19:02.161661+0000","last_fullsized":"2023-09-23T22:19:02.161661+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"187'10955","reported_seq":15847,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:20.087832+0000","last_change":"2023-09-23T22:17:15.752911+0000","last_active":"2023-09-23T22:19:20.087832+0000","last_peered":"2023-09-23T22:19:20.087832+0000","last_clean":"2023-09-23T22:19:20.087832+0000","last_became_active":"2023-09-23T22:17:15.728788+0000","last_became_peered":"2023-09-23T22:17:15.728788+0000","last_unstale":"2023-09-23T22:19:20.087832+0000","last_undegraded":"2023-09-23T22:19:20.087832+0000","last_fullsized":"2023-09-23T22:19:20.087832+0000","mapping_epoch":158,"log_start":"133'3363","ondisk_log_start":"133'3363","created":111,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7592,"log_dups_size":0,"ondisk_log_size":7592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:35.923920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63897600,"num_objects":975,"num_object_clones":0,"num_object_copies":2925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10955,"num_write_kb":381760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2374,"num_bytes_recovered":155582464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":536,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:18.292553+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:19:18.292553+0000","last_peered":"2023-09-23T22:19:18.292553+0000","last_clean":"2023-09-23T22:19:18.292553+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:19:18.292553+0000","last_undegraded":"2023-09-23T22:19:18.292553+0000","last_fullsized":"2023-09-23T22:19:18.292553+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":49,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.799120+0000","last_change":"2023-09-23T22:18:31.795915+0000","last_active":"2023-09-23T22:19:11.799120+0000","last_peered":"2023-09-23T22:19:11.799120+0000","last_clean":"2023-09-23T22:19:11.799120+0000","last_became_active":"2023-09-23T22:18:31.795608+0000","last_became_peered":"2023-09-23T22:18:31.795608+0000","last_unstale":"2023-09-23T22:19:11.799120+0000","last_undegraded":"2023-09-23T22:19:11.799120+0000","last_fullsized":"2023-09-23T22:19:11.799120+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:49.789927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":15,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161744+0000","last_change":"2023-09-23T22:18:57.721257+0000","last_active":"2023-09-23T22:19:02.161744+0000","last_peered":"2023-09-23T22:19:02.161744+0000","last_clean":"2023-09-23T22:19:02.161744+0000","last_became_active":"2023-09-23T22:18:57.720548+0000","last_became_peered":"2023-09-23T22:18:57.720548+0000","last_unstale":"2023-09-23T22:19:02.161744+0000","last_undegraded":"2023-09-23T22:19:02.161744+0000","last_fullsized":"2023-09-23T22:19:02.161744+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:10.709024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":451,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.162014+0000","last_change":"2023-09-23T22:18:57.727276+0000","last_active":"2023-09-23T22:19:02.162014+0000","last_peered":"2023-09-23T22:19:02.162014+0000","last_clean":"2023-09-23T22:19:02.162014+0000","last_became_active":"2023-09-23T22:18:57.726938+0000","last_became_peered":"2023-09-23T22:18:57.726938+0000","last_unstale":"2023-09-23T22:19:02.162014+0000","last_undegraded":"2023-09-23T22:19:02.162014+0000","last_fullsized":"2023-09-23T22:19:02.162014+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":76,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.168128+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:19:02.168128+0000","last_peered":"2023-09-23T22:19:02.168128+0000","last_clean":"2023-09-23T22:19:02.168128+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:19:02.168128+0000","last_undegraded":"2023-09-23T22:19:02.168128+0000","last_fullsized":"2023-09-23T22:19:02.168128+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":127,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:08.285966+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:19:08.285966+0000","last_peered":"2023-09-23T22:19:08.285966+0000","last_clean":"2023-09-23T22:19:08.285966+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:19:08.285966+0000","last_undegraded":"2023-09-23T22:19:08.285966+0000","last_fullsized":"2023-09-23T22:19:08.285966+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":40,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.802854+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:19:11.802854+0000","last_peered":"2023-09-23T22:19:11.802854+0000","last_clean":"2023-09-23T22:19:11.802854+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:19:11.802854+0000","last_undegraded":"2023-09-23T22:19:11.802854+0000","last_fullsized":"2023-09-23T22:19:11.802854+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"187'10935","reported_seq":14283,"reported_epoch":187,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:21.089720+0000","last_change":"2023-09-23T22:18:57.739452+0000","last_active":"2023-09-23T22:19:21.089720+0000","last_peered":"2023-09-23T22:19:21.089720+0000","last_clean":"2023-09-23T22:16:48.102227+0000","last_became_active":"2023-09-23T22:16:50.283490+0000","last_became_peered":"2023-09-23T22:16:50.283490+0000","last_unstale":"2023-09-23T22:19:21.089720+0000","last_undegraded":"2023-09-23T22:17:00.207659+0000","last_fullsized":"2023-09-23T22:16:50.146447+0000","mapping_epoch":147,"log_start":"139'4977","ondisk_log_start":"139'4977","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5958,"log_dups_size":0,"ondisk_log_size":5958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:48.954402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65077248,"num_objects":993,"num_object_clones":0,"num_object_copies":2979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4346,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10935,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1610,"num_bytes_recovered":105512960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5","objects":-3353},{"shards":"0,5,6","objects":984}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":451,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.162117+0000","last_change":"2023-09-23T22:18:57.742449+0000","last_active":"2023-09-23T22:19:02.162117+0000","last_peered":"2023-09-23T22:19:02.162117+0000","last_clean":"2023-09-23T22:19:02.162117+0000","last_became_active":"2023-09-23T22:18:57.742146+0000","last_became_peered":"2023-09-23T22:18:57.742146+0000","last_unstale":"2023-09-23T22:19:02.162117+0000","last_undegraded":"2023-09-23T22:19:02.162117+0000","last_fullsized":"2023-09-23T22:19:02.162117+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":199,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.791176+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:19:11.791176+0000","last_peered":"2023-09-23T22:19:11.791176+0000","last_clean":"2023-09-23T22:19:11.791176+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:19:11.791176+0000","last_undegraded":"2023-09-23T22:19:11.791176+0000","last_fullsized":"2023-09-23T22:19:11.791176+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":828,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-09-23T22:19:23.403751+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:19:23.403751+0000","last_peered":"2023-09-23T22:19:23.403751+0000","last_clean":"2023-09-23T22:19:23.403751+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:19:23.403751+0000","last_undegraded":"2023-09-23T22:19:23.403751+0000","last_fullsized":"2023-09-23T22:19:23.403751+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":184,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.789965+0000","last_change":"2023-09-23T22:18:57.722771+0000","last_active":"2023-09-23T22:19:11.789965+0000","last_peered":"2023-09-23T22:19:11.789965+0000","last_clean":"2023-09-23T22:19:11.789965+0000","last_became_active":"2023-09-23T22:18:57.722258+0000","last_became_peered":"2023-09-23T22:18:57.722258+0000","last_unstale":"2023-09-23T22:19:11.789965+0000","last_undegraded":"2023-09-23T22:19:11.789965+0000","last_fullsized":"2023-09-23T22:19:11.789965+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:18.267091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"187'10650","reported_seq":10726,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:22.431177+0000","last_change":"2023-09-23T22:15:14.828787+0000","last_active":"2023-09-23T22:19:22.431177+0000","last_peered":"2023-09-23T22:19:22.431177+0000","last_clean":"2023-09-23T22:19:22.431177+0000","last_became_active":"2023-09-23T22:15:14.828370+0000","last_became_peered":"2023-09-23T22:15:14.828370+0000","last_unstale":"2023-09-23T22:19:22.431177+0000","last_undegraded":"2023-09-23T22:19:22.431177+0000","last_fullsized":"2023-09-23T22:19:22.431177+0000","mapping_epoch":111,"log_start":"136'3970","ondisk_log_start":"136'3970","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6680,"log_dups_size":0,"ondisk_log_size":6680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:35.080268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58064896,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10650,"num_write_kb":369152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":455,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161672+0000","last_change":"2023-09-23T22:19:00.918399+0000","last_active":"2023-09-23T22:19:02.161672+0000","last_peered":"2023-09-23T22:19:02.161672+0000","last_clean":"2023-09-23T22:19:02.161672+0000","last_became_active":"2023-09-23T22:18:57.720614+0000","last_became_peered":"2023-09-23T22:18:57.720614+0000","last_unstale":"2023-09-23T22:19:02.161672+0000","last_undegraded":"2023-09-23T22:19:02.161672+0000","last_fullsized":"2023-09-23T22:19:02.161672+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":251,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.175377+0000","last_change":"2023-09-23T22:16:49.270015+0000","last_active":"2023-09-23T22:19:02.175377+0000","last_peered":"2023-09-23T22:19:02.175377+0000","last_clean":"2023-09-23T22:19:02.175377+0000","last_became_active":"2023-09-23T22:16:49.252726+0000","last_became_peered":"2023-09-23T22:16:49.252726+0000","last_unstale":"2023-09-23T22:19:02.175377+0000","last_undegraded":"2023-09-23T22:19:02.175377+0000","last_fullsized":"2023-09-23T22:19:02.175377+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:18.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":19,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161925+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:19:02.161925+0000","last_peered":"2023-09-23T22:19:02.161925+0000","last_clean":"2023-09-23T22:19:02.161925+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:19:02.161925+0000","last_undegraded":"2023-09-23T22:19:02.161925+0000","last_fullsized":"2023-09-23T22:19:02.161925+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":150,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129068+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:19:02.129068+0000","last_peered":"2023-09-23T22:19:02.129068+0000","last_clean":"2023-09-23T22:19:02.129068+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:19:02.129068+0000","last_undegraded":"2023-09-23T22:19:02.129068+0000","last_fullsized":"2023-09-23T22:19:02.129068+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"188'10764","reported_seq":10851,"reported_epoch":188,"state":"active+clean","last_fresh":"2023-09-23T22:19:23.878809+0000","last_change":"2023-09-23T22:15:14.825150+0000","last_active":"2023-09-23T22:19:23.878809+0000","last_peered":"2023-09-23T22:19:23.878809+0000","last_clean":"2023-09-23T22:19:23.878809+0000","last_became_active":"2023-09-23T22:15:14.824867+0000","last_became_peered":"2023-09-23T22:15:14.824867+0000","last_unstale":"2023-09-23T22:19:23.878809+0000","last_undegraded":"2023-09-23T22:19:23.878809+0000","last_fullsized":"2023-09-23T22:19:23.878809+0000","mapping_epoch":111,"log_start":"133'3700","ondisk_log_start":"133'3700","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7064,"log_dups_size":0,"ondisk_log_size":7064,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:30.053753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54132736,"num_objects":826,"num_object_clones":0,"num_object_copies":2478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10764,"num_write_kb":370880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":15,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.161816+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:19:02.161816+0000","last_peered":"2023-09-23T22:19:02.161816+0000","last_clean":"2023-09-23T22:19:02.161816+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:19:02.161816+0000","last_undegraded":"2023-09-23T22:19:02.161816+0000","last_fullsized":"2023-09-23T22:19:02.161816+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":171,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.173811+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:19:02.173811+0000","last_peered":"2023-09-23T22:19:02.173811+0000","last_clean":"2023-09-23T22:19:02.173811+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:19:02.173811+0000","last_undegraded":"2023-09-23T22:19:02.173811+0000","last_fullsized":"2023-09-23T22:19:02.173811+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":149,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.172573+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:19:02.172573+0000","last_peered":"2023-09-23T22:19:02.172573+0000","last_clean":"2023-09-23T22:19:02.172573+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:19:02.172573+0000","last_undegraded":"2023-09-23T22:19:02.172573+0000","last_fullsized":"2023-09-23T22:19:02.172573+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":162,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129212+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:19:02.129212+0000","last_peered":"2023-09-23T22:19:02.129212+0000","last_clean":"2023-09-23T22:19:02.129212+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:19:02.129212+0000","last_undegraded":"2023-09-23T22:19:02.129212+0000","last_fullsized":"2023-09-23T22:19:02.129212+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":146,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.172192+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:19:02.172192+0000","last_peered":"2023-09-23T22:19:02.172192+0000","last_clean":"2023-09-23T22:19:02.172192+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:19:02.172192+0000","last_undegraded":"2023-09-23T22:19:02.172192+0000","last_fullsized":"2023-09-23T22:19:02.172192+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.129241+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:19:02.129241+0000","last_peered":"2023-09-23T22:19:02.129241+0000","last_clean":"2023-09-23T22:19:02.129241+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:19:02.129241+0000","last_undegraded":"2023-09-23T22:19:02.129241+0000","last_fullsized":"2023-09-23T22:19:02.129241+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":168,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.790986+0000","last_change":"2023-09-23T22:16:49.145417+0000","last_active":"2023-09-23T22:19:11.790986+0000","last_peered":"2023-09-23T22:19:11.790986+0000","last_clean":"2023-09-23T22:19:11.790986+0000","last_became_active":"2023-09-23T22:16:49.145017+0000","last_became_peered":"2023-09-23T22:16:49.145017+0000","last_unstale":"2023-09-23T22:19:11.790986+0000","last_undegraded":"2023-09-23T22:19:11.790986+0000","last_fullsized":"2023-09-23T22:19:11.790986+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:47.762149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":210,"reported_epoch":187,"state":"active+clean","last_fresh":"2023-09-23T22:19:11.789935+0000","last_change":"2023-09-23T22:17:15.721123+0000","last_active":"2023-09-23T22:19:11.789935+0000","last_peered":"2023-09-23T22:19:11.789935+0000","last_clean":"2023-09-23T22:19:11.789935+0000","last_became_active":"2023-09-23T22:17:15.720812+0000","last_became_peered":"2023-09-23T22:17:15.720812+0000","last_unstale":"2023-09-23T22:19:11.789935+0000","last_undegraded":"2023-09-23T22:19:11.789935+0000","last_fullsized":"2023-09-23T22:19:11.789935+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.176278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":59,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.165037+0000","last_change":"2023-09-23T22:18:57.729455+0000","last_active":"2023-09-23T22:19:02.165037+0000","last_peered":"2023-09-23T22:19:02.165037+0000","last_clean":"2023-09-23T22:19:02.165037+0000","last_became_active":"2023-09-23T22:18:57.729088+0000","last_became_peered":"2023-09-23T22:18:57.729088+0000","last_unstale":"2023-09-23T22:19:02.165037+0000","last_undegraded":"2023-09-23T22:19:02.165037+0000","last_fullsized":"2023-09-23T22:19:02.165037+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:17.861334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":155,"reported_epoch":186,"state":"active+clean","last_fresh":"2023-09-23T22:19:02.128773+0000","last_change":"2023-09-23T22:16:49.164638+0000","last_active":"2023-09-23T22:19:02.128773+0000","last_peered":"2023-09-23T22:19:02.128773+0000","last_clean":"2023-09-23T22:19:02.128773+0000","last_became_active":"2023-09-23T22:16:49.164349+0000","last_became_peered":"2023-09-23T22:16:49.164349+0000","last_unstale":"2023-09-23T22:19:02.128773+0000","last_undegraded":"2023-09-23T22:19:02.128773+0000","last_fullsized":"2023-09-23T22:19:02.128773+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:26.536927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":982122520,"num_objects":14987,"num_object_clones":0,"num_object_copies":44961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15310,"num_objects_misplaced":516,"num_objects_unfound":0,"num_objects_dirty":14987,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172625,"num_write_kb":6003521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28814,"num_bytes_recovered":1888288768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5117325312,"data_stored":5117313096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108271,"ondisk_log_size":108271,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4892,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2098784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234180,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":501948,"kb_used_data":425524,"kb_used_omap":0,"kb_used_meta":76416,"kb_avail":93243204,"statfs":{"total":95995035648,"available":95481040896,"internally_reserved":0,"allocated":435736576,"data_stored":434919086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":78249984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":2.2080000000000002,"15min":1.105},"min":{"1min":0.23100000000000001,"5min":0.223,"15min":0.223},"max":{"1min":21.977,"5min":109.208,"15min":109.208},"last":9.7799999999999994},{"interface":"front","average":{"1min":2.548,"5min":2.536,"15min":1.2370000000000001},"min":{"1min":0.28199999999999997,"5min":0.217,"15min":0.217},"max":{"1min":11.698,"5min":109.13800000000001,"15min":109.13800000000001},"last":1.6579999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":4.4370000000000003,"5min":4.2199999999999998,"15min":1.784},"min":{"1min":0.254,"5min":0.246,"15min":0.246},"max":{"1min":27.800000000000001,"5min":123.79300000000001,"15min":123.79300000000001},"last":3.8199999999999998},{"interface":"front","average":{"1min":4.3490000000000002,"5min":4.569,"15min":1.9139999999999999},"min":{"1min":0.316,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":27.785,"5min":123.66500000000001,"15min":123.66500000000001},"last":1.7809999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":6.2619999999999996,"5min":6.1609999999999996,"15min":2.7250000000000001},"min":{"1min":0.78600000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":37.698,"5min":125.101,"15min":125.101},"last":3.8650000000000002},{"interface":"front","average":{"1min":5.4589999999999996,"5min":5.8220000000000001,"15min":2.3500000000000001},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":37.679000000000002,"5min":126.021,"15min":126.021},"last":3.0590000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":2.4710000000000001,"15min":1.2250000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.399,"5min":113.282,"15min":113.282},"last":2.8319999999999999},{"interface":"front","average":{"1min":2.9769999999999999,"5min":2.8180000000000001,"15min":1.3859999999999999},"min":{"1min":0.41299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.2809999999999997,"5min":104.74299999999999,"15min":104.74299999999999},"last":2.742}]},{"osd":4,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.3009999999999999,"15min":0.749},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.251},"max":{"1min":10.305999999999999,"5min":33.627000000000002,"15min":33.627000000000002},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.335,"5min":1.111,"15min":0.70299999999999996},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.1300000000000008,"5min":33.113,"15min":33.113},"last":0.312}]},{"osd":5,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.0109999999999999,"15min":0.65700000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.9219999999999997,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.946,"5min":1.5369999999999999,"15min":0.88200000000000001},"min":{"1min":0.35499999999999998,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":10.423999999999999,"5min":31.257999999999999,"15min":31.257999999999999},"last":0.38400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":0.999,"15min":0.68899999999999995},"min":{"1min":0.33700000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":7.9459999999999997,"5min":9.0139999999999993,"15min":9.0139999999999993},"last":0.371},{"interface":"front","average":{"1min":2.2930000000000001,"5min":1.3500000000000001,"15min":0.84199999999999997},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.308},"max":{"1min":10.347,"5min":13.362,"15min":13.362},"last":0.41499999999999998}]}]},{"osd":6,"up_from":0,"seq":197568495816,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795088,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":514192,"kb_used_data":347976,"kb_used_omap":7,"kb_used_meta":166200,"kb_avail":93230960,"statfs":{"total":95995035648,"available":95468503040,"internally_reserved":0,"allocated":356327424,"data_stored":355309142,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":170189172},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.79,"15min":0.83599999999999997},"min":{"1min":0.20399999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":10.705,"5min":61.890000000000001,"15min":61.890000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":3.7280000000000002,"5min":2.4220000000000002,"15min":1.0780000000000001},"min":{"1min":0.216,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":15.08,"5min":62.689999999999998,"15min":62.689999999999998},"last":6.2039999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":5.3970000000000002,"5min":2.1629999999999998,"15min":0.97299999999999998},"min":{"1min":0.29099999999999998,"5min":0.216,"15min":0.216},"max":{"1min":33.902000000000001,"5min":33.902000000000001,"15min":33.902000000000001},"last":2.6219999999999999},{"interface":"front","average":{"1min":7.593,"5min":2.302,"15min":1.042},"min":{"1min":0.48399999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":34.781999999999996,"5min":34.781999999999996,"15min":34.781999999999996},"last":2.8109999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:19:14 2023","interfaces":[{"interface":"back","average":{"1min":4.1349999999999998,"5min":2.1909999999999998,"15min":0.995},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":16.131,"5min":22.326000000000001,"15min":22.326000000000001},"last":2.8559999999999999},{"interface":"front","average":{"1min":2.9049999999999998,"5min":1.962,"15min":0.92700000000000005},"min":{"1min":0.25900000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":15.859999999999999,"5min":22.259,"15min":22.259},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:19:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":1.0289999999999999,"15min":0.78800000000000003},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.845000000000001,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.6560000000000001,"5min":1.5189999999999999,"15min":0.94899999999999995},"min":{"1min":0.33200000000000002,"5min":0.318,"15min":0.318},"max":{"1min":11.037000000000001,"5min":11.037000000000001,"15min":11.037000000000001},"last":2.8879999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9039999999999999,"5min":1.522,"15min":0.91700000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":11.125,"5min":11.125,"15min":11.125},"last":2.9239999999999999},{"interface":"front","average":{"1min":1.54,"5min":1.292,"15min":0.85399999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.1680000000000001,"5min":6.6130000000000004,"15min":6.6130000000000004},"last":0.54300000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4930000000000001,"5min":1.3660000000000001,"15min":0.88200000000000001},"min":{"1min":0.372,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.101,"5min":10.989000000000001,"15min":10.989000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.246,"15min":0.86399999999999999},"min":{"1min":0.35699999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.811999999999999,"5min":10.811999999999999,"15min":10.811999999999999},"last":0.63200000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.516,"15min":0.93600000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.9829999999999997,"5min":10.948,"15min":10.948},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.635,"5min":1.369,"15min":0.90900000000000003},"min":{"1min":0.39500000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":10.781000000000001,"5min":10.781000000000001,"15min":10.781000000000001},"last":0.59799999999999998}]}]},{"osd":0,"up_from":93,"seq":399431958622,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":932932,"kb_used_data":774652,"kb_used_omap":8,"kb_used_meta":158007,"kb_avail":92812220,"statfs":{"total":95995035648,"available":95039713280,"internally_reserved":0,"allocated":793243648,"data_stored":792236303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":161800042},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:19:12 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":2.4529999999999998,"15min":1.0429999999999999},"min":{"1min":0.23999999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":11.711,"5min":44.776000000000003,"15min":44.776000000000003},"last":0.41499999999999998},{"interface":"front","average":{"1min":4.6529999999999996,"5min":3.1000000000000001,"15min":1.276},"min":{"1min":0.23899999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":20.927,"5min":48.838000000000001,"15min":48.838000000000001},"last":0.26800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:18:40 2023","interfaces":[{"interface":"back","average":{"1min":3.9319999999999999,"5min":1.502,"15min":0.76900000000000002},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":25.893000000000001,"5min":25.893000000000001,"15min":25.893000000000001},"last":0.28399999999999997},{"interface":"front","average":{"1min":7.742,"5min":2.7810000000000001,"15min":1.2749999999999999},"min":{"1min":0.28999999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":66.665000000000006,"5min":66.665000000000006,"15min":66.665000000000006},"last":0.89800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:19:12 2023","interfaces":[{"interface":"back","average":{"1min":3.2589999999999999,"5min":2.4060000000000001,"15min":1.099},"min":{"1min":0.33400000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":13.663,"5min":35.496000000000002,"15min":35.496000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":4.2469999999999999,"5min":2.6779999999999999,"15min":1.1859999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":20.870999999999999,"5min":35.533000000000001,"15min":35.533000000000001},"last":0.441}]},{"osd":4,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.2880000000000003,"5min":2.2050000000000001,"15min":1.1679999999999999},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.217000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.9860000000000002,"5min":1.2569999999999999,"15min":0.85799999999999998},"min":{"1min":0.32700000000000001,"5min":0.308,"15min":0.308},"max":{"1min":22.209,"5min":22.209,"15min":22.209},"last":0.495}]},{"osd":5,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.6959999999999997,"5min":2.173,"15min":1.1799999999999999},"min":{"1min":0.36099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.372,"5min":12.372,"15min":12.372},"last":0.379},{"interface":"front","average":{"1min":2.6699999999999999,"5min":1.286,"15min":0.88200000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":20.390999999999998,"5min":20.390999999999998,"15min":20.390999999999998},"last":0.54700000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.5609999999999999,"5min":2.2149999999999999,"15min":1.1879999999999999},"min":{"1min":0.46400000000000002,"5min":0.379,"15min":0.379},"max":{"1min":10.275,"5min":10.275,"15min":10.275},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.867,"5min":1.3080000000000001,"15min":0.91000000000000003},"min":{"1min":0.38600000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":22.263000000000002,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.57999999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.1180000000000003,"5min":2.3079999999999998,"15min":1.252},"min":{"1min":0.44500000000000001,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":20.491,"5min":23.306000000000001,"15min":23.306000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":4.5170000000000003,"5min":2.6419999999999999,"15min":1.379},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.257999999999999,"5min":23.265999999999998,"15min":23.265999999999998},"last":0.52400000000000002}]}]},{"osd":2,"up_from":103,"seq":442381631561,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":561600,"kb_used_data":407672,"kb_used_omap":10,"kb_used_meta":153845,"kb_avail":93183552,"statfs":{"total":95995035648,"available":95419957248,"internally_reserved":0,"allocated":417456128,"data_stored":416454342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":157537452},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":4.7610000000000001,"5min":1.9370000000000001,"15min":0.86899999999999999},"min":{"1min":0.308,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":15.26,"5min":15.26,"15min":15.26},"last":0.38200000000000001},{"interface":"front","average":{"1min":4.7240000000000002,"5min":2.2749999999999999,"15min":1},"min":{"1min":0.21199999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":19.149999999999999,"5min":19.149999999999999,"15min":19.149999999999999},"last":3.4500000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":4.5199999999999996,"5min":3.9489999999999998,"15min":1.5980000000000001},"min":{"1min":0.32600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":19.869,"5min":157.636,"15min":157.636},"last":8.3889999999999993},{"interface":"front","average":{"1min":4.1710000000000003,"5min":4.0369999999999999,"15min":1.625},"min":{"1min":0.28100000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":17.768999999999998,"5min":155.63300000000001,"15min":155.63300000000001},"last":4.3559999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:19:05 2023","interfaces":[{"interface":"back","average":{"1min":4.1289999999999996,"5min":2.2949999999999999,"15min":0.99199999999999999},"min":{"1min":0.25,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":19.852,"5min":19.852,"15min":19.852},"last":2.032},{"interface":"front","average":{"1min":4.4340000000000002,"5min":2.4009999999999998,"15min":1.0589999999999999},"min":{"1min":0.27900000000000003,"5min":0.246,"15min":0.246},"max":{"1min":20.581,"5min":20.581,"15min":20.581},"last":3.5529999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":1.5289999999999999,"15min":0.90800000000000003},"min":{"1min":0.26800000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.452,"5min":11.244,"15min":11.244},"last":3.3199999999999998},{"interface":"front","average":{"1min":2.7029999999999998,"5min":1.544,"15min":0.93200000000000005},"min":{"1min":0.372,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.2959999999999994,"5min":9.2799999999999994,"15min":9.2799999999999994},"last":2.0859999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":1.671,"15min":0.97999999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.5009999999999994,"5min":11.291,"15min":11.291},"last":3.3690000000000002},{"interface":"front","average":{"1min":2.911,"5min":1.617,"15min":0.94299999999999995},"min":{"1min":0.36799999999999999,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":8.327,"5min":9.4499999999999993,"15min":9.4499999999999993},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":3.589,"5min":1.7130000000000001,"15min":1.014},"min":{"1min":0.38400000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":17.640999999999998,"5min":17.640999999999998,"15min":17.640999999999998},"last":3.4209999999999998},{"interface":"front","average":{"1min":2.8839999999999999,"5min":1.5169999999999999,"15min":1.0289999999999999},"min":{"1min":0.42299999999999999,"5min":0.375,"15min":0.375},"max":{"1min":11.875,"5min":11.875,"15min":11.875},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:18:31 2023","interfaces":[{"interface":"back","average":{"1min":3.8969999999999998,"5min":1.714,"15min":1.0780000000000001},"min":{"1min":0.38500000000000001,"5min":0.38,"15min":0.38},"max":{"1min":10.135,"5min":10.135,"15min":10.135},"last":3.6549999999999998},{"interface":"front","average":{"1min":3.46,"5min":1.9279999999999999,"15min":1.0669999999999999},"min":{"1min":0.51000000000000001,"5min":0.377,"15min":0.377},"max":{"1min":17.738,"5min":17.738,"15min":17.738},"last":3.4009999999999998}]}]},{"osd":3,"up_from":108,"seq":463856468035,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":324236,"kb_used_data":260676,"kb_used_omap":4,"kb_used_meta":63547,"kb_avail":93420916,"statfs":{"total":95995035648,"available":95663017984,"internally_reserved":0,"allocated":266932224,"data_stored":265921665,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":65072685},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":4.3410000000000002,"5min":2.738,"15min":1.1639999999999999},"min":{"1min":0.23300000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":32.731999999999999,"5min":77.834999999999994,"15min":77.834999999999994},"last":1.103},{"interface":"front","average":{"1min":4.8730000000000002,"5min":3.1019999999999999,"15min":1.2749999999999999},"min":{"1min":0.26500000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":32.786999999999999,"5min":75.558000000000007,"15min":75.558000000000007},"last":6.2130000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":7.4589999999999996,"5min":3.5379999999999998,"15min":1.4139999999999999},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":82.951999999999998,"5min":82.951999999999998,"15min":82.951999999999998},"last":8.9949999999999992},{"interface":"front","average":{"1min":9.4529999999999994,"5min":3.601,"15min":1.4730000000000001},"min":{"1min":0.22600000000000001,"5min":0.189,"15min":0.189},"max":{"1min":86.001000000000005,"5min":86.001000000000005,"15min":86.001000000000005},"last":4.7220000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":17.736000000000001,"5min":7.7190000000000003,"15min":2.835},"min":{"1min":0.28000000000000003,"5min":0.223,"15min":0.223},"max":{"1min":99.962999999999994,"5min":99.962999999999994,"15min":99.962999999999994},"last":8.9450000000000003},{"interface":"front","average":{"1min":18.577000000000002,"5min":7.1669999999999998,"15min":2.6720000000000002},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":100.429,"5min":100.429,"15min":100.429},"last":3.6019999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:19:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":1.3169999999999999,"15min":0.85599999999999998},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.7610000000000001,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.4630000000000001,"15min":0.92500000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.6200000000000001,"5min":12.036,"15min":12.036},"last":6.2610000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.397,"15min":0.88200000000000001},"min":{"1min":0.32800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.6529999999999996,"5min":14.292,"15min":14.292},"last":6.2869999999999999},{"interface":"front","average":{"1min":2.726,"5min":1.526,"15min":0.91500000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.484,"5min":11.56,"15min":11.56},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":2.6190000000000002,"5min":1.6100000000000001,"15min":0.97199999999999998},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.948,"5min":12.068,"15min":12.068},"last":6.3120000000000003},{"interface":"front","average":{"1min":2.0939999999999999,"5min":1.52,"15min":0.93000000000000005},"min":{"1min":0.35599999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.0660000000000007,"5min":11.586,"15min":11.586},"last":0.56299999999999994}]},{"osd":7,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":1.607,"15min":0.95599999999999996},"min":{"1min":0.39600000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":9.1630000000000003,"5min":11.624000000000001,"15min":11.624000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.504,"5min":1.405,"15min":0.92400000000000004},"min":{"1min":0.40400000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.7880000000000003,"5min":12.382,"15min":12.382},"last":0.60799999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019086,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":586920,"kb_used_data":483488,"kb_used_omap":0,"kb_used_meta":103424,"kb_avail":93158232,"statfs":{"total":95995035648,"available":95394029568,"internally_reserved":0,"allocated":495091712,"data_stored":494286103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":105906176},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,3,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.9970000000000001,"15min":1.0840000000000001},"min":{"1min":0.23000000000000001,"5min":0.218,"15min":0.218},"max":{"1min":7.5789999999999997,"5min":66.355000000000004,"15min":66.355000000000004},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.4299999999999999,"5min":2.2200000000000002,"15min":1.1699999999999999},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.5949999999999998,"5min":66.332999999999998,"15min":66.332999999999998},"last":0.51600000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":1.748,"15min":0.94599999999999995},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.625,"5min":38.878,"15min":38.878},"last":1.6919999999999999},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.4220000000000002,"15min":1.1899999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":10.236000000000001,"5min":38.853999999999999,"15min":38.853999999999999},"last":1.5149999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:18:40 2023","interfaces":[{"interface":"back","average":{"1min":8.1180000000000003,"5min":2.8780000000000001,"15min":1.3200000000000001},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.255},"max":{"1min":133.22900000000001,"5min":133.22900000000001,"15min":133.22900000000001},"last":0.73699999999999999},{"interface":"front","average":{"1min":8.4529999999999994,"5min":3.4510000000000001,"15min":1.4910000000000001},"min":{"1min":0.33000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":133.60300000000001,"5min":133.60300000000001,"15min":133.60300000000001},"last":1.8}]},{"osd":3,"last update":"Sat Sep 23 22:19:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4710000000000001,"5min":1.5489999999999999,"15min":0.91200000000000003},"min":{"1min":0.35199999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9109999999999996,"5min":7.9109999999999996,"15min":7.9109999999999996},"last":0.63800000000000001},{"interface":"front","average":{"1min":2.169,"5min":1.607,"15min":0.91900000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.8930000000000007,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":1.8169999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:19:09 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.93300000000000005,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":4.0629999999999997,"5min":8.9499999999999993,"15min":8.9499999999999993},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.86099999999999999,"15min":0.64300000000000002},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.0069999999999997,"5min":7.8789999999999996,"15min":7.8789999999999996},"last":3.6179999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:18:24 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.75,"15min":0.58599999999999997},"min":{"1min":0.309,"5min":0.251,"15min":0.251},"max":{"1min":3.6699999999999999,"5min":3.6699999999999999,"15min":3.6699999999999999},"last":0.502},{"interface":"front","average":{"1min":0.95099999999999996,"5min":0.84599999999999997,"15min":0.64600000000000002},"min":{"1min":0.32100000000000001,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":3.052,"5min":8.5210000000000008,"15min":8.5210000000000008},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.1180000000000001,"15min":0.76000000000000001},"min":{"1min":0.33100000000000002,"5min":0.307,"15min":0.307},"max":{"1min":5.3440000000000003,"5min":8.577,"15min":8.577},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.90400000000000003,"15min":0.69299999999999995},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.7799999999999998,"5min":5.6280000000000001,"15min":5.6280000000000001},"last":2.6859999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757452,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":931936,"kb_used_data":821656,"kb_used_omap":0,"kb_used_meta":110272,"kb_avail":92813216,"statfs":{"total":95995035648,"available":95040733184,"internally_reserved":0,"allocated":841375744,"data_stored":840574114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":112918528},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":3.113,"15min":1.4139999999999999},"min":{"1min":0.26800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.6849999999999996,"5min":133.22499999999999,"15min":133.22499999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.732,"5min":3.387,"15min":1.554},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.444,"5min":132.78200000000001,"15min":132.78200000000001},"last":2.1349999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":7.7859999999999996,"5min":4.5830000000000002,"15min":1.9219999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":83.759,"5min":83.759,"15min":83.759},"last":4.5439999999999996},{"interface":"front","average":{"1min":7.8920000000000003,"5min":4.6139999999999999,"15min":1.9179999999999999},"min":{"1min":0.30399999999999999,"5min":0.252,"15min":0.252},"max":{"1min":83.741,"5min":83.741,"15min":83.741},"last":4.5629999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:18:39 2023","interfaces":[{"interface":"back","average":{"1min":12.324999999999999,"5min":6.0250000000000004,"15min":2.383},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":104.254,"5min":104.254,"15min":104.254},"last":0.33200000000000002},{"interface":"front","average":{"1min":12.984,"5min":6.6600000000000001,"15min":2.6139999999999999},"min":{"1min":0.34699999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":104.217,"5min":104.217,"15min":104.217},"last":9.1910000000000007}]},{"osd":3,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.615,"5min":2.6389999999999998,"15min":1.2649999999999999},"min":{"1min":0.31900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":4.7549999999999999,"5min":88.106999999999999,"15min":88.106999999999999},"last":0.37},{"interface":"front","average":{"1min":2.3929999999999998,"5min":2.8610000000000002,"15min":1.3460000000000001},"min":{"1min":0.38800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1369999999999996,"5min":87.887,"15min":87.887},"last":0.433}]},{"osd":4,"last update":"Sat Sep 23 22:19:01 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.6559999999999999,"15min":0.86399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.4880000000000004,"5min":82.530000000000001,"15min":82.530000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.7070000000000001,"15min":0.91800000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.577,"5min":85.441000000000003,"15min":85.441000000000003},"last":0.75}]},{"osd":6,"last update":"Sat Sep 23 22:18:30 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.117,"15min":1.0509999999999999},"min":{"1min":0.32800000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":9.6519999999999992,"5min":114.83499999999999,"15min":114.83499999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.1230000000000002,"5min":1.9119999999999999,"15min":0.995},"min":{"1min":0.31,"5min":0.31,"15min":0.29699999999999999},"max":{"1min":7.4640000000000004,"5min":110.905,"15min":110.905},"last":0.38100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:18:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.1850000000000001,"15min":0.77200000000000002},"min":{"1min":0.35899999999999999,"5min":0.33500000000000002,"15min":0.32600000000000001},"max":{"1min":8.2159999999999993,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.1950000000000001,"5min":0.97599999999999998,"15min":0.69699999999999995},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.33200000000000002},"max":{"1min":5.2460000000000004,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":0.42299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":784007168,"data_stored":784007168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":340197376,"data_stored":340197376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":407179264,"data_stored":407175192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":262340608,"data_stored":262340608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":489029632,"data_stored":489029632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":833097728,"data_stored":833093656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1578041344,"data_stored":1578041344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":423432192,"data_stored":423428120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:19:27.758 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:27.838 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:27.928 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:27.980 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:27.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:27.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:28.088 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:28.147 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:28.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:28 smithi026 ceph-mon[215775]: pgmap v426: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 834 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 15148/39897 objects degraded (37.968%); 256/39897 objects misplaced (0.642%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:19:28.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15148/39897 objects degraded (37.968%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:28.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:28 smithi026 ceph-mon[223014]: pgmap v426: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 834 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 15148/39897 objects degraded (37.968%); 256/39897 objects misplaced (0.642%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:19:28.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15148/39897 objects degraded (37.968%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:28.325 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:28.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:28.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:28.436 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:28.481 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:28 smithi070 ceph-mon[192652]: pgmap v426: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 834 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 15148/39897 objects degraded (37.968%); 256/39897 objects misplaced (0.642%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:19:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15148/39897 objects degraded (37.968%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:28.713 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:28.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:28.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:28.824 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:28.909 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:29.033 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:29.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:29.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:29.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:19:29.140 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:29.237 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:29.449 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:19:29.449 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:19:29.541 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:29.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:29.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:29.662 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:29.738 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:29.849 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:29.938 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:19:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:19:29] "GET /metrics HTTP/1.1" 200 36391 "" "Prometheus/2.43.0" 2023-09-23T22:19:29.981 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:29.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:29.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:30.087 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:30.143 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:30.207 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:30.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:30.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:19:30.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:30.315 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:30.352 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:30.443 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:30.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:30.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:30.450 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:30 smithi070 ceph-mon[192652]: pgmap v427: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 771 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 15148/36897 objects degraded (41.055%); 256/36897 objects misplaced (0.694%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:19:30.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:30 smithi026 ceph-mon[215775]: pgmap v427: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 771 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 15148/36897 objects degraded (41.055%); 256/36897 objects misplaced (0.694%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:19:30.544 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:30 smithi026 ceph-mon[223014]: pgmap v427: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 771 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 15148/36897 objects degraded (41.055%); 256/36897 objects misplaced (0.694%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:19:30.551 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:30.590 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:30.683 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:19:30.698 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:30.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:30.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:30.748 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:30.805 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:30.834 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:30.902 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:30.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:30.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:31.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:31.074 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:31.294 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:19:31.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3483894306' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:31.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:31 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:31.542 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:31.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:31.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:31.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3483894306' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:31.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:31 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:31.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3483894306' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:31.552 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:31 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:31.647 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:31.690 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:31.950 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:31.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:31.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:32.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:32.092 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:32.306 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:32.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:32.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:32.409 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:32.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[223014]: pgmap v428: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 749 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 15114/35838 objects degraded (42.173%); 5.2 MiB/s, 85 objects/s recovering 2023-09-23T22:19:32.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:32.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:19:32.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[223014]: osdmap e189: 8 total, 8 up, 7 in 2023-09-23T22:19:32.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[223014]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[223014]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[223014]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[223014]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[215775]: pgmap v428: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 749 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 15114/35838 objects degraded (42.173%); 5.2 MiB/s, 85 objects/s recovering 2023-09-23T22:19:32.437 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:32.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:19:32.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[215775]: osdmap e189: 8 total, 8 up, 7 in 2023-09-23T22:19:32.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[215775]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[215775]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[215775]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.438 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:32 smithi026 ceph-mon[215775]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.477 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:32.538 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:19:32.538 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:19:32.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:32 smithi070 ceph-mon[192652]: pgmap v428: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 749 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 15114/35838 objects degraded (42.173%); 5.2 MiB/s, 85 objects/s recovering 2023-09-23T22:19:32.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:32 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:32.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:32 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:19:32.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:32 smithi070 ceph-mon[192652]: osdmap e189: 8 total, 8 up, 7 in 2023-09-23T22:19:32.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:32 smithi070 ceph-mon[192652]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:32 smithi070 ceph-mon[192652]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:32 smithi070 ceph-mon[192652]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:32 smithi070 ceph-mon[192652]: failed to encode map e189 with expected crc 2023-09-23T22:19:32.673 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:32.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:32.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:32.780 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:32.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:32.837 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:32.863 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:32.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:32.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:32.969 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:32.979 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:19:33.009 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:19:33.019 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:33.039 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:19:33.077 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:19:33.099 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:19:33.127 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:19:33.154 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:33.156 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:19:33.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:33.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:33.180 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:19:33.182 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:19:33.182 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:19:33.182 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:19:33.182 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:19:33.182 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:19:33.182 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:19:33.182 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:19:33.182 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:19:33.261 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:33.288 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:33.369 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:19:33.387 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:33.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:33.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:33.498 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:33.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:33 smithi070 ceph-mon[192652]: failed to encode map e190 with expected crc 2023-09-23T22:19:33.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:33 smithi070 ceph-mon[192652]: osdmap e190: 8 total, 8 up, 7 in 2023-09-23T22:19:33.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:33 smithi070 ceph-mon[192652]: failed to encode map e190 with expected crc 2023-09-23T22:19:33.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15114/35838 objects degraded (42.173%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:33.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:33 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/411631058' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:33.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:33.592 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:33.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[223014]: failed to encode map e190 with expected crc 2023-09-23T22:19:33.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[223014]: osdmap e190: 8 total, 8 up, 7 in 2023-09-23T22:19:33.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[223014]: failed to encode map e190 with expected crc 2023-09-23T22:19:33.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15114/35838 objects degraded (42.173%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:33.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/411631058' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:33.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[215775]: failed to encode map e190 with expected crc 2023-09-23T22:19:33.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[215775]: osdmap e190: 8 total, 8 up, 7 in 2023-09-23T22:19:33.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[215775]: failed to encode map e190 with expected crc 2023-09-23T22:19:33.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15114/35838 objects degraded (42.173%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:33.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:33 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/411631058' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:33.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:33.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:33.704 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:33.793 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:33.838 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:34.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:34 smithi070 ceph-mon[192652]: pgmap v431: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 628 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 14930/30003 objects degraded (49.762%); 6.7 MiB/s, 111 objects/s recovering 2023-09-23T22:19:34.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:34 smithi070 ceph-mon[192652]: osdmap e191: 8 total, 8 up, 7 in 2023-09-23T22:19:34.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:34 smithi070 ceph-mon[192652]: failed to encode map e191 with expected crc 2023-09-23T22:19:34.470 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:19:34.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:34 smithi026 ceph-mon[223014]: pgmap v431: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 628 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 14930/30003 objects degraded (49.762%); 6.7 MiB/s, 111 objects/s recovering 2023-09-23T22:19:34.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:34 smithi026 ceph-mon[223014]: osdmap e191: 8 total, 8 up, 7 in 2023-09-23T22:19:34.563 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:34 smithi026 ceph-mon[223014]: failed to encode map e191 with expected crc 2023-09-23T22:19:34.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:34 smithi026 ceph-mon[215775]: pgmap v431: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 628 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 14930/30003 objects degraded (49.762%); 6.7 MiB/s, 111 objects/s recovering 2023-09-23T22:19:34.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:34 smithi026 ceph-mon[215775]: osdmap e191: 8 total, 8 up, 7 in 2023-09-23T22:19:34.564 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:34 smithi026 ceph-mon[215775]: failed to encode map e191 with expected crc 2023-09-23T22:19:35.217 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:35.220 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:35.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:35.407 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:35.638 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:19:35.639 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:19:35.639 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:19:35.639 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:19:35.639 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:19:36.238 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:36.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:36.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:36.330 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:19:36.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:36.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:36.525 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:36 smithi026 ceph-mon[215775]: pgmap v433: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 599 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 10687/28626 objects degraded (37.333%); 4.6 MiB/s, 94 objects/s recovering 2023-09-23T22:19:36.527 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:36 smithi026 ceph-mon[223014]: pgmap v433: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 599 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 10687/28626 objects degraded (37.333%); 4.6 MiB/s, 94 objects/s recovering 2023-09-23T22:19:36.610 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:36.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:36 smithi070 ceph-mon[192652]: pgmap v433: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 92 active+clean; 599 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 10687/28626 objects degraded (37.333%); 4.6 MiB/s, 94 objects/s recovering 2023-09-23T22:19:36.716 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:36.772 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:36.788 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:36.800 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:36.998 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:37.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:37.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:37.105 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:37.236 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:37.393 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:37.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:37.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:37.514 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:37.562 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:37.693 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:19:37.711 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:19:37.744 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:37.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:37.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:37.937 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:38.528 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:19:38.579 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:19:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:19:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:19:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:19:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.579 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:14291 5.9 5:931fe9f1:::benchmark_data_smithi026_286012_object14290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.825342+0000", 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 252.75109059799999, 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15490252700000001, 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 14291 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.580 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825342+0000", 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825342+0000", 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825340+0000", 2023-09-23T22:19:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825456+0000", 2023-09-23T22:19:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115197 2023-09-23T22:19:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825472+0000", 2023-09-23T22:19:38.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6113e-05 2023-09-23T22:19:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825477+0000", 2023-09-23T22:19:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6930000000000004e-06 2023-09-23T22:19:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825530+0000", 2023-09-23T22:19:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2459e-05 2023-09-23T22:19:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825656+0000", 2023-09-23T22:19:38.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012608699999999999 2023-09-23T22:19:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:19:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825807+0000", 2023-09-23T22:19:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000150832 2023-09-23T22:19:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.826939+0000", 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011316690000000001 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.826984+0000", 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5592e-05 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.980153+0000", 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15316913200000001 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.980205+0000", 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2130999999999999e-05 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.588 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.980244+0000", 2023-09-23T22:19:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9028e-05 2023-09-23T22:19:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:14292 5.9 5:940e22e9:::benchmark_data_smithi026_286012_object14291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:19:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.825492+0000", 2023-09-23T22:19:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 252.750940464, 2023-09-23T22:19:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.154911729, 2023-09-23T22:19:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.590 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 14292 2023-09-23T22:19:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825492+0000", 2023-09-23T22:19:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825492+0000", 2023-09-23T22:19:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825491+0000", 2023-09-23T22:19:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:19:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825526+0000", 2023-09-23T22:19:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4508000000000002e-05 2023-09-23T22:19:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825534+0000", 2023-09-23T22:19:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8469999999999996e-06 2023-09-23T22:19:38.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.825539+0000", 2023-09-23T22:19:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3380000000000004e-06 2023-09-23T22:19:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.595 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.826334+0000", 2023-09-23T22:19:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00079483799999999997 2023-09-23T22:19:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.826460+0000", 2023-09-23T22:19:38.596 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012650699999999999 2023-09-23T22:19:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:19:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.826564+0000", 2023-09-23T22:19:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103786 2023-09-23T22:19:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.597 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.827112+0000", 2023-09-23T22:19:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00054744500000000003 2023-09-23T22:19:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.827446+0000", 2023-09-23T22:19:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00033380399999999999 2023-09-23T22:19:38.598 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.980304+0000", 2023-09-23T22:19:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15285805 2023-09-23T22:19:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.599 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.980365+0000", 2023-09-23T22:19:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1251e-05 2023-09-23T22:19:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.980404+0000", 2023-09-23T22:19:38.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8928999999999999e-05 2023-09-23T22:19:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19265 5.2 5:4c5a11fa:::benchmark_data_smithi026_286012_object19264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:19:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.623919+0000", 2023-09-23T22:19:38.601 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 249.95251379600001, 2023-09-23T22:19:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0646463700000002, 2023-09-23T22:19:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19265 2023-09-23T22:19:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.602 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.623919+0000", 2023-09-23T22:19:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.623919+0000", 2023-09-23T22:19:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.623917+0000", 2023-09-23T22:19:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:38.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.623972+0000", 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4429999999999999e-05 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.623986+0000", 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4394e-05 2023-09-23T22:19:38.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.623995+0000", 2023-09-23T22:19:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6249999999999996e-06 2023-09-23T22:19:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.625112+0000", 2023-09-23T22:19:38.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011170010000000001 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.625123+0000", 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1333e-05 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.621456+0000", 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99633342400000002 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.621470+0000", 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3631999999999999e-05 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.682053+0000", 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.060583075 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.608 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.682178+0000", 2023-09-23T22:19:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012459400000000001 2023-09-23T22:19:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:19:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.682327+0000", 2023-09-23T22:19:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014909400000000001 2023-09-23T22:19:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.683976+0000", 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001649595 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.686360+0000", 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023834519999999999 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.687595+0000", 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012354589999999999 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.687646+0000", 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1125000000000003e-05 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.688565+0000", 2023-09-23T22:19:38.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00091863399999999999 2023-09-23T22:19:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19274 5.3 5:cad49d88:::benchmark_data_smithi026_286012_object19273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:19:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.638745+0000", 2023-09-23T22:19:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 249.937687944, 2023-09-23T22:19:38.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0486302919999999, 2023-09-23T22:19:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19274 2023-09-23T22:19:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.613 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.638745+0000", 2023-09-23T22:19:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.638745+0000", 2023-09-23T22:19:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.638743+0000", 2023-09-23T22:19:38.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.638785+0000", 2023-09-23T22:19:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1695e-05 2023-09-23T22:19:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.638802+0000", 2023-09-23T22:19:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7544999999999999e-05 2023-09-23T22:19:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.638808+0000", 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0639999999999999e-06 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.638879+0000", 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0488999999999995e-05 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.638889+0000", 2023-09-23T22:19:38.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6379999999999999e-06 2023-09-23T22:19:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.621721+0000", 2023-09-23T22:19:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.9828325 2023-09-23T22:19:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.621731+0000", 2023-09-23T22:19:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0259e-05 2023-09-23T22:19:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.680199+0000", 2023-09-23T22:19:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.058467622 2023-09-23T22:19:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.680650+0000", 2023-09-23T22:19:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000451433 2023-09-23T22:19:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:19:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.680826+0000", 2023-09-23T22:19:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000175494 2023-09-23T22:19:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.684760+0000", 2023-09-23T22:19:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003933766 2023-09-23T22:19:38.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.686631+0000", 2023-09-23T22:19:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001871761 2023-09-23T22:19:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.686676+0000", 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5108999999999998e-05 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.686725+0000", 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8677999999999997e-05 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.687375+0000", 2023-09-23T22:19:38.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00064967900000000001 2023-09-23T22:19:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19283 5.3 5:c13d32f0:::benchmark_data_smithi026_286012_object19282:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:19:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.645666+0000", 2023-09-23T22:19:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 249.93076665000001, 2023-09-23T22:19:38.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0428579039999999, 2023-09-23T22:19:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19283 2023-09-23T22:19:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.626 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.645666+0000", 2023-09-23T22:19:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.645666+0000", 2023-09-23T22:19:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.645665+0000", 2023-09-23T22:19:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.645695+0000", 2023-09-23T22:19:38.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0246000000000001e-05 2023-09-23T22:19:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.645706+0000", 2023-09-23T22:19:38.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0407e-05 2023-09-23T22:19:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.645712+0000", 2023-09-23T22:19:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9789999999999998e-06 2023-09-23T22:19:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.645764+0000", 2023-09-23T22:19:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2318999999999998e-05 2023-09-23T22:19:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.645774+0000", 2023-09-23T22:19:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7010000000000008e-06 2023-09-23T22:19:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.621876+0000", 2023-09-23T22:19:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97610218599999998 2023-09-23T22:19:38.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.621882+0000", 2023-09-23T22:19:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3450000000000002e-06 2023-09-23T22:19:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.681483+0000", 2023-09-23T22:19:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.059601193 2023-09-23T22:19:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.681618+0000", 2023-09-23T22:19:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013500299999999999 2023-09-23T22:19:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:19:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.681784+0000", 2023-09-23T22:19:38.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000166093 2023-09-23T22:19:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.684766+0000", 2023-09-23T22:19:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029814479999999998 2023-09-23T22:19:38.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.687473+0000", 2023-09-23T22:19:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027074059999999999 2023-09-23T22:19:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.687915+0000", 2023-09-23T22:19:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044199000000000002 2023-09-23T22:19:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.687966+0000", 2023-09-23T22:19:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1187000000000003e-05 2023-09-23T22:19:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.688524+0000", 2023-09-23T22:19:38.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00055728500000000003 2023-09-23T22:19:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.641 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:36989 5.3 5:c1ea3a30:::benchmark_data_smithi026_286012_object36988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:19:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:43.919522+0000", 2023-09-23T22:19:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 234.656910055, 2023-09-23T22:19:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0369815090000001, 2023-09-23T22:19:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:19:38.642 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36989 2023-09-23T22:19:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919522+0000", 2023-09-23T22:19:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919522+0000", 2023-09-23T22:19:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919520+0000", 2023-09-23T22:19:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:19:38.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919697+0000", 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000177515 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919712+0000", 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4878999999999999e-05 2023-09-23T22:19:38.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919719+0000", 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8900000000000001e-06 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.920380+0000", 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00066100900000000005 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.920392+0000", 2023-09-23T22:19:38.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1892000000000001e-05 2023-09-23T22:19:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956428+0000", 2023-09-23T22:19:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0360360900000001 2023-09-23T22:19:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956504+0000", 2023-09-23T22:19:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5813000000000001e-05 2023-09-23T22:19:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:36993 5.3 5:cf430005:::benchmark_data_smithi026_286012_object36992:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:19:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:43.919737+0000", 2023-09-23T22:19:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 234.65669583600001, 2023-09-23T22:19:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0608951689999999, 2023-09-23T22:19:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:19:38.652 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36993 2023-09-23T22:19:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919737+0000", 2023-09-23T22:19:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919737+0000", 2023-09-23T22:19:38.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919736+0000", 2023-09-23T22:19:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:19:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919868+0000", 2023-09-23T22:19:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013176400000000001 2023-09-23T22:19:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919881+0000", 2023-09-23T22:19:38.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3098999999999999e-05 2023-09-23T22:19:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919886+0000", 2023-09-23T22:19:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.31e-06 2023-09-23T22:19:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.920420+0000", 2023-09-23T22:19:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00053391500000000004 2023-09-23T22:19:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.920427+0000", 2023-09-23T22:19:38.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.148e-06 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956540+0000", 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0361125579999999 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956959+0000", 2023-09-23T22:19:38.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00041942200000000002 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.980561+0000", 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.023601609999999999 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.980632+0000", 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0927999999999996e-05 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.660 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:36994 5.3 5:c9195e24:::benchmark_data_smithi026_286012_object36993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:43.919892+0000", 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 234.65654069300001, 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.061134585, 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.661 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36994 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919892+0000", 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919892+0000", 2023-09-23T22:19:38.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919891+0000", 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919925+0000", 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3699999999999999e-05 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919933+0000", 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1429999999999998e-06 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.919938+0000", 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4049999999999999e-06 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.923420+0000", 2023-09-23T22:19:38.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034815530000000001 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.923431+0000", 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0710000000000001e-05 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956995+0000", 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0335644530000001 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957161+0000", 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000166002 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957263+0000", 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010228 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962519+0000", 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052553870000000003 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.979484+0000", 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016965411999999999 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.980661+0000", 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011765370000000001 2023-09-23T22:19:38.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.980697+0000", 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5775999999999998e-05 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981026+0000", 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032977199999999999 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.668 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37030 5.3 5:cc9280f1:::benchmark_data_smithi026_286012_object37029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.032516+0000", 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 234.54391621400001, 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94895995499999997, 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37030 2023-09-23T22:19:38.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.032516+0000", 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.032516+0000", 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.032515+0000", 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.032541+0000", 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5573000000000001e-05 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.032553+0000", 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2588999999999999e-05 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.032558+0000", 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8940000000000001e-06 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.032619+0000", 2023-09-23T22:19:38.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0767999999999998e-05 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.032629+0000", 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0454e-05 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.958488+0000", 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92585907099999998 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.958631+0000", 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014214499999999999 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.958815+0000", 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000184265 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962531+0000", 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003716001 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.979525+0000", 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016994189 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981097+0000", 2023-09-23T22:19:38.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015719239999999999 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981130+0000", 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2631e-05 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981476+0000", 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034668200000000001 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.676 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37055 5.3 5:c4c16b4c:::benchmark_data_smithi026_286012_object37054:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.050626+0000", 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 234.5258068, 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93129797400000003, 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.677 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37055 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.050626+0000", 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.050626+0000", 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.050625+0000", 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.050677+0000", 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2664999999999998e-05 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.050689+0000", 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1851000000000001e-05 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.050694+0000", 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7430000000000002e-06 2023-09-23T22:19:38.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.050740+0000", 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5967000000000002e-05 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.050746+0000", 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0510000000000002e-06 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.960151+0000", 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90940474599999999 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.960284+0000", 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013316099999999999 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.960420+0000", 2023-09-23T22:19:38.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000136117 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964715+0000", 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042948229999999997 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.979761+0000", 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015045997 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981539+0000", 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017782550000000001 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981570+0000", 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1093999999999997e-05 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981924+0000", 2023-09-23T22:19:38.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035342000000000001 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37059 5.3 5:cb3099ab:::benchmark_data_smithi026_286012_object37058:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.054139+0000", 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 234.52229333899999, 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92833871400000001, 2023-09-23T22:19:38.685 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37059 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.054139+0000", 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.054139+0000", 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.054138+0000", 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:38.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.054204+0000", 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5220000000000002e-05 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.054221+0000", 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7730000000000001e-05 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.054242+0000", 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0176000000000001e-05 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.054297+0000", 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5053000000000003e-05 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.054306+0000", 2023-09-23T22:19:38.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8889999999999997e-06 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961316+0000", 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90700934200000005 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961453+0000", 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013753499999999999 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961655+0000", 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000201165 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.978926+0000", 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017271740000000001 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.979803+0000", 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00087706399999999999 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981970+0000", 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021667330000000001 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.982000+0000", 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9861000000000001e-05 2023-09-23T22:19:38.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.982478+0000", 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00047792700000000001 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37064 5.3 5:c904f5eb:::benchmark_data_smithi026_286012_object37063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.059398+0000", 2023-09-23T22:19:38.693 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 234.51703407700001, 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92522864400000004, 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37064 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059398+0000", 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059398+0000", 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059397+0000", 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:38.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059456+0000", 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8668000000000001e-05 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059473+0000", 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7516999999999998e-05 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059477+0000", 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2289999999999997e-06 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059543+0000", 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5371999999999997e-05 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059555+0000", 2023-09-23T22:19:38.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2319e-05 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962572+0000", 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90301729399999997 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962703+0000", 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013037500000000001 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962868+0000", 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000165754 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.978935+0000", 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016066858999999999 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981066+0000", 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002130828 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.983749+0000", 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026832150000000001 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.983803+0000", 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3789999999999998e-05 2023-09-23T22:19:38.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.984627+0000", 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00082396200000000002 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37065 5.3 5:cc27d6d1:::benchmark_data_smithi026_286012_object37064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.059495+0000", 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 234.51693713, 2023-09-23T22:19:38.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92592022500000004, 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37065 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059495+0000", 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059495+0000", 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059495+0000", 2023-09-23T22:19:38.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059519+0000", 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4281e-05 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059524+0000", 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7060000000000003e-06 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059527+0000", 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.858e-06 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059583+0000", 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6187999999999998e-05 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.059591+0000", 2023-09-23T22:19:38.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1559999999999995e-06 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963612+0000", 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90402128100000001 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963722+0000", 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109029 2023-09-23T22:19:38.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963828+0000", 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106795 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.979495+0000", 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01566675 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.981512+0000", 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020167280000000002 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.984693+0000", 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031809910000000002 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.984747+0000", 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4137e-05 2023-09-23T22:19:38.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.985416+0000", 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00066866199999999999 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:57197 5.3 5:ccadb28c:::benchmark_data_smithi026_286012_object57196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:04.898073+0000", 2023-09-23T22:19:38.709 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 213.678359812, 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.124934301, 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 57197 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.898073+0000", 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.898073+0000", 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.898072+0000", 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:19:38.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.898104+0000", 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1526999999999998e-05 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.898111+0000", 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2459999999999997e-06 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.898115+0000", 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3669999999999999e-06 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.901452+0000", 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033369680000000001 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.901641+0000", 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018924799999999999 2023-09-23T22:19:38.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.901813+0000", 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017127700000000001 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.904921+0000", 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031086429999999999 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.905690+0000", 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00076845199999999998 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.022073+0000", 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11638335800000001 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.022112+0000", 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8506e-05 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.023007+0000", 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00089522600000000005 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.716 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:57211 5.2 5:43ef9d2d:::benchmark_data_smithi026_286012_object57210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:04.908377+0000", 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 213.668055877, 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.115999862, 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 57211 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.908377+0000", 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.908377+0000", 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.908376+0000", 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.908406+0000", 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9740999999999999e-05 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.908416+0000", 2023-09-23T22:19:38.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0558e-05 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.908422+0000", 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9089999999999997e-06 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.908473+0000", 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1100000000000002e-05 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.908574+0000", 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010083599999999999 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.908658+0000", 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4311e-05 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.910329+0000", 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016706780000000001 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.910870+0000", 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00054117399999999995 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.024280+0000", 2023-09-23T22:19:38.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.113410312 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.024337+0000", 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6112000000000001e-05 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.024376+0000", 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9888000000000002e-05 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.723 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:57215 5.3 5:cf8e7dbc:::benchmark_data_smithi026_286012_object57214:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:04.913819+0000", 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 213.66261386799999, 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.129066344, 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.724 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 57215 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.913819+0000", 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.913819+0000", 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.913817+0000", 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.913862+0000", 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4617000000000003e-05 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.913879+0000", 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7054e-05 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.913887+0000", 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6369999999999999e-06 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.913938+0000", 2023-09-23T22:19:38.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1391000000000003e-05 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.914096+0000", 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015826999999999999 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.914216+0000", 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119629 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.915857+0000", 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001640877 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.916085+0000", 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022802 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.041898+0000", 2023-09-23T22:19:38.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12581283600000001 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.041939+0000", 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1378999999999998e-05 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.042885+0000", 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00094594299999999998 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.730 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:57226 5.3 5:c7dfa3f2:::benchmark_data_smithi026_286012_object57225:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:04.926215+0000", 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 213.65021761599999, 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11754753, 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.731 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 57226 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.926215+0000", 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.926215+0000", 2023-09-23T22:19:38.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.926214+0000", 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.926319+0000", 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105752 2023-09-23T22:19:38.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.926335+0000", 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5500999999999999e-05 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.926340+0000", 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2329999999999997e-06 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.926405+0000", 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4461000000000005e-05 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.926575+0000", 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017071400000000001 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.926761+0000", 2023-09-23T22:19:38.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018541299999999999 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.928438+0000", 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016771659999999999 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:04.928919+0000", 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048110399999999999 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.042935+0000", 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11401589500000001 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.043003+0000", 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7714999999999998e-05 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:05.043762+0000", 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00075975100000000002 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:80547 5.2 5:410ee4de:::benchmark_data_smithi026_286012_object80546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:31.760941+0000", 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 186.81549119100001, 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.118017671, 2023-09-23T22:19:38.738 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 80547 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.760941+0000", 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.760941+0000", 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.760940+0000", 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.760984+0000", 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4484000000000003e-05 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.761007+0000", 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2582000000000002e-05 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.761019+0000", 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2850000000000001e-05 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.761077+0000", 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7124999999999999e-05 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.761242+0000", 2023-09-23T22:19:38.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016529000000000001 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.761397+0000", 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015477700000000001 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.763799+0000", 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024022309999999999 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764009+0000", 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000209903 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.878883+0000", 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11487462800000001 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.878916+0000", 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2659e-05 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.878959+0000", 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2904999999999997e-05 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.745 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:80552 5.2 5:4512555d:::benchmark_data_smithi026_286012_object80551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:31.764714+0000", 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 186.81171822100001, 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114376037, 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 80552 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764714+0000", 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764714+0000", 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764712+0000", 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764795+0000", 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2855000000000005e-05 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764812+0000", 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7132000000000001e-05 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764819+0000", 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6850000000000004e-06 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.764870+0000", 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1319000000000001e-05 2023-09-23T22:19:38.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.765016+0000", 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014636099999999999 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.765171+0000", 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015509700000000001 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.768272+0000", 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031003659999999998 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.768984+0000", 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00071239999999999997 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.879002+0000", 2023-09-23T22:19:38.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.110018238 2023-09-23T22:19:38.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.879054+0000", 2023-09-23T22:19:38.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1671e-05 2023-09-23T22:19:38.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.879090+0000", 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.612e-05 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:80555 5.2 5:45a2bc00:::benchmark_data_smithi026_286012_object80554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:31.770281+0000", 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 186.80615177499999, 2023-09-23T22:19:38.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.108988424, 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 80555 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:19:38.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770281+0000", 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770281+0000", 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770279+0000", 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:19:38.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770315+0000", 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5935000000000001e-05 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770329+0000", 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3896999999999999e-05 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770334+0000", 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1490000000000002e-06 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770384+0000", 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9682000000000003e-05 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770551+0000", 2023-09-23T22:19:38.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000167331 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.770724+0000", 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017280299999999999 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.772888+0000", 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021638759999999999 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.772999+0000", 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011121 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.879212+0000", 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10621243299999999 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.879246+0000", 2023-09-23T22:19:38.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4388999999999999e-05 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:31.879269+0000", 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3034e-05 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:19:38.760 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:19:38.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:38 smithi026 ceph-mon[215775]: pgmap v434: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 495 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 9841/23613 objects degraded (41.676%); 457/23613 objects misplaced (1.935%); 5.1 MiB/s, 111 objects/s recovering 2023-09-23T22:19:38.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:38 smithi026 ceph-mon[215775]: from='client.54754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:38.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 9841/23613 objects degraded (41.676%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:19:38.761 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:38 smithi070 ceph-mon[192652]: pgmap v434: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 495 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 9841/23613 objects degraded (41.676%); 457/23613 objects misplaced (1.935%); 5.1 MiB/s, 111 objects/s recovering 2023-09-23T22:19:38.762 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:38 smithi070 ceph-mon[192652]: from='client.54754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:38.762 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 9841/23613 objects degraded (41.676%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:19:38.762 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:38 smithi026 ceph-mon[223014]: pgmap v434: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 495 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 9841/23613 objects degraded (41.676%); 457/23613 objects misplaced (1.935%); 5.1 MiB/s, 111 objects/s recovering 2023-09-23T22:19:38.762 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:38 smithi026 ceph-mon[223014]: from='client.54754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:19:38.762 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 9841/23613 objects degraded (41.676%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:19:39.303 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:19:39.389 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:39.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:39.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:39.497 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:39.522 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:39.606 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:39.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:39.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:39.617 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:39.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:39.622 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:19:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:19:39] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2023-09-23T22:19:39.623 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:39.718 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:39.765 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:39.822 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:39.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:39.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:39.874 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:39.925 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:39.952 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:40.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:40 smithi070 ceph-mon[192652]: pgmap v435: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 436 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 9841/20793 objects degraded (47.328%); 457/20793 objects misplaced (2.198%); 2.4 MiB/s, 86 objects/s recovering 2023-09-23T22:19:40.486 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":434,"stamp":"2023-09-23T22:19:37.131446+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":518723531,"num_objects":7871,"num_object_clones":0,"num_object_copies":23613,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":9841,"num_objects_misplaced":457,"num_objects_unfound":0,"num_objects_dirty":7871,"num_whiteouts":0,"num_read":6469,"num_read_kb":13097,"num_write":180915,"num_write_kb":6018662,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30340,"num_bytes_recovered":1975327138,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111417,"ondisk_log_size":111417,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":318,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3188648,"kb_used_data":2298160,"kb_used_omap":31,"kb_used_meta":890336,"kb_avail":653027416,"statfs":{"total":671965249536,"available":668700073984,"internally_reserved":0,"allocated":2353315840,"data_stored":2346722913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":911705015},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,8,3,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-267059200,"num_objects":-4075,"num_object_clones":0,"num_object_copies":-12225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-5273,"num_objects_misplaced":457,"num_objects_unfound":0,"num_objects_dirty":-4075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4075,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":668,"num_bytes_recovered":32112640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.005401"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":167,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.295455+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:19:31.295455+0000","last_peered":"2023-09-23T22:19:31.295455+0000","last_clean":"2023-09-23T22:19:31.295455+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:19:31.295455+0000","last_undegraded":"2023-09-23T22:19:31.295455+0000","last_fullsized":"2023-09-23T22:19:31.295455+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":170,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291759+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:19:32.291759+0000","last_peered":"2023-09-23T22:19:32.291759+0000","last_clean":"2023-09-23T22:19:32.291759+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:19:32.291759+0000","last_undegraded":"2023-09-23T22:19:32.291759+0000","last_fullsized":"2023-09-23T22:19:32.291759+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":130,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.311756+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:19:32.311756+0000","last_peered":"2023-09-23T22:19:32.311756+0000","last_clean":"2023-09-23T22:19:32.311756+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:19:32.311756+0000","last_undegraded":"2023-09-23T22:19:32.311756+0000","last_fullsized":"2023-09-23T22:19:32.311756+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":19,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309763+0000","last_change":"2023-09-23T22:18:57.724501+0000","last_active":"2023-09-23T22:19:32.309763+0000","last_peered":"2023-09-23T22:19:32.309763+0000","last_clean":"2023-09-23T22:19:32.309763+0000","last_became_active":"2023-09-23T22:18:57.724203+0000","last_became_peered":"2023-09-23T22:18:57.724203+0000","last_unstale":"2023-09-23T22:19:32.309763+0000","last_undegraded":"2023-09-23T22:19:32.309763+0000","last_fullsized":"2023-09-23T22:19:32.309763+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:52.168267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":210,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291148+0000","last_change":"2023-09-23T22:18:57.722574+0000","last_active":"2023-09-23T22:19:32.291148+0000","last_peered":"2023-09-23T22:19:32.291148+0000","last_clean":"2023-09-23T22:19:32.291148+0000","last_became_active":"2023-09-23T22:18:57.721800+0000","last_became_peered":"2023-09-23T22:18:57.721800+0000","last_unstale":"2023-09-23T22:19:32.291148+0000","last_undegraded":"2023-09-23T22:19:32.291148+0000","last_fullsized":"2023-09-23T22:19:32.291148+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:09.303838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":128,"reported_epoch":190,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:19:32.294070+0000","last_change":"2023-09-23T22:18:58.730170+0000","last_active":"2023-09-23T22:19:32.294070+0000","last_peered":"2023-09-23T22:19:32.294070+0000","last_clean":"2023-09-23T22:18:49.795932+0000","last_became_active":"2023-09-23T22:18:58.729860+0000","last_became_peered":"2023-09-23T22:18:58.729860+0000","last_unstale":"2023-09-23T22:19:32.294070+0000","last_undegraded":"2023-09-23T22:19:32.294070+0000","last_fullsized":"2023-09-23T22:19:32.294070+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:30.948774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,3],"avail_no_missing":["5","0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":57,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309097+0000","last_change":"2023-09-23T22:18:57.729038+0000","last_active":"2023-09-23T22:19:32.309097+0000","last_peered":"2023-09-23T22:19:32.309097+0000","last_clean":"2023-09-23T22:19:32.309097+0000","last_became_active":"2023-09-23T22:18:57.728650+0000","last_became_peered":"2023-09-23T22:18:57.728650+0000","last_unstale":"2023-09-23T22:19:32.309097+0000","last_undegraded":"2023-09-23T22:19:32.309097+0000","last_fullsized":"2023-09-23T22:19:32.309097+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:11.461883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":165,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291148+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:19:32.291148+0000","last_peered":"2023-09-23T22:19:32.291148+0000","last_clean":"2023-09-23T22:19:32.291148+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:19:32.291148+0000","last_undegraded":"2023-09-23T22:19:32.291148+0000","last_fullsized":"2023-09-23T22:19:32.291148+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":151,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.293233+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:19:32.293233+0000","last_peered":"2023-09-23T22:19:32.293233+0000","last_clean":"2023-09-23T22:19:32.293233+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:19:32.293233+0000","last_undegraded":"2023-09-23T22:19:32.293233+0000","last_fullsized":"2023-09-23T22:19:32.293233+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":115,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291256+0000","last_change":"2023-09-23T22:16:49.154341+0000","last_active":"2023-09-23T22:19:32.291256+0000","last_peered":"2023-09-23T22:19:32.291256+0000","last_clean":"2023-09-23T22:19:32.291256+0000","last_became_active":"2023-09-23T22:16:49.154041+0000","last_became_peered":"2023-09-23T22:16:49.154041+0000","last_unstale":"2023-09-23T22:19:32.291256+0000","last_undegraded":"2023-09-23T22:19:32.291256+0000","last_fullsized":"2023-09-23T22:19:32.291256+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:05.733262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":102,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291679+0000","last_change":"2023-09-23T22:15:14.829731+0000","last_active":"2023-09-23T22:19:32.291679+0000","last_peered":"2023-09-23T22:19:32.291679+0000","last_clean":"2023-09-23T22:19:32.291679+0000","last_became_active":"2023-09-23T22:15:14.829437+0000","last_became_peered":"2023-09-23T22:15:14.829437+0000","last_unstale":"2023-09-23T22:19:32.291679+0000","last_undegraded":"2023-09-23T22:19:32.291679+0000","last_fullsized":"2023-09-23T22:19:32.291679+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:45.456558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":157,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.311938+0000","last_change":"2023-09-23T22:18:57.723165+0000","last_active":"2023-09-23T22:19:32.311938+0000","last_peered":"2023-09-23T22:19:32.311938+0000","last_clean":"2023-09-23T22:19:32.311938+0000","last_became_active":"2023-09-23T22:18:57.722863+0000","last_became_peered":"2023-09-23T22:18:57.722863+0000","last_unstale":"2023-09-23T22:19:32.311938+0000","last_undegraded":"2023-09-23T22:19:32.311938+0000","last_fullsized":"2023-09-23T22:19:32.311938+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:19.645029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":174,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.294849+0000","last_change":"2023-09-23T22:16:49.265991+0000","last_active":"2023-09-23T22:19:31.294849+0000","last_peered":"2023-09-23T22:19:31.294849+0000","last_clean":"2023-09-23T22:19:31.294849+0000","last_became_active":"2023-09-23T22:16:49.251169+0000","last_became_peered":"2023-09-23T22:16:49.251169+0000","last_unstale":"2023-09-23T22:19:31.294849+0000","last_undegraded":"2023-09-23T22:19:31.294849+0000","last_fullsized":"2023-09-23T22:19:31.294849+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.502218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":158,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.294791+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:19:31.294791+0000","last_peered":"2023-09-23T22:19:31.294791+0000","last_clean":"2023-09-23T22:19:31.294791+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:19:31.294791+0000","last_undegraded":"2023-09-23T22:19:31.294791+0000","last_fullsized":"2023-09-23T22:19:31.294791+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":101,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.294868+0000","last_change":"2023-09-23T22:15:14.826143+0000","last_active":"2023-09-23T22:19:31.294868+0000","last_peered":"2023-09-23T22:19:31.294868+0000","last_clean":"2023-09-23T22:19:31.294868+0000","last_became_active":"2023-09-23T22:15:14.825780+0000","last_became_peered":"2023-09-23T22:15:14.825780+0000","last_unstale":"2023-09-23T22:19:31.294868+0000","last_undegraded":"2023-09-23T22:19:31.294868+0000","last_fullsized":"2023-09-23T22:19:31.294868+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:22.608676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":156,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291551+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:19:32.291551+0000","last_peered":"2023-09-23T22:19:32.291551+0000","last_clean":"2023-09-23T22:19:32.291551+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:19:32.291551+0000","last_undegraded":"2023-09-23T22:19:32.291551+0000","last_fullsized":"2023-09-23T22:19:32.291551+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":19,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.311404+0000","last_change":"2023-09-23T22:18:57.726112+0000","last_active":"2023-09-23T22:19:32.311404+0000","last_peered":"2023-09-23T22:19:32.311404+0000","last_clean":"2023-09-23T22:19:32.311404+0000","last_became_active":"2023-09-23T22:18:57.725558+0000","last_became_peered":"2023-09-23T22:18:57.725558+0000","last_unstale":"2023-09-23T22:19:32.311404+0000","last_undegraded":"2023-09-23T22:19:32.311404+0000","last_fullsized":"2023-09-23T22:19:32.311404+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:38.315664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":150,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.292978+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:19:32.292978+0000","last_peered":"2023-09-23T22:19:32.292978+0000","last_clean":"2023-09-23T22:19:32.292978+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:19:32.292978+0000","last_undegraded":"2023-09-23T22:19:32.292978+0000","last_fullsized":"2023-09-23T22:19:32.292978+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":113,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.296287+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:19:31.296287+0000","last_peered":"2023-09-23T22:19:31.296287+0000","last_clean":"2023-09-23T22:19:31.296287+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:19:31.296287+0000","last_undegraded":"2023-09-23T22:19:31.296287+0000","last_fullsized":"2023-09-23T22:19:31.296287+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":19,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309933+0000","last_change":"2023-09-23T22:18:57.726691+0000","last_active":"2023-09-23T22:19:32.309933+0000","last_peered":"2023-09-23T22:19:32.309933+0000","last_clean":"2023-09-23T22:19:32.309933+0000","last_became_active":"2023-09-23T22:18:57.726386+0000","last_became_peered":"2023-09-23T22:18:57.726386+0000","last_unstale":"2023-09-23T22:19:32.309933+0000","last_undegraded":"2023-09-23T22:19:32.309933+0000","last_fullsized":"2023-09-23T22:19:32.309933+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:34.829149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":160,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309242+0000","last_change":"2023-09-23T22:18:57.728767+0000","last_active":"2023-09-23T22:19:32.309242+0000","last_peered":"2023-09-23T22:19:32.309242+0000","last_clean":"2023-09-23T22:19:32.309242+0000","last_became_active":"2023-09-23T22:18:57.728377+0000","last_became_peered":"2023-09-23T22:18:57.728377+0000","last_unstale":"2023-09-23T22:19:32.309242+0000","last_undegraded":"2023-09-23T22:19:32.309242+0000","last_fullsized":"2023-09-23T22:19:32.309242+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:01:01.370904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":156,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291886+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:19:32.291886+0000","last_peered":"2023-09-23T22:19:32.291886+0000","last_clean":"2023-09-23T22:19:32.291886+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:19:32.291886+0000","last_undegraded":"2023-09-23T22:19:32.291886+0000","last_fullsized":"2023-09-23T22:19:32.291886+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"190'10978","reported_seq":22615,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.372794+0000","last_change":"2023-09-23T22:19:02.159483+0000","last_active":"2023-09-23T22:19:32.372794+0000","last_peered":"2023-09-23T22:19:32.372794+0000","last_clean":"2023-09-23T22:19:32.372794+0000","last_became_active":"2023-09-23T22:19:02.132959+0000","last_became_peered":"2023-09-23T22:19:02.132959+0000","last_unstale":"2023-09-23T22:19:32.372794+0000","last_undegraded":"2023-09-23T22:19:32.372794+0000","last_fullsized":"2023-09-23T22:19:32.372794+0000","mapping_epoch":185,"log_start":"136'4165","ondisk_log_start":"136'4165","created":111,"last_epoch_clean":186,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6813,"log_dups_size":0,"ondisk_log_size":6813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:20.126557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37224448,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10978,"num_write_kb":369472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5765,"num_bytes_recovered":377815040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":157,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.297487+0000","last_change":"2023-09-23T22:19:00.682889+0000","last_active":"2023-09-23T22:19:31.297487+0000","last_peered":"2023-09-23T22:19:31.297487+0000","last_clean":"2023-09-23T22:19:31.297487+0000","last_became_active":"2023-09-23T22:18:57.718002+0000","last_became_peered":"2023-09-23T22:18:57.718002+0000","last_unstale":"2023-09-23T22:19:31.297487+0000","last_undegraded":"2023-09-23T22:19:31.297487+0000","last_fullsized":"2023-09-23T22:19:31.297487+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:22.517277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":143,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.292795+0000","last_change":"2023-09-23T22:16:49.158580+0000","last_active":"2023-09-23T22:19:32.292795+0000","last_peered":"2023-09-23T22:19:32.292795+0000","last_clean":"2023-09-23T22:19:32.292795+0000","last_became_active":"2023-09-23T22:16:49.158266+0000","last_became_peered":"2023-09-23T22:16:49.158266+0000","last_unstale":"2023-09-23T22:19:32.292795+0000","last_undegraded":"2023-09-23T22:19:32.292795+0000","last_fullsized":"2023-09-23T22:19:32.292795+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:03.791658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"191'11095","reported_seq":16434,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T22:19:36.163943+0000","last_change":"2023-09-23T22:17:16.903782+0000","last_active":"2023-09-23T22:19:36.163943+0000","last_peered":"2023-09-23T22:19:36.163943+0000","last_clean":"2023-09-23T22:19:36.163943+0000","last_became_active":"2023-09-23T22:17:16.895165+0000","last_became_peered":"2023-09-23T22:17:16.895165+0000","last_unstale":"2023-09-23T22:19:36.163943+0000","last_undegraded":"2023-09-23T22:19:36.163943+0000","last_fullsized":"2023-09-23T22:19:36.163943+0000","mapping_epoch":159,"log_start":"136'3900","ondisk_log_start":"136'3900","created":111,"last_epoch_clean":160,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7195,"log_dups_size":0,"ondisk_log_size":7195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:31.886804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28639232,"num_objects":437,"num_object_clones":0,"num_object_copies":1311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11095,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2600,"num_bytes_recovered":170393600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":101,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.295633+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:19:31.295633+0000","last_peered":"2023-09-23T22:19:31.295633+0000","last_clean":"2023-09-23T22:19:31.295633+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:19:31.295633+0000","last_undegraded":"2023-09-23T22:19:31.295633+0000","last_fullsized":"2023-09-23T22:19:31.295633+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.296247+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:19:31.296247+0000","last_peered":"2023-09-23T22:19:31.296247+0000","last_clean":"2023-09-23T22:19:31.296247+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:19:31.296247+0000","last_undegraded":"2023-09-23T22:19:31.296247+0000","last_fullsized":"2023-09-23T22:19:31.296247+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"191'11067","reported_seq":17615,"reported_epoch":191,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:36.216151+0000","last_change":"2023-09-23T22:19:00.690164+0000","last_active":"2023-09-23T22:19:36.216151+0000","last_peered":"2023-09-23T22:19:36.216151+0000","last_clean":"2023-09-23T22:18:39.199855+0000","last_became_active":"2023-09-23T22:18:41.409775+0000","last_became_peered":"2023-09-23T22:18:41.409775+0000","last_unstale":"2023-09-23T22:19:36.216151+0000","last_undegraded":"2023-09-23T22:18:41.249089+0000","last_fullsized":"2023-09-23T22:18:41.215237+0000","mapping_epoch":178,"log_start":"136'3900","ondisk_log_start":"136'3900","created":111,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7167,"log_dups_size":0,"ondisk_log_size":7167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:30.052669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29294592,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5621,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11067,"num_write_kb":368448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3204,"num_bytes_recovered":209977344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0],"avail_no_missing":["2","0","7"],"object_location_counts":[{"shards":"0,2","objects":-1762},{"shards":"0,2,7","objects":447}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":116,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.292893+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:19:32.292893+0000","last_peered":"2023-09-23T22:19:32.292893+0000","last_clean":"2023-09-23T22:19:32.292893+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:19:32.292893+0000","last_undegraded":"2023-09-23T22:19:32.292893+0000","last_fullsized":"2023-09-23T22:19:32.292893+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":106,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.312012+0000","last_change":"2023-09-23T22:16:49.161676+0000","last_active":"2023-09-23T22:19:32.312012+0000","last_peered":"2023-09-23T22:19:32.312012+0000","last_clean":"2023-09-23T22:19:32.312012+0000","last_became_active":"2023-09-23T22:16:49.161247+0000","last_became_peered":"2023-09-23T22:16:49.161247+0000","last_unstale":"2023-09-23T22:19:32.312012+0000","last_undegraded":"2023-09-23T22:19:32.312012+0000","last_fullsized":"2023-09-23T22:19:32.312012+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:30.708371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.a","version":"191'11249","reported_seq":11339,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T22:19:36.232523+0000","last_change":"2023-09-23T22:15:14.826723+0000","last_active":"2023-09-23T22:19:36.232523+0000","last_peered":"2023-09-23T22:19:36.232523+0000","last_clean":"2023-09-23T22:19:36.232523+0000","last_became_active":"2023-09-23T22:15:14.826454+0000","last_became_peered":"2023-09-23T22:15:14.826454+0000","last_unstale":"2023-09-23T22:19:36.232523+0000","last_undegraded":"2023-09-23T22:19:36.232523+0000","last_fullsized":"2023-09-23T22:19:36.232523+0000","mapping_epoch":111,"log_start":"136'4100","ondisk_log_start":"136'4100","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7149,"log_dups_size":0,"ondisk_log_size":7149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:26:11.166077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29163520,"num_objects":445,"num_object_clones":0,"num_object_copies":1335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11249,"num_write_kb":374208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":161,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.293177+0000","last_change":"2023-09-23T22:16:49.164833+0000","last_active":"2023-09-23T22:19:32.293177+0000","last_peered":"2023-09-23T22:19:32.293177+0000","last_clean":"2023-09-23T22:19:32.293177+0000","last_became_active":"2023-09-23T22:16:49.163833+0000","last_became_peered":"2023-09-23T22:16:49.163833+0000","last_unstale":"2023-09-23T22:19:32.293177+0000","last_undegraded":"2023-09-23T22:19:32.293177+0000","last_fullsized":"2023-09-23T22:19:32.293177+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.891395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":44,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309538+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:19:32.309538+0000","last_peered":"2023-09-23T22:19:32.309538+0000","last_clean":"2023-09-23T22:19:32.309538+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:19:32.309538+0000","last_undegraded":"2023-09-23T22:19:32.309538+0000","last_fullsized":"2023-09-23T22:19:32.309538+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":455,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.308196+0000","last_change":"2023-09-23T22:18:57.721431+0000","last_active":"2023-09-23T22:19:32.308196+0000","last_peered":"2023-09-23T22:19:32.308196+0000","last_clean":"2023-09-23T22:19:32.308196+0000","last_became_active":"2023-09-23T22:18:57.721043+0000","last_became_peered":"2023-09-23T22:18:57.721043+0000","last_unstale":"2023-09-23T22:19:32.308196+0000","last_undegraded":"2023-09-23T22:19:32.308196+0000","last_fullsized":"2023-09-23T22:19:32.308196+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"191'11376","reported_seq":11467,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T22:19:36.229252+0000","last_change":"2023-09-23T22:15:14.824965+0000","last_active":"2023-09-23T22:19:36.229252+0000","last_peered":"2023-09-23T22:19:36.229252+0000","last_clean":"2023-09-23T22:19:36.229252+0000","last_became_active":"2023-09-23T22:15:14.824626+0000","last_became_peered":"2023-09-23T22:15:14.824626+0000","last_unstale":"2023-09-23T22:19:36.229252+0000","last_undegraded":"2023-09-23T22:19:36.229252+0000","last_fullsized":"2023-09-23T22:19:36.229252+0000","mapping_epoch":111,"log_start":"136'4230","ondisk_log_start":"136'4230","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7146,"log_dups_size":0,"ondisk_log_size":7146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:34.613455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27197464,"num_objects":416,"num_object_clones":0,"num_object_copies":1248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":416,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11376,"num_write_kb":377281,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":202,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291505+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:19:32.291505+0000","last_peered":"2023-09-23T22:19:32.291505+0000","last_clean":"2023-09-23T22:19:32.291505+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:19:32.291505+0000","last_undegraded":"2023-09-23T22:19:32.291505+0000","last_fullsized":"2023-09-23T22:19:32.291505+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.294703+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:19:31.294703+0000","last_peered":"2023-09-23T22:19:31.294703+0000","last_clean":"2023-09-23T22:19:31.294703+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:19:31.294703+0000","last_undegraded":"2023-09-23T22:19:31.294703+0000","last_fullsized":"2023-09-23T22:19:31.294703+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":453,"reported_epoch":190,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:19:32.308805+0000","last_change":"2023-09-23T22:18:57.743507+0000","last_active":"2023-09-23T22:19:32.308805+0000","last_peered":"2023-09-23T22:19:32.308805+0000","last_clean":"2023-09-23T22:08:39.036645+0000","last_became_active":"2023-09-23T22:18:57.742130+0000","last_became_peered":"2023-09-23T22:18:57.742130+0000","last_unstale":"2023-09-23T22:19:32.308805+0000","last_undegraded":"2023-09-23T22:18:57.716066+0000","last_fullsized":"2023-09-23T22:19:32.308805+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":120,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":["1","6","7"],"object_location_counts":[{"shards":"0,1,7","objects":-1},{"shards":"1,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.c","version":"190'11144","reported_seq":13456,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.879073+0000","last_change":"2023-09-23T22:16:30.157766+0000","last_active":"2023-09-23T22:19:32.879073+0000","last_peered":"2023-09-23T22:19:32.879073+0000","last_clean":"2023-09-23T22:19:32.879073+0000","last_became_active":"2023-09-23T22:16:30.150277+0000","last_became_peered":"2023-09-23T22:16:30.150277+0000","last_unstale":"2023-09-23T22:19:32.879073+0000","last_undegraded":"2023-09-23T22:19:32.879073+0000","last_fullsized":"2023-09-23T22:19:32.879073+0000","mapping_epoch":140,"log_start":"145'5725","ondisk_log_start":"145'5725","created":111,"last_epoch_clean":141,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5419,"log_dups_size":0,"ondisk_log_size":5419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:20.415978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37093376,"num_objects":566,"num_object_clones":0,"num_object_copies":1698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11144,"num_write_kb":374720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1098,"num_bytes_recovered":71958528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":106,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.295606+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:19:31.295606+0000","last_peered":"2023-09-23T22:19:31.295606+0000","last_clean":"2023-09-23T22:19:31.295606+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:19:31.295606+0000","last_undegraded":"2023-09-23T22:19:31.295606+0000","last_fullsized":"2023-09-23T22:19:31.295606+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":165,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.295072+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:19:31.295072+0000","last_peered":"2023-09-23T22:19:31.295072+0000","last_clean":"2023-09-23T22:19:31.295072+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:19:31.295072+0000","last_undegraded":"2023-09-23T22:19:31.295072+0000","last_fullsized":"2023-09-23T22:19:31.295072+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"191'11277","reported_seq":23147,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T22:19:35.132138+0000","last_change":"2023-09-23T22:19:33.384024+0000","last_active":"2023-09-23T22:19:35.132138+0000","last_peered":"2023-09-23T22:19:35.132138+0000","last_clean":"2023-09-23T22:19:35.132138+0000","last_became_active":"2023-09-23T22:19:33.294252+0000","last_became_peered":"2023-09-23T22:19:33.294252+0000","last_unstale":"2023-09-23T22:19:35.132138+0000","last_undegraded":"2023-09-23T22:19:35.132138+0000","last_fullsized":"2023-09-23T22:19:35.132138+0000","mapping_epoch":190,"log_start":"133'3624","ondisk_log_start":"133'3624","created":111,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7653,"log_dups_size":0,"ondisk_log_size":7653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:29.118680+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32440320,"num_objects":495,"num_object_clones":0,"num_object_copies":1485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11277,"num_write_kb":376704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5877,"num_bytes_recovered":377552896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":141,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.296141+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:19:31.296141+0000","last_peered":"2023-09-23T22:19:31.296141+0000","last_clean":"2023-09-23T22:19:31.296141+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:19:31.296141+0000","last_undegraded":"2023-09-23T22:19:31.296141+0000","last_fullsized":"2023-09-23T22:19:31.296141+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":158,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.295708+0000","last_change":"2023-09-23T22:16:49.165118+0000","last_active":"2023-09-23T22:19:31.295708+0000","last_peered":"2023-09-23T22:19:31.295708+0000","last_clean":"2023-09-23T22:19:31.295708+0000","last_became_active":"2023-09-23T22:16:49.164802+0000","last_became_peered":"2023-09-23T22:16:49.164802+0000","last_unstale":"2023-09-23T22:19:31.295708+0000","last_undegraded":"2023-09-23T22:19:31.295708+0000","last_fullsized":"2023-09-23T22:19:31.295708+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:23.079142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":455,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.311480+0000","last_change":"2023-09-23T22:18:57.726339+0000","last_active":"2023-09-23T22:19:32.311480+0000","last_peered":"2023-09-23T22:19:32.311480+0000","last_clean":"2023-09-23T22:19:32.311480+0000","last_became_active":"2023-09-23T22:18:57.725773+0000","last_became_peered":"2023-09-23T22:18:57.725773+0000","last_unstale":"2023-09-23T22:19:32.311480+0000","last_undegraded":"2023-09-23T22:19:32.311480+0000","last_fullsized":"2023-09-23T22:19:32.311480+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"190'11360","reported_seq":15212,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.344347+0000","last_change":"2023-09-23T22:15:59.633734+0000","last_active":"2023-09-23T22:19:32.344347+0000","last_peered":"2023-09-23T22:19:32.344347+0000","last_clean":"2023-09-23T22:19:32.344347+0000","last_became_active":"2023-09-23T22:15:59.605894+0000","last_became_peered":"2023-09-23T22:15:59.605894+0000","last_unstale":"2023-09-23T22:19:32.344347+0000","last_undegraded":"2023-09-23T22:19:32.344347+0000","last_fullsized":"2023-09-23T22:19:32.344347+0000","mapping_epoch":132,"log_start":"139'4547","ondisk_log_start":"139'4547","created":111,"last_epoch_clean":133,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6813,"log_dups_size":0,"ondisk_log_size":6813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:22.683442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37224448,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11360,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1867,"num_bytes_recovered":122355712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":157,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291043+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:19:32.291043+0000","last_peered":"2023-09-23T22:19:32.291043+0000","last_clean":"2023-09-23T22:19:32.291043+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:19:32.291043+0000","last_undegraded":"2023-09-23T22:19:32.291043+0000","last_fullsized":"2023-09-23T22:19:32.291043+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":130,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.312830+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:19:32.312830+0000","last_peered":"2023-09-23T22:19:32.312830+0000","last_clean":"2023-09-23T22:19:32.312830+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:19:32.312830+0000","last_undegraded":"2023-09-23T22:19:32.312830+0000","last_fullsized":"2023-09-23T22:19:32.312830+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":455,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.308770+0000","last_change":"2023-09-23T22:18:57.725616+0000","last_active":"2023-09-23T22:19:32.308770+0000","last_peered":"2023-09-23T22:19:32.308770+0000","last_clean":"2023-09-23T22:19:32.308770+0000","last_became_active":"2023-09-23T22:18:57.725224+0000","last_became_peered":"2023-09-23T22:18:57.725224+0000","last_unstale":"2023-09-23T22:19:32.308770+0000","last_undegraded":"2023-09-23T22:19:32.308770+0000","last_fullsized":"2023-09-23T22:19:32.308770+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":738,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.311408+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:19:32.311408+0000","last_peered":"2023-09-23T22:19:32.311408+0000","last_clean":"2023-09-23T22:19:32.311408+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:19:32.311408+0000","last_undegraded":"2023-09-23T22:19:32.311408+0000","last_fullsized":"2023-09-23T22:19:32.311408+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":168,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.293222+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:19:32.293222+0000","last_peered":"2023-09-23T22:19:32.293222+0000","last_clean":"2023-09-23T22:19:32.293222+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:19:32.293222+0000","last_undegraded":"2023-09-23T22:19:32.293222+0000","last_fullsized":"2023-09-23T22:19:32.293222+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":112,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.299623+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:19:32.299623+0000","last_peered":"2023-09-23T22:19:32.299623+0000","last_clean":"2023-09-23T22:19:32.299623+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:19:32.299623+0000","last_undegraded":"2023-09-23T22:19:32.299623+0000","last_fullsized":"2023-09-23T22:19:32.299623+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"191'11276","reported_seq":18754,"reported_epoch":191,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:19:36.193043+0000","last_change":"2023-09-23T22:18:59.033329+0000","last_active":"2023-09-23T22:19:36.193043+0000","last_peered":"2023-09-23T22:19:36.193043+0000","last_clean":"2023-09-23T22:16:48.130011+0000","last_became_active":"2023-09-23T22:18:58.953635+0000","last_became_peered":"2023-09-23T22:18:58.953635+0000","last_unstale":"2023-09-23T22:19:36.193043+0000","last_undegraded":"2023-09-23T22:18:58.948229+0000","last_fullsized":"2023-09-23T22:19:36.193043+0000","mapping_epoch":183,"log_start":"142'5313","ondisk_log_start":"142'5313","created":111,"last_epoch_clean":138,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5963,"log_dups_size":0,"ondisk_log_size":5963,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:59.274843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30932992,"num_objects":472,"num_object_clones":0,"num_object_copies":1416,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":472,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":472,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11276,"num_write_kb":375936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3666,"num_bytes_recovered":240189440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,1],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5,6","objects":472}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":455,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.308610+0000","last_change":"2023-09-23T22:18:57.742555+0000","last_active":"2023-09-23T22:19:32.308610+0000","last_peered":"2023-09-23T22:19:32.308610+0000","last_clean":"2023-09-23T22:19:32.308610+0000","last_became_active":"2023-09-23T22:18:57.742205+0000","last_became_peered":"2023-09-23T22:18:57.742205+0000","last_unstale":"2023-09-23T22:19:32.308610+0000","last_undegraded":"2023-09-23T22:19:32.308610+0000","last_fullsized":"2023-09-23T22:19:32.308610+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"190'11423","reported_seq":15531,"reported_epoch":191,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:19:33.384357+0000","last_change":"2023-09-23T22:19:31.604408+0000","last_active":"2023-09-23T22:19:33.384357+0000","last_peered":"2023-09-23T22:19:33.384357+0000","last_clean":"2023-09-23T22:16:50.502857+0000","last_became_active":"2023-09-23T22:18:57.873881+0000","last_became_peered":"2023-09-23T22:18:57.873881+0000","last_unstale":"2023-09-23T22:19:33.384357+0000","last_undegraded":"2023-09-23T22:19:33.384357+0000","last_fullsized":"2023-09-23T22:18:57.704742+0000","mapping_epoch":182,"log_start":"156'6040","ondisk_log_start":"156'6040","created":111,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5383,"log_dups_size":0,"ondisk_log_size":5383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:58.431178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35848192,"num_objects":547,"num_object_clones":0,"num_object_copies":1641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11423,"num_write_kb":383040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1991,"num_bytes_recovered":124190720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[1,4],"avail_no_missing":["1","4","6"],"object_location_counts":[{"shards":"1,4","objects":547}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":923,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.292943+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:19:32.292943+0000","last_peered":"2023-09-23T22:19:32.292943+0000","last_clean":"2023-09-23T22:19:32.292943+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:19:32.292943+0000","last_undegraded":"2023-09-23T22:19:32.292943+0000","last_fullsized":"2023-09-23T22:19:32.292943+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":199,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.294722+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:19:31.294722+0000","last_peered":"2023-09-23T22:19:31.294722+0000","last_clean":"2023-09-23T22:19:31.294722+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:19:31.294722+0000","last_undegraded":"2023-09-23T22:19:31.294722+0000","last_fullsized":"2023-09-23T22:19:31.294722+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":165,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.292881+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:19:32.292881+0000","last_peered":"2023-09-23T22:19:32.292881+0000","last_clean":"2023-09-23T22:19:32.292881+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:19:32.292881+0000","last_undegraded":"2023-09-23T22:19:32.292881+0000","last_fullsized":"2023-09-23T22:19:32.292881+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":455,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.311271+0000","last_change":"2023-09-23T22:18:57.725923+0000","last_active":"2023-09-23T22:19:32.311271+0000","last_peered":"2023-09-23T22:19:32.311271+0000","last_clean":"2023-09-23T22:19:32.311271+0000","last_became_active":"2023-09-23T22:18:57.725260+0000","last_became_peered":"2023-09-23T22:18:57.725260+0000","last_unstale":"2023-09-23T22:19:32.311271+0000","last_undegraded":"2023-09-23T22:19:32.311271+0000","last_fullsized":"2023-09-23T22:19:32.311271+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"191'11218","reported_seq":11309,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T22:19:36.243246+0000","last_change":"2023-09-23T22:15:14.823394+0000","last_active":"2023-09-23T22:19:36.243246+0000","last_peered":"2023-09-23T22:19:36.243246+0000","last_clean":"2023-09-23T22:19:36.243246+0000","last_became_active":"2023-09-23T22:15:14.823003+0000","last_became_peered":"2023-09-23T22:15:14.823003+0000","last_unstale":"2023-09-23T22:19:36.243246+0000","last_undegraded":"2023-09-23T22:19:36.243246+0000","last_fullsized":"2023-09-23T22:19:36.243246+0000","mapping_epoch":111,"log_start":"136'4000","ondisk_log_start":"136'4000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7218,"log_dups_size":0,"ondisk_log_size":7218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:13.496813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29491200,"num_objects":450,"num_object_clones":0,"num_object_copies":1350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11218,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":532,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.299976+0000","last_change":"2023-09-23T22:15:51.379227+0000","last_active":"2023-09-23T22:19:32.299976+0000","last_peered":"2023-09-23T22:19:32.299976+0000","last_clean":"2023-09-23T22:19:32.299976+0000","last_became_active":"2023-09-23T22:15:43.928977+0000","last_became_peered":"2023-09-23T22:15:43.928977+0000","last_unstale":"2023-09-23T22:19:32.299976+0000","last_undegraded":"2023-09-23T22:19:32.299976+0000","last_fullsized":"2023-09-23T22:19:32.299976+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:04.398533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":140,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.313134+0000","last_change":"2023-09-23T22:16:49.251412+0000","last_active":"2023-09-23T22:19:32.313134+0000","last_peered":"2023-09-23T22:19:32.313134+0000","last_clean":"2023-09-23T22:19:32.313134+0000","last_became_active":"2023-09-23T22:16:49.251097+0000","last_became_peered":"2023-09-23T22:16:49.251097+0000","last_unstale":"2023-09-23T22:19:32.313134+0000","last_undegraded":"2023-09-23T22:19:32.313134+0000","last_fullsized":"2023-09-23T22:19:32.313134+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:41.601435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":196,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.312912+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:19:32.312912+0000","last_peered":"2023-09-23T22:19:32.312912+0000","last_clean":"2023-09-23T22:19:32.312912+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:19:32.312912+0000","last_undegraded":"2023-09-23T22:19:32.312912+0000","last_fullsized":"2023-09-23T22:19:32.312912+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":776,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.292973+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:19:32.292973+0000","last_peered":"2023-09-23T22:19:32.292973+0000","last_clean":"2023-09-23T22:19:32.292973+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:19:32.292973+0000","last_undegraded":"2023-09-23T22:19:32.292973+0000","last_fullsized":"2023-09-23T22:19:32.292973+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":455,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309153+0000","last_change":"2023-09-23T22:18:57.743602+0000","last_active":"2023-09-23T22:19:32.309153+0000","last_peered":"2023-09-23T22:19:32.309153+0000","last_clean":"2023-09-23T22:19:32.309153+0000","last_became_active":"2023-09-23T22:18:57.743004+0000","last_became_peered":"2023-09-23T22:18:57.743004+0000","last_unstale":"2023-09-23T22:19:32.309153+0000","last_undegraded":"2023-09-23T22:19:32.309153+0000","last_fullsized":"2023-09-23T22:19:32.309153+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":115,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291091+0000","last_change":"2023-09-23T22:16:49.155156+0000","last_active":"2023-09-23T22:19:32.291091+0000","last_peered":"2023-09-23T22:19:32.291091+0000","last_clean":"2023-09-23T22:19:32.291091+0000","last_became_active":"2023-09-23T22:16:49.154923+0000","last_became_peered":"2023-09-23T22:16:49.154923+0000","last_unstale":"2023-09-23T22:19:32.291091+0000","last_undegraded":"2023-09-23T22:19:32.291091+0000","last_fullsized":"2023-09-23T22:19:32.291091+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:33.532238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":76,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.295218+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:19:31.295218+0000","last_peered":"2023-09-23T22:19:31.295218+0000","last_clean":"2023-09-23T22:19:31.295218+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:19:31.295218+0000","last_undegraded":"2023-09-23T22:19:31.295218+0000","last_fullsized":"2023-09-23T22:19:31.295218+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"191'11319","reported_seq":11410,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T22:19:36.182013+0000","last_change":"2023-09-23T22:15:14.824102+0000","last_active":"2023-09-23T22:19:36.182013+0000","last_peered":"2023-09-23T22:19:36.182013+0000","last_clean":"2023-09-23T22:19:36.182013+0000","last_became_active":"2023-09-23T22:15:14.823790+0000","last_became_peered":"2023-09-23T22:15:14.823790+0000","last_unstale":"2023-09-23T22:19:36.182013+0000","last_undegraded":"2023-09-23T22:19:36.182013+0000","last_fullsized":"2023-09-23T22:19:36.182013+0000","mapping_epoch":111,"log_start":"136'4100","ondisk_log_start":"136'4100","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7219,"log_dups_size":0,"ondisk_log_size":7219,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:24.517495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28508160,"num_objects":435,"num_object_clones":0,"num_object_copies":1305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11319,"num_write_kb":376128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":455,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309268+0000","last_change":"2023-09-23T22:18:57.720920+0000","last_active":"2023-09-23T22:19:32.309268+0000","last_peered":"2023-09-23T22:19:32.309268+0000","last_clean":"2023-09-23T22:19:32.309268+0000","last_became_active":"2023-09-23T22:18:57.720584+0000","last_became_peered":"2023-09-23T22:18:57.720584+0000","last_unstale":"2023-09-23T22:19:32.309268+0000","last_undegraded":"2023-09-23T22:19:32.309268+0000","last_fullsized":"2023-09-23T22:19:32.309268+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"191'11482","reported_seq":16378,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T22:19:35.121967+0000","last_change":"2023-09-23T22:17:15.752911+0000","last_active":"2023-09-23T22:19:35.121967+0000","last_peered":"2023-09-23T22:19:35.121967+0000","last_clean":"2023-09-23T22:19:35.121967+0000","last_became_active":"2023-09-23T22:17:15.728788+0000","last_became_peered":"2023-09-23T22:17:15.728788+0000","last_unstale":"2023-09-23T22:19:35.121967+0000","last_undegraded":"2023-09-23T22:19:35.121967+0000","last_fullsized":"2023-09-23T22:19:35.121967+0000","mapping_epoch":158,"log_start":"135'3863","ondisk_log_start":"135'3863","created":111,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7619,"log_dups_size":0,"ondisk_log_size":7619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:35.923920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29360128,"num_objects":448,"num_object_clones":0,"num_object_copies":1344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11482,"num_write_kb":381760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2374,"num_bytes_recovered":155582464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":539,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.313270+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:19:32.313270+0000","last_peered":"2023-09-23T22:19:32.313270+0000","last_clean":"2023-09-23T22:19:32.313270+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:19:32.313270+0000","last_undegraded":"2023-09-23T22:19:32.313270+0000","last_fullsized":"2023-09-23T22:19:32.313270+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.308472+0000","last_change":"2023-09-23T22:18:31.795915+0000","last_active":"2023-09-23T22:19:32.308472+0000","last_peered":"2023-09-23T22:19:32.308472+0000","last_clean":"2023-09-23T22:19:32.308472+0000","last_became_active":"2023-09-23T22:18:31.795608+0000","last_became_peered":"2023-09-23T22:18:31.795608+0000","last_unstale":"2023-09-23T22:19:32.308472+0000","last_undegraded":"2023-09-23T22:19:32.308472+0000","last_fullsized":"2023-09-23T22:19:32.308472+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:49.789927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309682+0000","last_change":"2023-09-23T22:18:57.721257+0000","last_active":"2023-09-23T22:19:32.309682+0000","last_peered":"2023-09-23T22:19:32.309682+0000","last_clean":"2023-09-23T22:19:32.309682+0000","last_became_active":"2023-09-23T22:18:57.720548+0000","last_became_peered":"2023-09-23T22:18:57.720548+0000","last_unstale":"2023-09-23T22:19:32.309682+0000","last_undegraded":"2023-09-23T22:19:32.309682+0000","last_fullsized":"2023-09-23T22:19:32.309682+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:10.709024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":455,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309847+0000","last_change":"2023-09-23T22:18:57.727276+0000","last_active":"2023-09-23T22:19:32.309847+0000","last_peered":"2023-09-23T22:19:32.309847+0000","last_clean":"2023-09-23T22:19:32.309847+0000","last_became_active":"2023-09-23T22:18:57.726938+0000","last_became_peered":"2023-09-23T22:18:57.726938+0000","last_unstale":"2023-09-23T22:19:32.309847+0000","last_undegraded":"2023-09-23T22:19:32.309847+0000","last_fullsized":"2023-09-23T22:19:32.309847+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":80,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.308620+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:19:32.308620+0000","last_peered":"2023-09-23T22:19:32.308620+0000","last_clean":"2023-09-23T22:19:32.308620+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:19:32.308620+0000","last_undegraded":"2023-09-23T22:19:32.308620+0000","last_fullsized":"2023-09-23T22:19:32.308620+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.312050+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:19:32.312050+0000","last_peered":"2023-09-23T22:19:32.312050+0000","last_clean":"2023-09-23T22:19:32.312050+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:19:32.312050+0000","last_undegraded":"2023-09-23T22:19:32.312050+0000","last_fullsized":"2023-09-23T22:19:32.312050+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":44,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309472+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:19:32.309472+0000","last_peered":"2023-09-23T22:19:32.309472+0000","last_clean":"2023-09-23T22:19:32.309472+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:19:32.309472+0000","last_undegraded":"2023-09-23T22:19:32.309472+0000","last_fullsized":"2023-09-23T22:19:32.309472+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"191'11446","reported_seq":15081,"reported_epoch":191,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:19:36.205366+0000","last_change":"2023-09-23T22:19:31.622755+0000","last_active":"2023-09-23T22:19:36.205366+0000","last_peered":"2023-09-23T22:19:36.205366+0000","last_clean":"2023-09-23T22:16:48.102227+0000","last_became_active":"2023-09-23T22:16:50.283490+0000","last_became_peered":"2023-09-23T22:16:50.283490+0000","last_unstale":"2023-09-23T22:19:36.205366+0000","last_undegraded":"2023-09-23T22:17:00.207659+0000","last_fullsized":"2023-09-23T22:16:50.146447+0000","mapping_epoch":147,"log_start":"142'5477","ondisk_log_start":"142'5477","created":111,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":5969,"log_dups_size":0,"ondisk_log_size":5969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:48.954402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31588352,"num_objects":482,"num_object_clones":0,"num_object_copies":1446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3747,"num_objects_misplaced":457,"num_objects_unfound":0,"num_objects_dirty":482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11446,"num_write_kb":381696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1752,"num_bytes_recovered":114819072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5","objects":-3722},{"shards":"0,5,6","objects":457}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":455,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.308380+0000","last_change":"2023-09-23T22:18:57.742449+0000","last_active":"2023-09-23T22:19:32.308380+0000","last_peered":"2023-09-23T22:19:32.308380+0000","last_clean":"2023-09-23T22:19:32.308380+0000","last_became_active":"2023-09-23T22:18:57.742146+0000","last_became_peered":"2023-09-23T22:18:57.742146+0000","last_unstale":"2023-09-23T22:19:32.308380+0000","last_undegraded":"2023-09-23T22:19:32.308380+0000","last_fullsized":"2023-09-23T22:19:32.308380+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":202,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291937+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:19:32.291937+0000","last_peered":"2023-09-23T22:19:32.291937+0000","last_clean":"2023-09-23T22:19:32.291937+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:19:32.291937+0000","last_undegraded":"2023-09-23T22:19:32.291937+0000","last_fullsized":"2023-09-23T22:19:32.291937+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":831,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.312477+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:19:32.312477+0000","last_peered":"2023-09-23T22:19:32.312477+0000","last_clean":"2023-09-23T22:19:32.312477+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:19:32.312477+0000","last_undegraded":"2023-09-23T22:19:32.312477+0000","last_fullsized":"2023-09-23T22:19:32.312477+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":187,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291044+0000","last_change":"2023-09-23T22:18:57.722771+0000","last_active":"2023-09-23T22:19:32.291044+0000","last_peered":"2023-09-23T22:19:32.291044+0000","last_clean":"2023-09-23T22:19:32.291044+0000","last_became_active":"2023-09-23T22:18:57.722258+0000","last_became_peered":"2023-09-23T22:18:57.722258+0000","last_unstale":"2023-09-23T22:19:32.291044+0000","last_undegraded":"2023-09-23T22:19:32.291044+0000","last_fullsized":"2023-09-23T22:19:32.291044+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:18.267091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"190'10986","reported_seq":11065,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.349497+0000","last_change":"2023-09-23T22:15:14.828787+0000","last_active":"2023-09-23T22:19:32.349497+0000","last_peered":"2023-09-23T22:19:32.349497+0000","last_clean":"2023-09-23T22:19:32.349497+0000","last_became_active":"2023-09-23T22:15:14.828370+0000","last_became_peered":"2023-09-23T22:15:14.828370+0000","last_unstale":"2023-09-23T22:19:32.349497+0000","last_undegraded":"2023-09-23T22:19:32.349497+0000","last_fullsized":"2023-09-23T22:19:32.349497+0000","mapping_epoch":111,"log_start":"136'4170","ondisk_log_start":"136'4170","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":6816,"log_dups_size":0,"ondisk_log_size":6816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:35.080268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36044800,"num_objects":550,"num_object_clones":0,"num_object_copies":1650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10986,"num_write_kb":369152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":459,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.308561+0000","last_change":"2023-09-23T22:19:00.918399+0000","last_active":"2023-09-23T22:19:32.308561+0000","last_peered":"2023-09-23T22:19:32.308561+0000","last_clean":"2023-09-23T22:19:32.308561+0000","last_became_active":"2023-09-23T22:18:57.720614+0000","last_became_peered":"2023-09-23T22:18:57.720614+0000","last_unstale":"2023-09-23T22:19:32.308561+0000","last_undegraded":"2023-09-23T22:19:32.308561+0000","last_fullsized":"2023-09-23T22:19:32.308561+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":255,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.300796+0000","last_change":"2023-09-23T22:16:49.270015+0000","last_active":"2023-09-23T22:19:32.300796+0000","last_peered":"2023-09-23T22:19:32.300796+0000","last_clean":"2023-09-23T22:19:32.300796+0000","last_became_active":"2023-09-23T22:16:49.252726+0000","last_became_peered":"2023-09-23T22:16:49.252726+0000","last_unstale":"2023-09-23T22:19:32.300796+0000","last_undegraded":"2023-09-23T22:19:32.300796+0000","last_fullsized":"2023-09-23T22:19:32.300796+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:18.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":23,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309800+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:19:32.309800+0000","last_peered":"2023-09-23T22:19:32.309800+0000","last_clean":"2023-09-23T22:19:32.309800+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:19:32.309800+0000","last_undegraded":"2023-09-23T22:19:32.309800+0000","last_fullsized":"2023-09-23T22:19:32.309800+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":153,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.296181+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:19:31.296181+0000","last_peered":"2023-09-23T22:19:31.296181+0000","last_clean":"2023-09-23T22:19:31.296181+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:19:31.296181+0000","last_undegraded":"2023-09-23T22:19:31.296181+0000","last_fullsized":"2023-09-23T22:19:31.296181+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"191'11073","reported_seq":11164,"reported_epoch":191,"state":"active+clean","last_fresh":"2023-09-23T22:19:33.848431+0000","last_change":"2023-09-23T22:15:14.825150+0000","last_active":"2023-09-23T22:19:33.848431+0000","last_peered":"2023-09-23T22:19:33.848431+0000","last_clean":"2023-09-23T22:19:33.848431+0000","last_became_active":"2023-09-23T22:15:14.824867+0000","last_became_peered":"2023-09-23T22:15:14.824867+0000","last_unstale":"2023-09-23T22:19:33.848431+0000","last_undegraded":"2023-09-23T22:19:33.848431+0000","last_fullsized":"2023-09-23T22:19:33.848431+0000","mapping_epoch":111,"log_start":"136'4000","ondisk_log_start":"136'4000","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:15:13.805294+0000","last_clean_scrub_stamp":"2023-09-23T22:15:13.805294+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:30.053753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33882112,"num_objects":517,"num_object_clones":0,"num_object_copies":1551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11073,"num_write_kb":370880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":19,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309402+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:19:32.309402+0000","last_peered":"2023-09-23T22:19:32.309402+0000","last_clean":"2023-09-23T22:19:32.309402+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:19:32.309402+0000","last_undegraded":"2023-09-23T22:19:32.309402+0000","last_fullsized":"2023-09-23T22:19:32.309402+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":175,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.300616+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:19:32.300616+0000","last_peered":"2023-09-23T22:19:32.300616+0000","last_clean":"2023-09-23T22:19:32.300616+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:19:32.300616+0000","last_undegraded":"2023-09-23T22:19:32.300616+0000","last_fullsized":"2023-09-23T22:19:32.300616+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":153,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.293123+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:19:32.293123+0000","last_peered":"2023-09-23T22:19:32.293123+0000","last_clean":"2023-09-23T22:19:32.293123+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:19:32.293123+0000","last_undegraded":"2023-09-23T22:19:32.293123+0000","last_fullsized":"2023-09-23T22:19:32.293123+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":165,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.294936+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:19:31.294936+0000","last_peered":"2023-09-23T22:19:31.294936+0000","last_clean":"2023-09-23T22:19:31.294936+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:19:31.294936+0000","last_undegraded":"2023-09-23T22:19:31.294936+0000","last_fullsized":"2023-09-23T22:19:31.294936+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":150,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.293178+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:19:32.293178+0000","last_peered":"2023-09-23T22:19:32.293178+0000","last_clean":"2023-09-23T22:19:32.293178+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:19:32.293178+0000","last_undegraded":"2023-09-23T22:19:32.293178+0000","last_fullsized":"2023-09-23T22:19:32.293178+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":24,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.295525+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:19:31.295525+0000","last_peered":"2023-09-23T22:19:31.295525+0000","last_clean":"2023-09-23T22:19:31.295525+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:19:31.295525+0000","last_undegraded":"2023-09-23T22:19:31.295525+0000","last_fullsized":"2023-09-23T22:19:31.295525+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":171,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291779+0000","last_change":"2023-09-23T22:16:49.145417+0000","last_active":"2023-09-23T22:19:32.291779+0000","last_peered":"2023-09-23T22:19:32.291779+0000","last_clean":"2023-09-23T22:19:32.291779+0000","last_became_active":"2023-09-23T22:16:49.145017+0000","last_became_peered":"2023-09-23T22:16:49.145017+0000","last_unstale":"2023-09-23T22:19:32.291779+0000","last_undegraded":"2023-09-23T22:19:32.291779+0000","last_fullsized":"2023-09-23T22:19:32.291779+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:47.762149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":213,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.291111+0000","last_change":"2023-09-23T22:17:15.721123+0000","last_active":"2023-09-23T22:19:32.291111+0000","last_peered":"2023-09-23T22:19:32.291111+0000","last_clean":"2023-09-23T22:19:32.291111+0000","last_became_active":"2023-09-23T22:17:15.720812+0000","last_became_peered":"2023-09-23T22:17:15.720812+0000","last_unstale":"2023-09-23T22:19:32.291111+0000","last_undegraded":"2023-09-23T22:19:32.291111+0000","last_fullsized":"2023-09-23T22:19:32.291111+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.176278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":63,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T22:19:32.309021+0000","last_change":"2023-09-23T22:18:57.729455+0000","last_active":"2023-09-23T22:19:32.309021+0000","last_peered":"2023-09-23T22:19:32.309021+0000","last_clean":"2023-09-23T22:19:32.309021+0000","last_became_active":"2023-09-23T22:18:57.729088+0000","last_became_peered":"2023-09-23T22:18:57.729088+0000","last_unstale":"2023-09-23T22:19:32.309021+0000","last_undegraded":"2023-09-23T22:19:32.309021+0000","last_fullsized":"2023-09-23T22:19:32.309021+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:17.861334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":158,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T22:19:31.294499+0000","last_change":"2023-09-23T22:16:49.164638+0000","last_active":"2023-09-23T22:19:31.294499+0000","last_peered":"2023-09-23T22:19:31.294499+0000","last_clean":"2023-09-23T22:19:31.294499+0000","last_became_active":"2023-09-23T22:16:49.164349+0000","last_became_peered":"2023-09-23T22:16:49.164349+0000","last_unstale":"2023-09-23T22:19:31.294499+0000","last_undegraded":"2023-09-23T22:19:31.294499+0000","last_fullsized":"2023-09-23T22:19:31.294499+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:26.536927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":513933336,"num_objects":7843,"num_object_clones":0,"num_object_copies":23529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9840,"num_objects_misplaced":457,"num_objects_unfound":0,"num_objects_dirty":7843,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":179769,"num_write_kb":6003521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30194,"num_bytes_recovered":1964834816,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3842322432,"data_stored":3842310216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108615,"ondisk_log_size":108615,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4892,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2098784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234182,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":404192,"kb_used_data":318488,"kb_used_omap":0,"kb_used_meta":85696,"kb_avail":93340960,"statfs":{"total":95995035648,"available":95581143040,"internally_reserved":0,"allocated":326131712,"data_stored":325302152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":87752704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.7320000000000002,"5min":2.2080000000000002,"15min":1.105},"min":{"1min":0.23100000000000001,"5min":0.223,"15min":0.223},"max":{"1min":21.977,"5min":109.208,"15min":109.208},"last":0.316},{"interface":"front","average":{"1min":2.548,"5min":2.536,"15min":1.2370000000000001},"min":{"1min":0.28199999999999997,"5min":0.217,"15min":0.217},"max":{"1min":11.698,"5min":109.13800000000001,"15min":109.13800000000001},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":4.4370000000000003,"5min":4.2199999999999998,"15min":1.784},"min":{"1min":0.254,"5min":0.246,"15min":0.246},"max":{"1min":27.800000000000001,"5min":123.79300000000001,"15min":123.79300000000001},"last":52.262999999999998},{"interface":"front","average":{"1min":4.3490000000000002,"5min":4.569,"15min":1.9139999999999999},"min":{"1min":0.316,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":27.785,"5min":123.66500000000001,"15min":123.66500000000001},"last":60.207000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":6.2619999999999996,"5min":6.1609999999999996,"15min":2.7250000000000001},"min":{"1min":0.78600000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":37.698,"5min":125.101,"15min":125.101},"last":48.113999999999997},{"interface":"front","average":{"1min":5.4589999999999996,"5min":5.8220000000000001,"15min":2.3500000000000001},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":37.679000000000002,"5min":126.021,"15min":126.021},"last":48.533000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":2.4710000000000001,"15min":1.2250000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.399,"5min":113.282,"15min":113.282},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.9769999999999999,"5min":2.8180000000000001,"15min":1.3859999999999999},"min":{"1min":0.41299999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.2809999999999997,"5min":104.74299999999999,"15min":104.74299999999999},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.3009999999999999,"15min":0.749},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.251},"max":{"1min":10.305999999999999,"5min":33.627000000000002,"15min":33.627000000000002},"last":0.245},{"interface":"front","average":{"1min":1.335,"5min":1.111,"15min":0.70299999999999996},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.1300000000000008,"5min":33.113,"15min":33.113},"last":0.35599999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":1.0109999999999999,"15min":0.65700000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.9219999999999997,"5min":8.9120000000000008,"15min":8.9120000000000008},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.946,"5min":1.5369999999999999,"15min":0.88200000000000001},"min":{"1min":0.35499999999999998,"5min":0.34200000000000003,"15min":0.29199999999999998},"max":{"1min":10.423999999999999,"5min":31.257999999999999,"15min":31.257999999999999},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":0.999,"15min":0.68899999999999995},"min":{"1min":0.33700000000000002,"5min":0.33000000000000002,"15min":0.315},"max":{"1min":7.9459999999999997,"5min":9.0139999999999993,"15min":9.0139999999999993},"last":0.372},{"interface":"front","average":{"1min":2.2930000000000001,"5min":1.3500000000000001,"15min":0.84199999999999997},"min":{"1min":0.38700000000000001,"5min":0.38400000000000001,"15min":0.308},"max":{"1min":10.347,"5min":13.362,"15min":13.362},"last":0.40100000000000002}]}]},{"osd":6,"up_from":0,"seq":197568495818,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795090,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":379704,"kb_used_data":221680,"kb_used_omap":7,"kb_used_meta":158008,"kb_avail":93365448,"statfs":{"total":95995035648,"available":95606218752,"internally_reserved":0,"allocated":227000320,"data_stored":225966232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":161800564},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.79,"15min":0.83599999999999997},"min":{"1min":0.20399999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":10.705,"5min":61.890000000000001,"15min":61.890000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":3.7280000000000002,"5min":2.4220000000000002,"15min":1.0780000000000001},"min":{"1min":0.216,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":15.08,"5min":62.689999999999998,"15min":62.689999999999998},"last":0.47699999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:18:37 2023","interfaces":[{"interface":"back","average":{"1min":5.3970000000000002,"5min":2.1629999999999998,"15min":0.97299999999999998},"min":{"1min":0.29099999999999998,"5min":0.216,"15min":0.216},"max":{"1min":33.902000000000001,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":7.593,"5min":2.302,"15min":1.042},"min":{"1min":0.48399999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":34.781999999999996,"5min":34.781999999999996,"15min":34.781999999999996},"last":0.58799999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:19:14 2023","interfaces":[{"interface":"back","average":{"1min":4.1349999999999998,"5min":2.1909999999999998,"15min":0.995},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":16.131,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.78800000000000003},{"interface":"front","average":{"1min":2.9049999999999998,"5min":1.962,"15min":0.92700000000000005},"min":{"1min":0.25900000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":15.859999999999999,"5min":22.259,"15min":22.259},"last":11.005000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:19:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":1.0289999999999999,"15min":0.78800000000000003},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.845000000000001,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.6560000000000001,"5min":1.5189999999999999,"15min":0.94899999999999995},"min":{"1min":0.33200000000000002,"5min":0.318,"15min":0.318},"max":{"1min":11.037000000000001,"5min":11.037000000000001,"15min":11.037000000000001},"last":0.55500000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9039999999999999,"5min":1.522,"15min":0.91700000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":11.125,"5min":11.125,"15min":11.125},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.54,"5min":1.292,"15min":0.85399999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.1680000000000001,"5min":6.6130000000000004,"15min":6.6130000000000004},"last":10.872999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4930000000000001,"5min":1.3660000000000001,"15min":0.88200000000000001},"min":{"1min":0.372,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.101,"5min":10.989000000000001,"15min":10.989000000000001},"last":10.971},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.246,"15min":0.86399999999999999},"min":{"1min":0.35699999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.811999999999999,"5min":10.811999999999999,"15min":10.811999999999999},"last":2.9529999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.516,"15min":0.93600000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.9829999999999997,"5min":10.948,"15min":10.948},"last":10.933999999999999},{"interface":"front","average":{"1min":1.635,"5min":1.369,"15min":0.90900000000000003},"min":{"1min":0.39500000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":10.781000000000001,"5min":10.781000000000001,"15min":10.781000000000001},"last":0.65100000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958625,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":675264,"kb_used_data":499192,"kb_used_omap":8,"kb_used_meta":176055,"kb_avail":93069888,"statfs":{"total":95995035648,"available":95303565312,"internally_reserved":0,"allocated":511172608,"data_stored":510137931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":180281194},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,7],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:19:12 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":2.4529999999999998,"15min":1.0429999999999999},"min":{"1min":0.23999999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":11.711,"5min":44.776000000000003,"15min":44.776000000000003},"last":0.437},{"interface":"front","average":{"1min":4.6529999999999996,"5min":3.1000000000000001,"15min":1.276},"min":{"1min":0.23899999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":20.927,"5min":48.838000000000001,"15min":48.838000000000001},"last":12.641999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:18:40 2023","interfaces":[{"interface":"back","average":{"1min":3.9319999999999999,"5min":1.502,"15min":0.76900000000000002},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":25.893000000000001,"5min":25.893000000000001,"15min":25.893000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":7.742,"5min":2.7810000000000001,"15min":1.2749999999999999},"min":{"1min":0.28999999999999998,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":66.665000000000006,"5min":66.665000000000006,"15min":66.665000000000006},"last":12.602}]},{"osd":3,"last update":"Sat Sep 23 22:19:12 2023","interfaces":[{"interface":"back","average":{"1min":3.2589999999999999,"5min":2.4060000000000001,"15min":1.099},"min":{"1min":0.33400000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":13.663,"5min":35.496000000000002,"15min":35.496000000000002},"last":0.46600000000000003},{"interface":"front","average":{"1min":4.2469999999999999,"5min":2.6779999999999999,"15min":1.1859999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":20.870999999999999,"5min":35.533000000000001,"15min":35.533000000000001},"last":0.67600000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.2880000000000003,"5min":2.2050000000000001,"15min":1.1679999999999999},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.217000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":10.032999999999999},{"interface":"front","average":{"1min":2.9860000000000002,"5min":1.2569999999999999,"15min":0.85799999999999998},"min":{"1min":0.32700000000000001,"5min":0.308,"15min":0.308},"max":{"1min":22.209,"5min":22.209,"15min":22.209},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.6959999999999997,"5min":2.173,"15min":1.1799999999999999},"min":{"1min":0.36099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.372,"5min":12.372,"15min":12.372},"last":9.9920000000000009},{"interface":"front","average":{"1min":2.6699999999999999,"5min":1.286,"15min":0.88200000000000001},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":20.390999999999998,"5min":20.390999999999998,"15min":20.390999999999998},"last":0.54200000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.5609999999999999,"5min":2.2149999999999999,"15min":1.1879999999999999},"min":{"1min":0.46400000000000002,"5min":0.379,"15min":0.379},"max":{"1min":10.275,"5min":10.275,"15min":10.275},"last":10.087},{"interface":"front","average":{"1min":2.867,"5min":1.3080000000000001,"15min":0.91000000000000003},"min":{"1min":0.38600000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":22.263000000000002,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:18:55 2023","interfaces":[{"interface":"back","average":{"1min":4.1180000000000003,"5min":2.3079999999999998,"15min":1.252},"min":{"1min":0.44500000000000001,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":20.491,"5min":23.306000000000001,"15min":23.306000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":4.5170000000000003,"5min":2.6419999999999999,"15min":1.379},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.257999999999999,"5min":23.265999999999998,"15min":23.265999999999998},"last":10.061}]}]},{"osd":2,"up_from":103,"seq":442381631564,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381432,"kb_used_data":211184,"kb_used_omap":10,"kb_used_meta":170229,"kb_avail":93363720,"statfs":{"total":95995035648,"available":95604449280,"internally_reserved":0,"allocated":216252416,"data_stored":215227055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":174314668},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,3,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":3.1629999999999998,"5min":2.5030000000000001,"15min":1.0569999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":19.344000000000001,"5min":19.344000000000001,"15min":19.344000000000001},"last":7.2009999999999996},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.7210000000000001,"15min":1.149},"min":{"1min":0.25700000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":9.4779999999999998,"5min":19.149999999999999,"15min":19.149999999999999},"last":3.1480000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":3.8220000000000001,"5min":4.6289999999999996,"15min":1.825},"min":{"1min":0.31900000000000001,"5min":0.222,"15min":0.222},"max":{"1min":16.141999999999999,"5min":157.636,"15min":157.636},"last":1.327},{"interface":"front","average":{"1min":3.7509999999999999,"5min":4.7030000000000003,"15min":1.847},"min":{"1min":0.29499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":20.783999999999999,"5min":155.63300000000001,"15min":155.63300000000001},"last":3.371}]},{"osd":3,"last update":"Sat Sep 23 22:19:05 2023","interfaces":[{"interface":"back","average":{"1min":4.1289999999999996,"5min":2.2949999999999999,"15min":0.99199999999999999},"min":{"1min":0.25,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":19.852,"5min":19.852,"15min":19.852},"last":7.5119999999999996},{"interface":"front","average":{"1min":4.4340000000000002,"5min":2.4009999999999998,"15min":1.0589999999999999},"min":{"1min":0.27900000000000003,"5min":0.246,"15min":0.246},"max":{"1min":20.581,"5min":20.581,"15min":20.581},"last":1.2709999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9019999999999999,"5min":1.79,"15min":0.995},"min":{"1min":0.34200000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.3689999999999998,"5min":11.244,"15min":11.244},"last":3.246},{"interface":"front","average":{"1min":2.1320000000000001,"5min":1.845,"15min":1.0329999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8659999999999997,"5min":9.2799999999999994,"15min":9.2799999999999994},"last":7.4820000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.893,"15min":1.054},"min":{"1min":0.311,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.3219999999999992,"5min":11.291,"15min":11.291},"last":3.319},{"interface":"front","average":{"1min":2.3610000000000002,"5min":1.968,"15min":1.0600000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.915,"5min":9.4499999999999993,"15min":9.4499999999999993},"last":7.5659999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.008,"15min":1.1120000000000001},"min":{"1min":0.36699999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.4280000000000008,"5min":17.640999999999998,"15min":17.640999999999998},"last":3.3460000000000001},{"interface":"front","average":{"1min":2.2559999999999998,"5min":1.8109999999999999,"15min":1.127},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.891,"5min":11.875,"15min":11.875},"last":7.5389999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":2.0019999999999998,"15min":1.1739999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":7.8019999999999996,"5min":10.135,"15min":10.135},"last":1.502},{"interface":"front","average":{"1min":2.3610000000000002,"5min":2.2719999999999998,"15min":1.1819999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.377},"max":{"1min":9.4529999999999994,"5min":17.738,"15min":17.738},"last":3.282}]}]},{"osd":3,"up_from":108,"seq":463856468037,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":266748,"kb_used_data":199028,"kb_used_omap":4,"kb_used_meta":67643,"kb_avail":93478404,"statfs":{"total":95995035648,"available":95721885696,"internally_reserved":0,"allocated":203804672,"data_stored":202770785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":69266989},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":4.3410000000000002,"5min":2.738,"15min":1.1639999999999999},"min":{"1min":0.23300000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":32.731999999999999,"5min":77.834999999999994,"15min":77.834999999999994},"last":7.3079999999999998},{"interface":"front","average":{"1min":4.8730000000000002,"5min":3.1019999999999999,"15min":1.2749999999999999},"min":{"1min":0.26500000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":32.786999999999999,"5min":75.558000000000007,"15min":75.558000000000007},"last":10.798}]},{"osd":1,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":7.4589999999999996,"5min":3.5379999999999998,"15min":1.4139999999999999},"min":{"1min":0.246,"5min":0.218,"15min":0.218},"max":{"1min":82.951999999999998,"5min":82.951999999999998,"15min":82.951999999999998},"last":12.445},{"interface":"front","average":{"1min":9.4529999999999994,"5min":3.601,"15min":1.4730000000000001},"min":{"1min":0.22600000000000001,"5min":0.189,"15min":0.189},"max":{"1min":86.001000000000005,"5min":86.001000000000005,"15min":86.001000000000005},"last":6.4560000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":17.736000000000001,"5min":7.7190000000000003,"15min":2.835},"min":{"1min":0.28000000000000003,"5min":0.223,"15min":0.223},"max":{"1min":99.962999999999994,"5min":99.962999999999994,"15min":99.962999999999994},"last":1.5329999999999999},{"interface":"front","average":{"1min":18.577000000000002,"5min":7.1669999999999998,"15min":2.6720000000000002},"min":{"1min":0.28799999999999998,"5min":0.251,"15min":0.251},"max":{"1min":100.429,"5min":100.429,"15min":100.429},"last":6.8520000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:19:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":1.3169999999999999,"15min":0.85599999999999998},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.7610000000000001,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":6.242},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.4630000000000001,"15min":0.92500000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.6200000000000001,"5min":12.036,"15min":12.036},"last":10.720000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.397,"15min":0.88200000000000001},"min":{"1min":0.32800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.6529999999999996,"5min":14.292,"15min":14.292},"last":10.664},{"interface":"front","average":{"1min":2.726,"5min":1.526,"15min":0.91500000000000004},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.484,"5min":11.56,"15min":11.56},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":2.6190000000000002,"5min":1.6100000000000001,"15min":0.97199999999999998},"min":{"1min":0.38300000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.948,"5min":12.068,"15min":12.068},"last":10.760999999999999},{"interface":"front","average":{"1min":2.0939999999999999,"5min":1.52,"15min":0.93000000000000005},"min":{"1min":0.35599999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.0660000000000007,"5min":11.586,"15min":11.586},"last":0.45200000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:19:08 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":1.607,"15min":0.95599999999999996},"min":{"1min":0.39600000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":9.1630000000000003,"5min":11.624000000000001,"15min":11.624000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.504,"5min":1.405,"15min":0.92400000000000004},"min":{"1min":0.40400000000000003,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":6.7880000000000003,"5min":12.382,"15min":12.382},"last":6.2729999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019088,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":408524,"kb_used_data":294468,"kb_used_omap":0,"kb_used_meta":114048,"kb_avail":93336628,"statfs":{"total":95995035648,"available":95576707072,"internally_reserved":0,"allocated":301535232,"data_stored":300717582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":116785152},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.9970000000000001,"15min":1.0840000000000001},"min":{"1min":0.23000000000000001,"5min":0.218,"15min":0.218},"max":{"1min":7.5789999999999997,"5min":66.355000000000004,"15min":66.355000000000004},"last":4.9279999999999999},{"interface":"front","average":{"1min":1.4299999999999999,"5min":2.2200000000000002,"15min":1.1699999999999999},"min":{"1min":0.29799999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.5949999999999998,"5min":66.332999999999998,"15min":66.332999999999998},"last":4.9400000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":1.748,"15min":0.94599999999999995},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":13.625,"5min":38.878,"15min":38.878},"last":2.8460000000000001},{"interface":"front","average":{"1min":2.6110000000000002,"5min":2.4220000000000002,"15min":1.1899999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":10.236000000000001,"5min":38.853999999999999,"15min":38.853999999999999},"last":3.4969999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:18:40 2023","interfaces":[{"interface":"back","average":{"1min":8.1180000000000003,"5min":2.8780000000000001,"15min":1.3200000000000001},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.255},"max":{"1min":133.22900000000001,"5min":133.22900000000001,"15min":133.22900000000001},"last":3.8650000000000002},{"interface":"front","average":{"1min":8.4529999999999994,"5min":3.4510000000000001,"15min":1.4910000000000001},"min":{"1min":0.33000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":133.60300000000001,"5min":133.60300000000001,"15min":133.60300000000001},"last":5.1390000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:19:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4710000000000001,"5min":1.5489999999999999,"15min":0.91200000000000003},"min":{"1min":0.35199999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9109999999999996,"5min":7.9109999999999996,"15min":7.9109999999999996},"last":3.569},{"interface":"front","average":{"1min":2.169,"5min":1.607,"15min":0.91900000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.8930000000000007,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":3.7480000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:19:09 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.93300000000000005,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":4.0629999999999997,"5min":8.9499999999999993,"15min":8.9499999999999993},"last":1.2350000000000001},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.86099999999999999,"15min":0.64300000000000002},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.0069999999999997,"5min":7.8789999999999996,"15min":7.8789999999999996},"last":1.2569999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.82399999999999995,"15min":0.61799999999999999},"min":{"1min":0.35899999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.754,"5min":3.754,"15min":3.754},"last":1.7470000000000001},{"interface":"front","average":{"1min":1.0760000000000001,"5min":0.94699999999999995,"15min":0.68000000000000005},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":6.7279999999999998,"5min":8.5210000000000008,"15min":8.5210000000000008},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:18:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":1.1180000000000001,"15min":0.76000000000000001},"min":{"1min":0.33100000000000002,"5min":0.307,"15min":0.307},"max":{"1min":5.3440000000000003,"5min":8.577,"15min":8.577},"last":0.95499999999999996},{"interface":"front","average":{"1min":1.0109999999999999,"5min":0.90400000000000003,"15min":0.69299999999999995},"min":{"1min":0.38500000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.7799999999999998,"5min":5.6280000000000001,"15min":5.6280000000000001},"last":0.39700000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757455,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":672784,"kb_used_data":554120,"kb_used_omap":0,"kb_used_meta":118656,"kb_avail":93072368,"statfs":{"total":95995035648,"available":95306104832,"internally_reserved":0,"allocated":567418880,"data_stored":566601176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":121503744},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:18:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":3.113,"15min":1.4139999999999999},"min":{"1min":0.26800000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.6849999999999996,"5min":133.22499999999999,"15min":133.22499999999999},"last":4.5679999999999996},{"interface":"front","average":{"1min":1.732,"5min":3.387,"15min":1.554},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.444,"5min":132.78200000000001,"15min":132.78200000000001},"last":4.3499999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":7.7859999999999996,"5min":4.5830000000000002,"15min":1.9219999999999999},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":83.759,"5min":83.759,"15min":83.759},"last":4.3049999999999997},{"interface":"front","average":{"1min":7.8920000000000003,"5min":4.6139999999999999,"15min":1.9179999999999999},"min":{"1min":0.30399999999999999,"5min":0.252,"15min":0.252},"max":{"1min":83.741,"5min":83.741,"15min":83.741},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:18:39 2023","interfaces":[{"interface":"back","average":{"1min":12.324999999999999,"5min":6.0250000000000004,"15min":2.383},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":104.254,"5min":104.254,"15min":104.254},"last":4.4160000000000004},{"interface":"front","average":{"1min":12.984,"5min":6.6600000000000001,"15min":2.6139999999999999},"min":{"1min":0.34699999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":104.217,"5min":104.217,"15min":104.217},"last":0.47199999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.615,"5min":2.6389999999999998,"15min":1.2649999999999999},"min":{"1min":0.31900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":4.7549999999999999,"5min":88.106999999999999,"15min":88.106999999999999},"last":4.4900000000000002},{"interface":"front","average":{"1min":2.3929999999999998,"5min":2.8610000000000002,"15min":1.3460000000000001},"min":{"1min":0.38800000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1369999999999996,"5min":87.887,"15min":87.887},"last":4.4029999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:19:01 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.6559999999999999,"15min":0.86399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.4880000000000004,"5min":82.530000000000001,"15min":82.530000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.7070000000000001,"15min":0.91800000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.577,"5min":85.441000000000003,"15min":85.441000000000003},"last":4.4580000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:19:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":2.137,"15min":1.0669999999999999},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5350000000000001,"5min":114.83499999999999,"15min":114.83499999999999},"last":4.5350000000000001},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.9710000000000001,"15min":1.0209999999999999},"min":{"1min":0.36299999999999999,"5min":0.31,"15min":0.31},"max":{"1min":4.5220000000000002,"5min":110.905,"15min":110.905},"last":4.5220000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:18:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4980000000000002,"5min":1.1850000000000001,"15min":0.77200000000000002},"min":{"1min":0.35899999999999999,"5min":0.33500000000000002,"15min":0.32600000000000001},"max":{"1min":8.2159999999999993,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":4.3760000000000003},{"interface":"front","average":{"1min":1.1950000000000001,"5min":0.97599999999999998,"15min":0.69699999999999995},"min":{"1min":0.371,"5min":0.36499999999999999,"15min":0.33200000000000002},"max":{"1min":5.2460000000000004,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":1.536}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":502792192,"data_stored":502792192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":213778432,"data_stored":213778432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":204607488,"data_stored":204603416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":198967296,"data_stored":198967296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":298516480,"data_stored":298516480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":558239744,"data_stored":558235672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1554513920,"data_stored":1554513920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":310906880,"data_stored":310902808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:19:40.493 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:19:40.512 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:40.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:40.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:40.624 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:40.652 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:40.695 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:40.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:40.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:40.704 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:40.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:40 smithi026 ceph-mon[215775]: pgmap v435: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 436 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 9841/20793 objects degraded (47.328%); 457/20793 objects misplaced (2.198%); 2.4 MiB/s, 86 objects/s recovering 2023-09-23T22:19:40.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:40 smithi026 ceph-mon[223014]: pgmap v435: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 436 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 9841/20793 objects degraded (47.328%); 457/20793 objects misplaced (2.198%); 2.4 MiB/s, 86 objects/s recovering 2023-09-23T22:19:40.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:40.828 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:41.066 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:41.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:41.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:41.173 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:41.271 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:41.497 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:41.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:19:41.550 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:19:41.551 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:19:41.604 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:41.652 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:41.818 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:41.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:41.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:41.925 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:41.951 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:41.992 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:41.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:41.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:42.096 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:42.182 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:42.311 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:42.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:42.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:42.334 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:19:42.423 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:19:42.427 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:42.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:42 smithi026 ceph-mon[223014]: pgmap v436: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 416 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 9841/19833 objects degraded (49.619%); 457/19833 objects misplaced (2.304%); 2.2 MiB/s, 76 objects/s recovering 2023-09-23T22:19:42.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2970538611' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:19:42.465 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:42 smithi026 ceph-mon[215775]: pgmap v436: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 416 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 9841/19833 objects degraded (49.619%); 457/19833 objects misplaced (2.304%); 2.2 MiB/s, 76 objects/s recovering 2023-09-23T22:19:42.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2970538611' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:19:42.524 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:42.602 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:42.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:42.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:42.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:42 smithi070 ceph-mon[192652]: pgmap v436: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 416 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 445 op/s; 9841/19833 objects degraded (49.619%); 457/19833 objects misplaced (2.304%); 2.2 MiB/s, 76 objects/s recovering 2023-09-23T22:19:42.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2970538611' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:19:42.705 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:42.783 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:42.785 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:42.936 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:42.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:42.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:43.043 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:43.079 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:43.158 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:43.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:43.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:43.264 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:43.347 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:43.713 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:43.715 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:43.715 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2970538611' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:19:43.716 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[215775]: osdmap e192: 8 total, 8 up, 7 in 2023-09-23T22:19:43.716 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[215775]: failed to encode map e192 with expected crc 2023-09-23T22:19:43.716 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 9841/19833 objects degraded (49.619%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:19:43.716 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:43.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2970538611' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:19:43.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[223014]: osdmap e192: 8 total, 8 up, 7 in 2023-09-23T22:19:43.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[223014]: failed to encode map e192 with expected crc 2023-09-23T22:19:43.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 9841/19833 objects degraded (49.619%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:19:43.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:43.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:43.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:43 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:19:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2970538611' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:19:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:43 smithi070 ceph-mon[192652]: osdmap e192: 8 total, 8 up, 7 in 2023-09-23T22:19:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:43 smithi070 ceph-mon[192652]: failed to encode map e192 with expected crc 2023-09-23T22:19:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 9841/19833 objects degraded (49.619%), 4 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:19:43.903 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:44.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:44.525 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:44.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:44.611 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:44.615 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:19:44.616 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:19:44.616 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:19:44.616 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:19:44.616 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:19:44.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:44.671 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:44.892 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:44.894 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:44 smithi070 ceph-mon[192652]: pgmap v438: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 304 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 9674/14481 objects degraded (66.805%); 305/14481 objects misplaced (2.106%); 4.0 MiB/s, 101 objects/s recovering 2023-09-23T22:19:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:44.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:44.995 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:45.021 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:45.034 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:44 smithi026 ceph-mon[215775]: pgmap v438: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 304 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 9674/14481 objects degraded (66.805%); 305/14481 objects misplaced (2.106%); 4.0 MiB/s, 101 objects/s recovering 2023-09-23T22:19:45.034 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:44 smithi026 ceph-mon[223014]: pgmap v438: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 304 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 9674/14481 objects degraded (66.805%); 305/14481 objects misplaced (2.106%); 4.0 MiB/s, 101 objects/s recovering 2023-09-23T22:19:45.084 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:45.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:45.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:45.192 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:45.314 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:45.358 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:19:45.377 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:19:45.389 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:19:45.436 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:45.437 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:19:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:45.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:45.478 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:19:45.520 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:19:45.549 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:45.553 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:19:45.602 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:45.612 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:19:45.649 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:19:45.671 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:45.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:45.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:45.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:19:45.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:19:45.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:19:45.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:19:45.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:19:45.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:19:45.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:19:45.677 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:19:45.777 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:45.852 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:19:45.863 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:45.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3086422214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:45.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3086422214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:46.091 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3086422214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:19:46.634 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:19:46.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:46 smithi070 ceph-mon[192652]: pgmap v439: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 271 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 9674/12870 objects degraded (75.167%); 305/12870 objects misplaced (2.370%); 3.9 MiB/s, 110 objects/s recovering 2023-09-23T22:19:46.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:46 smithi026 ceph-mon[215775]: pgmap v439: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 271 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 9674/12870 objects degraded (75.167%); 305/12870 objects misplaced (2.370%); 3.9 MiB/s, 110 objects/s recovering 2023-09-23T22:19:46.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:46 smithi026 ceph-mon[223014]: pgmap v439: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 271 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 9674/12870 objects degraded (75.167%); 305/12870 objects misplaced (2.370%); 3.9 MiB/s, 110 objects/s recovering 2023-09-23T22:19:47.297 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:47.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:47.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:47.404 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:47.432 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:47.699 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:19:47.700 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:19:47.810 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:47.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:47.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:47.933 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:47 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 9674/12870 objects degraded (75.167%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:47.934 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:47 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 9674/12870 objects degraded (75.167%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:47.934 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:48.030 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:48.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:47 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 9674/12870 objects degraded (75.167%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:48.131 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:48.169 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:48.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:48.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:48.285 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:48.328 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:48.410 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:48.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:48.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:48.472 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:19:48.514 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:48.605 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:48.816 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:48 smithi026 ceph-mon[215775]: pgmap v440: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 166 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 9535/7815 objects degraded (122.009%); 108/7815 objects misplaced (1.382%); 4.1 MiB/s, 105 objects/s recovering 2023-09-23T22:19:48.816 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:48 smithi026 ceph-mon[223014]: pgmap v440: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 166 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 9535/7815 objects degraded (122.009%); 108/7815 objects misplaced (1.382%); 4.1 MiB/s, 105 objects/s recovering 2023-09-23T22:19:48.873 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:48.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:48.994 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:49.108 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:49.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:48 smithi070 ceph-mon[192652]: pgmap v440: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 166 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 526 op/s; 9535/7815 objects degraded (122.009%); 108/7815 objects misplaced (1.382%); 4.1 MiB/s, 105 objects/s recovering 2023-09-23T22:19:49.168 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:19:49.168 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 9535/7815 objects degraded (122.009%), 4 pgs degraded, 2 pgs undersized","count":6},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":573,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":192,"num_osds":8,"num_up_osds":8,"osd_up_since":1695507239,"num_in_osds":7,"osd_in_since":1695507408,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":93},{"state_name":"active+recovery_wait+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":99,"num_pools":5,"num_objects":2605,"data_bytes":173610955,"bytes_used":2301063168,"bytes_avail":669664186368,"bytes_total":671965249536,"degraded_objects":9535,"degraded_total":7815,"degraded_ratio":1.220089571337172,"misplaced_objects":108,"misplaced_total":7815,"misplaced_ratio":0.013819577735124759,"recovering_objects_per_sec":105,"recovering_bytes_per_sec":4290640,"recovering_keys_per_sec":0,"num_objects_recovered":1056,"num_bytes_recovered":42926080,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":526},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T22:14:59.012277+0000","services":{}},"progress_events":{"2fcb4d05-59c9-4a0a-bf68-bf36661520e1":{"message":"Global Recovery Event (4m)\n [==========================..] (remaining: 16s)","progress":0.93939393758773804,"add_to_ceph_s":true},"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:19:49.558 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 93805 objects 2023-09-23T22:19:49.558 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :161.235 2023-09-23T22:19:49.695 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T22:19:49.695 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T22:19:49.935 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:19:49.966 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:19:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:19:49] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2023-09-23T22:19:49.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:49 smithi026 ceph-mon[223014]: pgmap v441: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 100 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 9535/4650 objects degraded (205.054%); 108/4650 objects misplaced (2.323%); 4.1 MiB/s, 106 objects/s recovering 2023-09-23T22:19:49.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:49 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/522483202' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:19:49.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:49 smithi026 ceph-mon[215775]: pgmap v441: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 100 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 9535/4650 objects degraded (205.054%); 108/4650 objects misplaced (2.323%); 4.1 MiB/s, 106 objects/s recovering 2023-09-23T22:19:49.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:49 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/522483202' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:19:50.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:49 smithi070 ceph-mon[192652]: pgmap v441: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 100 MiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 537 op/s; 9535/4650 objects degraded (205.054%); 108/4650 objects misplaced (2.323%); 4.1 MiB/s, 106 objects/s recovering 2023-09-23T22:19:50.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:49 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/522483202' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:19:50.987 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:50.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:50.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:51.091 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:19:51.126 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:51.176 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:19:51.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:51.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:19:51.282 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:51.310 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:51.353 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:51.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:51.360 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:51.460 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:51.486 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:51.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:51.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:51.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:51.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:51.662 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:52.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:52 smithi026 ceph-mon[223014]: pgmap v442: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 82 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 9535/3792 objects degraded (251.450%); 108/3792 objects misplaced (2.848%); 4.1 MiB/s, 106 objects/s recovering 2023-09-23T22:19:52.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:52 smithi026 ceph-mon[215775]: pgmap v442: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 82 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 9535/3792 objects degraded (251.450%); 108/3792 objects misplaced (2.848%); 4.1 MiB/s, 106 objects/s recovering 2023-09-23T22:19:52.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:52 smithi070 ceph-mon[192652]: pgmap v442: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 93 active+clean; 82 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 9535/3792 objects degraded (251.450%); 108/3792 objects misplaced (2.848%); 4.1 MiB/s, 106 objects/s recovering 2023-09-23T22:19:52.865 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:52.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:52.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:52.969 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:19:53.001 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:53.059 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:19:53.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:19:53.074 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:19:53.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:53.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:19:53.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:53.229 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:53.325 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:53.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:53.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:53.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 9535/3792 objects degraded (251.450%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:53.345 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 9535/3792 objects degraded (251.450%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:53.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:53.371 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:53.424 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:19:53.447 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:53.485 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 9535/3792 objects degraded (251.450%), 4 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:53.670 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:19:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:53.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:19:53.774 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:53.806 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:53.854 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:53.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:53.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:53.960 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:53.987 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:54.029 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:54.036 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:54.137 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:19:54.163 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:54.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:54 smithi026 ceph-mon[223014]: pgmap v443: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 9.9 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 9251/342 objects degraded (2704.971%); 3.1 MiB/s, 105 objects/s recovering 2023-09-23T22:19:54.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1180400300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:54.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:54 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:54.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:54.205 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:19:54.208 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:19:54.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:54.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:19:54.312 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:54.338 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:54.379 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:54.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:54.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:54.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:54 smithi026 ceph-mon[215775]: pgmap v443: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 9.9 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 9251/342 objects degraded (2704.971%); 3.1 MiB/s, 105 objects/s recovering 2023-09-23T22:19:54.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1180400300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:54.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:54 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:54.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:54.486 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:19:54.523 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:54 smithi070 ceph-mon[192652]: pgmap v443: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 9.9 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 9251/342 objects degraded (2704.971%); 3.1 MiB/s, 105 objects/s recovering 2023-09-23T22:19:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1180400300' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:54 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:19:54.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:19:55.231 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:19:55.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:55.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:19:55.336 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:19:55.369 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:55.418 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:19:55.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:55.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:19:55.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:19:55.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:19:55.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: osdmap e193: 8 total, 8 up, 7 in 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[215775]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:19:55.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: osdmap e193: 8 total, 8 up, 7 in 2023-09-23T22:19:55.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:55 smithi026 ceph-mon[223014]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:19:55.557 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:55.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:19:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:19:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: osdmap e193: 8 total, 8 up, 7 in 2023-09-23T22:19:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: failed to encode map e193 with expected crc 2023-09-23T22:19:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:55 smithi070 ceph-mon[192652]: failed to encode map e193 with expected crc 2023-09-23T22:19:56.114 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:19:56.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:56.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:19:56.218 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:19:56.251 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:56.298 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:19:56.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:19:56.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:19:56.406 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:19:56.434 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:56 smithi026 ceph-mon[215775]: pgmap v445: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 9251/84 objects degraded (11013.095%); 3.4 MiB/s, 144 objects/s recovering 2023-09-23T22:19:56.434 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:56 smithi026 ceph-mon[223014]: pgmap v445: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 9251/84 objects degraded (11013.095%); 3.4 MiB/s, 144 objects/s recovering 2023-09-23T22:19:56.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:19:56.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:56 smithi070 ceph-mon[192652]: pgmap v445: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.6 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 9251/84 objects degraded (11013.095%); 3.4 MiB/s, 144 objects/s recovering 2023-09-23T22:19:58.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:58 smithi026 ceph-mon[223014]: pgmap v446: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 8710/84 objects degraded (10369.048%); 1.3 MiB/s, 164 objects/s recovering 2023-09-23T22:19:58.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:19:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:58.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:58 smithi026 ceph-mon[215775]: pgmap v446: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 8710/84 objects degraded (10369.048%); 1.3 MiB/s, 164 objects/s recovering 2023-09-23T22:19:58.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:19:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:58 smithi070 ceph-mon[192652]: pgmap v446: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 257 op/s; 8710/84 objects degraded (10369.048%); 1.3 MiB/s, 164 objects/s recovering 2023-09-23T22:19:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:19:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:19:59.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:19:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:19:59] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T22:20:00.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:00.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:00.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:00.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:00.226 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:00.291 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:00.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:00.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:00.395 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:00.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[223014]: pgmap v447: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 8710/84 objects degraded (10369.048%); 1.3 MiB/s, 254 objects/s recovering 2023-09-23T22:20:00.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[223014]: Health detail: HEALTH_WARN Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 3 pgs undersized 2023-09-23T22:20:00.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[223014]: [WRN] PG_DEGRADED: Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 3 pgs undersized 2023-09-23T22:20:00.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[223014]: pg 1.8 is active+recovery_wait+degraded, acting [0,7,1] 2023-09-23T22:20:00.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[223014]: pg 5.2 is stuck undersized for 3m, current state active+recovering+undersized+degraded+remapped, last acting [0,5] 2023-09-23T22:20:00.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[223014]: pg 5.5 is stuck undersized for 61s, current state active+recovering+undersized+remapped, last acting [1,4] 2023-09-23T22:20:00.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[223014]: pg 5.b is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2023-09-23T22:20:00.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[215775]: pgmap v447: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 8710/84 objects degraded (10369.048%); 1.3 MiB/s, 254 objects/s recovering 2023-09-23T22:20:00.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[215775]: Health detail: HEALTH_WARN Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 3 pgs undersized 2023-09-23T22:20:00.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[215775]: [WRN] PG_DEGRADED: Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 3 pgs undersized 2023-09-23T22:20:00.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[215775]: pg 1.8 is active+recovery_wait+degraded, acting [0,7,1] 2023-09-23T22:20:00.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[215775]: pg 5.2 is stuck undersized for 3m, current state active+recovering+undersized+degraded+remapped, last acting [0,5] 2023-09-23T22:20:00.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[215775]: pg 5.5 is stuck undersized for 61s, current state active+recovering+undersized+remapped, last acting [1,4] 2023-09-23T22:20:00.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:00 smithi026 ceph-mon[215775]: pg 5.b is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2023-09-23T22:20:00.428 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:00.489 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:00.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:00.592 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:00.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:00 smithi070 ceph-mon[192652]: pgmap v447: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 152 op/s; 8710/84 objects degraded (10369.048%); 1.3 MiB/s, 254 objects/s recovering 2023-09-23T22:20:00.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:00 smithi070 ceph-mon[192652]: Health detail: HEALTH_WARN Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 3 pgs undersized 2023-09-23T22:20:00.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:00 smithi070 ceph-mon[192652]: [WRN] PG_DEGRADED: Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 3 pgs undersized 2023-09-23T22:20:00.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:00 smithi070 ceph-mon[192652]: pg 1.8 is active+recovery_wait+degraded, acting [0,7,1] 2023-09-23T22:20:00.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:00 smithi070 ceph-mon[192652]: pg 5.2 is stuck undersized for 3m, current state active+recovering+undersized+degraded+remapped, last acting [0,5] 2023-09-23T22:20:00.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:00 smithi070 ceph-mon[192652]: pg 5.5 is stuck undersized for 61s, current state active+recovering+undersized+remapped, last acting [1,4] 2023-09-23T22:20:00.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:00 smithi070 ceph-mon[192652]: pg 5.b is stuck undersized for 77s, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,0] 2023-09-23T22:20:00.626 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:00.677 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:00.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:00.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:00.785 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:00.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:00.859 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:00.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:00.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:00.965 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:00.991 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:01.033 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:01.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:01.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:01.140 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:01.172 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:01.226 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:01.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:01.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:01.330 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:01.362 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:01.412 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:01.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:01.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:01.517 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:01.546 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:01.598 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:01.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:01.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:01.702 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:01.729 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:01.772 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:01.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:01.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:01.878 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:01.908 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:01.960 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:01.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:01.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:02.064 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:02.091 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:02.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:02.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:02.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:02.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:02 smithi026 ceph-mon[223014]: pgmap v448: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 4.6 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 123 op/s; 8710/84 objects degraded (10369.048%); 1.3 MiB/s, 254 objects/s recovering 2023-09-23T22:20:02.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:02 smithi026 ceph-mon[215775]: pgmap v448: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 4.6 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 123 op/s; 8710/84 objects degraded (10369.048%); 1.3 MiB/s, 254 objects/s recovering 2023-09-23T22:20:02.239 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:02.266 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:02.307 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:02.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:02.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:02.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:02 smithi070 ceph-mon[192652]: pgmap v448: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 93 active+clean; 4.6 MiB data, 1.5 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 123 op/s; 8710/84 objects degraded (10369.048%); 1.3 MiB/s, 254 objects/s recovering 2023-09-23T22:20:02.413 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:02.439 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:02.480 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:02.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:02.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:02.586 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:02.617 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:02.670 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:02.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:02.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:02.774 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:02.806 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:02.858 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:02.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:02.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:02.963 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:02.993 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:03.048 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:03.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:03.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:03.153 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:03.184 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:03.231 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:03.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:03.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:03.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:03.367 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:03.413 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:03.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:03.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:03.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:20:03.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:20:03.522 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:03.550 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:03.596 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:03.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:03.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 8710/84 objects degraded (10369.048%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:20:03.700 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:03.728 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:03.772 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:03.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:03.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:03.908 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:03.957 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:03.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:04.060 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:04.089 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:04.139 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:04.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:04.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:04.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:04.273 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:04.926 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:04.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:04.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:05.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:05.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:04 smithi026 ceph-mon[223014]: pgmap v449: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 8094/84 objects degraded (9635.714%); 0 B/s, 276 objects/s recovering 2023-09-23T22:20:05.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:04 smithi026 ceph-mon[223014]: osdmap e194: 8 total, 8 up, 7 in 2023-09-23T22:20:05.058 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:04 smithi026 ceph-mon[215775]: pgmap v449: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 8094/84 objects degraded (9635.714%); 0 B/s, 276 objects/s recovering 2023-09-23T22:20:05.058 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:04 smithi026 ceph-mon[215775]: osdmap e194: 8 total, 8 up, 7 in 2023-09-23T22:20:05.060 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:05.114 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:05.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:05.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:05.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:04 smithi070 ceph-mon[192652]: pgmap v449: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 8094/84 objects degraded (9635.714%); 0 B/s, 276 objects/s recovering 2023-09-23T22:20:05.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:04 smithi070 ceph-mon[192652]: osdmap e194: 8 total, 8 up, 7 in 2023-09-23T22:20:05.219 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:05.251 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:05.304 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:05.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:05.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:05.407 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:05.437 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:05.483 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:05.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:05.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:05.591 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:05.618 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:05.660 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:05.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:05.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:05.767 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:05.796 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:05.851 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:05.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:05.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:05.954 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:05.981 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:06.023 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:06.026 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:05 smithi070 ceph-mon[192652]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.027 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:05 smithi070 ceph-mon[192652]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.027 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:05 smithi070 ceph-mon[192652]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.027 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:05 smithi070 ceph-mon[192652]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.027 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:05 smithi070 ceph-mon[192652]: failed to encode map e195 with expected crc 2023-09-23T22:20:06.027 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:05 smithi070 ceph-mon[192652]: osdmap e195: 8 total, 8 up, 7 in 2023-09-23T22:20:06.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:06.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:06.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[215775]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[215775]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[215775]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[215775]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[215775]: failed to encode map e195 with expected crc 2023-09-23T22:20:06.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[215775]: osdmap e195: 8 total, 8 up, 7 in 2023-09-23T22:20:06.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[223014]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[223014]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[223014]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[223014]: failed to encode map e194 with expected crc 2023-09-23T22:20:06.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[223014]: failed to encode map e195 with expected crc 2023-09-23T22:20:06.070 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:05 smithi026 ceph-mon[223014]: osdmap e195: 8 total, 8 up, 7 in 2023-09-23T22:20:06.129 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:06.157 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:06 smithi070 ceph-mon[192652]: pgmap v451: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 8094/84 objects degraded (9635.714%); 0 B/s, 418 objects/s recovering 2023-09-23T22:20:07.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:06 smithi070 ceph-mon[192652]: failed to encode map e195 with expected crc 2023-09-23T22:20:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:06 smithi070 ceph-mon[192652]: failed to encode map e195 with expected crc 2023-09-23T22:20:07.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:06 smithi026 ceph-mon[223014]: pgmap v451: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 8094/84 objects degraded (9635.714%); 0 B/s, 418 objects/s recovering 2023-09-23T22:20:07.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:06 smithi026 ceph-mon[223014]: failed to encode map e195 with expected crc 2023-09-23T22:20:07.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:06 smithi026 ceph-mon[223014]: failed to encode map e195 with expected crc 2023-09-23T22:20:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:06 smithi026 ceph-mon[215775]: pgmap v451: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 93 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 8094/84 objects degraded (9635.714%); 0 B/s, 418 objects/s recovering 2023-09-23T22:20:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:06 smithi026 ceph-mon[215775]: failed to encode map e195 with expected crc 2023-09-23T22:20:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:06 smithi026 ceph-mon[215775]: failed to encode map e195 with expected crc 2023-09-23T22:20:07.328 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:07.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:07.433 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:07.464 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:07.508 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:07.513 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:07.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:07.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:07.620 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:07.661 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:07.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:07.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:07.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:07.849 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:07.877 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:07.920 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:07.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:07.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:07.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:07 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 8094/84 objects degraded (9635.714%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:20:07.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:07 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 8094/84 objects degraded (9635.714%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:20:08.009 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:20:08.009 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:20:08.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:08.059 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:07 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 8094/84 objects degraded (9635.714%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:20:08.772 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:20:08.772 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:20:08.772 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:20:08.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:20:08.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:20:08.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:20:08.775 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:20:08.797 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:20:08.798 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:20:08.798 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:20:08.798 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:20:08.798 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:20:08.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:20:08.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:20:08.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:20:08.799 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:20:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:08 smithi026 ceph-mon[215775]: pgmap v453: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 6796/84 objects degraded (8090.476%); 0 B/s, 480 objects/s recovering 2023-09-23T22:20:08.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:08 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3232152393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:08.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:08 smithi026 ceph-mon[223014]: pgmap v453: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 6796/84 objects degraded (8090.476%); 0 B/s, 480 objects/s recovering 2023-09-23T22:20:08.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:08 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3232152393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:08 smithi070 ceph-mon[192652]: pgmap v453: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 6796/84 objects degraded (8090.476%); 0 B/s, 480 objects/s recovering 2023-09-23T22:20:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:08 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3232152393' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:09.345 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:20:09.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:20:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:20:09] "GET /metrics HTTP/1.1" 200 36369 "" "Prometheus/2.43.0" 2023-09-23T22:20:10.045 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_0' removed 2023-09-23T22:20:10.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:10.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:10.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[223014]: pgmap v454: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 6796/84 objects degraded (8090.476%); 0 B/s, 480 objects/s recovering 2023-09-23T22:20:10.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1088155667' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:20:10.165 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:20:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[215775]: pgmap v454: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 6796/84 objects degraded (8090.476%); 0 B/s, 480 objects/s recovering 2023-09-23T22:20:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1088155667' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:20:10.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:20:10.167 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:20:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:10 smithi070 ceph-mon[192652]: pgmap v454: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 94 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 6796/84 objects degraded (8090.476%); 0 B/s, 480 objects/s recovering 2023-09-23T22:20:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1088155667' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:20:10.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:20:10.775 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:10.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:10.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:10.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:10.907 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:11.104 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:11.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:11.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:11.209 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:11.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:20:11.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:11 smithi026 ceph-mon[215775]: osdmap e196: 8 total, 8 up, 7 in 2023-09-23T22:20:11.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:11 smithi026 ceph-mon[215775]: failed to encode map e196 with expected crc 2023-09-23T22:20:11.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:20:11.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:11 smithi026 ceph-mon[223014]: osdmap e196: 8 total, 8 up, 7 in 2023-09-23T22:20:11.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:11 smithi026 ceph-mon[223014]: failed to encode map e196 with expected crc 2023-09-23T22:20:11.246 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:11.288 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:11.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:20:11.294 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:11 smithi070 ceph-mon[192652]: osdmap e196: 8 total, 8 up, 7 in 2023-09-23T22:20:11.294 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:11 smithi070 ceph-mon[192652]: failed to encode map e196 with expected crc 2023-09-23T22:20:11.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:11.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:11.395 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:11.421 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:11.461 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:11.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:11.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:11.518 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:20:11.519 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:20:11.519 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T22:20:11.519 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T22:20:11.568 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:11.594 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:11.618 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:11.635 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:11.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:11.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:11.650 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:11.741 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:11.768 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:11.810 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:11.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:11.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:11.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:11.943 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:11.985 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:11.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:11.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:12.091 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:12.122 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:12.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[215775]: 1.4 scrub starts 2023-09-23T22:20:12.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[215775]: failed to encode map e196 with expected crc 2023-09-23T22:20:12.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[215775]: failed to encode map e196 with expected crc 2023-09-23T22:20:12.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[215775]: failed to encode map e197 with expected crc 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[215775]: failed to encode map e197 with expected crc 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[215775]: osdmap e197: 8 total, 8 up, 7 in 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[215775]: failed to encode map e197 with expected crc 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[215775]: pgmap v457: 83 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 81 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[223014]: 1.4 scrub starts 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[223014]: failed to encode map e196 with expected crc 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[223014]: failed to encode map e196 with expected crc 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[223014]: failed to encode map e197 with expected crc 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[223014]: failed to encode map e197 with expected crc 2023-09-23T22:20:12.169 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[223014]: osdmap e197: 8 total, 8 up, 7 in 2023-09-23T22:20:12.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[223014]: failed to encode map e197 with expected crc 2023-09-23T22:20:12.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:12 smithi026 ceph-mon[223014]: pgmap v457: 83 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 81 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:20:12.229 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:12.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:12.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:12.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:20:12.333 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:12.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:12 smithi070 ceph-mon[192652]: 1.4 scrub starts 2023-09-23T22:20:12.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:12 smithi070 ceph-mon[192652]: failed to encode map e196 with expected crc 2023-09-23T22:20:12.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:12 smithi070 ceph-mon[192652]: failed to encode map e196 with expected crc 2023-09-23T22:20:12.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:12 smithi070 ceph-mon[192652]: failed to encode map e197 with expected crc 2023-09-23T22:20:12.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:12 smithi070 ceph-mon[192652]: failed to encode map e197 with expected crc 2023-09-23T22:20:12.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:12 smithi070 ceph-mon[192652]: osdmap e197: 8 total, 8 up, 7 in 2023-09-23T22:20:12.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:12 smithi070 ceph-mon[192652]: failed to encode map e197 with expected crc 2023-09-23T22:20:12.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:12 smithi070 ceph-mon[192652]: pgmap v457: 83 pgs: 1 active+remapped+backfill_wait, 1 active+recovery_wait+degraded, 81 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T22:20:12.360 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:12.402 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:12.410 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:20:12.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:12.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:12.413 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:20:12.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:12.545 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:13.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:13 smithi070 ceph-mon[192652]: failed to encode map e197 with expected crc 2023-09-23T22:20:13.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:13 smithi070 ceph-mon[192652]: failed to encode map e198 with expected crc 2023-09-23T22:20:13.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:13 smithi070 ceph-mon[192652]: failed to encode map e198 with expected crc 2023-09-23T22:20:13.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:13 smithi070 ceph-mon[192652]: osdmap e198: 8 total, 8 up, 7 in 2023-09-23T22:20:13.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:13 smithi070 ceph-mon[192652]: failed to encode map e198 with expected crc 2023-09-23T22:20:13.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:13 smithi070 ceph-mon[192652]: from='client.44945 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:13.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:20:13.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[223014]: failed to encode map e197 with expected crc 2023-09-23T22:20:13.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[223014]: failed to encode map e198 with expected crc 2023-09-23T22:20:13.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[223014]: failed to encode map e198 with expected crc 2023-09-23T22:20:13.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[223014]: osdmap e198: 8 total, 8 up, 7 in 2023-09-23T22:20:13.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[223014]: failed to encode map e198 with expected crc 2023-09-23T22:20:13.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[223014]: from='client.44945 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:13.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:20:13.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[215775]: failed to encode map e197 with expected crc 2023-09-23T22:20:13.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[215775]: failed to encode map e198 with expected crc 2023-09-23T22:20:13.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[215775]: failed to encode map e198 with expected crc 2023-09-23T22:20:13.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[215775]: osdmap e198: 8 total, 8 up, 7 in 2023-09-23T22:20:13.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[215775]: failed to encode map e198 with expected crc 2023-09-23T22:20:13.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[215775]: from='client.44945 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:13.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:20:13.637 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":458,"stamp":"2023-09-23T22:20:12.035617+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6468,"num_read_kb":13096,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":147,"num_bytes_recovered":10755010,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2802,"ondisk_log_size":2802,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":294,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1454232,"kb_used_data":505824,"kb_used_omap":31,"kb_used_meta":948320,"kb_avail":654761832,"statfs":{"total":671965249536,"available":670476115968,"internally_reserved":0,"allocated":517963776,"data_stored":511197645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":971080631},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":-8094,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-187612,"num_write_kb":-6003521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-36544,"num_bytes_recovered":-2020998624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.823571"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":172,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.222150+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:20:04.222150+0000","last_peered":"2023-09-23T22:20:04.222150+0000","last_clean":"2023-09-23T22:20:04.222150+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:20:04.222150+0000","last_undegraded":"2023-09-23T22:20:04.222150+0000","last_fullsized":"2023-09-23T22:20:04.222150+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":174,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.213564+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:20:04.213564+0000","last_peered":"2023-09-23T22:20:04.213564+0000","last_clean":"2023-09-23T22:20:04.213564+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:20:04.213564+0000","last_undegraded":"2023-09-23T22:20:04.213564+0000","last_fullsized":"2023-09-23T22:20:04.213564+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":135,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-09-23T22:20:05.219905+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:20:05.219905+0000","last_peered":"2023-09-23T22:20:05.219905+0000","last_clean":"2023-09-23T22:20:05.219905+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:20:05.219905+0000","last_undegraded":"2023-09-23T22:20:05.219905+0000","last_fullsized":"2023-09-23T22:20:05.219905+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":25,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031880+0000","last_change":"2023-09-23T22:18:57.724501+0000","last_active":"2023-09-23T22:20:10.031880+0000","last_peered":"2023-09-23T22:20:10.031880+0000","last_clean":"2023-09-23T22:20:10.031880+0000","last_became_active":"2023-09-23T22:18:57.724203+0000","last_became_peered":"2023-09-23T22:18:57.724203+0000","last_unstale":"2023-09-23T22:20:10.031880+0000","last_undegraded":"2023-09-23T22:20:10.031880+0000","last_fullsized":"2023-09-23T22:20:10.031880+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:52.168267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":214,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.214552+0000","last_change":"2023-09-23T22:18:57.722574+0000","last_active":"2023-09-23T22:20:04.214552+0000","last_peered":"2023-09-23T22:20:04.214552+0000","last_clean":"2023-09-23T22:20:04.214552+0000","last_became_active":"2023-09-23T22:18:57.721800+0000","last_became_peered":"2023-09-23T22:18:57.721800+0000","last_unstale":"2023-09-23T22:20:04.214552+0000","last_undegraded":"2023-09-23T22:20:04.214552+0000","last_fullsized":"2023-09-23T22:20:04.214552+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:09.303838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":132,"reported_epoch":194,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:20:04.606871+0000","last_change":"2023-09-23T22:18:58.730170+0000","last_active":"2023-09-23T22:20:04.606871+0000","last_peered":"2023-09-23T22:20:04.606871+0000","last_clean":"2023-09-23T22:18:49.795932+0000","last_became_active":"2023-09-23T22:18:58.729860+0000","last_became_peered":"2023-09-23T22:18:58.729860+0000","last_unstale":"2023-09-23T22:20:04.606871+0000","last_undegraded":"2023-09-23T22:20:04.606871+0000","last_fullsized":"2023-09-23T22:20:04.606871+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":152,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:30.948774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,3],"avail_no_missing":["5","0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":63,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031589+0000","last_change":"2023-09-23T22:18:57.729038+0000","last_active":"2023-09-23T22:20:10.031589+0000","last_peered":"2023-09-23T22:20:10.031589+0000","last_clean":"2023-09-23T22:20:10.031589+0000","last_became_active":"2023-09-23T22:18:57.728650+0000","last_became_peered":"2023-09-23T22:18:57.728650+0000","last_unstale":"2023-09-23T22:20:10.031589+0000","last_undegraded":"2023-09-23T22:20:10.031589+0000","last_fullsized":"2023-09-23T22:20:10.031589+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:11.461883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":169,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.214616+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:20:04.214616+0000","last_peered":"2023-09-23T22:20:04.214616+0000","last_clean":"2023-09-23T22:20:04.214616+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:20:04.214616+0000","last_undegraded":"2023-09-23T22:20:04.214616+0000","last_fullsized":"2023-09-23T22:20:04.214616+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":155,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606720+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:20:04.606720+0000","last_peered":"2023-09-23T22:20:04.606720+0000","last_clean":"2023-09-23T22:20:04.606720+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:20:04.606720+0000","last_undegraded":"2023-09-23T22:20:04.606720+0000","last_fullsized":"2023-09-23T22:20:04.606720+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":119,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.214200+0000","last_change":"2023-09-23T22:16:49.154341+0000","last_active":"2023-09-23T22:20:04.214200+0000","last_peered":"2023-09-23T22:20:04.214200+0000","last_clean":"2023-09-23T22:20:04.214200+0000","last_became_active":"2023-09-23T22:16:49.154041+0000","last_became_peered":"2023-09-23T22:16:49.154041+0000","last_unstale":"2023-09-23T22:20:04.214200+0000","last_undegraded":"2023-09-23T22:20:04.214200+0000","last_fullsized":"2023-09-23T22:20:04.214200+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:05.733262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":107,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-09-23T22:20:05.215900+0000","last_change":"2023-09-23T22:15:14.829731+0000","last_active":"2023-09-23T22:20:05.215900+0000","last_peered":"2023-09-23T22:20:05.215900+0000","last_clean":"2023-09-23T22:20:05.215900+0000","last_became_active":"2023-09-23T22:15:14.829437+0000","last_became_peered":"2023-09-23T22:15:14.829437+0000","last_unstale":"2023-09-23T22:20:05.215900+0000","last_undegraded":"2023-09-23T22:20:05.215900+0000","last_fullsized":"2023-09-23T22:20:05.215900+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:45.456558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":162,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.211974+0000","last_change":"2023-09-23T22:18:57.723165+0000","last_active":"2023-09-23T22:20:04.211974+0000","last_peered":"2023-09-23T22:20:04.211974+0000","last_clean":"2023-09-23T22:20:04.211974+0000","last_became_active":"2023-09-23T22:18:57.722863+0000","last_became_peered":"2023-09-23T22:18:57.722863+0000","last_unstale":"2023-09-23T22:20:04.211974+0000","last_undegraded":"2023-09-23T22:20:04.211974+0000","last_fullsized":"2023-09-23T22:20:04.211974+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:19.645029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":179,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221729+0000","last_change":"2023-09-23T22:16:49.265991+0000","last_active":"2023-09-23T22:20:04.221729+0000","last_peered":"2023-09-23T22:20:04.221729+0000","last_clean":"2023-09-23T22:20:04.221729+0000","last_became_active":"2023-09-23T22:16:49.251169+0000","last_became_peered":"2023-09-23T22:16:49.251169+0000","last_unstale":"2023-09-23T22:20:04.221729+0000","last_undegraded":"2023-09-23T22:20:04.221729+0000","last_fullsized":"2023-09-23T22:20:04.221729+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.502218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":163,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221427+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:20:04.221427+0000","last_peered":"2023-09-23T22:20:04.221427+0000","last_clean":"2023-09-23T22:20:04.221427+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:20:04.221427+0000","last_undegraded":"2023-09-23T22:20:04.221427+0000","last_fullsized":"2023-09-23T22:20:04.221427+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":106,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221512+0000","last_change":"2023-09-23T22:15:14.826143+0000","last_active":"2023-09-23T22:20:04.221512+0000","last_peered":"2023-09-23T22:20:04.221512+0000","last_clean":"2023-09-23T22:20:04.221512+0000","last_became_active":"2023-09-23T22:15:14.825780+0000","last_became_peered":"2023-09-23T22:15:14.825780+0000","last_unstale":"2023-09-23T22:20:04.221512+0000","last_undegraded":"2023-09-23T22:20:04.221512+0000","last_fullsized":"2023-09-23T22:20:04.221512+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:22.608676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":160,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.213253+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:20:04.213253+0000","last_peered":"2023-09-23T22:20:04.213253+0000","last_clean":"2023-09-23T22:20:04.213253+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:20:04.213253+0000","last_undegraded":"2023-09-23T22:20:04.213253+0000","last_fullsized":"2023-09-23T22:20:04.213253+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":25,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.035230+0000","last_change":"2023-09-23T22:18:57.726112+0000","last_active":"2023-09-23T22:20:10.035230+0000","last_peered":"2023-09-23T22:20:10.035230+0000","last_clean":"2023-09-23T22:20:10.035230+0000","last_became_active":"2023-09-23T22:18:57.725558+0000","last_became_peered":"2023-09-23T22:18:57.725558+0000","last_unstale":"2023-09-23T22:20:10.035230+0000","last_undegraded":"2023-09-23T22:20:10.035230+0000","last_fullsized":"2023-09-23T22:20:10.035230+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:38.315664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":154,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606704+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:20:04.606704+0000","last_peered":"2023-09-23T22:20:04.606704+0000","last_clean":"2023-09-23T22:20:04.606704+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:20:04.606704+0000","last_undegraded":"2023-09-23T22:20:04.606704+0000","last_fullsized":"2023-09-23T22:20:04.606704+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":118,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221745+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:20:04.221745+0000","last_peered":"2023-09-23T22:20:04.221745+0000","last_clean":"2023-09-23T22:20:04.221745+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:20:04.221745+0000","last_undegraded":"2023-09-23T22:20:04.221745+0000","last_fullsized":"2023-09-23T22:20:04.221745+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":25,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.032106+0000","last_change":"2023-09-23T22:18:57.726691+0000","last_active":"2023-09-23T22:20:10.032106+0000","last_peered":"2023-09-23T22:20:10.032106+0000","last_clean":"2023-09-23T22:20:10.032106+0000","last_became_active":"2023-09-23T22:18:57.726386+0000","last_became_peered":"2023-09-23T22:18:57.726386+0000","last_unstale":"2023-09-23T22:20:10.032106+0000","last_undegraded":"2023-09-23T22:20:10.032106+0000","last_fullsized":"2023-09-23T22:20:10.032106+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:34.829149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":166,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.029834+0000","last_change":"2023-09-23T22:18:57.728767+0000","last_active":"2023-09-23T22:20:10.029834+0000","last_peered":"2023-09-23T22:20:10.029834+0000","last_clean":"2023-09-23T22:20:10.029834+0000","last_became_active":"2023-09-23T22:18:57.728377+0000","last_became_peered":"2023-09-23T22:18:57.728377+0000","last_unstale":"2023-09-23T22:20:10.029834+0000","last_undegraded":"2023-09-23T22:20:10.029834+0000","last_fullsized":"2023-09-23T22:20:10.029834+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:01:01.370904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":160,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.213852+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:20:04.213852+0000","last_peered":"2023-09-23T22:20:04.213852+0000","last_clean":"2023-09-23T22:20:04.213852+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:20:04.213852+0000","last_undegraded":"2023-09-23T22:20:04.213852+0000","last_fullsized":"2023-09-23T22:20:04.213852+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":162,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221728+0000","last_change":"2023-09-23T22:19:00.682889+0000","last_active":"2023-09-23T22:20:04.221728+0000","last_peered":"2023-09-23T22:20:04.221728+0000","last_clean":"2023-09-23T22:20:04.221728+0000","last_became_active":"2023-09-23T22:18:57.718002+0000","last_became_peered":"2023-09-23T22:18:57.718002+0000","last_unstale":"2023-09-23T22:20:04.221728+0000","last_undegraded":"2023-09-23T22:20:04.221728+0000","last_fullsized":"2023-09-23T22:20:04.221728+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:22.517277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":147,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606256+0000","last_change":"2023-09-23T22:16:49.158580+0000","last_active":"2023-09-23T22:20:04.606256+0000","last_peered":"2023-09-23T22:20:04.606256+0000","last_clean":"2023-09-23T22:20:04.606256+0000","last_became_active":"2023-09-23T22:16:49.158266+0000","last_became_peered":"2023-09-23T22:16:49.158266+0000","last_unstale":"2023-09-23T22:20:04.606256+0000","last_undegraded":"2023-09-23T22:20:04.606256+0000","last_fullsized":"2023-09-23T22:20:04.606256+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:03.791658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":106,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.222237+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:20:04.222237+0000","last_peered":"2023-09-23T22:20:04.222237+0000","last_clean":"2023-09-23T22:20:04.222237+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:20:04.222237+0000","last_undegraded":"2023-09-23T22:20:04.222237+0000","last_fullsized":"2023-09-23T22:20:04.222237+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":162,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221655+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:20:04.221655+0000","last_peered":"2023-09-23T22:20:04.221655+0000","last_clean":"2023-09-23T22:20:04.221655+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:20:04.221655+0000","last_undegraded":"2023-09-23T22:20:04.221655+0000","last_fullsized":"2023-09-23T22:20:04.221655+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":120,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606554+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:20:04.606554+0000","last_peered":"2023-09-23T22:20:04.606554+0000","last_clean":"2023-09-23T22:20:04.606554+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:20:04.606554+0000","last_undegraded":"2023-09-23T22:20:04.606554+0000","last_fullsized":"2023-09-23T22:20:04.606554+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":110,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.212061+0000","last_change":"2023-09-23T22:16:49.161676+0000","last_active":"2023-09-23T22:20:04.212061+0000","last_peered":"2023-09-23T22:20:04.212061+0000","last_clean":"2023-09-23T22:20:04.212061+0000","last_became_active":"2023-09-23T22:16:49.161247+0000","last_became_peered":"2023-09-23T22:16:49.161247+0000","last_unstale":"2023-09-23T22:20:04.212061+0000","last_undegraded":"2023-09-23T22:20:04.212061+0000","last_fullsized":"2023-09-23T22:20:04.212061+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:30.708371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":165,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606587+0000","last_change":"2023-09-23T22:16:49.164833+0000","last_active":"2023-09-23T22:20:04.606587+0000","last_peered":"2023-09-23T22:20:04.606587+0000","last_clean":"2023-09-23T22:20:04.606587+0000","last_became_active":"2023-09-23T22:16:49.163833+0000","last_became_peered":"2023-09-23T22:16:49.163833+0000","last_unstale":"2023-09-23T22:20:04.606587+0000","last_undegraded":"2023-09-23T22:20:04.606587+0000","last_fullsized":"2023-09-23T22:20:04.606587+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.891395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":50,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031962+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:20:10.031962+0000","last_peered":"2023-09-23T22:20:10.031962+0000","last_clean":"2023-09-23T22:20:10.031962+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:20:10.031962+0000","last_undegraded":"2023-09-23T22:20:10.031962+0000","last_fullsized":"2023-09-23T22:20:10.031962+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":461,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.028994+0000","last_change":"2023-09-23T22:18:57.721431+0000","last_active":"2023-09-23T22:20:10.028994+0000","last_peered":"2023-09-23T22:20:10.028994+0000","last_clean":"2023-09-23T22:20:10.028994+0000","last_became_active":"2023-09-23T22:18:57.721043+0000","last_became_peered":"2023-09-23T22:18:57.721043+0000","last_unstale":"2023-09-23T22:20:10.028994+0000","last_undegraded":"2023-09-23T22:20:10.028994+0000","last_fullsized":"2023-09-23T22:20:10.028994+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":206,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.214409+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:20:04.214409+0000","last_peered":"2023-09-23T22:20:04.214409+0000","last_clean":"2023-09-23T22:20:04.214409+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:20:04.214409+0000","last_undegraded":"2023-09-23T22:20:04.214409+0000","last_fullsized":"2023-09-23T22:20:04.214409+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221906+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:20:04.221906+0000","last_peered":"2023-09-23T22:20:04.221906+0000","last_clean":"2023-09-23T22:20:04.221906+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:20:04.221906+0000","last_undegraded":"2023-09-23T22:20:04.221906+0000","last_fullsized":"2023-09-23T22:20:04.221906+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":465,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.029098+0000","last_change":"2023-09-23T22:20:09.831700+0000","last_active":"2023-09-23T22:20:10.029098+0000","last_peered":"2023-09-23T22:20:10.029098+0000","last_clean":"2023-09-23T22:20:10.029098+0000","last_became_active":"2023-09-23T22:18:57.742130+0000","last_became_peered":"2023-09-23T22:18:57.742130+0000","last_unstale":"2023-09-23T22:20:10.029098+0000","last_undegraded":"2023-09-23T22:20:10.029098+0000","last_fullsized":"2023-09-23T22:20:10.029098+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":195,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":111,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.222321+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:20:04.222321+0000","last_peered":"2023-09-23T22:20:04.222321+0000","last_clean":"2023-09-23T22:20:04.222321+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:20:04.222321+0000","last_undegraded":"2023-09-23T22:20:04.222321+0000","last_fullsized":"2023-09-23T22:20:04.222321+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":170,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221830+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:20:04.221830+0000","last_peered":"2023-09-23T22:20:04.221830+0000","last_clean":"2023-09-23T22:20:04.221830+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:20:04.221830+0000","last_undegraded":"2023-09-23T22:20:04.221830+0000","last_fullsized":"2023-09-23T22:20:04.221830+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":146,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221337+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:20:04.221337+0000","last_peered":"2023-09-23T22:20:04.221337+0000","last_clean":"2023-09-23T22:20:04.221337+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:20:04.221337+0000","last_undegraded":"2023-09-23T22:20:04.221337+0000","last_fullsized":"2023-09-23T22:20:04.221337+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":163,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.222080+0000","last_change":"2023-09-23T22:16:49.165118+0000","last_active":"2023-09-23T22:20:04.222080+0000","last_peered":"2023-09-23T22:20:04.222080+0000","last_clean":"2023-09-23T22:20:04.222080+0000","last_became_active":"2023-09-23T22:16:49.164802+0000","last_became_peered":"2023-09-23T22:16:49.164802+0000","last_unstale":"2023-09-23T22:20:04.222080+0000","last_undegraded":"2023-09-23T22:20:04.222080+0000","last_fullsized":"2023-09-23T22:20:04.222080+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:23.079142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":461,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.035302+0000","last_change":"2023-09-23T22:18:57.726339+0000","last_active":"2023-09-23T22:20:10.035302+0000","last_peered":"2023-09-23T22:20:10.035302+0000","last_clean":"2023-09-23T22:20:10.035302+0000","last_became_active":"2023-09-23T22:18:57.725773+0000","last_became_peered":"2023-09-23T22:18:57.725773+0000","last_unstale":"2023-09-23T22:20:10.035302+0000","last_undegraded":"2023-09-23T22:20:10.035302+0000","last_fullsized":"2023-09-23T22:20:10.035302+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":161,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.214418+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:20:04.214418+0000","last_peered":"2023-09-23T22:20:04.214418+0000","last_clean":"2023-09-23T22:20:04.214418+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:20:04.214418+0000","last_undegraded":"2023-09-23T22:20:04.214418+0000","last_fullsized":"2023-09-23T22:20:04.214418+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":135,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.213015+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:20:04.213015+0000","last_peered":"2023-09-23T22:20:04.213015+0000","last_clean":"2023-09-23T22:20:04.213015+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:20:04.213015+0000","last_undegraded":"2023-09-23T22:20:04.213015+0000","last_fullsized":"2023-09-23T22:20:04.213015+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":461,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031048+0000","last_change":"2023-09-23T22:18:57.725616+0000","last_active":"2023-09-23T22:20:10.031048+0000","last_peered":"2023-09-23T22:20:10.031048+0000","last_clean":"2023-09-23T22:20:10.031048+0000","last_became_active":"2023-09-23T22:18:57.725224+0000","last_became_peered":"2023-09-23T22:18:57.725224+0000","last_unstale":"2023-09-23T22:20:10.031048+0000","last_undegraded":"2023-09-23T22:20:10.031048+0000","last_fullsized":"2023-09-23T22:20:10.031048+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":742,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.213728+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:20:04.213728+0000","last_peered":"2023-09-23T22:20:04.213728+0000","last_clean":"2023-09-23T22:20:04.213728+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:20:04.213728+0000","last_undegraded":"2023-09-23T22:20:04.213728+0000","last_fullsized":"2023-09-23T22:20:04.213728+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":172,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606767+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:20:04.606767+0000","last_peered":"2023-09-23T22:20:04.606767+0000","last_clean":"2023-09-23T22:20:04.606767+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:20:04.606767+0000","last_undegraded":"2023-09-23T22:20:04.606767+0000","last_fullsized":"2023-09-23T22:20:04.606767+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-09-23T22:20:05.224836+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:20:05.224836+0000","last_peered":"2023-09-23T22:20:05.224836+0000","last_clean":"2023-09-23T22:20:05.224836+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:20:05.224836+0000","last_undegraded":"2023-09-23T22:20:05.224836+0000","last_fullsized":"2023-09-23T22:20:05.224836+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":461,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.030281+0000","last_change":"2023-09-23T22:18:57.742555+0000","last_active":"2023-09-23T22:20:10.030281+0000","last_peered":"2023-09-23T22:20:10.030281+0000","last_clean":"2023-09-23T22:20:10.030281+0000","last_became_active":"2023-09-23T22:18:57.742205+0000","last_became_peered":"2023-09-23T22:18:57.742205+0000","last_unstale":"2023-09-23T22:20:10.030281+0000","last_undegraded":"2023-09-23T22:20:10.030281+0000","last_fullsized":"2023-09-23T22:20:10.030281+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":927,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606335+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:20:04.606335+0000","last_peered":"2023-09-23T22:20:04.606335+0000","last_clean":"2023-09-23T22:20:04.606335+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:20:04.606335+0000","last_undegraded":"2023-09-23T22:20:04.606335+0000","last_fullsized":"2023-09-23T22:20:04.606335+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":204,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221338+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:20:04.221338+0000","last_peered":"2023-09-23T22:20:04.221338+0000","last_clean":"2023-09-23T22:20:04.221338+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:20:04.221338+0000","last_undegraded":"2023-09-23T22:20:04.221338+0000","last_fullsized":"2023-09-23T22:20:04.221338+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":169,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606338+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:20:04.606338+0000","last_peered":"2023-09-23T22:20:04.606338+0000","last_clean":"2023-09-23T22:20:04.606338+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:20:04.606338+0000","last_undegraded":"2023-09-23T22:20:04.606338+0000","last_fullsized":"2023-09-23T22:20:04.606338+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":461,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.035096+0000","last_change":"2023-09-23T22:18:57.725923+0000","last_active":"2023-09-23T22:20:10.035096+0000","last_peered":"2023-09-23T22:20:10.035096+0000","last_clean":"2023-09-23T22:20:10.035096+0000","last_became_active":"2023-09-23T22:18:57.725260+0000","last_became_peered":"2023-09-23T22:18:57.725260+0000","last_unstale":"2023-09-23T22:20:10.035096+0000","last_undegraded":"2023-09-23T22:20:10.035096+0000","last_fullsized":"2023-09-23T22:20:10.035096+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":538,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-09-23T22:20:05.223291+0000","last_change":"2023-09-23T22:15:51.379227+0000","last_active":"2023-09-23T22:20:05.223291+0000","last_peered":"2023-09-23T22:20:05.223291+0000","last_clean":"2023-09-23T22:20:05.223291+0000","last_became_active":"2023-09-23T22:15:43.928977+0000","last_became_peered":"2023-09-23T22:15:43.928977+0000","last_unstale":"2023-09-23T22:20:05.223291+0000","last_undegraded":"2023-09-23T22:20:05.223291+0000","last_fullsized":"2023-09-23T22:20:05.223291+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:04.398533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":144,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.212116+0000","last_change":"2023-09-23T22:16:49.251412+0000","last_active":"2023-09-23T22:20:04.212116+0000","last_peered":"2023-09-23T22:20:04.212116+0000","last_clean":"2023-09-23T22:20:04.212116+0000","last_became_active":"2023-09-23T22:16:49.251097+0000","last_became_peered":"2023-09-23T22:16:49.251097+0000","last_unstale":"2023-09-23T22:20:04.212116+0000","last_undegraded":"2023-09-23T22:20:04.212116+0000","last_fullsized":"2023-09-23T22:20:04.212116+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:41.601435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":200,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.213089+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:20:04.213089+0000","last_peered":"2023-09-23T22:20:04.213089+0000","last_clean":"2023-09-23T22:20:04.213089+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:20:04.213089+0000","last_undegraded":"2023-09-23T22:20:04.213089+0000","last_fullsized":"2023-09-23T22:20:04.213089+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":780,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606383+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:20:04.606383+0000","last_peered":"2023-09-23T22:20:04.606383+0000","last_clean":"2023-09-23T22:20:04.606383+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:20:04.606383+0000","last_undegraded":"2023-09-23T22:20:04.606383+0000","last_fullsized":"2023-09-23T22:20:04.606383+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":461,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.029685+0000","last_change":"2023-09-23T22:18:57.743602+0000","last_active":"2023-09-23T22:20:10.029685+0000","last_peered":"2023-09-23T22:20:10.029685+0000","last_clean":"2023-09-23T22:20:10.029685+0000","last_became_active":"2023-09-23T22:18:57.743004+0000","last_became_peered":"2023-09-23T22:18:57.743004+0000","last_unstale":"2023-09-23T22:20:10.029685+0000","last_undegraded":"2023-09-23T22:20:10.029685+0000","last_fullsized":"2023-09-23T22:20:10.029685+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.213818+0000","last_change":"2023-09-23T22:16:49.155156+0000","last_active":"2023-09-23T22:20:04.213818+0000","last_peered":"2023-09-23T22:20:04.213818+0000","last_clean":"2023-09-23T22:20:04.213818+0000","last_became_active":"2023-09-23T22:16:49.154923+0000","last_became_peered":"2023-09-23T22:16:49.154923+0000","last_unstale":"2023-09-23T22:20:04.213818+0000","last_undegraded":"2023-09-23T22:20:04.213818+0000","last_fullsized":"2023-09-23T22:20:04.213818+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:33.532238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.222094+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:20:04.222094+0000","last_peered":"2023-09-23T22:20:04.222094+0000","last_clean":"2023-09-23T22:20:04.222094+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:20:04.222094+0000","last_undegraded":"2023-09-23T22:20:04.222094+0000","last_fullsized":"2023-09-23T22:20:04.222094+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":461,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031737+0000","last_change":"2023-09-23T22:18:57.720920+0000","last_active":"2023-09-23T22:20:10.031737+0000","last_peered":"2023-09-23T22:20:10.031737+0000","last_clean":"2023-09-23T22:20:10.031737+0000","last_became_active":"2023-09-23T22:18:57.720584+0000","last_became_peered":"2023-09-23T22:18:57.720584+0000","last_unstale":"2023-09-23T22:20:10.031737+0000","last_undegraded":"2023-09-23T22:20:10.031737+0000","last_fullsized":"2023-09-23T22:20:10.031737+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":543,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.212280+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:20:04.212280+0000","last_peered":"2023-09-23T22:20:04.212280+0000","last_clean":"2023-09-23T22:20:04.212280+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:20:04.212280+0000","last_undegraded":"2023-09-23T22:20:04.212280+0000","last_fullsized":"2023-09-23T22:20:04.212280+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":59,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.030131+0000","last_change":"2023-09-23T22:18:31.795915+0000","last_active":"2023-09-23T22:20:10.030131+0000","last_peered":"2023-09-23T22:20:10.030131+0000","last_clean":"2023-09-23T22:20:10.030131+0000","last_became_active":"2023-09-23T22:18:31.795608+0000","last_became_peered":"2023-09-23T22:18:31.795608+0000","last_unstale":"2023-09-23T22:20:10.030131+0000","last_undegraded":"2023-09-23T22:20:10.030131+0000","last_fullsized":"2023-09-23T22:20:10.030131+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:49.789927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":25,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031808+0000","last_change":"2023-09-23T22:18:57.721257+0000","last_active":"2023-09-23T22:20:10.031808+0000","last_peered":"2023-09-23T22:20:10.031808+0000","last_clean":"2023-09-23T22:20:10.031808+0000","last_became_active":"2023-09-23T22:18:57.720548+0000","last_became_peered":"2023-09-23T22:18:57.720548+0000","last_unstale":"2023-09-23T22:20:10.031808+0000","last_undegraded":"2023-09-23T22:20:10.031808+0000","last_fullsized":"2023-09-23T22:20:10.031808+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:10.709024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":461,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.032036+0000","last_change":"2023-09-23T22:18:57.727276+0000","last_active":"2023-09-23T22:20:10.032036+0000","last_peered":"2023-09-23T22:20:10.032036+0000","last_clean":"2023-09-23T22:20:10.032036+0000","last_became_active":"2023-09-23T22:18:57.726938+0000","last_became_peered":"2023-09-23T22:18:57.726938+0000","last_unstale":"2023-09-23T22:20:10.032036+0000","last_undegraded":"2023-09-23T22:20:10.032036+0000","last_fullsized":"2023-09-23T22:20:10.032036+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":86,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.030937+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:20:10.030937+0000","last_peered":"2023-09-23T22:20:10.030937+0000","last_clean":"2023-09-23T22:20:10.030937+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:20:10.030937+0000","last_undegraded":"2023-09-23T22:20:10.030937+0000","last_fullsized":"2023-09-23T22:20:10.030937+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":135,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.214067+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:20:04.214067+0000","last_peered":"2023-09-23T22:20:04.214067+0000","last_clean":"2023-09-23T22:20:04.214067+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:20:04.214067+0000","last_undegraded":"2023-09-23T22:20:04.214067+0000","last_fullsized":"2023-09-23T22:20:04.214067+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":50,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031923+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:20:10.031923+0000","last_peered":"2023-09-23T22:20:10.031923+0000","last_clean":"2023-09-23T22:20:10.031923+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:20:10.031923+0000","last_undegraded":"2023-09-23T22:20:10.031923+0000","last_fullsized":"2023-09-23T22:20:10.031923+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":461,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.030231+0000","last_change":"2023-09-23T22:18:57.742449+0000","last_active":"2023-09-23T22:20:10.030231+0000","last_peered":"2023-09-23T22:20:10.030231+0000","last_clean":"2023-09-23T22:20:10.030231+0000","last_became_active":"2023-09-23T22:18:57.742146+0000","last_became_peered":"2023-09-23T22:18:57.742146+0000","last_unstale":"2023-09-23T22:20:10.030231+0000","last_undegraded":"2023-09-23T22:20:10.030231+0000","last_fullsized":"2023-09-23T22:20:10.030231+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":206,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.213932+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:20:04.213932+0000","last_peered":"2023-09-23T22:20:04.213932+0000","last_clean":"2023-09-23T22:20:04.213932+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:20:04.213932+0000","last_undegraded":"2023-09-23T22:20:04.213932+0000","last_fullsized":"2023-09-23T22:20:04.213932+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":835,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.212486+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:20:04.212486+0000","last_peered":"2023-09-23T22:20:04.212486+0000","last_clean":"2023-09-23T22:20:04.212486+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:20:04.212486+0000","last_undegraded":"2023-09-23T22:20:04.212486+0000","last_fullsized":"2023-09-23T22:20:04.212486+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":192,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.214483+0000","last_change":"2023-09-23T22:18:57.722771+0000","last_active":"2023-09-23T22:20:04.214483+0000","last_peered":"2023-09-23T22:20:04.214483+0000","last_clean":"2023-09-23T22:20:04.214483+0000","last_became_active":"2023-09-23T22:18:57.722258+0000","last_became_peered":"2023-09-23T22:18:57.722258+0000","last_unstale":"2023-09-23T22:20:04.214483+0000","last_undegraded":"2023-09-23T22:20:04.214483+0000","last_fullsized":"2023-09-23T22:20:04.214483+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:18.267091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":466,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.478928+0000","last_change":"2023-09-23T22:20:11.037967+0000","last_active":"2023-09-23T22:20:10.478928+0000","last_peered":"2023-09-23T22:20:10.478928+0000","last_clean":"2023-09-23T22:20:10.478928+0000","last_became_active":"2023-09-23T22:18:57.720614+0000","last_became_peered":"2023-09-23T22:18:57.720614+0000","last_unstale":"2023-09-23T22:20:10.478928+0000","last_undegraded":"2023-09-23T22:20:10.478928+0000","last_fullsized":"2023-09-23T22:20:10.478928+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":260,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-09-23T22:20:05.224273+0000","last_change":"2023-09-23T22:16:49.270015+0000","last_active":"2023-09-23T22:20:05.224273+0000","last_peered":"2023-09-23T22:20:05.224273+0000","last_clean":"2023-09-23T22:20:05.224273+0000","last_became_active":"2023-09-23T22:16:49.252726+0000","last_became_peered":"2023-09-23T22:16:49.252726+0000","last_unstale":"2023-09-23T22:20:05.224273+0000","last_undegraded":"2023-09-23T22:20:05.224273+0000","last_fullsized":"2023-09-23T22:20:05.224273+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:18.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":29,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031960+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:20:10.031960+0000","last_peered":"2023-09-23T22:20:10.031960+0000","last_clean":"2023-09-23T22:20:10.031960+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:20:10.031960+0000","last_undegraded":"2023-09-23T22:20:10.031960+0000","last_fullsized":"2023-09-23T22:20:10.031960+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":158,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221447+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:20:04.221447+0000","last_peered":"2023-09-23T22:20:04.221447+0000","last_clean":"2023-09-23T22:20:04.221447+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:20:04.221447+0000","last_undegraded":"2023-09-23T22:20:04.221447+0000","last_fullsized":"2023-09-23T22:20:04.221447+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":25,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031877+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:20:10.031877+0000","last_peered":"2023-09-23T22:20:10.031877+0000","last_clean":"2023-09-23T22:20:10.031877+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:20:10.031877+0000","last_undegraded":"2023-09-23T22:20:10.031877+0000","last_fullsized":"2023-09-23T22:20:10.031877+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":180,"reported_epoch":195,"state":"active+clean","last_fresh":"2023-09-23T22:20:05.224249+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:20:05.224249+0000","last_peered":"2023-09-23T22:20:05.224249+0000","last_clean":"2023-09-23T22:20:05.224249+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:20:05.224249+0000","last_undegraded":"2023-09-23T22:20:05.224249+0000","last_fullsized":"2023-09-23T22:20:05.224249+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":157,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606706+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:20:04.606706+0000","last_peered":"2023-09-23T22:20:04.606706+0000","last_clean":"2023-09-23T22:20:04.606706+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:20:04.606706+0000","last_undegraded":"2023-09-23T22:20:04.606706+0000","last_fullsized":"2023-09-23T22:20:04.606706+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":170,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.222154+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:20:04.222154+0000","last_peered":"2023-09-23T22:20:04.222154+0000","last_clean":"2023-09-23T22:20:04.222154+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:20:04.222154+0000","last_undegraded":"2023-09-23T22:20:04.222154+0000","last_fullsized":"2023-09-23T22:20:04.222154+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":154,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.606656+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:20:04.606656+0000","last_peered":"2023-09-23T22:20:04.606656+0000","last_clean":"2023-09-23T22:20:04.606656+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:20:04.606656+0000","last_undegraded":"2023-09-23T22:20:04.606656+0000","last_fullsized":"2023-09-23T22:20:04.606656+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":29,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.222235+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:20:04.222235+0000","last_peered":"2023-09-23T22:20:04.222235+0000","last_clean":"2023-09-23T22:20:04.222235+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:20:04.222235+0000","last_undegraded":"2023-09-23T22:20:04.222235+0000","last_fullsized":"2023-09-23T22:20:04.222235+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":176,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.213713+0000","last_change":"2023-09-23T22:16:49.145417+0000","last_active":"2023-09-23T22:20:04.213713+0000","last_peered":"2023-09-23T22:20:04.213713+0000","last_clean":"2023-09-23T22:20:04.213713+0000","last_became_active":"2023-09-23T22:16:49.145017+0000","last_became_peered":"2023-09-23T22:16:49.145017+0000","last_unstale":"2023-09-23T22:20:04.213713+0000","last_undegraded":"2023-09-23T22:20:04.213713+0000","last_fullsized":"2023-09-23T22:20:04.213713+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:47.762149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":217,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.214073+0000","last_change":"2023-09-23T22:17:15.721123+0000","last_active":"2023-09-23T22:20:04.214073+0000","last_peered":"2023-09-23T22:20:04.214073+0000","last_clean":"2023-09-23T22:20:04.214073+0000","last_became_active":"2023-09-23T22:17:15.720812+0000","last_became_peered":"2023-09-23T22:17:15.720812+0000","last_unstale":"2023-09-23T22:20:04.214073+0000","last_undegraded":"2023-09-23T22:20:04.214073+0000","last_fullsized":"2023-09-23T22:20:04.214073+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.176278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":69,"reported_epoch":196,"state":"active+clean","last_fresh":"2023-09-23T22:20:10.031544+0000","last_change":"2023-09-23T22:18:57.729455+0000","last_active":"2023-09-23T22:20:10.031544+0000","last_peered":"2023-09-23T22:20:10.031544+0000","last_clean":"2023-09-23T22:20:10.031544+0000","last_became_active":"2023-09-23T22:18:57.729088+0000","last_became_peered":"2023-09-23T22:18:57.729088+0000","last_unstale":"2023-09-23T22:20:10.031544+0000","last_undegraded":"2023-09-23T22:20:10.031544+0000","last_fullsized":"2023-09-23T22:20:10.031544+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:17.861334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":163,"reported_epoch":194,"state":"active+clean","last_fresh":"2023-09-23T22:20:04.221046+0000","last_change":"2023-09-23T22:16:49.164638+0000","last_active":"2023-09-23T22:20:04.221046+0000","last_peered":"2023-09-23T22:20:04.221046+0000","last_clean":"2023-09-23T22:20:04.221046+0000","last_became_active":"2023-09-23T22:16:49.164349+0000","last_became_peered":"2023-09-23T22:16:49.164349+0000","last_unstale":"2023-09-23T22:20:04.221046+0000","last_undegraded":"2023-09-23T22:20:04.221046+0000","last_fullsized":"2023-09-23T22:20:04.221046+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:26.536927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4892,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234189,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":236876,"kb_used_data":152772,"kb_used_omap":0,"kb_used_meta":84096,"kb_avail":93508276,"statfs":{"total":95995035648,"available":95752474624,"internally_reserved":0,"allocated":156438528,"data_stored":155596850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":86114304},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":2.4780000000000002,"15min":1.1970000000000001},"min":{"1min":0.23999999999999999,"5min":0.223,"15min":0.223},"max":{"1min":9.7799999999999994,"5min":109.208,"15min":109.208},"last":0.497},{"interface":"front","average":{"1min":1.752,"5min":2.7650000000000001,"15min":1.3149999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":7.4489999999999998,"5min":109.13800000000001,"15min":109.13800000000001},"last":0.53500000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:20:09 2023","interfaces":[{"interface":"back","average":{"1min":9.1620000000000008,"5min":5.9320000000000004,"15min":2.3570000000000002},"min":{"1min":0.25600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":101.145,"5min":123.79300000000001,"15min":123.79300000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":9.4949999999999992,"5min":6.2809999999999997,"15min":2.508},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":101.13,"5min":123.66500000000001,"15min":123.66500000000001},"last":0.67300000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":13.066000000000001,"5min":8.5730000000000004,"15min":3.5289999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":105.34399999999999,"5min":125.101,"15min":125.101},"last":0.76300000000000001},{"interface":"front","average":{"1min":12.569000000000001,"5min":8.2129999999999992,"15min":3.1469999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":105.63200000000001,"5min":126.021,"15min":126.021},"last":0.80400000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:20:09 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":2.5619999999999998,"15min":1.2549999999999999},"min":{"1min":0.308,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8319999999999999,"5min":113.282,"15min":113.282},"last":0.879},{"interface":"front","average":{"1min":1.2370000000000001,"5min":2.931,"15min":1.4239999999999999},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.7910000000000004,"5min":104.74299999999999,"15min":104.74299999999999},"last":0.86699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":6.8159999999999998,"5min":2.5489999999999999,"15min":1.173},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":88.543000000000006,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.82799999999999996},{"interface":"front","average":{"1min":6.3959999999999999,"5min":2.286,"15min":1.095},"min":{"1min":0.312,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":88.495999999999995,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.73999999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":6.9299999999999997,"5min":2.2909999999999999,"15min":1.0860000000000001},"min":{"1min":0.27500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":93.816000000000003,"5min":93.816000000000003,"15min":93.816000000000003},"last":0.77700000000000002},{"interface":"front","average":{"1min":7.0010000000000003,"5min":2.8149999999999999,"15min":1.3129999999999999},"min":{"1min":0.34399999999999997,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":95.579999999999998,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.89800000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.97399999999999998,"15min":0.67700000000000005},"min":{"1min":0.317,"5min":0.317,"15min":0.315},"max":{"1min":0.66600000000000004,"5min":9.0139999999999993,"15min":9.0139999999999993},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.871,"5min":1.3939999999999999,"15min":0.86199999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":3.7869999999999999,"5min":13.362,"15min":13.362},"last":0.92600000000000005}]}]},{"osd":6,"up_from":0,"seq":197568495825,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795097,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":165056,"kb_used_data":7032,"kb_used_omap":7,"kb_used_meta":158008,"kb_avail":93580096,"statfs":{"total":95995035648,"available":95826018304,"internally_reserved":0,"allocated":7200768,"data_stored":6135516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":161800564},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":1.952,"5min":1.79,"15min":0.83599999999999997},"min":{"1min":0.20399999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":10.705,"5min":61.890000000000001,"15min":61.890000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":3.7280000000000002,"5min":2.4220000000000002,"15min":1.0780000000000001},"min":{"1min":0.216,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":15.08,"5min":62.689999999999998,"15min":62.689999999999998},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":2.4079999999999999,"15min":1.0549999999999999},"min":{"1min":0.223,"5min":0.216,"15min":0.216},"max":{"1min":5.2539999999999996,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":4.0620000000000003,"5min":3.032,"15min":1.2849999999999999},"min":{"1min":0.309,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.015000000000001,"5min":34.781999999999996,"15min":34.781999999999996},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 22:19:14 2023","interfaces":[{"interface":"back","average":{"1min":4.1349999999999998,"5min":2.1909999999999998,"15min":0.995},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":16.131,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.38},{"interface":"front","average":{"1min":2.9049999999999998,"5min":1.962,"15min":0.92700000000000005},"min":{"1min":0.25900000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":15.859999999999999,"5min":22.259,"15min":22.259},"last":0.45500000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:19:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":1.0289999999999999,"15min":0.78800000000000003},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":10.845000000000001,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":2.6560000000000001,"5min":1.5189999999999999,"15min":0.94899999999999995},"min":{"1min":0.33200000000000002,"5min":0.318,"15min":0.318},"max":{"1min":11.037000000000001,"5min":11.037000000000001,"15min":11.037000000000001},"last":0.73999999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9039999999999999,"5min":1.522,"15min":0.91700000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":11.125,"5min":11.125,"15min":11.125},"last":0.82799999999999996},{"interface":"front","average":{"1min":1.54,"5min":1.292,"15min":0.85399999999999998},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.1680000000000001,"5min":6.6130000000000004,"15min":6.6130000000000004},"last":0.76000000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":1.4930000000000001,"5min":1.3660000000000001,"15min":0.88200000000000001},"min":{"1min":0.372,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":5.101,"5min":10.989000000000001,"15min":10.989000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.6659999999999999,"5min":1.246,"15min":0.86399999999999999},"min":{"1min":0.35699999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":10.811999999999999,"5min":10.811999999999999,"15min":10.811999999999999},"last":0.88500000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:19:10 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":1.516,"15min":0.93600000000000005},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.9829999999999997,"5min":10.948,"15min":10.948},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.635,"5min":1.369,"15min":0.90900000000000003},"min":{"1min":0.39500000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":10.781000000000001,"5min":10.781000000000001,"15min":10.781000000000001},"last":0.84199999999999997}]}]},{"osd":0,"up_from":93,"seq":399431958632,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":195408,"kb_used_data":7048,"kb_used_omap":8,"kb_used_meta":188343,"kb_avail":93549744,"statfs":{"total":95995035648,"available":95794937856,"internally_reserved":0,"allocated":7217152,"data_stored":6143636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":192864106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:19:12 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":2.4529999999999998,"15min":1.0429999999999999},"min":{"1min":0.23999999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":11.711,"5min":44.776000000000003,"15min":44.776000000000003},"last":0.27900000000000003},{"interface":"front","average":{"1min":4.6529999999999996,"5min":3.1000000000000001,"15min":1.276},"min":{"1min":0.23899999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":20.927,"5min":48.838000000000001,"15min":48.838000000000001},"last":0.318}]},{"osd":2,"last update":"Sat Sep 23 22:19:41 2023","interfaces":[{"interface":"back","average":{"1min":3.2090000000000001,"5min":2.0630000000000002,"15min":0.95599999999999996},"min":{"1min":0.28399999999999997,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":13.625999999999999,"5min":25.893000000000001,"15min":25.893000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":4.2910000000000004,"5min":3.5350000000000001,"15min":1.526},"min":{"1min":0.36199999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.16,"5min":66.665000000000006,"15min":66.665000000000006},"last":0.39800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:19:12 2023","interfaces":[{"interface":"back","average":{"1min":3.2589999999999999,"5min":2.4060000000000001,"15min":1.099},"min":{"1min":0.33400000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":13.663,"5min":35.496000000000002,"15min":35.496000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":4.2469999999999999,"5min":2.6779999999999999,"15min":1.1859999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":20.870999999999999,"5min":35.533000000000001,"15min":35.533000000000001},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.6389999999999998,"15min":1.319},"min":{"1min":0.34200000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.032999999999999,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.274,"15min":0.86199999999999999},"min":{"1min":0.313,"5min":0.308,"15min":0.308},"max":{"1min":4.0270000000000001,"5min":22.209,"15min":22.209},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":2.609,"15min":1.3300000000000001},"min":{"1min":0.34799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9920000000000009,"5min":12.372,"15min":12.372},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.73899999999999999,"5min":1.2909999999999999,"15min":0.88500000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.0030000000000001,"5min":20.390999999999998,"15min":20.390999999999998},"last":0.61399999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.6680000000000001,"15min":1.3440000000000001},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.379},"max":{"1min":10.087,"5min":10.275,"15min":10.275},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.133,"5min":1.401,"15min":0.93700000000000006},"min":{"1min":0.34999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.4039999999999999,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":2.524,"15min":1.329},"min":{"1min":0.43099999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":4.1369999999999996,"5min":23.306000000000001,"15min":23.306000000000001},"last":0.438},{"interface":"front","average":{"1min":3.0019999999999998,"5min":3.0840000000000001,"15min":1.5289999999999999},"min":{"1min":0.47199999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.061,"5min":23.265999999999998,"15min":23.265999999999998},"last":0.51900000000000002}]}]},{"osd":2,"up_from":103,"seq":442381631571,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":182460,"kb_used_data":2356,"kb_used_omap":10,"kb_used_meta":180085,"kb_avail":93562692,"statfs":{"total":95995035648,"available":95808196608,"internally_reserved":0,"allocated":2412544,"data_stored":1359654,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":184407212},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":3.1629999999999998,"5min":2.5030000000000001,"15min":1.0569999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":19.344000000000001,"5min":19.344000000000001,"15min":19.344000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.7210000000000001,"15min":1.149},"min":{"1min":0.25700000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":9.4779999999999998,"5min":19.149999999999999,"15min":19.149999999999999},"last":0.38500000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":3.8220000000000001,"5min":4.6289999999999996,"15min":1.825},"min":{"1min":0.31900000000000001,"5min":0.222,"15min":0.222},"max":{"1min":16.141999999999999,"5min":157.636,"15min":157.636},"last":0.32800000000000001},{"interface":"front","average":{"1min":3.7509999999999999,"5min":4.7030000000000003,"15min":1.847},"min":{"1min":0.29499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":20.783999999999999,"5min":155.63300000000001,"15min":155.63300000000001},"last":0.54000000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:20:06 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":2.5950000000000002,"15min":1.0920000000000001},"min":{"1min":0.27100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.9429999999999996,"5min":19.852,"15min":19.852},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.8730000000000002,"15min":1.216},"min":{"1min":0.28100000000000003,"5min":0.26000000000000001,"15min":0.246},"max":{"1min":20.234999999999999,"5min":20.581,"15min":20.581},"last":0.41399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9019999999999999,"5min":1.79,"15min":0.995},"min":{"1min":0.34200000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.3689999999999998,"5min":11.244,"15min":11.244},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.1320000000000001,"5min":1.845,"15min":1.0329999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8659999999999997,"5min":9.2799999999999994,"15min":9.2799999999999994},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.893,"15min":1.054},"min":{"1min":0.311,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.3219999999999992,"5min":11.291,"15min":11.291},"last":0.60699999999999998},{"interface":"front","average":{"1min":2.3610000000000002,"5min":1.968,"15min":1.0600000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.915,"5min":9.4499999999999993,"15min":9.4499999999999993},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.008,"15min":1.1120000000000001},"min":{"1min":0.36699999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.4280000000000008,"5min":17.640999999999998,"15min":17.640999999999998},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.2559999999999998,"5min":1.8109999999999999,"15min":1.127},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.891,"5min":11.875,"15min":11.875},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":2.0019999999999998,"15min":1.1739999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":7.8019999999999996,"5min":10.135,"15min":10.135},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.3610000000000002,"5min":2.2719999999999998,"15min":1.1819999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.377},"max":{"1min":9.4529999999999994,"5min":17.738,"15min":17.738},"last":0.72099999999999997}]}]},{"osd":3,"up_from":108,"seq":463856468044,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":189180,"kb_used_data":95284,"kb_used_omap":4,"kb_used_meta":93883,"kb_avail":93555972,"statfs":{"total":95995035648,"available":95801315328,"internally_reserved":0,"allocated":97570816,"data_stored":96505716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":96136749},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":4.8399999999999999,"5min":3.6309999999999998,"15min":1.462},"min":{"1min":0.23000000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":48.676000000000002,"5min":77.834999999999994,"15min":77.834999999999994},"last":48.676000000000002},{"interface":"front","average":{"1min":4.8970000000000002,"5min":4.0090000000000003,"15min":1.577},"min":{"1min":0.26600000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":48.774000000000001,"5min":75.558000000000007,"15min":75.558000000000007},"last":48.774000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6779999999999999,"5min":4.2030000000000003,"15min":1.6359999999999999},"min":{"1min":0.24099999999999999,"5min":0.218,"15min":0.218},"max":{"1min":12.445,"5min":82.951999999999998,"15min":82.951999999999998},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.4119999999999999,"5min":4.0019999999999998,"15min":1.6060000000000001},"min":{"1min":0.22800000000000001,"5min":0.189,"15min":0.189},"max":{"1min":11.970000000000001,"5min":86.001000000000005,"15min":86.001000000000005},"last":0.22800000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":8.2799999999999994,"15min":3.0219999999999998},"min":{"1min":0.27700000000000002,"5min":0.223,"15min":0.223},"max":{"1min":11.708,"5min":99.962999999999994,"15min":99.962999999999994},"last":0.29399999999999998},{"interface":"front","average":{"1min":3.238,"5min":7.7300000000000004,"15min":2.8599999999999999},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":11.994,"5min":100.429,"15min":100.429},"last":0.28199999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:19:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5110000000000001,"5min":1.3169999999999999,"15min":0.85599999999999998},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.7610000000000001,"5min":6.7610000000000001,"15min":6.7610000000000001},"last":0.498},{"interface":"front","average":{"1min":2.0299999999999998,"5min":1.4630000000000001,"15min":0.92500000000000004},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.6200000000000001,"5min":12.036,"15min":12.036},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":4.4249999999999998,"5min":2.157,"15min":1.135},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":49.695999999999998,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.435},{"interface":"front","average":{"1min":4.7279999999999998,"5min":2.3500000000000001,"15min":1.1899999999999999},"min":{"1min":0.41599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":49.668999999999997,"5min":49.668999999999997,"15min":49.668999999999997},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.841,"15min":1.0489999999999999},"min":{"1min":0.441,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.760999999999999,"5min":12.068,"15min":12.068},"last":0.55000000000000004},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.8140000000000001,"15min":1.028},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":1.9039999999999999,"15min":1.0549999999999999},"min":{"1min":0.39600000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":11.596,"5min":11.624000000000001,"15min":11.624000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.419,"5min":1.752,"15min":1.04},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":11.945,"5min":12.382,"15min":12.382},"last":0.47199999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019095,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":123528,"kb_used_data":2112,"kb_used_omap":0,"kb_used_meta":121408,"kb_avail":93621624,"statfs":{"total":95995035648,"available":95868542976,"internally_reserved":0,"allocated":2162688,"data_stored":1324815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":124321792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":2.488,"15min":1.246},"min":{"1min":0.30099999999999999,"5min":0.218,"15min":0.218},"max":{"1min":27.942,"5min":66.355000000000004,"15min":66.355000000000004},"last":0.45600000000000002},{"interface":"front","average":{"1min":3.1339999999999999,"5min":2.7210000000000001,"15min":1.3360000000000001},"min":{"1min":0.315,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":27.956,"5min":66.332999999999998,"15min":66.332999999999998},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7570000000000001,"5min":2.1819999999999999,"15min":1.0940000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":19.053000000000001,"5min":38.878,"15min":38.878},"last":0.30599999999999999},{"interface":"front","average":{"1min":3.024,"5min":2.9100000000000001,"15min":1.3540000000000001},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":19.170999999999999,"5min":38.853999999999999,"15min":38.853999999999999},"last":0.54500000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:19:41 2023","interfaces":[{"interface":"back","average":{"1min":9.6750000000000007,"5min":4.7050000000000001,"15min":1.929},"min":{"1min":0.32400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":138.06100000000001,"5min":138.06100000000001,"15min":138.06100000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":10.814,"5min":5.5119999999999996,"15min":2.1779999999999999},"min":{"1min":0.309,"5min":0.308,"15min":0.308},"max":{"1min":138.15199999999999,"5min":138.15199999999999,"15min":138.15199999999999},"last":0.32900000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:19:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4710000000000001,"5min":1.5489999999999999,"15min":0.91200000000000003},"min":{"1min":0.35199999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9109999999999996,"5min":7.9109999999999996,"15min":7.9109999999999996},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.169,"5min":1.607,"15min":0.91900000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":8.8930000000000007,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:19:09 2023","interfaces":[{"interface":"back","average":{"1min":0.83499999999999996,"5min":0.93300000000000005,"15min":0.63},"min":{"1min":0.29599999999999999,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":4.0629999999999997,"5min":8.9499999999999993,"15min":8.9499999999999993},"last":0.437},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.86099999999999999,"15min":0.64300000000000002},"min":{"1min":0.29499999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.0069999999999997,"5min":7.8789999999999996,"15min":7.8789999999999996},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.82399999999999995,"15min":0.61799999999999999},"min":{"1min":0.35899999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.754,"5min":3.754,"15min":3.754},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.0760000000000001,"5min":0.94699999999999995,"15min":0.68000000000000005},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":6.7279999999999998,"5min":8.5210000000000008,"15min":8.5210000000000008},"last":0.58899999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.228,"15min":0.79200000000000004},"min":{"1min":0.36299999999999999,"5min":0.307,"15min":0.307},"max":{"1min":6.7480000000000002,"5min":8.577,"15min":8.577},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.95999999999999996,"15min":0.71699999999999997},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2679999999999998,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.504}]}]},{"osd":5,"up_from":38,"seq":163208757462,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":361724,"kb_used_data":239220,"kb_used_omap":0,"kb_used_meta":122496,"kb_avail":93383428,"statfs":{"total":95995035648,"available":95624630272,"internally_reserved":0,"allocated":244961280,"data_stored":244131458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":125435904},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8439999999999999,"5min":3.7749999999999999,"15min":1.6319999999999999},"min":{"1min":0.23000000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":28.741,"5min":133.22499999999999,"15min":133.22499999999999},"last":0.372},{"interface":"front","average":{"1min":3.3719999999999999,"5min":3.9609999999999999,"15min":1.7350000000000001},"min":{"1min":0.27100000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":28.873999999999999,"5min":132.78200000000001,"15min":132.78200000000001},"last":0.45300000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":4.8230000000000004,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.8120000000000003,"5min":83.759,"15min":83.759},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.3560000000000001,"5min":4.7699999999999996,"15min":1.9710000000000001},"min":{"1min":0.28399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":4.5629999999999997,"5min":83.741,"15min":83.741},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:19:40 2023","interfaces":[{"interface":"back","average":{"1min":8.5419999999999998,"5min":7.6210000000000004,"15min":2.915},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":78.137,"5min":104.254,"15min":104.254},"last":0.41899999999999998},{"interface":"front","average":{"1min":10.172000000000001,"5min":8.5760000000000005,"15min":3.2530000000000001},"min":{"1min":0.34000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":78.061000000000007,"5min":104.217,"15min":104.217},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":2.964,"15min":1.3740000000000001},"min":{"1min":0.37,"5min":0.246,"15min":0.246},"max":{"1min":9.9459999999999997,"5min":88.106999999999999,"15min":88.106999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.149,"5min":3.173,"15min":1.45},"min":{"1min":0.40699999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.7350000000000003,"5min":87.887,"15min":87.887},"last":0.58399999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.645,"15min":0.87},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.3399999999999999,"5min":82.530000000000001,"15min":82.530000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.78,"15min":0.94799999999999995},"min":{"1min":0.29399999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.4580000000000002,"5min":85.441000000000003,"15min":85.441000000000003},"last":0.5}]},{"osd":6,"last update":"Sat Sep 23 22:19:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":2.137,"15min":1.0669999999999999},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5350000000000001,"5min":114.83499999999999,"15min":114.83499999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.9710000000000001,"15min":1.0209999999999999},"min":{"1min":0.36299999999999999,"5min":0.31,"15min":0.31},"max":{"1min":4.5220000000000002,"5min":110.905,"15min":110.905},"last":0.623}]},{"osd":7,"last update":"Sat Sep 23 22:19:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.2969999999999999,"15min":0.81200000000000006},"min":{"1min":0.34999999999999998,"5min":0.33500000000000002,"15min":0.32600000000000001},"max":{"1min":4.3760000000000003,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.97199999999999998,"15min":0.70199999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33200000000000002},"max":{"1min":2.379,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":0.57599999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:20:13.783 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:13.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:13.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:13.887 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:13.919 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:13.926 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:13.968 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:13.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:13.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:13.985 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:14.073 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:14.101 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:14.143 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:14.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:14.248 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:14.272 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:14 smithi070 ceph-mon[192652]: failed to encode map e198 with expected crc 2023-09-23T22:20:14.272 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:14 smithi070 ceph-mon[192652]: pgmap v459: 83 pgs: 1 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:20:14.272 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:14 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T22:20:14.274 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:14.295 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:14 smithi026 ceph-mon[215775]: failed to encode map e198 with expected crc 2023-09-23T22:20:14.295 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:14 smithi026 ceph-mon[215775]: pgmap v459: 83 pgs: 1 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:20:14.295 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:14 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T22:20:14.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:14 smithi026 ceph-mon[223014]: failed to encode map e198 with expected crc 2023-09-23T22:20:14.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:14 smithi026 ceph-mon[223014]: pgmap v459: 83 pgs: 1 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail 2023-09-23T22:20:14.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:14 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T22:20:14.316 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:14.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:14.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:14.421 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:14.478 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:14.549 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:20:14.549 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:20:14.549 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:20:14.550 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:20:14.550 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:20:15.054 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:20:15.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T22:20:15.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:20:15.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T22:20:15.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: osdmap e199: 8 total, 8 up, 7 in 2023-09-23T22:20:15.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.255 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.255 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.255 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4240579796' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:20:15.255 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:20:15.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T22:20:15.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:20:15.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T22:20:15.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: osdmap e199: 8 total, 8 up, 7 in 2023-09-23T22:20:15.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.256 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4240579796' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:20:15.257 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:15 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:20:15.286 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:15.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:15.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:15.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T22:20:15.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: osdmap e199: 8 total, 8 up, 7 in 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: failed to encode map e199 with expected crc 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4240579796' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:20:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:15 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:20:15.391 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:15.420 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:16.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:20:16.040 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[215775]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.073 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:16.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:16.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:16.159 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:20:16.177 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:16.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:16.256 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:16.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:16.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:16.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:20:16.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[223014]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[223014]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:20:16.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[223014]: osdmap e200: 8 total, 8 up, 7 in 2023-09-23T22:20:16.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[223014]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[223014]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[223014]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[223014]: pgmap v462: 83 pgs: 1 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-09-23T22:20:16.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[215775]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:20:16.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[215775]: osdmap e200: 8 total, 8 up, 7 in 2023-09-23T22:20:16.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[215775]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[215775]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[215775]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:16 smithi026 ceph-mon[215775]: pgmap v462: 83 pgs: 1 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-09-23T22:20:16.363 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:16.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:16 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:20:16.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:16 smithi070 ceph-mon[192652]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:16 smithi070 ceph-mon[192652]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:16 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:20:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:16 smithi070 ceph-mon[192652]: osdmap e200: 8 total, 8 up, 7 in 2023-09-23T22:20:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:16 smithi070 ceph-mon[192652]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:16 smithi070 ceph-mon[192652]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:16 smithi070 ceph-mon[192652]: failed to encode map e200 with expected crc 2023-09-23T22:20:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:16 smithi070 ceph-mon[192652]: pgmap v462: 83 pgs: 1 active+remapped+backfill_wait, 82 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-09-23T22:20:16.395 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:16.442 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:16.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:16.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:16.546 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:16.578 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:16.626 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:16.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:16.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:16.644 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:20:16.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:16.761 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:16.804 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:16.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:16.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:16.909 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:16.935 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:16.977 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:16.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:16.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:17.085 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:17.110 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:17.152 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:17.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:17.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:17.245 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:17.257 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:17.290 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:17.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:17 smithi026 ceph-mon[223014]: osdmap e201: 8 total, 8 up, 7 in 2023-09-23T22:20:17.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:17 smithi026 ceph-mon[223014]: failed to encode map e201 with expected crc 2023-09-23T22:20:17.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:17 smithi026 ceph-mon[223014]: failed to encode map e201 with expected crc 2023-09-23T22:20:17.323 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:17 smithi026 ceph-mon[223014]: failed to encode map e201 with expected crc 2023-09-23T22:20:17.323 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:17 smithi026 ceph-mon[215775]: osdmap e201: 8 total, 8 up, 7 in 2023-09-23T22:20:17.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:17 smithi026 ceph-mon[215775]: failed to encode map e201 with expected crc 2023-09-23T22:20:17.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:17 smithi026 ceph-mon[215775]: failed to encode map e201 with expected crc 2023-09-23T22:20:17.324 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:17 smithi026 ceph-mon[215775]: failed to encode map e201 with expected crc 2023-09-23T22:20:17.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:17 smithi070 ceph-mon[192652]: osdmap e201: 8 total, 8 up, 7 in 2023-09-23T22:20:17.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:17 smithi070 ceph-mon[192652]: failed to encode map e201 with expected crc 2023-09-23T22:20:17.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:17 smithi070 ceph-mon[192652]: failed to encode map e201 with expected crc 2023-09-23T22:20:17.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:17 smithi070 ceph-mon[192652]: failed to encode map e201 with expected crc 2023-09-23T22:20:17.556 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:20:17.577 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:20:17.598 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:20:17.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:20:17.636 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:20:17.648 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:17.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:17.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:17.656 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:20:17.676 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:20:17.697 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:20:17.698 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:20:17.698 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:20:17.698 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:20:17.698 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:20:17.698 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:20:17.699 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:20:17.699 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:20:17.699 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:20:17.752 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:17.779 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:17.785 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:17.821 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:17.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:17.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:17.926 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:17.952 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:17.994 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:18.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:18.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:18.101 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:18.131 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:18.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:20:18.430 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:18.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:18.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:18.466 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_1' created 2023-09-23T22:20:18.534 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:18.562 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:18.606 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:18.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:18.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:18.714 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:18.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[223014]: failed to encode map e201 with expected crc 2023-09-23T22:20:18.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[223014]: failed to encode map e201 with expected crc 2023-09-23T22:20:18.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[223014]: failed to encode map e202 with expected crc 2023-09-23T22:20:18.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[223014]: failed to encode map e202 with expected crc 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[223014]: osdmap e202: 8 total, 8 up, 7 in 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[223014]: pgmap v465: 83 pgs: 2 activating+undersized+degraded+remapped, 7 activating+undersized+remapped, 74 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 2/84 objects misplaced (2.381%) 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2390305130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[215775]: failed to encode map e201 with expected crc 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[215775]: failed to encode map e201 with expected crc 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[215775]: failed to encode map e202 with expected crc 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[215775]: failed to encode map e202 with expected crc 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[215775]: osdmap e202: 8 total, 8 up, 7 in 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[215775]: pgmap v465: 83 pgs: 2 activating+undersized+degraded+remapped, 7 activating+undersized+remapped, 74 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 2/84 objects misplaced (2.381%) 2023-09-23T22:20:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2390305130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:18.738 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:18 smithi070 ceph-mon[192652]: failed to encode map e201 with expected crc 2023-09-23T22:20:18.738 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:18 smithi070 ceph-mon[192652]: failed to encode map e201 with expected crc 2023-09-23T22:20:18.738 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:18 smithi070 ceph-mon[192652]: failed to encode map e202 with expected crc 2023-09-23T22:20:18.738 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:18 smithi070 ceph-mon[192652]: failed to encode map e202 with expected crc 2023-09-23T22:20:18.738 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:18 smithi070 ceph-mon[192652]: osdmap e202: 8 total, 8 up, 7 in 2023-09-23T22:20:18.738 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:18 smithi070 ceph-mon[192652]: pgmap v465: 83 pgs: 2 activating+undersized+degraded+remapped, 7 activating+undersized+remapped, 74 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 2/84 objects misplaced (2.381%) 2023-09-23T22:20:18.738 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2390305130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:18.740 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:18.782 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:18.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:18.887 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:18.912 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:18.954 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:18.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:19.059 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:19.085 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:19.125 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:19.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:19.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:19.230 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:19.255 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:19.279 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T22:20:19.296 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:19.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:19.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:19.404 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:19.430 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:19.476 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:19.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:19.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:19.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1930518266' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:20:19.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: osdmap e203: 8 total, 8 up, 7 in 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[223014]: failed to encode map e202 with expected crc 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1930518266' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: osdmap e203: 8 total, 8 up, 7 in 2023-09-23T22:20:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:19 smithi026 ceph-mon[215775]: failed to encode map e202 with expected crc 2023-09-23T22:20:19.584 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:19.613 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1930518266' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T22:20:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: osdmap e203: 8 total, 8 up, 7 in 2023-09-23T22:20:19.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: failed to encode map e203 with expected crc 2023-09-23T22:20:19.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:19 smithi070 ceph-mon[192652]: failed to encode map e202 with expected crc 2023-09-23T22:20:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:20:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:20:19] "GET /metrics HTTP/1.1" 200 36369 "" "Prometheus/2.43.0" 2023-09-23T22:20:20.296 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:20.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:20.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:20.403 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:20.430 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:20.475 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:20.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:20.581 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:20.607 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:20.649 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:20.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:20.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:20.706 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:20.709 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[223014]: pgmap v467: 99 pgs: 4 creating+peering, 12 unknown, 2 peering, 1 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 76 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 2/84 objects degraded (2.381%); 1/84 objects misplaced (1.190%); 78 KiB/s, 0 objects/s recovering 2023-09-23T22:20:20.709 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[223014]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.709 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[223014]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.709 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[223014]: osdmap e204: 8 total, 8 up, 7 in 2023-09-23T22:20:20.709 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[223014]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.709 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[223014]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.710 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[215775]: pgmap v467: 99 pgs: 4 creating+peering, 12 unknown, 2 peering, 1 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 76 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 2/84 objects degraded (2.381%); 1/84 objects misplaced (1.190%); 78 KiB/s, 0 objects/s recovering 2023-09-23T22:20:20.710 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[215775]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.710 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[215775]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.710 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[215775]: osdmap e204: 8 total, 8 up, 7 in 2023-09-23T22:20:20.710 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[215775]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.710 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:20 smithi026 ceph-mon[215775]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.736 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:20:20.737 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:20:20.754 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:20.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:20:20.777 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:20 smithi070 ceph-mon[192652]: pgmap v467: 99 pgs: 4 creating+peering, 12 unknown, 2 peering, 1 activating+undersized+degraded+remapped, 4 activating+undersized+remapped, 76 active+clean; 4.6 MiB data, 1.4 GiB used, 624 GiB / 626 GiB avail; 2/84 objects degraded (2.381%); 1/84 objects misplaced (1.190%); 78 KiB/s, 0 objects/s recovering 2023-09-23T22:20:20.777 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:20 smithi070 ceph-mon[192652]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.778 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:20 smithi070 ceph-mon[192652]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.778 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:20 smithi070 ceph-mon[192652]: osdmap e204: 8 total, 8 up, 7 in 2023-09-23T22:20:20.778 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:20 smithi070 ceph-mon[192652]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.778 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:20 smithi070 ceph-mon[192652]: failed to encode map e204 with expected crc 2023-09-23T22:20:20.780 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:20.821 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:20.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:20.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:20.834 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:20.926 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:20.956 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:21.129 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:21.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:21.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:21.171 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:20:21.174 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:20:21.232 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:21.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:21.300 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:21.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:21.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:21.407 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:21.433 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:21.480 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:21.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:21.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:21.587 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:21.615 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:21.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:21 smithi070 ceph-mon[192652]: failed to encode map e205 with expected crc 2023-09-23T22:20:21.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:21 smithi070 ceph-mon[192652]: osdmap e205: 8 total, 8 up, 7 in 2023-09-23T22:20:21.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:20:21.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:21 smithi026 ceph-mon[223014]: failed to encode map e205 with expected crc 2023-09-23T22:20:21.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:21 smithi026 ceph-mon[223014]: osdmap e205: 8 total, 8 up, 7 in 2023-09-23T22:20:21.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:20:21.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:21 smithi026 ceph-mon[215775]: failed to encode map e205 with expected crc 2023-09-23T22:20:21.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:21 smithi026 ceph-mon[215775]: osdmap e205: 8 total, 8 up, 7 in 2023-09-23T22:20:21.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:20:21.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:20:22.719 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:22.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[215775]: pgmap v470: 99 pgs: 4 creating+peering, 9 unknown, 2 peering, 2 activating+undersized+remapped, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 141 KiB/s, 0 objects/s recovering 2023-09-23T22:20:22.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[215775]: from='client.44969 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:22.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T22:20:22.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.721 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[223014]: pgmap v470: 99 pgs: 4 creating+peering, 9 unknown, 2 peering, 2 activating+undersized+remapped, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 141 KiB/s, 0 objects/s recovering 2023-09-23T22:20:22.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[223014]: from='client.44969 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:22.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T22:20:22.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.722 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:22 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:22.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:22.824 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:22.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:22 smithi070 ceph-mon[192652]: pgmap v470: 99 pgs: 4 creating+peering, 9 unknown, 2 peering, 2 activating+undersized+remapped, 82 active+clean; 4.6 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 141 KiB/s, 0 objects/s recovering 2023-09-23T22:20:22.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:22 smithi070 ceph-mon[192652]: from='client.44969 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:22.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:22 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T22:20:22.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:22 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:22 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:22 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.850 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:22 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:22.852 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:22.895 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:22.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:22.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:22.922 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":470,"stamp":"2023-09-23T22:20:21.145343+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6468,"num_read_kb":13096,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":11345378,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2802,"ondisk_log_size":2802,"up":270,"acting":268,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":278,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1388312,"kb_used_data":432672,"kb_used_omap":31,"kb_used_meta":955552,"kb_avail":654827752,"statfs":{"total":671965249536,"available":670543618048,"internally_reserved":0,"allocated":443056128,"data_stored":436187967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":978486199},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3,"num_objects_misplaced":-2,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":590368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.092084"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":179,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499968+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:20:16.499968+0000","last_peered":"2023-09-23T22:20:16.499968+0000","last_clean":"2023-09-23T22:20:16.499968+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:20:16.499968+0000","last_undegraded":"2023-09-23T22:20:16.499968+0000","last_fullsized":"2023-09-23T22:20:16.499968+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.066009+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:20:17.066009+0000","last_peered":"2023-09-23T22:20:17.066009+0000","last_clean":"2023-09-23T22:20:17.066009+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:20:17.066009+0000","last_undegraded":"2023-09-23T22:20:17.066009+0000","last_fullsized":"2023-09-23T22:20:17.066009+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":142,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.054781+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:20:16.054781+0000","last_peered":"2023-09-23T22:20:16.054781+0000","last_clean":"2023-09-23T22:20:16.054781+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:20:16.054781+0000","last_undegraded":"2023-09-23T22:20:16.054781+0000","last_fullsized":"2023-09-23T22:20:16.054781+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":29,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.052124+0000","last_change":"2023-09-23T22:18:57.724501+0000","last_active":"2023-09-23T22:20:15.052124+0000","last_peered":"2023-09-23T22:20:15.052124+0000","last_clean":"2023-09-23T22:20:15.052124+0000","last_became_active":"2023-09-23T22:18:57.724203+0000","last_became_peered":"2023-09-23T22:18:57.724203+0000","last_unstale":"2023-09-23T22:20:15.052124+0000","last_undegraded":"2023-09-23T22:20:15.052124+0000","last_fullsized":"2023-09-23T22:20:15.052124+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:52.168267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":222,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065247+0000","last_change":"2023-09-23T22:18:57.722574+0000","last_active":"2023-09-23T22:20:17.065247+0000","last_peered":"2023-09-23T22:20:17.065247+0000","last_clean":"2023-09-23T22:20:17.065247+0000","last_became_active":"2023-09-23T22:18:57.721800+0000","last_became_peered":"2023-09-23T22:18:57.721800+0000","last_unstale":"2023-09-23T22:20:17.065247+0000","last_undegraded":"2023-09-23T22:20:17.065247+0000","last_fullsized":"2023-09-23T22:20:17.065247+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:09.303838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":159,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.462137+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:20:18.462137+0000","last_peered":"2023-09-23T22:20:18.462137+0000","last_clean":"2023-09-23T22:20:18.462137+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:20:18.462137+0000","last_undegraded":"2023-09-23T22:20:18.462137+0000","last_fullsized":"2023-09-23T22:20:18.462137+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":67,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.054544+0000","last_change":"2023-09-23T22:18:57.729038+0000","last_active":"2023-09-23T22:20:15.054544+0000","last_peered":"2023-09-23T22:20:15.054544+0000","last_clean":"2023-09-23T22:20:15.054544+0000","last_became_active":"2023-09-23T22:18:57.728650+0000","last_became_peered":"2023-09-23T22:18:57.728650+0000","last_unstale":"2023-09-23T22:20:15.054544+0000","last_undegraded":"2023-09-23T22:20:15.054544+0000","last_fullsized":"2023-09-23T22:20:15.054544+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:11.461883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":177,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065247+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:20:17.065247+0000","last_peered":"2023-09-23T22:20:17.065247+0000","last_clean":"2023-09-23T22:20:17.065247+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:20:17.065247+0000","last_undegraded":"2023-09-23T22:20:17.065247+0000","last_fullsized":"2023-09-23T22:20:17.065247+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":164,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.461864+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:20:18.461864+0000","last_peered":"2023-09-23T22:20:18.461864+0000","last_clean":"2023-09-23T22:20:18.461864+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:20:18.461864+0000","last_undegraded":"2023-09-23T22:20:18.461864+0000","last_fullsized":"2023-09-23T22:20:18.461864+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":127,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065424+0000","last_change":"2023-09-23T22:16:49.154341+0000","last_active":"2023-09-23T22:20:17.065424+0000","last_peered":"2023-09-23T22:20:17.065424+0000","last_clean":"2023-09-23T22:20:17.065424+0000","last_became_active":"2023-09-23T22:16:49.154041+0000","last_became_peered":"2023-09-23T22:16:49.154041+0000","last_unstale":"2023-09-23T22:20:17.065424+0000","last_undegraded":"2023-09-23T22:20:17.065424+0000","last_fullsized":"2023-09-23T22:20:17.065424+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:05.733262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":115,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065945+0000","last_change":"2023-09-23T22:15:14.829731+0000","last_active":"2023-09-23T22:20:17.065945+0000","last_peered":"2023-09-23T22:20:17.065945+0000","last_clean":"2023-09-23T22:20:17.065945+0000","last_became_active":"2023-09-23T22:15:14.829437+0000","last_became_peered":"2023-09-23T22:15:14.829437+0000","last_unstale":"2023-09-23T22:20:17.065945+0000","last_undegraded":"2023-09-23T22:20:17.065945+0000","last_fullsized":"2023-09-23T22:20:17.065945+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:45.456558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":169,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.054947+0000","last_change":"2023-09-23T22:18:57.723165+0000","last_active":"2023-09-23T22:20:16.054947+0000","last_peered":"2023-09-23T22:20:16.054947+0000","last_clean":"2023-09-23T22:20:16.054947+0000","last_became_active":"2023-09-23T22:18:57.722863+0000","last_became_peered":"2023-09-23T22:18:57.722863+0000","last_unstale":"2023-09-23T22:20:16.054947+0000","last_undegraded":"2023-09-23T22:20:16.054947+0000","last_fullsized":"2023-09-23T22:20:16.054947+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:19.645029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":186,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499831+0000","last_change":"2023-09-23T22:16:49.265991+0000","last_active":"2023-09-23T22:20:16.499831+0000","last_peered":"2023-09-23T22:20:16.499831+0000","last_clean":"2023-09-23T22:20:16.499831+0000","last_became_active":"2023-09-23T22:16:49.251169+0000","last_became_peered":"2023-09-23T22:16:49.251169+0000","last_unstale":"2023-09-23T22:20:16.499831+0000","last_undegraded":"2023-09-23T22:20:16.499831+0000","last_fullsized":"2023-09-23T22:20:16.499831+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.502218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499786+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:20:16.499786+0000","last_peered":"2023-09-23T22:20:16.499786+0000","last_clean":"2023-09-23T22:20:16.499786+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:20:16.499786+0000","last_undegraded":"2023-09-23T22:20:16.499786+0000","last_fullsized":"2023-09-23T22:20:16.499786+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":113,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499814+0000","last_change":"2023-09-23T22:15:14.826143+0000","last_active":"2023-09-23T22:20:16.499814+0000","last_peered":"2023-09-23T22:20:16.499814+0000","last_clean":"2023-09-23T22:20:16.499814+0000","last_became_active":"2023-09-23T22:15:14.825780+0000","last_became_peered":"2023-09-23T22:15:14.825780+0000","last_unstale":"2023-09-23T22:20:16.499814+0000","last_undegraded":"2023-09-23T22:20:16.499814+0000","last_fullsized":"2023-09-23T22:20:16.499814+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:22.608676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":168,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065758+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:20:17.065758+0000","last_peered":"2023-09-23T22:20:17.065758+0000","last_clean":"2023-09-23T22:20:17.065758+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:20:17.065758+0000","last_undegraded":"2023-09-23T22:20:17.065758+0000","last_fullsized":"2023-09-23T22:20:17.065758+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":29,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.048534+0000","last_change":"2023-09-23T22:18:57.726112+0000","last_active":"2023-09-23T22:20:15.048534+0000","last_peered":"2023-09-23T22:20:15.048534+0000","last_clean":"2023-09-23T22:20:15.048534+0000","last_became_active":"2023-09-23T22:18:57.725558+0000","last_became_peered":"2023-09-23T22:18:57.725558+0000","last_unstale":"2023-09-23T22:20:15.048534+0000","last_undegraded":"2023-09-23T22:20:15.048534+0000","last_fullsized":"2023-09-23T22:20:15.048534+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:38.315664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":163,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.461554+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:20:18.461554+0000","last_peered":"2023-09-23T22:20:18.461554+0000","last_clean":"2023-09-23T22:20:18.461554+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:20:18.461554+0000","last_undegraded":"2023-09-23T22:20:18.461554+0000","last_fullsized":"2023-09-23T22:20:18.461554+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":125,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499781+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:20:16.499781+0000","last_peered":"2023-09-23T22:20:16.499781+0000","last_clean":"2023-09-23T22:20:16.499781+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:20:16.499781+0000","last_undegraded":"2023-09-23T22:20:16.499781+0000","last_fullsized":"2023-09-23T22:20:16.499781+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":29,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.051808+0000","last_change":"2023-09-23T22:18:57.726691+0000","last_active":"2023-09-23T22:20:15.051808+0000","last_peered":"2023-09-23T22:20:15.051808+0000","last_clean":"2023-09-23T22:20:15.051808+0000","last_became_active":"2023-09-23T22:18:57.726386+0000","last_became_peered":"2023-09-23T22:18:57.726386+0000","last_unstale":"2023-09-23T22:20:15.051808+0000","last_undegraded":"2023-09-23T22:20:15.051808+0000","last_fullsized":"2023-09-23T22:20:15.051808+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:34.829149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":170,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.050247+0000","last_change":"2023-09-23T22:18:57.728767+0000","last_active":"2023-09-23T22:20:15.050247+0000","last_peered":"2023-09-23T22:20:15.050247+0000","last_clean":"2023-09-23T22:20:15.050247+0000","last_became_active":"2023-09-23T22:18:57.728377+0000","last_became_peered":"2023-09-23T22:18:57.728377+0000","last_unstale":"2023-09-23T22:20:15.050247+0000","last_undegraded":"2023-09-23T22:20:15.050247+0000","last_fullsized":"2023-09-23T22:20:15.050247+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:01:01.370904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":168,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065873+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:20:17.065873+0000","last_peered":"2023-09-23T22:20:17.065873+0000","last_clean":"2023-09-23T22:20:17.065873+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:20:17.065873+0000","last_undegraded":"2023-09-23T22:20:17.065873+0000","last_fullsized":"2023-09-23T22:20:17.065873+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":9,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:20:19.470064+0000","last_change":"2023-09-23T22:20:19.470064+0000","last_active":"2023-09-23T22:20:19.470064+0000","last_peered":"2023-09-23T22:20:19.470064+0000","last_clean":"2023-09-23T22:20:19.470064+0000","last_became_active":"2023-09-23T22:20:19.469807+0000","last_became_peered":"2023-09-23T22:20:19.469807+0000","last_unstale":"2023-09-23T22:20:19.470064+0000","last_undegraded":"2023-09-23T22:20:19.470064+0000","last_fullsized":"2023-09-23T22:20:19.470064+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:45.351484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":169,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.500321+0000","last_change":"2023-09-23T22:19:00.682889+0000","last_active":"2023-09-23T22:20:16.500321+0000","last_peered":"2023-09-23T22:20:16.500321+0000","last_clean":"2023-09-23T22:20:16.500321+0000","last_became_active":"2023-09-23T22:18:57.718002+0000","last_became_peered":"2023-09-23T22:18:57.718002+0000","last_unstale":"2023-09-23T22:20:16.500321+0000","last_undegraded":"2023-09-23T22:20:16.500321+0000","last_fullsized":"2023-09-23T22:20:16.500321+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:22.517277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":156,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.460721+0000","last_change":"2023-09-23T22:16:49.158580+0000","last_active":"2023-09-23T22:20:18.460721+0000","last_peered":"2023-09-23T22:20:18.460721+0000","last_clean":"2023-09-23T22:20:18.460721+0000","last_became_active":"2023-09-23T22:16:49.158266+0000","last_became_peered":"2023-09-23T22:16:49.158266+0000","last_unstale":"2023-09-23T22:20:18.460721+0000","last_undegraded":"2023-09-23T22:20:18.460721+0000","last_fullsized":"2023-09-23T22:20:18.460721+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:03.791658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"0'0","reported_seq":1,"reported_epoch":203,"state":"creating+peering","last_fresh":"2023-09-23T22:20:18.454985+0000","last_change":"2023-09-23T22:20:18.454985+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.454985+0000","last_undegraded":"2023-09-23T22:20:18.454985+0000","last_fullsized":"2023-09-23T22:20:18.454985+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:25.959782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":113,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.500166+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:20:16.500166+0000","last_peered":"2023-09-23T22:20:16.500166+0000","last_clean":"2023-09-23T22:20:16.500166+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:20:16.500166+0000","last_undegraded":"2023-09-23T22:20:16.500166+0000","last_fullsized":"2023-09-23T22:20:16.500166+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":169,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499702+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:20:16.499702+0000","last_peered":"2023-09-23T22:20:16.499702+0000","last_clean":"2023-09-23T22:20:16.499702+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:20:16.499702+0000","last_undegraded":"2023-09-23T22:20:16.499702+0000","last_fullsized":"2023-09-23T22:20:16.499702+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:20:18.446560+0000","last_change":"2023-09-23T22:20:18.446560+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.446560+0000","last_undegraded":"2023-09-23T22:20:18.446560+0000","last_fullsized":"2023-09-23T22:20:18.446560+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":129,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.461535+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:20:18.461535+0000","last_peered":"2023-09-23T22:20:18.461535+0000","last_clean":"2023-09-23T22:20:18.461535+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:20:18.461535+0000","last_undegraded":"2023-09-23T22:20:18.461535+0000","last_fullsized":"2023-09-23T22:20:18.461535+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":117,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.055023+0000","last_change":"2023-09-23T22:16:49.161676+0000","last_active":"2023-09-23T22:20:16.055023+0000","last_peered":"2023-09-23T22:20:16.055023+0000","last_clean":"2023-09-23T22:20:16.055023+0000","last_became_active":"2023-09-23T22:16:49.161247+0000","last_became_peered":"2023-09-23T22:16:49.161247+0000","last_unstale":"2023-09-23T22:20:16.055023+0000","last_undegraded":"2023-09-23T22:20:16.055023+0000","last_fullsized":"2023-09-23T22:20:16.055023+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:30.708371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:20:18.446560+0000","last_change":"2023-09-23T22:20:18.446560+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.446560+0000","last_undegraded":"2023-09-23T22:20:18.446560+0000","last_fullsized":"2023-09-23T22:20:18.446560+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":174,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.462040+0000","last_change":"2023-09-23T22:16:49.164833+0000","last_active":"2023-09-23T22:20:18.462040+0000","last_peered":"2023-09-23T22:20:18.462040+0000","last_clean":"2023-09-23T22:20:18.462040+0000","last_became_active":"2023-09-23T22:16:49.163833+0000","last_became_peered":"2023-09-23T22:16:49.163833+0000","last_unstale":"2023-09-23T22:20:18.462040+0000","last_undegraded":"2023-09-23T22:20:18.462040+0000","last_fullsized":"2023-09-23T22:20:18.462040+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.891395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":54,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.050447+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:20:15.050447+0000","last_peered":"2023-09-23T22:20:15.050447+0000","last_clean":"2023-09-23T22:20:15.050447+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:20:15.050447+0000","last_undegraded":"2023-09-23T22:20:15.050447+0000","last_fullsized":"2023-09-23T22:20:15.050447+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:20:18.446560+0000","last_change":"2023-09-23T22:20:18.446560+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.446560+0000","last_undegraded":"2023-09-23T22:20:18.446560+0000","last_fullsized":"2023-09-23T22:20:18.446560+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":476,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.468018+0000","last_change":"2023-09-23T22:20:18.461035+0000","last_active":"2023-09-23T22:20:18.468018+0000","last_peered":"2023-09-23T22:20:18.468018+0000","last_clean":"2023-09-23T22:20:18.468018+0000","last_became_active":"2023-09-23T22:20:18.460615+0000","last_became_peered":"2023-09-23T22:20:18.460615+0000","last_unstale":"2023-09-23T22:20:18.468018+0000","last_undegraded":"2023-09-23T22:20:18.468018+0000","last_fullsized":"2023-09-23T22:20:18.468018+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":214,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065540+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:20:17.065540+0000","last_peered":"2023-09-23T22:20:17.065540+0000","last_clean":"2023-09-23T22:20:17.065540+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:20:17.065540+0000","last_undegraded":"2023-09-23T22:20:17.065540+0000","last_fullsized":"2023-09-23T22:20:17.065540+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499727+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:20:16.499727+0000","last_peered":"2023-09-23T22:20:16.499727+0000","last_clean":"2023-09-23T22:20:16.499727+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:20:16.499727+0000","last_undegraded":"2023-09-23T22:20:16.499727+0000","last_fullsized":"2023-09-23T22:20:16.499727+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:20:18.446560+0000","last_change":"2023-09-23T22:20:18.446560+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.446560+0000","last_undegraded":"2023-09-23T22:20:18.446560+0000","last_fullsized":"2023-09-23T22:20:18.446560+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":482,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.472653+0000","last_change":"2023-09-23T22:20:18.472306+0000","last_active":"2023-09-23T22:20:18.472653+0000","last_peered":"2023-09-23T22:20:18.472653+0000","last_clean":"2023-09-23T22:20:18.472653+0000","last_became_active":"2023-09-23T22:20:18.471844+0000","last_became_peered":"2023-09-23T22:20:18.471844+0000","last_unstale":"2023-09-23T22:20:18.472653+0000","last_undegraded":"2023-09-23T22:20:18.472653+0000","last_fullsized":"2023-09-23T22:20:18.472653+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":118,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499709+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:20:16.499709+0000","last_peered":"2023-09-23T22:20:16.499709+0000","last_clean":"2023-09-23T22:20:16.499709+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:20:16.499709+0000","last_undegraded":"2023-09-23T22:20:16.499709+0000","last_fullsized":"2023-09-23T22:20:16.499709+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":177,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.500053+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:20:16.500053+0000","last_peered":"2023-09-23T22:20:16.500053+0000","last_clean":"2023-09-23T22:20:16.500053+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:20:16.500053+0000","last_undegraded":"2023-09-23T22:20:16.500053+0000","last_fullsized":"2023-09-23T22:20:16.500053+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":1,"reported_epoch":203,"state":"creating+peering","last_fresh":"2023-09-23T22:20:18.454623+0000","last_change":"2023-09-23T22:20:18.454623+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.454623+0000","last_undegraded":"2023-09-23T22:20:18.454623+0000","last_fullsized":"2023-09-23T22:20:18.454623+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:16.691256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":153,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499596+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:20:16.499596+0000","last_peered":"2023-09-23T22:20:16.499596+0000","last_clean":"2023-09-23T22:20:16.499596+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:20:16.499596+0000","last_undegraded":"2023-09-23T22:20:16.499596+0000","last_fullsized":"2023-09-23T22:20:16.499596+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.500292+0000","last_change":"2023-09-23T22:16:49.165118+0000","last_active":"2023-09-23T22:20:16.500292+0000","last_peered":"2023-09-23T22:20:16.500292+0000","last_clean":"2023-09-23T22:20:16.500292+0000","last_became_active":"2023-09-23T22:16:49.164802+0000","last_became_peered":"2023-09-23T22:16:49.164802+0000","last_unstale":"2023-09-23T22:20:16.500292+0000","last_undegraded":"2023-09-23T22:20:16.500292+0000","last_fullsized":"2023-09-23T22:20:16.500292+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:23.079142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"0'0","reported_seq":9,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:20:19.473637+0000","last_change":"2023-09-23T22:20:19.473637+0000","last_active":"2023-09-23T22:20:19.473637+0000","last_peered":"2023-09-23T22:20:19.473637+0000","last_clean":"2023-09-23T22:20:19.473637+0000","last_became_active":"2023-09-23T22:20:19.473385+0000","last_became_peered":"2023-09-23T22:20:19.473385+0000","last_unstale":"2023-09-23T22:20:19.473637+0000","last_undegraded":"2023-09-23T22:20:19.473637+0000","last_fullsized":"2023-09-23T22:20:19.473637+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:01.911861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":478,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.473837+0000","last_change":"2023-09-23T22:20:18.473341+0000","last_active":"2023-09-23T22:20:18.473837+0000","last_peered":"2023-09-23T22:20:18.473837+0000","last_clean":"2023-09-23T22:20:18.473837+0000","last_became_active":"2023-09-23T22:20:18.472892+0000","last_became_peered":"2023-09-23T22:20:18.472892+0000","last_unstale":"2023-09-23T22:20:18.473837+0000","last_undegraded":"2023-09-23T22:20:18.473837+0000","last_fullsized":"2023-09-23T22:20:18.473837+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":169,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.061846+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:20:17.061846+0000","last_peered":"2023-09-23T22:20:17.061846+0000","last_clean":"2023-09-23T22:20:17.061846+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:20:17.061846+0000","last_undegraded":"2023-09-23T22:20:17.061846+0000","last_fullsized":"2023-09-23T22:20:17.061846+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":142,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.056069+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:20:16.056069+0000","last_peered":"2023-09-23T22:20:16.056069+0000","last_clean":"2023-09-23T22:20:16.056069+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:20:16.056069+0000","last_undegraded":"2023-09-23T22:20:16.056069+0000","last_fullsized":"2023-09-23T22:20:16.056069+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:20:18.446560+0000","last_change":"2023-09-23T22:20:18.446560+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.446560+0000","last_undegraded":"2023-09-23T22:20:18.446560+0000","last_fullsized":"2023-09-23T22:20:18.446560+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":476,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.471037+0000","last_change":"2023-09-23T22:20:18.468073+0000","last_active":"2023-09-23T22:20:18.471037+0000","last_peered":"2023-09-23T22:20:18.471037+0000","last_clean":"2023-09-23T22:20:18.471037+0000","last_became_active":"2023-09-23T22:20:18.467701+0000","last_became_peered":"2023-09-23T22:20:18.467701+0000","last_unstale":"2023-09-23T22:20:18.471037+0000","last_undegraded":"2023-09-23T22:20:18.471037+0000","last_fullsized":"2023-09-23T22:20:18.471037+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":749,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.057485+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:20:16.057485+0000","last_peered":"2023-09-23T22:20:16.057485+0000","last_clean":"2023-09-23T22:20:16.057485+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:20:16.057485+0000","last_undegraded":"2023-09-23T22:20:16.057485+0000","last_fullsized":"2023-09-23T22:20:16.057485+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":181,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.461648+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:20:18.461648+0000","last_peered":"2023-09-23T22:20:18.461648+0000","last_clean":"2023-09-23T22:20:18.461648+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:20:18.461648+0000","last_undegraded":"2023-09-23T22:20:18.461648+0000","last_fullsized":"2023-09-23T22:20:18.461648+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":122,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T22:20:14.053811+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:20:14.053811+0000","last_peered":"2023-09-23T22:20:14.053811+0000","last_clean":"2023-09-23T22:20:14.053811+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:20:14.053811+0000","last_undegraded":"2023-09-23T22:20:14.053811+0000","last_fullsized":"2023-09-23T22:20:14.053811+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":1,"reported_epoch":203,"state":"creating+peering","last_fresh":"2023-09-23T22:20:18.454058+0000","last_change":"2023-09-23T22:20:18.454058+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.454058+0000","last_undegraded":"2023-09-23T22:20:18.454058+0000","last_fullsized":"2023-09-23T22:20:18.454058+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:08.337445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":478,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.469676+0000","last_change":"2023-09-23T22:20:18.469369+0000","last_active":"2023-09-23T22:20:18.469676+0000","last_peered":"2023-09-23T22:20:18.469676+0000","last_clean":"2023-09-23T22:20:18.469676+0000","last_became_active":"2023-09-23T22:20:18.469020+0000","last_became_peered":"2023-09-23T22:20:18.469020+0000","last_unstale":"2023-09-23T22:20:18.469676+0000","last_undegraded":"2023-09-23T22:20:18.469676+0000","last_fullsized":"2023-09-23T22:20:18.469676+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":936,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.461114+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:20:18.461114+0000","last_peered":"2023-09-23T22:20:18.461114+0000","last_clean":"2023-09-23T22:20:18.461114+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:20:18.461114+0000","last_undegraded":"2023-09-23T22:20:18.461114+0000","last_fullsized":"2023-09-23T22:20:18.461114+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":211,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499674+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:20:16.499674+0000","last_peered":"2023-09-23T22:20:16.499674+0000","last_clean":"2023-09-23T22:20:16.499674+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:20:16.499674+0000","last_undegraded":"2023-09-23T22:20:16.499674+0000","last_fullsized":"2023-09-23T22:20:16.499674+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":178,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.462185+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:20:18.462185+0000","last_peered":"2023-09-23T22:20:18.462185+0000","last_clean":"2023-09-23T22:20:18.462185+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:20:18.462185+0000","last_undegraded":"2023-09-23T22:20:18.462185+0000","last_fullsized":"2023-09-23T22:20:18.462185+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":1,"reported_epoch":203,"state":"creating+peering","last_fresh":"2023-09-23T22:20:18.455037+0000","last_change":"2023-09-23T22:20:18.455037+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.455037+0000","last_undegraded":"2023-09-23T22:20:18.455037+0000","last_fullsized":"2023-09-23T22:20:18.455037+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:47.461541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":469,"reported_epoch":201,"state":"activating+undersized+remapped","last_fresh":"2023-09-23T22:20:16.053291+0000","last_change":"2023-09-23T22:20:16.053291+0000","last_active":"2023-09-23T22:20:14.043689+0000","last_peered":"2023-09-23T22:20:12.489790+0000","last_clean":"2023-09-23T22:20:12.489790+0000","last_became_active":"2023-09-23T22:18:57.725260+0000","last_became_peered":"2023-09-23T22:18:57.725260+0000","last_unstale":"2023-09-23T22:20:16.053291+0000","last_undegraded":"2023-09-23T22:20:16.053291+0000","last_fullsized":"2023-09-23T22:20:16.053100+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7],"avail_no_missing":["0","1","4","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":542,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T22:20:14.052730+0000","last_change":"2023-09-23T22:15:51.379227+0000","last_active":"2023-09-23T22:20:14.052730+0000","last_peered":"2023-09-23T22:20:14.052730+0000","last_clean":"2023-09-23T22:20:14.052730+0000","last_became_active":"2023-09-23T22:15:43.928977+0000","last_became_peered":"2023-09-23T22:15:43.928977+0000","last_unstale":"2023-09-23T22:20:14.052730+0000","last_undegraded":"2023-09-23T22:20:14.052730+0000","last_fullsized":"2023-09-23T22:20:14.052730+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:04.398533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":151,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.050917+0000","last_change":"2023-09-23T22:16:49.251412+0000","last_active":"2023-09-23T22:20:16.050917+0000","last_peered":"2023-09-23T22:20:16.050917+0000","last_clean":"2023-09-23T22:20:16.050917+0000","last_became_active":"2023-09-23T22:16:49.251097+0000","last_became_peered":"2023-09-23T22:16:49.251097+0000","last_unstale":"2023-09-23T22:20:16.050917+0000","last_undegraded":"2023-09-23T22:20:16.050917+0000","last_fullsized":"2023-09-23T22:20:16.050917+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:41.601435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":207,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.056156+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:20:16.056156+0000","last_peered":"2023-09-23T22:20:16.056156+0000","last_clean":"2023-09-23T22:20:16.056156+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:20:16.056156+0000","last_undegraded":"2023-09-23T22:20:16.056156+0000","last_fullsized":"2023-09-23T22:20:16.056156+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"0'0","reported_seq":9,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T22:20:19.470871+0000","last_change":"2023-09-23T22:20:19.470871+0000","last_active":"2023-09-23T22:20:19.470871+0000","last_peered":"2023-09-23T22:20:19.470871+0000","last_clean":"2023-09-23T22:20:19.470871+0000","last_became_active":"2023-09-23T22:20:19.470597+0000","last_became_peered":"2023-09-23T22:20:19.470597+0000","last_unstale":"2023-09-23T22:20:19.470871+0000","last_undegraded":"2023-09-23T22:20:19.470871+0000","last_fullsized":"2023-09-23T22:20:19.470871+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:32.388985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":789,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.462639+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:20:18.462639+0000","last_peered":"2023-09-23T22:20:18.462639+0000","last_clean":"2023-09-23T22:20:18.462639+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:20:18.462639+0000","last_undegraded":"2023-09-23T22:20:18.462639+0000","last_fullsized":"2023-09-23T22:20:18.462639+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":465,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.050086+0000","last_change":"2023-09-23T22:18:57.743602+0000","last_active":"2023-09-23T22:20:15.050086+0000","last_peered":"2023-09-23T22:20:15.050086+0000","last_clean":"2023-09-23T22:20:15.050086+0000","last_became_active":"2023-09-23T22:18:57.743004+0000","last_became_peered":"2023-09-23T22:18:57.743004+0000","last_unstale":"2023-09-23T22:20:15.050086+0000","last_undegraded":"2023-09-23T22:20:15.050086+0000","last_fullsized":"2023-09-23T22:20:15.050086+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":128,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065392+0000","last_change":"2023-09-23T22:16:49.155156+0000","last_active":"2023-09-23T22:20:17.065392+0000","last_peered":"2023-09-23T22:20:17.065392+0000","last_clean":"2023-09-23T22:20:17.065392+0000","last_became_active":"2023-09-23T22:16:49.154923+0000","last_became_peered":"2023-09-23T22:16:49.154923+0000","last_unstale":"2023-09-23T22:20:17.065392+0000","last_undegraded":"2023-09-23T22:20:17.065392+0000","last_fullsized":"2023-09-23T22:20:17.065392+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:33.532238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":88,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.500164+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:20:16.500164+0000","last_peered":"2023-09-23T22:20:16.500164+0000","last_clean":"2023-09-23T22:20:16.500164+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:20:16.500164+0000","last_undegraded":"2023-09-23T22:20:16.500164+0000","last_fullsized":"2023-09-23T22:20:16.500164+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:20:18.446560+0000","last_change":"2023-09-23T22:20:18.446560+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.446560+0000","last_undegraded":"2023-09-23T22:20:18.446560+0000","last_fullsized":"2023-09-23T22:20:18.446560+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":512,"reported_epoch":202,"state":"peering","last_fresh":"2023-09-23T22:20:17.058539+0000","last_change":"2023-09-23T22:20:17.058539+0000","last_active":"2023-09-23T22:20:17.058367+0000","last_peered":"2023-09-23T22:20:16.499237+0000","last_clean":"2023-09-23T22:18:49.795111+0000","last_became_active":"2023-09-23T22:20:16.498941+0000","last_became_peered":"2023-09-23T22:20:16.498941+0000","last_unstale":"2023-09-23T22:20:17.058539+0000","last_undegraded":"2023-09-23T22:20:17.058539+0000","last_fullsized":"2023-09-23T22:20:17.058539+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":550,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.051050+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:20:16.051050+0000","last_peered":"2023-09-23T22:20:16.051050+0000","last_clean":"2023-09-23T22:20:16.051050+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:20:16.051050+0000","last_undegraded":"2023-09-23T22:20:16.051050+0000","last_fullsized":"2023-09-23T22:20:16.051050+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":63,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.050503+0000","last_change":"2023-09-23T22:18:31.795915+0000","last_active":"2023-09-23T22:20:15.050503+0000","last_peered":"2023-09-23T22:20:15.050503+0000","last_clean":"2023-09-23T22:20:15.050503+0000","last_became_active":"2023-09-23T22:18:31.795608+0000","last_became_peered":"2023-09-23T22:18:31.795608+0000","last_unstale":"2023-09-23T22:20:15.050503+0000","last_undegraded":"2023-09-23T22:20:15.050503+0000","last_fullsized":"2023-09-23T22:20:15.050503+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:49.789927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":29,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.050330+0000","last_change":"2023-09-23T22:18:57.721257+0000","last_active":"2023-09-23T22:20:15.050330+0000","last_peered":"2023-09-23T22:20:15.050330+0000","last_clean":"2023-09-23T22:20:15.050330+0000","last_became_active":"2023-09-23T22:18:57.720548+0000","last_became_peered":"2023-09-23T22:18:57.720548+0000","last_unstale":"2023-09-23T22:20:15.050330+0000","last_undegraded":"2023-09-23T22:20:15.050330+0000","last_fullsized":"2023-09-23T22:20:15.050330+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:10.709024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:20:18.446560+0000","last_change":"2023-09-23T22:20:18.446560+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.446560+0000","last_undegraded":"2023-09-23T22:20:18.446560+0000","last_fullsized":"2023-09-23T22:20:18.446560+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":468,"reported_epoch":201,"state":"activating+undersized+remapped","last_fresh":"2023-09-23T22:20:16.054899+0000","last_change":"2023-09-23T22:20:16.055121+0000","last_active":"2023-09-23T22:20:14.044262+0000","last_peered":"2023-09-23T22:20:12.490080+0000","last_clean":"2023-09-23T22:20:12.490080+0000","last_became_active":"2023-09-23T22:18:57.726938+0000","last_became_peered":"2023-09-23T22:18:57.726938+0000","last_unstale":"2023-09-23T22:20:16.054899+0000","last_undegraded":"2023-09-23T22:20:16.054899+0000","last_fullsized":"2023-09-23T22:20:16.054899+0000","mapping_epoch":200,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3],"avail_no_missing":["0","1","3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":90,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.048880+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:20:15.048880+0000","last_peered":"2023-09-23T22:20:15.048880+0000","last_clean":"2023-09-23T22:20:15.048880+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:20:15.048880+0000","last_undegraded":"2023-09-23T22:20:15.048880+0000","last_fullsized":"2023-09-23T22:20:15.048880+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":142,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.057287+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:20:16.057287+0000","last_peered":"2023-09-23T22:20:16.057287+0000","last_clean":"2023-09-23T22:20:16.057287+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:20:16.057287+0000","last_undegraded":"2023-09-23T22:20:16.057287+0000","last_fullsized":"2023-09-23T22:20:16.057287+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.050331+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:20:15.050331+0000","last_peered":"2023-09-23T22:20:15.050331+0000","last_clean":"2023-09-23T22:20:15.050331+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:20:15.050331+0000","last_undegraded":"2023-09-23T22:20:15.050331+0000","last_fullsized":"2023-09-23T22:20:15.050331+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:20:18.446560+0000","last_change":"2023-09-23T22:20:18.446560+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.446560+0000","last_undegraded":"2023-09-23T22:20:18.446560+0000","last_fullsized":"2023-09-23T22:20:18.446560+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":468,"reported_epoch":202,"state":"peering","last_fresh":"2023-09-23T22:20:17.057747+0000","last_change":"2023-09-23T22:20:17.057747+0000","last_active":"2023-09-23T22:20:15.049104+0000","last_peered":"2023-09-23T22:20:12.489808+0000","last_clean":"2023-09-23T22:20:12.489808+0000","last_became_active":"2023-09-23T22:18:57.742146+0000","last_became_peered":"2023-09-23T22:18:57.742146+0000","last_unstale":"2023-09-23T22:20:17.057747+0000","last_undegraded":"2023-09-23T22:20:17.057747+0000","last_fullsized":"2023-09-23T22:20:17.057747+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":214,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065868+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:20:17.065868+0000","last_peered":"2023-09-23T22:20:17.065868+0000","last_clean":"2023-09-23T22:20:17.065868+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:20:17.065868+0000","last_undegraded":"2023-09-23T22:20:17.065868+0000","last_fullsized":"2023-09-23T22:20:17.065868+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":842,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.055801+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:20:16.055801+0000","last_peered":"2023-09-23T22:20:16.055801+0000","last_clean":"2023-09-23T22:20:16.055801+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:20:16.055801+0000","last_undegraded":"2023-09-23T22:20:16.055801+0000","last_fullsized":"2023-09-23T22:20:16.055801+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":200,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.061964+0000","last_change":"2023-09-23T22:18:57.722771+0000","last_active":"2023-09-23T22:20:17.061964+0000","last_peered":"2023-09-23T22:20:17.061964+0000","last_clean":"2023-09-23T22:20:17.061964+0000","last_became_active":"2023-09-23T22:18:57.722258+0000","last_became_peered":"2023-09-23T22:18:57.722258+0000","last_unstale":"2023-09-23T22:20:17.061964+0000","last_undegraded":"2023-09-23T22:20:17.061964+0000","last_fullsized":"2023-09-23T22:20:17.061964+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:18.267091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:20:18.446560+0000","last_change":"2023-09-23T22:20:18.446560+0000","last_active":"2023-09-23T22:20:18.446560+0000","last_peered":"2023-09-23T22:20:18.446560+0000","last_clean":"2023-09-23T22:20:18.446560+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:20:18.446560+0000","last_undegraded":"2023-09-23T22:20:18.446560+0000","last_fullsized":"2023-09-23T22:20:18.446560+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":481,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.470887+0000","last_change":"2023-09-23T22:20:18.468192+0000","last_active":"2023-09-23T22:20:18.470887+0000","last_peered":"2023-09-23T22:20:18.470887+0000","last_clean":"2023-09-23T22:20:18.470887+0000","last_became_active":"2023-09-23T22:20:18.467795+0000","last_became_peered":"2023-09-23T22:20:18.467795+0000","last_unstale":"2023-09-23T22:20:18.470887+0000","last_undegraded":"2023-09-23T22:20:18.470887+0000","last_fullsized":"2023-09-23T22:20:18.470887+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":816370,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":264,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T22:20:14.053310+0000","last_change":"2023-09-23T22:16:49.270015+0000","last_active":"2023-09-23T22:20:14.053310+0000","last_peered":"2023-09-23T22:20:14.053310+0000","last_clean":"2023-09-23T22:20:14.053310+0000","last_became_active":"2023-09-23T22:16:49.252726+0000","last_became_peered":"2023-09-23T22:16:49.252726+0000","last_unstale":"2023-09-23T22:20:14.053310+0000","last_undegraded":"2023-09-23T22:20:14.053310+0000","last_fullsized":"2023-09-23T22:20:14.053310+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:18.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":33,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.050997+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:20:15.050997+0000","last_peered":"2023-09-23T22:20:15.050997+0000","last_clean":"2023-09-23T22:20:15.050997+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:20:15.050997+0000","last_undegraded":"2023-09-23T22:20:15.050997+0000","last_fullsized":"2023-09-23T22:20:15.050997+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":165,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499591+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:20:16.499591+0000","last_peered":"2023-09-23T22:20:16.499591+0000","last_clean":"2023-09-23T22:20:16.499591+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:20:16.499591+0000","last_undegraded":"2023-09-23T22:20:16.499591+0000","last_fullsized":"2023-09-23T22:20:16.499591+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":29,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.050246+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:20:15.050246+0000","last_peered":"2023-09-23T22:20:15.050246+0000","last_clean":"2023-09-23T22:20:15.050246+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:20:15.050246+0000","last_undegraded":"2023-09-23T22:20:15.050246+0000","last_fullsized":"2023-09-23T22:20:15.050246+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":184,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T22:20:14.052289+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:20:14.052289+0000","last_peered":"2023-09-23T22:20:14.052289+0000","last_clean":"2023-09-23T22:20:14.052289+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:20:14.052289+0000","last_undegraded":"2023-09-23T22:20:14.052289+0000","last_fullsized":"2023-09-23T22:20:14.052289+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":166,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.461934+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:20:18.461934+0000","last_peered":"2023-09-23T22:20:18.461934+0000","last_clean":"2023-09-23T22:20:18.461934+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:20:18.461934+0000","last_undegraded":"2023-09-23T22:20:18.461934+0000","last_fullsized":"2023-09-23T22:20:18.461934+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":177,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499872+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:20:16.499872+0000","last_peered":"2023-09-23T22:20:16.499872+0000","last_clean":"2023-09-23T22:20:16.499872+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:20:16.499872+0000","last_undegraded":"2023-09-23T22:20:16.499872+0000","last_fullsized":"2023-09-23T22:20:16.499872+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":163,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T22:20:18.461665+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:20:18.461665+0000","last_peered":"2023-09-23T22:20:18.461665+0000","last_clean":"2023-09-23T22:20:18.461665+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:20:18.461665+0000","last_undegraded":"2023-09-23T22:20:18.461665+0000","last_fullsized":"2023-09-23T22:20:18.461665+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":36,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499775+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:20:16.499775+0000","last_peered":"2023-09-23T22:20:16.499775+0000","last_clean":"2023-09-23T22:20:16.499775+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:20:16.499775+0000","last_undegraded":"2023-09-23T22:20:16.499775+0000","last_fullsized":"2023-09-23T22:20:16.499775+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":184,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.066192+0000","last_change":"2023-09-23T22:16:49.145417+0000","last_active":"2023-09-23T22:20:17.066192+0000","last_peered":"2023-09-23T22:20:17.066192+0000","last_clean":"2023-09-23T22:20:17.066192+0000","last_became_active":"2023-09-23T22:16:49.145017+0000","last_became_peered":"2023-09-23T22:16:49.145017+0000","last_unstale":"2023-09-23T22:20:17.066192+0000","last_undegraded":"2023-09-23T22:20:17.066192+0000","last_fullsized":"2023-09-23T22:20:17.066192+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:47.762149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":225,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T22:20:17.065343+0000","last_change":"2023-09-23T22:17:15.721123+0000","last_active":"2023-09-23T22:20:17.065343+0000","last_peered":"2023-09-23T22:20:17.065343+0000","last_clean":"2023-09-23T22:20:17.065343+0000","last_became_active":"2023-09-23T22:17:15.720812+0000","last_became_peered":"2023-09-23T22:17:15.720812+0000","last_unstale":"2023-09-23T22:20:17.065343+0000","last_undegraded":"2023-09-23T22:20:17.065343+0000","last_fullsized":"2023-09-23T22:20:17.065343+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.176278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":73,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T22:20:15.054458+0000","last_change":"2023-09-23T22:18:57.729455+0000","last_active":"2023-09-23T22:20:15.054458+0000","last_peered":"2023-09-23T22:20:15.054458+0000","last_clean":"2023-09-23T22:20:15.054458+0000","last_became_active":"2023-09-23T22:18:57.729088+0000","last_became_peered":"2023-09-23T22:18:57.729088+0000","last_unstale":"2023-09-23T22:20:15.054458+0000","last_undegraded":"2023-09-23T22:20:15.054458+0000","last_fullsized":"2023-09-23T22:20:15.054458+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:17.861334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T22:20:16.499489+0000","last_change":"2023-09-23T22:16:49.164638+0000","last_active":"2023-09-23T22:20:16.499489+0000","last_peered":"2023-09-23T22:20:16.499489+0000","last_clean":"2023-09-23T22:20:16.499489+0000","last_became_active":"2023-09-23T22:16:49.164349+0000","last_became_peered":"2023-09-23T22:16:49.164349+0000","last_unstale":"2023-09-23T22:20:16.499489+0000","last_undegraded":"2023-09-23T22:20:16.499489+0000","last_fullsized":"2023-09-23T22:20:16.499489+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:26.536927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":21,"acting":21,"num_store_stats":3},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4892,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2951840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2703360,"data_stored":2689152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":31,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234191,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":221132,"kb_used_data":135364,"kb_used_omap":0,"kb_used_meta":85760,"kb_avail":93524020,"statfs":{"total":95995035648,"available":95768596480,"internally_reserved":0,"allocated":138612736,"data_stored":137768224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":87818240},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":2.4780000000000002,"15min":1.1970000000000001},"min":{"1min":0.23999999999999999,"5min":0.223,"15min":0.223},"max":{"1min":9.7799999999999994,"5min":109.208,"15min":109.208},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.752,"5min":2.7650000000000001,"15min":1.3149999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":7.4489999999999998,"5min":109.13800000000001,"15min":109.13800000000001},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:20:09 2023","interfaces":[{"interface":"back","average":{"1min":9.1620000000000008,"5min":5.9320000000000004,"15min":2.3570000000000002},"min":{"1min":0.25600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":101.145,"5min":123.79300000000001,"15min":123.79300000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":9.4949999999999992,"5min":6.2809999999999997,"15min":2.508},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":101.13,"5min":123.66500000000001,"15min":123.66500000000001},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":13.066000000000001,"5min":8.5730000000000004,"15min":3.5289999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":105.34399999999999,"5min":125.101,"15min":125.101},"last":0.58399999999999996},{"interface":"front","average":{"1min":12.569000000000001,"5min":8.2129999999999992,"15min":3.1469999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":105.63200000000001,"5min":126.021,"15min":126.021},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:20:09 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":2.5619999999999998,"15min":1.2549999999999999},"min":{"1min":0.308,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8319999999999999,"5min":113.282,"15min":113.282},"last":0.505},{"interface":"front","average":{"1min":1.2370000000000001,"5min":2.931,"15min":1.4239999999999999},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.7910000000000004,"5min":104.74299999999999,"15min":104.74299999999999},"last":0.51700000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":6.8159999999999998,"5min":2.5489999999999999,"15min":1.173},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":88.543000000000006,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.49099999999999999},{"interface":"front","average":{"1min":6.3959999999999999,"5min":2.286,"15min":1.095},"min":{"1min":0.312,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":88.495999999999995,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":6.9299999999999997,"5min":2.2909999999999999,"15min":1.0860000000000001},"min":{"1min":0.27500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":93.816000000000003,"5min":93.816000000000003,"15min":93.816000000000003},"last":0.55500000000000005},{"interface":"front","average":{"1min":7.0010000000000003,"5min":2.8149999999999999,"15min":1.3129999999999999},"min":{"1min":0.34399999999999997,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":95.579999999999998,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.56499999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.97399999999999998,"15min":0.67700000000000005},"min":{"1min":0.317,"5min":0.317,"15min":0.315},"max":{"1min":0.66600000000000004,"5min":9.0139999999999993,"15min":9.0139999999999993},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.871,"5min":1.3939999999999999,"15min":0.86199999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":3.7869999999999999,"5min":13.362,"15min":13.362},"last":0.54700000000000004}]}]},{"osd":6,"up_from":0,"seq":197568495827,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795099,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":165140,"kb_used_data":7116,"kb_used_omap":7,"kb_used_meta":158008,"kb_avail":93580012,"statfs":{"total":95995035648,"available":95825932288,"internally_reserved":0,"allocated":7286784,"data_stored":6193668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":161800564},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.9299999999999999,"15min":0.88},"min":{"1min":0.21199999999999999,"5min":0.20399999999999999,"15min":0.19400000000000001},"max":{"1min":5.8840000000000003,"5min":61.890000000000001,"15min":61.890000000000001},"last":0.373},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.8140000000000001,"15min":1.2050000000000001},"min":{"1min":0.216,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":9.2669999999999995,"5min":62.689999999999998,"15min":62.689999999999998},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":2.4079999999999999,"15min":1.0549999999999999},"min":{"1min":0.223,"5min":0.216,"15min":0.216},"max":{"1min":5.2539999999999996,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":4.0620000000000003,"5min":3.032,"15min":1.2849999999999999},"min":{"1min":0.309,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.015000000000001,"5min":34.781999999999996,"15min":34.781999999999996},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:20:14 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.4580000000000002,"15min":1.0840000000000001},"min":{"1min":0.35899999999999999,"5min":0.252,"15min":0.252},"max":{"1min":6.4199999999999999,"5min":22.326000000000001,"15min":22.326000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.2229999999999999,"5min":2.3239999999999998,"15min":1.048},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.23000000000000001},"max":{"1min":11.005000000000001,"5min":22.259,"15min":22.259},"last":0.41999999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:20:14 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.0820000000000001,"15min":0.80300000000000005},"min":{"1min":0.29899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.4669999999999996,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.438},{"interface":"front","average":{"1min":1.5489999999999999,"5min":1.702,"15min":1.008},"min":{"1min":0.47999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":6.0720000000000001,"5min":11.037000000000001,"15min":11.037000000000001},"last":0.45600000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.7350000000000001,"15min":0.98399999999999999},"min":{"1min":0.442,"5min":0.31,"15min":0.31},"max":{"1min":6.0410000000000004,"5min":11.125,"15min":11.125},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.5980000000000001,"15min":0.95399999999999996},"min":{"1min":0.33800000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.872999999999999,"5min":10.872999999999999,"15min":10.872999999999999},"last":0.53300000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.6779999999999999,"15min":0.98099999999999998},"min":{"1min":0.373,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.971,"5min":10.989000000000001,"15min":10.989000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.234,"5min":1.3700000000000001,"15min":0.90200000000000002},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.34999999999999998},"max":{"1min":5.4189999999999996,"5min":10.811999999999999,"15min":10.811999999999999},"last":0.50600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.804,"15min":1.0269999999999999},"min":{"1min":0.33800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.933999999999999,"5min":10.948,"15min":10.948},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.4319999999999999,"15min":0.92600000000000005},"min":{"1min":0.38600000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.4930000000000003,"5min":10.781000000000001,"15min":10.781000000000001},"last":0.56299999999999994}]}]},{"osd":0,"up_from":93,"seq":399431958633,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":195452,"kb_used_data":7092,"kb_used_omap":8,"kb_used_meta":188343,"kb_avail":93549700,"statfs":{"total":95995035648,"available":95794892800,"internally_reserved":0,"allocated":7262208,"data_stored":6176575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":192864106},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":2.5750000000000002,"15min":1.0860000000000001},"min":{"1min":0.20399999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.9710000000000001,"5min":44.776000000000003,"15min":44.776000000000003},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.5289999999999999,"5min":3.5259999999999998,"15min":1.4199999999999999},"min":{"1min":0.23799999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.641999999999999,"5min":48.838000000000001,"15min":48.838000000000001},"last":0.73299999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:19:41 2023","interfaces":[{"interface":"back","average":{"1min":3.2090000000000001,"5min":2.0630000000000002,"15min":0.95599999999999996},"min":{"1min":0.28399999999999997,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":13.625999999999999,"5min":25.893000000000001,"15min":25.893000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":4.2910000000000004,"5min":3.5350000000000001,"15min":1.526},"min":{"1min":0.36199999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.16,"5min":66.665000000000006,"15min":66.665000000000006},"last":0.42799999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.5659999999999998,"15min":1.1519999999999999},"min":{"1min":0.27300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.1269999999999998,"5min":35.496000000000002,"15min":35.496000000000002},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.722,"5min":2.9350000000000001,"15min":1.2709999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.8630000000000004,"5min":35.533000000000001,"15min":35.533000000000001},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.6389999999999998,"15min":1.319},"min":{"1min":0.34200000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.032999999999999,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.274,"15min":0.86199999999999999},"min":{"1min":0.313,"5min":0.308,"15min":0.308},"max":{"1min":4.0270000000000001,"5min":22.209,"15min":22.209},"last":0.66900000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":2.609,"15min":1.3300000000000001},"min":{"1min":0.34799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9920000000000009,"5min":12.372,"15min":12.372},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.73899999999999999,"5min":1.2909999999999999,"15min":0.88500000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.0030000000000001,"5min":20.390999999999998,"15min":20.390999999999998},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.6680000000000001,"15min":1.3440000000000001},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.379},"max":{"1min":10.087,"5min":10.275,"15min":10.275},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.133,"5min":1.401,"15min":0.93700000000000006},"min":{"1min":0.34999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.4039999999999999,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.62}]},{"osd":7,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":2.524,"15min":1.329},"min":{"1min":0.43099999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":4.1369999999999996,"5min":23.306000000000001,"15min":23.306000000000001},"last":0.77600000000000002},{"interface":"front","average":{"1min":3.0019999999999998,"5min":3.0840000000000001,"15min":1.5289999999999999},"min":{"1min":0.47199999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.061,"5min":23.265999999999998,"15min":23.265999999999998},"last":0.67800000000000005}]}]},{"osd":2,"up_from":103,"seq":442381631572,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":182500,"kb_used_data":2396,"kb_used_omap":10,"kb_used_meta":180085,"kb_avail":93562652,"statfs":{"total":95995035648,"available":95808155648,"internally_reserved":0,"allocated":2453504,"data_stored":1391892,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":184407212},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":3.1629999999999998,"5min":2.5030000000000001,"15min":1.0569999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":19.344000000000001,"5min":19.344000000000001,"15min":19.344000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.5899999999999999,"5min":2.7210000000000001,"15min":1.149},"min":{"1min":0.25700000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":9.4779999999999998,"5min":19.149999999999999,"15min":19.149999999999999},"last":0.38500000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":3.8220000000000001,"5min":4.6289999999999996,"15min":1.825},"min":{"1min":0.31900000000000001,"5min":0.222,"15min":0.222},"max":{"1min":16.141999999999999,"5min":157.636,"15min":157.636},"last":0.32800000000000001},{"interface":"front","average":{"1min":3.7509999999999999,"5min":4.7030000000000003,"15min":1.847},"min":{"1min":0.29499999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":20.783999999999999,"5min":155.63300000000001,"15min":155.63300000000001},"last":0.54000000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:20:06 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":2.5950000000000002,"15min":1.0920000000000001},"min":{"1min":0.27100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.9429999999999996,"5min":19.852,"15min":19.852},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.8730000000000002,"15min":1.216},"min":{"1min":0.28100000000000003,"5min":0.26000000000000001,"15min":0.246},"max":{"1min":20.234999999999999,"5min":20.581,"15min":20.581},"last":0.41399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.9019999999999999,"5min":1.79,"15min":0.995},"min":{"1min":0.34200000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.3689999999999998,"5min":11.244,"15min":11.244},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.1320000000000001,"5min":1.845,"15min":1.0329999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8659999999999997,"5min":9.2799999999999994,"15min":9.2799999999999994},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":1.746,"5min":1.893,"15min":1.054},"min":{"1min":0.311,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.3219999999999992,"5min":11.291,"15min":11.291},"last":0.60699999999999998},{"interface":"front","average":{"1min":2.3610000000000002,"5min":1.968,"15min":1.0600000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.915,"5min":9.4499999999999993,"15min":9.4499999999999993},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1400000000000001,"5min":2.008,"15min":1.1120000000000001},"min":{"1min":0.36699999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.4280000000000008,"5min":17.640999999999998,"15min":17.640999999999998},"last":0.64800000000000002},{"interface":"front","average":{"1min":2.2559999999999998,"5min":1.8109999999999999,"15min":1.127},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.891,"5min":11.875,"15min":11.875},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:19:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":2.0019999999999998,"15min":1.1739999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":7.8019999999999996,"5min":10.135,"15min":10.135},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.3610000000000002,"5min":2.2719999999999998,"15min":1.1819999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.377},"max":{"1min":9.4529999999999994,"5min":17.738,"15min":17.738},"last":0.72099999999999997}]}]},{"osd":3,"up_from":108,"seq":463856468046,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":172640,"kb_used_data":78744,"kb_used_omap":4,"kb_used_meta":93883,"kb_avail":93572512,"statfs":{"total":95995035648,"available":95818252288,"internally_reserved":0,"allocated":80633856,"data_stored":79533884,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":96136749},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":4.8399999999999999,"5min":3.6309999999999998,"15min":1.462},"min":{"1min":0.23000000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":48.676000000000002,"5min":77.834999999999994,"15min":77.834999999999994},"last":0.35699999999999998},{"interface":"front","average":{"1min":4.8970000000000002,"5min":4.0090000000000003,"15min":1.577},"min":{"1min":0.26600000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":48.774000000000001,"5min":75.558000000000007,"15min":75.558000000000007},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6779999999999999,"5min":4.2030000000000003,"15min":1.6359999999999999},"min":{"1min":0.24099999999999999,"5min":0.218,"15min":0.218},"max":{"1min":12.445,"5min":82.951999999999998,"15min":82.951999999999998},"last":0.377},{"interface":"front","average":{"1min":2.4119999999999999,"5min":4.0019999999999998,"15min":1.6060000000000001},"min":{"1min":0.22800000000000001,"5min":0.189,"15min":0.189},"max":{"1min":11.970000000000001,"5min":86.001000000000005,"15min":86.001000000000005},"last":0.29699999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":8.2799999999999994,"15min":3.0219999999999998},"min":{"1min":0.27700000000000002,"5min":0.223,"15min":0.223},"max":{"1min":11.708,"5min":99.962999999999994,"15min":99.962999999999994},"last":0.312},{"interface":"front","average":{"1min":3.238,"5min":7.7300000000000004,"15min":2.8599999999999999},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":11.994,"5min":100.429,"15min":100.429},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:20:14 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":1.6020000000000001,"15min":0.95099999999999996},"min":{"1min":0.29699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.911,"5min":11.911,"15min":11.911},"last":0.434},{"interface":"front","average":{"1min":1.881,"5min":1.708,"15min":1.006},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.720000000000001,"5min":12.036,"15min":12.036},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":4.4249999999999998,"5min":2.157,"15min":1.135},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":49.695999999999998,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.48099999999999998},{"interface":"front","average":{"1min":4.7279999999999998,"5min":2.3500000000000001,"15min":1.1899999999999999},"min":{"1min":0.41599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":49.668999999999997,"5min":49.668999999999997,"15min":49.668999999999997},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.841,"15min":1.0489999999999999},"min":{"1min":0.441,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.760999999999999,"5min":12.068,"15min":12.068},"last":0.50900000000000001},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.8140000000000001,"15min":1.028},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":0.58099999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":1.9039999999999999,"15min":1.0549999999999999},"min":{"1min":0.39600000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":11.596,"5min":11.624000000000001,"15min":11.624000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.419,"5min":1.752,"15min":1.04},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":11.945,"5min":12.382,"15min":12.382},"last":0.46500000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019097,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":125876,"kb_used_data":2156,"kb_used_omap":0,"kb_used_meta":123712,"kb_avail":93619276,"statfs":{"total":95995035648,"available":95866138624,"internally_reserved":0,"allocated":2207744,"data_stored":1364188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":126681088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":2.488,"15min":1.246},"min":{"1min":0.30099999999999999,"5min":0.218,"15min":0.218},"max":{"1min":27.942,"5min":66.355000000000004,"15min":66.355000000000004},"last":0.629},{"interface":"front","average":{"1min":3.1339999999999999,"5min":2.7210000000000001,"15min":1.3360000000000001},"min":{"1min":0.315,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":27.956,"5min":66.332999999999998,"15min":66.332999999999998},"last":0.57999999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7570000000000001,"5min":2.1819999999999999,"15min":1.0940000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":19.053000000000001,"5min":38.878,"15min":38.878},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.024,"5min":2.9100000000000001,"15min":1.3540000000000001},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":19.170999999999999,"5min":38.853999999999999,"15min":38.853999999999999},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:19:41 2023","interfaces":[{"interface":"back","average":{"1min":9.6750000000000007,"5min":4.7050000000000001,"15min":1.929},"min":{"1min":0.32400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":138.06100000000001,"5min":138.06100000000001,"15min":138.06100000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":10.814,"5min":5.5119999999999996,"15min":2.1779999999999999},"min":{"1min":0.309,"5min":0.308,"15min":0.308},"max":{"1min":138.15199999999999,"5min":138.15199999999999,"15min":138.15199999999999},"last":0.65400000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.7989999999999999,"15min":0.995},"min":{"1min":0.33700000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6340000000000003,"5min":8.6340000000000003,"15min":8.6340000000000003},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.476,"5min":1.7869999999999999,"15min":0.97899999999999998},"min":{"1min":0.39500000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.1529999999999996,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.077,"15min":0.68200000000000005},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":6.6989999999999998,"5min":8.9499999999999993,"15min":8.9499999999999993},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.92100000000000004,"15min":0.66300000000000003},"min":{"1min":0.313,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.6179999999999999,"5min":7.8789999999999996,"15min":7.8789999999999996},"last":0.53900000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:19:26 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.82399999999999995,"15min":0.61799999999999999},"min":{"1min":0.35899999999999999,"5min":0.251,"15min":0.251},"max":{"1min":3.754,"5min":3.754,"15min":3.754},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.0760000000000001,"5min":0.94699999999999995,"15min":0.68000000000000005},"min":{"1min":0.34399999999999997,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":6.7279999999999998,"5min":8.5210000000000008,"15min":8.5210000000000008},"last":0.47299999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.228,"15min":0.79200000000000004},"min":{"1min":0.36299999999999999,"5min":0.307,"15min":0.307},"max":{"1min":6.7480000000000002,"5min":8.577,"15min":8.577},"last":0.442},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.95999999999999996,"15min":0.71699999999999997},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2679999999999998,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.55800000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757464,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":325572,"kb_used_data":199804,"kb_used_omap":0,"kb_used_meta":125760,"kb_avail":93419580,"statfs":{"total":95995035648,"available":95661649920,"internally_reserved":0,"allocated":204599296,"data_stored":203759536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":128778240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8439999999999999,"5min":3.7749999999999999,"15min":1.6319999999999999},"min":{"1min":0.23000000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":28.741,"5min":133.22499999999999,"15min":133.22499999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":3.3719999999999999,"5min":3.9609999999999999,"15min":1.7350000000000001},"min":{"1min":0.27100000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":28.873999999999999,"5min":132.78200000000001,"15min":132.78200000000001},"last":0.59899999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":4.8230000000000004,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.8120000000000003,"5min":83.759,"15min":83.759},"last":0.624},{"interface":"front","average":{"1min":1.3560000000000001,"5min":4.7699999999999996,"15min":1.9710000000000001},"min":{"1min":0.28399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":4.5629999999999997,"5min":83.741,"15min":83.741},"last":0.61599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:19:40 2023","interfaces":[{"interface":"back","average":{"1min":8.5419999999999998,"5min":7.6210000000000004,"15min":2.915},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":78.137,"5min":104.254,"15min":104.254},"last":0.495},{"interface":"front","average":{"1min":10.172000000000001,"5min":8.5760000000000005,"15min":3.2530000000000001},"min":{"1min":0.34000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":78.061000000000007,"5min":104.217,"15min":104.217},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":2.964,"15min":1.3740000000000001},"min":{"1min":0.37,"5min":0.246,"15min":0.246},"max":{"1min":9.9459999999999997,"5min":88.106999999999999,"15min":88.106999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.149,"5min":3.173,"15min":1.45},"min":{"1min":0.40699999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.7350000000000003,"5min":87.887,"15min":87.887},"last":0.64400000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.645,"15min":0.87},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.3399999999999999,"5min":82.530000000000001,"15min":82.530000000000001},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.78,"15min":0.94799999999999995},"min":{"1min":0.29399999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.4580000000000002,"5min":85.441000000000003,"15min":85.441000000000003},"last":0.40500000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:19:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":2.137,"15min":1.0669999999999999},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5350000000000001,"5min":114.83499999999999,"15min":114.83499999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.9710000000000001,"15min":1.0209999999999999},"min":{"1min":0.36299999999999999,"5min":0.31,"15min":0.31},"max":{"1min":4.5220000000000002,"5min":110.905,"15min":110.905},"last":0.434}]},{"osd":7,"last update":"Sat Sep 23 22:19:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.2969999999999999,"15min":0.81200000000000006},"min":{"1min":0.34999999999999998,"5min":0.33500000000000002,"15min":0.32600000000000001},"max":{"1min":4.3760000000000003,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.97199999999999998,"15min":0.70199999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33200000000000002},"max":{"1min":2.379,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":0.44900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:20:22.929 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:20:22.999 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:23.026 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:23.121 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:23.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:23.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:23.227 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:23.260 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:23.458 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:23.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:23.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:23.563 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:23.591 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:23.888 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:23.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:23.992 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:24.014 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:24.018 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:24.162 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:24.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:24.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:24.234 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:24.262 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:24.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:24.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:24.476 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:24.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:24.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:24.580 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:24.609 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:24.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[223014]: pgmap v471: 99 pgs: 4 creating+peering, 2 unknown, 1 peering, 92 active+clean; 4.6 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:20:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/772116789' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:20:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[215775]: pgmap v471: 99 pgs: 4 creating+peering, 2 unknown, 1 peering, 92 active+clean; 4.6 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:20:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/772116789' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:20:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:25.028 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:20:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:24 smithi070 ceph-mon[192652]: pgmap v471: 99 pgs: 4 creating+peering, 2 unknown, 1 peering, 92 active+clean; 4.6 MiB data, 1.3 GiB used, 624 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T22:20:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/772116789' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:20:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:25 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:20:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/772116789' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:20:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:25 smithi070 ceph-mon[192652]: osdmap e206: 8 total, 8 up, 7 in 2023-09-23T22:20:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:25 smithi070 ceph-mon[192652]: failed to encode map e206 with expected crc 2023-09-23T22:20:26.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:25 smithi070 ceph-mon[192652]: failed to encode map e206 with expected crc 2023-09-23T22:20:26.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:20:26.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/772116789' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:20:26.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[215775]: osdmap e206: 8 total, 8 up, 7 in 2023-09-23T22:20:26.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[215775]: failed to encode map e206 with expected crc 2023-09-23T22:20:26.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[215775]: failed to encode map e206 with expected crc 2023-09-23T22:20:26.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:20:26.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/772116789' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:20:26.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[223014]: osdmap e206: 8 total, 8 up, 7 in 2023-09-23T22:20:26.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[223014]: failed to encode map e206 with expected crc 2023-09-23T22:20:26.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:25 smithi026 ceph-mon[223014]: failed to encode map e206 with expected crc 2023-09-23T22:20:26.769 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:26.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:26.874 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:26.910 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:26.958 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:26.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:26.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:27.064 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:27.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:26 smithi026 ceph-mon[215775]: pgmap v473: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-09-23T22:20:27.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:26 smithi026 ceph-mon[215775]: failed to encode map e206 with expected crc 2023-09-23T22:20:27.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:26 smithi026 ceph-mon[215775]: failed to encode map e206 with expected crc 2023-09-23T22:20:27.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:26 smithi026 ceph-mon[223014]: pgmap v473: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-09-23T22:20:27.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:26 smithi026 ceph-mon[223014]: failed to encode map e206 with expected crc 2023-09-23T22:20:27.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:26 smithi026 ceph-mon[223014]: failed to encode map e206 with expected crc 2023-09-23T22:20:27.101 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:26 smithi070 ceph-mon[192652]: pgmap v473: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-09-23T22:20:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:26 smithi070 ceph-mon[192652]: failed to encode map e206 with expected crc 2023-09-23T22:20:27.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:26 smithi070 ceph-mon[192652]: failed to encode map e206 with expected crc 2023-09-23T22:20:27.411 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:27.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:27.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:27.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:27.548 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:27.596 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:27.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:27.703 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:27.735 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:27.805 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T22:20:27.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:27 smithi070 ceph-mon[192652]: failed to encode map e206 with expected crc 2023-09-23T22:20:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2504923872' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:20:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/359763280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:27.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:27 smithi026 ceph-mon[223014]: failed to encode map e206 with expected crc 2023-09-23T22:20:27.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2504923872' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:20:27.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/359763280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:27.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:27 smithi026 ceph-mon[215775]: failed to encode map e206 with expected crc 2023-09-23T22:20:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2504923872' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:20:27.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/359763280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:28.053 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:20:28.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:20:28.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:20:28.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:20:28.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:20:28.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:20:28.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:20:28.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:20:28.054 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:20:28.054 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:20:28.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:20:28.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:20:28.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:20:28.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:20:28.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:20:28.055 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:20:28.645 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:20:28.646 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:20:28.646 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:20:28.646 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:20:28.787 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:28.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:28.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:28.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:28.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[215775]: pgmap v474: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 33 KiB/s, 0 objects/s recovering 2023-09-23T22:20:28.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[215775]: failed to encode map e207 with expected crc 2023-09-23T22:20:28.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2504923872' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:20:28.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[215775]: osdmap e207: 8 total, 8 up, 7 in 2023-09-23T22:20:28.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[215775]: failed to encode map e207 with expected crc 2023-09-23T22:20:28.917 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[215775]: failed to encode map e207 with expected crc 2023-09-23T22:20:28.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[223014]: pgmap v474: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 33 KiB/s, 0 objects/s recovering 2023-09-23T22:20:28.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[223014]: failed to encode map e207 with expected crc 2023-09-23T22:20:28.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2504923872' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:20:28.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[223014]: osdmap e207: 8 total, 8 up, 7 in 2023-09-23T22:20:28.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[223014]: failed to encode map e207 with expected crc 2023-09-23T22:20:28.918 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:28 smithi026 ceph-mon[223014]: failed to encode map e207 with expected crc 2023-09-23T22:20:28.921 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:29.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:28 smithi070 ceph-mon[192652]: pgmap v474: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail; 33 KiB/s, 0 objects/s recovering 2023-09-23T22:20:29.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:28 smithi070 ceph-mon[192652]: failed to encode map e207 with expected crc 2023-09-23T22:20:29.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:28 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2504923872' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:20:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:28 smithi070 ceph-mon[192652]: osdmap e207: 8 total, 8 up, 7 in 2023-09-23T22:20:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:28 smithi070 ceph-mon[192652]: failed to encode map e207 with expected crc 2023-09-23T22:20:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:28 smithi070 ceph-mon[192652]: failed to encode map e207 with expected crc 2023-09-23T22:20:29.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:20:29.871 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:20:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:20:29] "GET /metrics HTTP/1.1" 200 36315 "" "Prometheus/2.43.0" 2023-09-23T22:20:29.880 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:29.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:29.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:29.985 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:30.016 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:30.066 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:30.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:30.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:30.173 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:30.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:30.244 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:30.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:30.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:30.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:30.365 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:20:30.365 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":614,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":207,"num_osds":8,"num_up_osds":8,"osd_up_since":1695507239,"num_in_osds":7,"osd_in_since":1695507408,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":99}],"num_pgs":99,"num_pools":5,"num_objects":28,"data_bytes":4790195,"bytes_used":1372131328,"bytes_avail":670593118208,"bytes_total":671965249536},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T22:14:59.012277+0000","services":{}},"progress_events":{"2fcb4d05-59c9-4a0a-bf68-bf36661520e1":{"message":"Global Recovery Event (4m)\n [==========================..] (remaining: 22s)","progress":0.92929291725158691,"add_to_ceph_s":true},"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:20:30.378 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:30.379 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:20:30.419 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:30.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:30.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:30.524 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:30.552 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:30.949 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:30.952 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:30.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:30.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:30.978 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T22:20:30.978 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:20:31.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:30 smithi026 ceph-mon[223014]: pgmap v476: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-09-23T22:20:31.044 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/159390314' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:20:31.044 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:30 smithi026 ceph-mon[215775]: pgmap v476: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-09-23T22:20:31.044 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/159390314' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:20:31.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:31.087 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:31.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:30 smithi070 ceph-mon[192652]: pgmap v476: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-09-23T22:20:31.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/159390314' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:20:31.197 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:31.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:31.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:31.302 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:31.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:31.333 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T22:20:31.335 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:20:31.335 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:20:31.379 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:31.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:31.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:31.384 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T22:20:31.384 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:20:31.384 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_355573 2023-09-23T22:20:31.385 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:20:31.385 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:20:31.405 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:31.484 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:31.515 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:31.569 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:31.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:31.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:31.675 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:31.702 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:31.871 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:20:32.090 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:32.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:32.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:32.195 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:32.357 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:32.384 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 2050 2034 127.118 127.125 0.00412199 0.00783098 2023-09-23T22:20:32.569 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:32.739 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:32.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:32.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:32.847 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:32.883 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:32.943 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:32.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:32.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:32.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:32 smithi026 ceph-mon[223014]: pgmap v477: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-09-23T22:20:32.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:32 smithi026 ceph-mon[215775]: pgmap v477: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-09-23T22:20:33.049 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:33.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:32 smithi070 ceph-mon[192652]: pgmap v477: 99 pgs: 99 active+clean; 4.6 MiB data, 1.3 GiB used, 625 GiB / 626 GiB avail 2023-09-23T22:20:33.083 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:33.149 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:33.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:33.256 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:33.321 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:33.384 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 4975 4959 154.958 182.812 0.00355013 0.00643836 2023-09-23T22:20:34.384 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 7975 7959 165.801 187.5 0.00417846 0.00602219 2023-09-23T22:20:34.637 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:34.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:34.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:34.749 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:34.841 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:35.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:34 smithi026 ceph-mon[215775]: pgmap v478: 99 pgs: 99 active+clean; 21 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 2.1 MiB/s wr, 33 op/s 2023-09-23T22:20:35.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:34 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:35.042 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:34 smithi026 ceph-mon[223014]: pgmap v478: 99 pgs: 99 active+clean; 21 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 2.1 MiB/s wr, 33 op/s 2023-09-23T22:20:35.042 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:34 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:35.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:34 smithi070 ceph-mon[192652]: pgmap v478: 99 pgs: 99 active+clean; 21 MiB data, 1.2 GiB used, 625 GiB / 626 GiB avail; 2.1 MiB/s wr, 33 op/s 2023-09-23T22:20:35.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:34 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:35.158 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:35.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:35.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:35.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:35.330 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:35.384 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 10030 10014 156.443 128.438 0.00636651 0.00638465 2023-09-23T22:20:35.716 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:20:35.717 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:20:35.717 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:20:35.717 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:20:35.717 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:20:35.764 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:35.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:35.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:35.813 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:20:35.869 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:35.906 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:35.969 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:35.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:35.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1829480961' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:20:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:35 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:20:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1829480961' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:20:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:35 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:20:36.076 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:36.106 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1829480961' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:20:36.106 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:35 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:20:36.109 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:36.167 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:36.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:36.173 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:36.273 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:36.332 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:36.385 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 12431 12415 155.164 150.062 0.00499154 0.00644037 2023-09-23T22:20:37.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:36 smithi070 ceph-mon[192652]: pgmap v479: 99 pgs: 99 active+clean; 162 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 16 MiB/s wr, 251 op/s 2023-09-23T22:20:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:36 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:20:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:36 smithi070 ceph-mon[192652]: failed to encode map e208 with expected crc 2023-09-23T22:20:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:36 smithi070 ceph-mon[192652]: failed to encode map e208 with expected crc 2023-09-23T22:20:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:20:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:36 smithi070 ceph-mon[192652]: osdmap e208: 8 total, 8 up, 7 in 2023-09-23T22:20:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[215775]: pgmap v479: 99 pgs: 99 active+clean; 162 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 16 MiB/s wr, 251 op/s 2023-09-23T22:20:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:20:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[215775]: failed to encode map e208 with expected crc 2023-09-23T22:20:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[215775]: failed to encode map e208 with expected crc 2023-09-23T22:20:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:20:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[215775]: osdmap e208: 8 total, 8 up, 7 in 2023-09-23T22:20:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[223014]: pgmap v479: 99 pgs: 99 active+clean; 162 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 16 MiB/s wr, 251 op/s 2023-09-23T22:20:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:20:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[223014]: failed to encode map e208 with expected crc 2023-09-23T22:20:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[223014]: failed to encode map e208 with expected crc 2023-09-23T22:20:37.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:20:37.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:36 smithi026 ceph-mon[223014]: osdmap e208: 8 total, 8 up, 7 in 2023-09-23T22:20:37.384 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 14960 14944 155.645 158.062 0.00620207 0.00641654 2023-09-23T22:20:38.123 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:37 smithi070 ceph-mon[192652]: 1.a scrub starts 2023-09-23T22:20:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:37 smithi070 ceph-mon[192652]: 1.a scrub ok 2023-09-23T22:20:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:37 smithi070 ceph-mon[192652]: 1.7 scrub starts 2023-09-23T22:20:38.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:37 smithi070 ceph-mon[192652]: 1.7 scrub ok 2023-09-23T22:20:38.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:38.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:37 smithi026 ceph-mon[215775]: 1.a scrub starts 2023-09-23T22:20:38.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:37 smithi026 ceph-mon[215775]: 1.a scrub ok 2023-09-23T22:20:38.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:37 smithi026 ceph-mon[215775]: 1.7 scrub starts 2023-09-23T22:20:38.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:37 smithi026 ceph-mon[215775]: 1.7 scrub ok 2023-09-23T22:20:38.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:37 smithi026 ceph-mon[223014]: 1.a scrub starts 2023-09-23T22:20:38.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:37 smithi026 ceph-mon[223014]: 1.a scrub ok 2023-09-23T22:20:38.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:37 smithi026 ceph-mon[223014]: 1.7 scrub starts 2023-09-23T22:20:38.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:37 smithi026 ceph-mon[223014]: 1.7 scrub ok 2023-09-23T22:20:38.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:38.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:38.297 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:38.385 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 16832 16816 150.123 117 0.00341802 0.00660911 2023-09-23T22:20:39.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:38 smithi026 ceph-mon[215775]: pgmap v481: 99 pgs: 99 active+clean; 512 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s 2023-09-23T22:20:39.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:38 smithi026 ceph-mon[215775]: 1.5 scrub starts 2023-09-23T22:20:39.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:38 smithi026 ceph-mon[215775]: 1.5 scrub ok 2023-09-23T22:20:39.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:38 smithi026 ceph-mon[223014]: pgmap v481: 99 pgs: 99 active+clean; 512 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s 2023-09-23T22:20:39.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:38 smithi026 ceph-mon[223014]: 1.5 scrub starts 2023-09-23T22:20:39.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:38 smithi026 ceph-mon[223014]: 1.5 scrub ok 2023-09-23T22:20:39.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:20:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:38 smithi070 ceph-mon[192652]: pgmap v481: 99 pgs: 99 active+clean; 512 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 54 MiB/s wr, 865 op/s 2023-09-23T22:20:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:38 smithi070 ceph-mon[192652]: 1.5 scrub starts 2023-09-23T22:20:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:38 smithi070 ceph-mon[192652]: 1.5 scrub ok 2023-09-23T22:20:39.206 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:39.357 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:39.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:39.385 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 18319 18303 142.974 92.9375 0.00720005 0.00698561 2023-09-23T22:20:39.470 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:39.500 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:39.554 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:39.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:39.659 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:39.700 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:20:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:20:39] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2023-09-23T22:20:39.742 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[223014]: 1.1 scrub starts 2023-09-23T22:20:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[223014]: 1.1 scrub ok 2023-09-23T22:20:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 7]}]: dispatch 2023-09-23T22:20:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-09-23T22:20:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 7]}]: dispatch 2023-09-23T22:20:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]: dispatch 2023-09-23T22:20:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:39.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[223014]: 1.3 scrub starts 2023-09-23T22:20:39.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[223014]: 1.3 scrub ok 2023-09-23T22:20:39.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[215775]: 1.1 scrub starts 2023-09-23T22:20:39.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[215775]: 1.1 scrub ok 2023-09-23T22:20:39.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 7]}]: dispatch 2023-09-23T22:20:39.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-09-23T22:20:39.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 7]}]: dispatch 2023-09-23T22:20:39.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]: dispatch 2023-09-23T22:20:39.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:39.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[215775]: 1.3 scrub starts 2023-09-23T22:20:39.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:39 smithi026 ceph-mon[215775]: 1.3 scrub ok 2023-09-23T22:20:40.013 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:40.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:39 smithi070 ceph-mon[192652]: 1.1 scrub starts 2023-09-23T22:20:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:39 smithi070 ceph-mon[192652]: 1.1 scrub ok 2023-09-23T22:20:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 7]}]: dispatch 2023-09-23T22:20:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-09-23T22:20:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 7]}]: dispatch 2023-09-23T22:20:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]: dispatch 2023-09-23T22:20:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:39 smithi070 ceph-mon[192652]: 1.3 scrub starts 2023-09-23T22:20:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:39 smithi070 ceph-mon[192652]: 1.3 scrub ok 2023-09-23T22:20:40.124 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:40.156 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:40.208 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:40.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:40.314 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:40.384 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:40.385 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 20011 19995 138.837 105.75 0.0102314 0.00719637 2023-09-23T22:20:41.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: pgmap v482: 99 pgs: 99 active+clean; 874 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: 1.8 scrub starts 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: 1.8 scrub ok 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: 1.0 scrub starts 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: 1.0 scrub ok 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 7]}]': finished 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 7]}]': finished 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: osdmap e209: 8 total, 8 up, 7 in 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: failed to encode map e209 with expected crc 2023-09-23T22:20:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:40 smithi070 ceph-mon[192652]: failed to encode map e209 with expected crc 2023-09-23T22:20:41.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: pgmap v482: 99 pgs: 99 active+clean; 874 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T22:20:41.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: 1.8 scrub starts 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: 1.8 scrub ok 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: 1.0 scrub starts 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: 1.0 scrub ok 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 7]}]': finished 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 7]}]': finished 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: osdmap e209: 8 total, 8 up, 7 in 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: failed to encode map e209 with expected crc 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[223014]: failed to encode map e209 with expected crc 2023-09-23T22:20:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: pgmap v482: 99 pgs: 99 active+clean; 874 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: 1.8 scrub starts 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: 1.8 scrub ok 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: 1.0 scrub starts 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: 1.0 scrub ok 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 7]}]': finished 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [3, 7]}]': finished 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1c", "id": [3, 5]}]': finished 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: osdmap e209: 8 total, 8 up, 7 in 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: failed to encode map e209 with expected crc 2023-09-23T22:20:41.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:40 smithi026 ceph-mon[215775]: failed to encode map e209 with expected crc 2023-09-23T22:20:41.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:20:41.386 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 15 21792 21777 136.068 111.375 0.00854652 0.00734384 2023-09-23T22:20:41.543 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:20:41.554 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:20:41.594 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:41.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:41.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:41.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:41.779 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:42.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: from='client.54838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:42.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: 1.9 scrub starts 2023-09-23T22:20:42.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: osdmap e210: 8 total, 8 up, 7 in 2023-09-23T22:20:42.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.045 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: 1.9 scrub ok 2023-09-23T22:20:42.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:42.046 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:42.046 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: from='client.54838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: 1.9 scrub starts 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: osdmap e210: 8 total, 8 up, 7 in 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: 1.9 scrub ok 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:42.047 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:42.058 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:42.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: from='client.54838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: 1.9 scrub starts 2023-09-23T22:20:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: osdmap e210: 8 total, 8 up, 7 in 2023-09-23T22:20:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: failed to encode map e210 with expected crc 2023-09-23T22:20:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: 1.9 scrub ok 2023-09-23T22:20:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:42.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:42.257 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:42.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 23394 23378 132.782 100.062 0.00780024 0.00752459 2023-09-23T22:20:42.896 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:42.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:42.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:43.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:43.047 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":483,"stamp":"2023-09-23T22:20:39.829176+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":915949069,"num_objects":13932,"num_object_clones":0,"num_object_copies":41796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13932,"num_whiteouts":0,"num_read":6424,"num_read_kb":13013,"num_write":15028,"num_write_kb":904777,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":11184369,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":16706,"ondisk_log_size":16706,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3612584,"kb_used_data":2640048,"kb_used_omap":31,"kb_used_meta":972320,"kb_avail":652603480,"statfs":{"total":671965249536,"available":668265963520,"internally_reserved":0,"allocated":2703409152,"data_stored":2696471066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":995656631},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":911158874,"num_objects":13904,"num_object_clones":0,"num_object_copies":41712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13904,"num_whiteouts":0,"num_read":-44,"num_read_kb":-83,"num_write":13882,"num_write_kb":889636,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":-161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.680895"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":185,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974733+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:20:27.974733+0000","last_peered":"2023-09-23T22:20:27.974733+0000","last_clean":"2023-09-23T22:20:27.974733+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:20:27.974733+0000","last_undegraded":"2023-09-23T22:20:27.974733+0000","last_fullsized":"2023-09-23T22:20:27.974733+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":187,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.775101+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:20:27.775101+0000","last_peered":"2023-09-23T22:20:27.775101+0000","last_clean":"2023-09-23T22:20:27.775101+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:20:27.775101+0000","last_undegraded":"2023-09-23T22:20:27.775101+0000","last_fullsized":"2023-09-23T22:20:27.775101+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":148,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.032189+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:20:28.032189+0000","last_peered":"2023-09-23T22:20:28.032189+0000","last_clean":"2023-09-23T22:20:28.032189+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:20:28.032189+0000","last_undegraded":"2023-09-23T22:20:28.032189+0000","last_fullsized":"2023-09-23T22:20:28.032189+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":36,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033584+0000","last_change":"2023-09-23T22:18:57.724501+0000","last_active":"2023-09-23T22:20:28.033584+0000","last_peered":"2023-09-23T22:20:28.033584+0000","last_clean":"2023-09-23T22:20:28.033584+0000","last_became_active":"2023-09-23T22:18:57.724203+0000","last_became_peered":"2023-09-23T22:18:57.724203+0000","last_unstale":"2023-09-23T22:20:28.033584+0000","last_undegraded":"2023-09-23T22:20:28.033584+0000","last_fullsized":"2023-09-23T22:20:28.033584+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:52.168267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":227,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.774544+0000","last_change":"2023-09-23T22:18:57.722574+0000","last_active":"2023-09-23T22:20:27.774544+0000","last_peered":"2023-09-23T22:20:27.774544+0000","last_clean":"2023-09-23T22:20:27.774544+0000","last_became_active":"2023-09-23T22:18:57.721800+0000","last_became_peered":"2023-09-23T22:18:57.721800+0000","last_unstale":"2023-09-23T22:20:27.774544+0000","last_undegraded":"2023-09-23T22:20:27.774544+0000","last_fullsized":"2023-09-23T22:20:27.774544+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:09.303838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":162,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.151005+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:20:25.151005+0000","last_peered":"2023-09-23T22:20:25.151005+0000","last_clean":"2023-09-23T22:20:25.151005+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:20:25.151005+0000","last_undegraded":"2023-09-23T22:20:25.151005+0000","last_fullsized":"2023-09-23T22:20:25.151005+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":74,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033958+0000","last_change":"2023-09-23T22:18:57.729038+0000","last_active":"2023-09-23T22:20:28.033958+0000","last_peered":"2023-09-23T22:20:28.033958+0000","last_clean":"2023-09-23T22:20:28.033958+0000","last_became_active":"2023-09-23T22:18:57.728650+0000","last_became_peered":"2023-09-23T22:18:57.728650+0000","last_unstale":"2023-09-23T22:20:28.033958+0000","last_undegraded":"2023-09-23T22:20:28.033958+0000","last_fullsized":"2023-09-23T22:20:28.033958+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:11.461883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":182,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.774521+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:20:27.774521+0000","last_peered":"2023-09-23T22:20:27.774521+0000","last_clean":"2023-09-23T22:20:27.774521+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:20:27.774521+0000","last_undegraded":"2023-09-23T22:20:27.774521+0000","last_fullsized":"2023-09-23T22:20:27.774521+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":167,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150789+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:20:25.150789+0000","last_peered":"2023-09-23T22:20:25.150789+0000","last_clean":"2023-09-23T22:20:25.150789+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:20:25.150789+0000","last_undegraded":"2023-09-23T22:20:25.150789+0000","last_fullsized":"2023-09-23T22:20:25.150789+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":132,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.774566+0000","last_change":"2023-09-23T22:16:49.154341+0000","last_active":"2023-09-23T22:20:27.774566+0000","last_peered":"2023-09-23T22:20:27.774566+0000","last_clean":"2023-09-23T22:20:27.774566+0000","last_became_active":"2023-09-23T22:16:49.154041+0000","last_became_peered":"2023-09-23T22:16:49.154041+0000","last_unstale":"2023-09-23T22:20:27.774566+0000","last_undegraded":"2023-09-23T22:20:27.774566+0000","last_fullsized":"2023-09-23T22:20:27.774566+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:05.733262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":120,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.775029+0000","last_change":"2023-09-23T22:15:14.829731+0000","last_active":"2023-09-23T22:20:27.775029+0000","last_peered":"2023-09-23T22:20:27.775029+0000","last_clean":"2023-09-23T22:20:27.775029+0000","last_became_active":"2023-09-23T22:15:14.829437+0000","last_became_peered":"2023-09-23T22:15:14.829437+0000","last_unstale":"2023-09-23T22:20:27.775029+0000","last_undegraded":"2023-09-23T22:20:27.775029+0000","last_fullsized":"2023-09-23T22:20:27.775029+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:45.456558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":175,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.032046+0000","last_change":"2023-09-23T22:18:57.723165+0000","last_active":"2023-09-23T22:20:28.032046+0000","last_peered":"2023-09-23T22:20:28.032046+0000","last_clean":"2023-09-23T22:20:28.032046+0000","last_became_active":"2023-09-23T22:18:57.722863+0000","last_became_peered":"2023-09-23T22:18:57.722863+0000","last_unstale":"2023-09-23T22:20:28.032046+0000","last_undegraded":"2023-09-23T22:20:28.032046+0000","last_fullsized":"2023-09-23T22:20:28.032046+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:19.645029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":192,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974832+0000","last_change":"2023-09-23T22:16:49.265991+0000","last_active":"2023-09-23T22:20:27.974832+0000","last_peered":"2023-09-23T22:20:27.974832+0000","last_clean":"2023-09-23T22:20:27.974832+0000","last_became_active":"2023-09-23T22:16:49.251169+0000","last_became_peered":"2023-09-23T22:16:49.251169+0000","last_unstale":"2023-09-23T22:20:27.974832+0000","last_undegraded":"2023-09-23T22:20:27.974832+0000","last_fullsized":"2023-09-23T22:20:27.974832+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.502218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":176,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974708+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:20:27.974708+0000","last_peered":"2023-09-23T22:20:27.974708+0000","last_clean":"2023-09-23T22:20:27.974708+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:20:27.974708+0000","last_undegraded":"2023-09-23T22:20:27.974708+0000","last_fullsized":"2023-09-23T22:20:27.974708+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":119,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974810+0000","last_change":"2023-09-23T22:15:14.826143+0000","last_active":"2023-09-23T22:20:27.974810+0000","last_peered":"2023-09-23T22:20:27.974810+0000","last_clean":"2023-09-23T22:20:27.974810+0000","last_became_active":"2023-09-23T22:15:14.825780+0000","last_became_peered":"2023-09-23T22:15:14.825780+0000","last_unstale":"2023-09-23T22:20:27.974810+0000","last_undegraded":"2023-09-23T22:20:27.974810+0000","last_fullsized":"2023-09-23T22:20:27.974810+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:22.608676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":173,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.775030+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:20:27.775030+0000","last_peered":"2023-09-23T22:20:27.775030+0000","last_clean":"2023-09-23T22:20:27.775030+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:20:27.775030+0000","last_undegraded":"2023-09-23T22:20:27.775030+0000","last_fullsized":"2023-09-23T22:20:27.775030+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":36,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033753+0000","last_change":"2023-09-23T22:18:57.726112+0000","last_active":"2023-09-23T22:20:28.033753+0000","last_peered":"2023-09-23T22:20:28.033753+0000","last_clean":"2023-09-23T22:20:28.033753+0000","last_became_active":"2023-09-23T22:18:57.725558+0000","last_became_peered":"2023-09-23T22:18:57.725558+0000","last_unstale":"2023-09-23T22:20:28.033753+0000","last_undegraded":"2023-09-23T22:20:28.033753+0000","last_fullsized":"2023-09-23T22:20:28.033753+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:38.315664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":166,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150877+0000","last_change":"2023-09-23T22:14:00.794669+0000","last_active":"2023-09-23T22:20:25.150877+0000","last_peered":"2023-09-23T22:20:25.150877+0000","last_clean":"2023-09-23T22:20:25.150877+0000","last_became_active":"2023-09-23T22:14:00.794371+0000","last_became_peered":"2023-09-23T22:14:00.794371+0000","last_unstale":"2023-09-23T22:20:25.150877+0000","last_undegraded":"2023-09-23T22:20:25.150877+0000","last_fullsized":"2023-09-23T22:20:25.150877+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.867700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":131,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974634+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:20:27.974634+0000","last_peered":"2023-09-23T22:20:27.974634+0000","last_clean":"2023-09-23T22:20:27.974634+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:20:27.974634+0000","last_undegraded":"2023-09-23T22:20:27.974634+0000","last_fullsized":"2023-09-23T22:20:27.974634+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":36,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033788+0000","last_change":"2023-09-23T22:18:57.726691+0000","last_active":"2023-09-23T22:20:28.033788+0000","last_peered":"2023-09-23T22:20:28.033788+0000","last_clean":"2023-09-23T22:20:28.033788+0000","last_became_active":"2023-09-23T22:18:57.726386+0000","last_became_peered":"2023-09-23T22:18:57.726386+0000","last_unstale":"2023-09-23T22:20:28.033788+0000","last_undegraded":"2023-09-23T22:20:28.033788+0000","last_fullsized":"2023-09-23T22:20:28.033788+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:34.829149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":177,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033173+0000","last_change":"2023-09-23T22:18:57.728767+0000","last_active":"2023-09-23T22:20:28.033173+0000","last_peered":"2023-09-23T22:20:28.033173+0000","last_clean":"2023-09-23T22:20:28.033173+0000","last_became_active":"2023-09-23T22:18:57.728377+0000","last_became_peered":"2023-09-23T22:18:57.728377+0000","last_unstale":"2023-09-23T22:20:28.033173+0000","last_undegraded":"2023-09-23T22:20:28.033173+0000","last_fullsized":"2023-09-23T22:20:28.033173+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:01:01.370904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":173,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.775499+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:20:27.775499+0000","last_peered":"2023-09-23T22:20:27.775499+0000","last_clean":"2023-09-23T22:20:27.775499+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:20:27.775499+0000","last_undegraded":"2023-09-23T22:20:27.775499+0000","last_fullsized":"2023-09-23T22:20:27.775499+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"207'625","reported_seq":637,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:35.144237+0000","last_change":"2023-09-23T22:20:19.470064+0000","last_active":"2023-09-23T22:20:35.144237+0000","last_peered":"2023-09-23T22:20:35.144237+0000","last_clean":"2023-09-23T22:20:35.144237+0000","last_became_active":"2023-09-23T22:20:19.469807+0000","last_became_peered":"2023-09-23T22:20:19.469807+0000","last_unstale":"2023-09-23T22:20:35.144237+0000","last_undegraded":"2023-09-23T22:20:35.144237+0000","last_fullsized":"2023-09-23T22:20:35.144237+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":625,"log_dups_size":0,"ondisk_log_size":625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:45.351484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40960000,"num_objects":625,"num_object_clones":0,"num_object_copies":1875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":625,"num_write_kb":40000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":175,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.975149+0000","last_change":"2023-09-23T22:19:00.682889+0000","last_active":"2023-09-23T22:20:27.975149+0000","last_peered":"2023-09-23T22:20:27.975149+0000","last_clean":"2023-09-23T22:20:27.975149+0000","last_became_active":"2023-09-23T22:18:57.718002+0000","last_became_peered":"2023-09-23T22:18:57.718002+0000","last_unstale":"2023-09-23T22:20:27.975149+0000","last_undegraded":"2023-09-23T22:20:27.975149+0000","last_fullsized":"2023-09-23T22:20:27.975149+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:22.517277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150588+0000","last_change":"2023-09-23T22:16:49.158580+0000","last_active":"2023-09-23T22:20:25.150588+0000","last_peered":"2023-09-23T22:20:25.150588+0000","last_clean":"2023-09-23T22:20:25.150588+0000","last_became_active":"2023-09-23T22:16:49.158266+0000","last_became_peered":"2023-09-23T22:16:49.158266+0000","last_unstale":"2023-09-23T22:20:25.150588+0000","last_undegraded":"2023-09-23T22:20:25.150588+0000","last_fullsized":"2023-09-23T22:20:25.150588+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:03.791658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"208'1099","reported_seq":1113,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:38.900636+0000","last_change":"2023-09-23T22:20:19.474568+0000","last_active":"2023-09-23T22:20:38.900636+0000","last_peered":"2023-09-23T22:20:38.900636+0000","last_clean":"2023-09-23T22:20:38.900636+0000","last_became_active":"2023-09-23T22:20:19.474222+0000","last_became_peered":"2023-09-23T22:20:19.474222+0000","last_unstale":"2023-09-23T22:20:38.900636+0000","last_undegraded":"2023-09-23T22:20:38.900636+0000","last_fullsized":"2023-09-23T22:20:38.900636+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1099,"log_dups_size":0,"ondisk_log_size":1099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:25.959782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72024064,"num_objects":1099,"num_object_clones":0,"num_object_copies":3297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1099,"num_write_kb":70336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":119,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.975200+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:20:27.975200+0000","last_peered":"2023-09-23T22:20:27.975200+0000","last_clean":"2023-09-23T22:20:27.975200+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:20:27.975200+0000","last_undegraded":"2023-09-23T22:20:27.975200+0000","last_fullsized":"2023-09-23T22:20:27.975200+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":175,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974555+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:20:27.974555+0000","last_peered":"2023-09-23T22:20:27.974555+0000","last_clean":"2023-09-23T22:20:27.974555+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:20:27.974555+0000","last_undegraded":"2023-09-23T22:20:27.974555+0000","last_fullsized":"2023-09-23T22:20:27.974555+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"208'776","reported_seq":790,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:36.272848+0000","last_change":"2023-09-23T22:20:19.468974+0000","last_active":"2023-09-23T22:20:36.272848+0000","last_peered":"2023-09-23T22:20:36.272848+0000","last_clean":"2023-09-23T22:20:36.272848+0000","last_became_active":"2023-09-23T22:20:19.468672+0000","last_became_peered":"2023-09-23T22:20:19.468672+0000","last_unstale":"2023-09-23T22:20:36.272848+0000","last_undegraded":"2023-09-23T22:20:36.272848+0000","last_fullsized":"2023-09-23T22:20:36.272848+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":776,"log_dups_size":0,"ondisk_log_size":776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:39.575600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50855936,"num_objects":776,"num_object_clones":0,"num_object_copies":2328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":776,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":776,"num_write_kb":49664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":132,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150829+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:20:25.150829+0000","last_peered":"2023-09-23T22:20:25.150829+0000","last_clean":"2023-09-23T22:20:25.150829+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:20:25.150829+0000","last_undegraded":"2023-09-23T22:20:25.150829+0000","last_fullsized":"2023-09-23T22:20:25.150829+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":123,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.032327+0000","last_change":"2023-09-23T22:16:49.161676+0000","last_active":"2023-09-23T22:20:28.032327+0000","last_peered":"2023-09-23T22:20:28.032327+0000","last_clean":"2023-09-23T22:20:28.032327+0000","last_became_active":"2023-09-23T22:16:49.161247+0000","last_became_peered":"2023-09-23T22:16:49.161247+0000","last_unstale":"2023-09-23T22:20:28.032327+0000","last_undegraded":"2023-09-23T22:20:28.032327+0000","last_fullsized":"2023-09-23T22:20:28.032327+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:30.708371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"208'795","reported_seq":809,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:36.238056+0000","last_change":"2023-09-23T22:20:19.471789+0000","last_active":"2023-09-23T22:20:36.238056+0000","last_peered":"2023-09-23T22:20:36.238056+0000","last_clean":"2023-09-23T22:20:36.238056+0000","last_became_active":"2023-09-23T22:20:19.471481+0000","last_became_peered":"2023-09-23T22:20:19.471481+0000","last_unstale":"2023-09-23T22:20:36.238056+0000","last_undegraded":"2023-09-23T22:20:36.238056+0000","last_fullsized":"2023-09-23T22:20:36.238056+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":795,"log_dups_size":0,"ondisk_log_size":795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:43.740017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52101120,"num_objects":795,"num_object_clones":0,"num_object_copies":2385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":795,"num_write_kb":50880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":177,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150755+0000","last_change":"2023-09-23T22:16:49.164833+0000","last_active":"2023-09-23T22:20:25.150755+0000","last_peered":"2023-09-23T22:20:25.150755+0000","last_clean":"2023-09-23T22:20:25.150755+0000","last_became_active":"2023-09-23T22:16:49.163833+0000","last_became_peered":"2023-09-23T22:16:49.163833+0000","last_unstale":"2023-09-23T22:20:25.150755+0000","last_undegraded":"2023-09-23T22:20:25.150755+0000","last_fullsized":"2023-09-23T22:20:25.150755+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.891395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":61,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033441+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:20:28.033441+0000","last_peered":"2023-09-23T22:20:28.033441+0000","last_clean":"2023-09-23T22:20:28.033441+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:20:28.033441+0000","last_undegraded":"2023-09-23T22:20:28.033441+0000","last_fullsized":"2023-09-23T22:20:28.033441+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"208'979","reported_seq":992,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:37.447625+0000","last_change":"2023-09-23T22:20:19.469349+0000","last_active":"2023-09-23T22:20:37.447625+0000","last_peered":"2023-09-23T22:20:37.447625+0000","last_clean":"2023-09-23T22:20:37.447625+0000","last_became_active":"2023-09-23T22:20:19.469086+0000","last_became_peered":"2023-09-23T22:20:19.469086+0000","last_unstale":"2023-09-23T22:20:37.447625+0000","last_undegraded":"2023-09-23T22:20:37.447625+0000","last_fullsized":"2023-09-23T22:20:37.447625+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":979,"log_dups_size":0,"ondisk_log_size":979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:28.871375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64159744,"num_objects":979,"num_object_clones":0,"num_object_copies":2937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":979,"num_write_kb":62656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":481,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.774388+0000","last_change":"2023-09-23T22:20:18.461035+0000","last_active":"2023-09-23T22:20:27.774388+0000","last_peered":"2023-09-23T22:20:27.774388+0000","last_clean":"2023-09-23T22:20:27.774388+0000","last_became_active":"2023-09-23T22:20:18.460615+0000","last_became_peered":"2023-09-23T22:20:18.460615+0000","last_unstale":"2023-09-23T22:20:27.774388+0000","last_undegraded":"2023-09-23T22:20:27.774388+0000","last_fullsized":"2023-09-23T22:20:27.774388+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":219,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.775054+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:20:27.775054+0000","last_peered":"2023-09-23T22:20:27.775054+0000","last_clean":"2023-09-23T22:20:27.775054+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:20:27.775054+0000","last_undegraded":"2023-09-23T22:20:27.775054+0000","last_fullsized":"2023-09-23T22:20:27.775054+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":176,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974740+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:20:27.974740+0000","last_peered":"2023-09-23T22:20:27.974740+0000","last_clean":"2023-09-23T22:20:27.974740+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:20:27.974740+0000","last_undegraded":"2023-09-23T22:20:27.974740+0000","last_fullsized":"2023-09-23T22:20:27.974740+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"208'744","reported_seq":758,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:36.278102+0000","last_change":"2023-09-23T22:20:19.470036+0000","last_active":"2023-09-23T22:20:36.278102+0000","last_peered":"2023-09-23T22:20:36.278102+0000","last_clean":"2023-09-23T22:20:36.278102+0000","last_became_active":"2023-09-23T22:20:19.468632+0000","last_became_peered":"2023-09-23T22:20:19.468632+0000","last_unstale":"2023-09-23T22:20:36.278102+0000","last_undegraded":"2023-09-23T22:20:36.278102+0000","last_fullsized":"2023-09-23T22:20:36.278102+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":744,"log_dups_size":0,"ondisk_log_size":744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:04.458920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48758784,"num_objects":744,"num_object_clones":0,"num_object_copies":2232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":744,"num_write_kb":47616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":485,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150483+0000","last_change":"2023-09-23T22:20:18.472306+0000","last_active":"2023-09-23T22:20:25.150483+0000","last_peered":"2023-09-23T22:20:25.150483+0000","last_clean":"2023-09-23T22:20:25.150483+0000","last_became_active":"2023-09-23T22:20:18.471844+0000","last_became_peered":"2023-09-23T22:20:18.471844+0000","last_unstale":"2023-09-23T22:20:25.150483+0000","last_undegraded":"2023-09-23T22:20:25.150483+0000","last_fullsized":"2023-09-23T22:20:25.150483+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974632+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:20:27.974632+0000","last_peered":"2023-09-23T22:20:27.974632+0000","last_clean":"2023-09-23T22:20:27.974632+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:20:27.974632+0000","last_undegraded":"2023-09-23T22:20:27.974632+0000","last_fullsized":"2023-09-23T22:20:27.974632+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":183,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974944+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:20:27.974944+0000","last_peered":"2023-09-23T22:20:27.974944+0000","last_clean":"2023-09-23T22:20:27.974944+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:20:27.974944+0000","last_undegraded":"2023-09-23T22:20:27.974944+0000","last_fullsized":"2023-09-23T22:20:27.974944+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"208'1067","reported_seq":1081,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:38.907198+0000","last_change":"2023-09-23T22:20:19.469847+0000","last_active":"2023-09-23T22:20:38.907198+0000","last_peered":"2023-09-23T22:20:38.907198+0000","last_clean":"2023-09-23T22:20:38.907198+0000","last_became_active":"2023-09-23T22:20:19.469569+0000","last_became_peered":"2023-09-23T22:20:19.469569+0000","last_unstale":"2023-09-23T22:20:38.907198+0000","last_undegraded":"2023-09-23T22:20:38.907198+0000","last_fullsized":"2023-09-23T22:20:38.907198+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1067,"log_dups_size":0,"ondisk_log_size":1067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:16.691256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69926912,"num_objects":1067,"num_object_clones":0,"num_object_copies":3201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1067,"num_write_kb":68288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":159,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974429+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:20:27.974429+0000","last_peered":"2023-09-23T22:20:27.974429+0000","last_clean":"2023-09-23T22:20:27.974429+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:20:27.974429+0000","last_undegraded":"2023-09-23T22:20:27.974429+0000","last_fullsized":"2023-09-23T22:20:27.974429+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":176,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.975125+0000","last_change":"2023-09-23T22:16:49.165118+0000","last_active":"2023-09-23T22:20:27.975125+0000","last_peered":"2023-09-23T22:20:27.975125+0000","last_clean":"2023-09-23T22:20:27.975125+0000","last_became_active":"2023-09-23T22:16:49.164802+0000","last_became_peered":"2023-09-23T22:16:49.164802+0000","last_unstale":"2023-09-23T22:20:27.975125+0000","last_undegraded":"2023-09-23T22:20:27.975125+0000","last_fullsized":"2023-09-23T22:20:27.975125+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:23.079142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"207'580","reported_seq":592,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:35.164322+0000","last_change":"2023-09-23T22:20:19.473637+0000","last_active":"2023-09-23T22:20:35.164322+0000","last_peered":"2023-09-23T22:20:35.164322+0000","last_clean":"2023-09-23T22:20:35.164322+0000","last_became_active":"2023-09-23T22:20:19.473385+0000","last_became_peered":"2023-09-23T22:20:19.473385+0000","last_unstale":"2023-09-23T22:20:35.164322+0000","last_undegraded":"2023-09-23T22:20:35.164322+0000","last_fullsized":"2023-09-23T22:20:35.164322+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":580,"log_dups_size":0,"ondisk_log_size":580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:01.911861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38010880,"num_objects":580,"num_object_clones":0,"num_object_copies":1740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":580,"num_write_kb":37120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":481,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150574+0000","last_change":"2023-09-23T22:20:18.473341+0000","last_active":"2023-09-23T22:20:25.150574+0000","last_peered":"2023-09-23T22:20:25.150574+0000","last_clean":"2023-09-23T22:20:25.150574+0000","last_became_active":"2023-09-23T22:20:18.472892+0000","last_became_peered":"2023-09-23T22:20:18.472892+0000","last_unstale":"2023-09-23T22:20:25.150574+0000","last_undegraded":"2023-09-23T22:20:25.150574+0000","last_fullsized":"2023-09-23T22:20:25.150574+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":175,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.911300+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:20:28.911300+0000","last_peered":"2023-09-23T22:20:28.911300+0000","last_clean":"2023-09-23T22:20:28.911300+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:20:28.911300+0000","last_undegraded":"2023-09-23T22:20:28.911300+0000","last_fullsized":"2023-09-23T22:20:28.911300+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":148,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.032666+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:20:28.032666+0000","last_peered":"2023-09-23T22:20:28.032666+0000","last_clean":"2023-09-23T22:20:28.032666+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:20:28.032666+0000","last_undegraded":"2023-09-23T22:20:28.032666+0000","last_fullsized":"2023-09-23T22:20:28.032666+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"208'769","reported_seq":783,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:36.241148+0000","last_change":"2023-09-23T22:20:19.471690+0000","last_active":"2023-09-23T22:20:36.241148+0000","last_peered":"2023-09-23T22:20:36.241148+0000","last_clean":"2023-09-23T22:20:36.241148+0000","last_became_active":"2023-09-23T22:20:19.471393+0000","last_became_peered":"2023-09-23T22:20:19.471393+0000","last_unstale":"2023-09-23T22:20:36.241148+0000","last_undegraded":"2023-09-23T22:20:36.241148+0000","last_fullsized":"2023-09-23T22:20:36.241148+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":769,"log_dups_size":0,"ondisk_log_size":769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:23.131932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50397184,"num_objects":769,"num_object_clones":0,"num_object_copies":2307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":769,"num_write_kb":49216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":493,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:36.790446+0000","last_change":"2023-09-23T22:20:36.790562+0000","last_active":"2023-09-23T22:20:36.790446+0000","last_peered":"2023-09-23T22:20:36.790446+0000","last_clean":"2023-09-23T22:20:36.790446+0000","last_became_active":"2023-09-23T22:20:18.467701+0000","last_became_peered":"2023-09-23T22:20:18.467701+0000","last_unstale":"2023-09-23T22:20:36.790446+0000","last_undegraded":"2023-09-23T22:20:36.790446+0000","last_fullsized":"2023-09-23T22:20:36.790446+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:31.210808+0000","scrub_duration":0.0070908769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":755,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.032685+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:20:28.032685+0000","last_peered":"2023-09-23T22:20:28.032685+0000","last_clean":"2023-09-23T22:20:28.032685+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:20:28.032685+0000","last_undegraded":"2023-09-23T22:20:28.032685+0000","last_fullsized":"2023-09-23T22:20:28.032685+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":184,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150960+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:20:25.150960+0000","last_peered":"2023-09-23T22:20:25.150960+0000","last_clean":"2023-09-23T22:20:25.150960+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:20:25.150960+0000","last_undegraded":"2023-09-23T22:20:25.150960+0000","last_fullsized":"2023-09-23T22:20:25.150960+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":130,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.031802+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:20:28.031802+0000","last_peered":"2023-09-23T22:20:28.031802+0000","last_clean":"2023-09-23T22:20:28.031802+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:20:28.031802+0000","last_undegraded":"2023-09-23T22:20:28.031802+0000","last_fullsized":"2023-09-23T22:20:28.031802+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"208'1083","reported_seq":1097,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:38.909148+0000","last_change":"2023-09-23T22:20:19.469302+0000","last_active":"2023-09-23T22:20:38.909148+0000","last_peered":"2023-09-23T22:20:38.909148+0000","last_clean":"2023-09-23T22:20:38.909148+0000","last_became_active":"2023-09-23T22:20:19.469029+0000","last_became_peered":"2023-09-23T22:20:19.469029+0000","last_unstale":"2023-09-23T22:20:38.909148+0000","last_undegraded":"2023-09-23T22:20:38.909148+0000","last_fullsized":"2023-09-23T22:20:38.909148+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1083,"log_dups_size":0,"ondisk_log_size":1083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:08.337445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70975488,"num_objects":1083,"num_object_clones":0,"num_object_copies":3249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1083,"num_write_kb":69312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":481,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150503+0000","last_change":"2023-09-23T22:20:18.469369+0000","last_active":"2023-09-23T22:20:25.150503+0000","last_peered":"2023-09-23T22:20:25.150503+0000","last_clean":"2023-09-23T22:20:25.150503+0000","last_became_active":"2023-09-23T22:20:18.469020+0000","last_became_peered":"2023-09-23T22:20:18.469020+0000","last_unstale":"2023-09-23T22:20:25.150503+0000","last_undegraded":"2023-09-23T22:20:25.150503+0000","last_fullsized":"2023-09-23T22:20:25.150503+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":939,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150687+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:20:25.150687+0000","last_peered":"2023-09-23T22:20:25.150687+0000","last_clean":"2023-09-23T22:20:25.150687+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:20:25.150687+0000","last_undegraded":"2023-09-23T22:20:25.150687+0000","last_fullsized":"2023-09-23T22:20:25.150687+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":217,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974693+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:20:27.974693+0000","last_peered":"2023-09-23T22:20:27.974693+0000","last_clean":"2023-09-23T22:20:27.974693+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:20:27.974693+0000","last_undegraded":"2023-09-23T22:20:27.974693+0000","last_fullsized":"2023-09-23T22:20:27.974693+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":181,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150859+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:20:25.150859+0000","last_peered":"2023-09-23T22:20:25.150859+0000","last_clean":"2023-09-23T22:20:25.150859+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:20:25.150859+0000","last_undegraded":"2023-09-23T22:20:25.150859+0000","last_fullsized":"2023-09-23T22:20:25.150859+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"208'1089","reported_seq":1103,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:38.889060+0000","last_change":"2023-09-23T22:20:19.473261+0000","last_active":"2023-09-23T22:20:38.889060+0000","last_peered":"2023-09-23T22:20:38.889060+0000","last_clean":"2023-09-23T22:20:38.889060+0000","last_became_active":"2023-09-23T22:20:19.472983+0000","last_became_peered":"2023-09-23T22:20:19.472983+0000","last_unstale":"2023-09-23T22:20:38.889060+0000","last_undegraded":"2023-09-23T22:20:38.889060+0000","last_fullsized":"2023-09-23T22:20:38.889060+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1089,"log_dups_size":0,"ondisk_log_size":1089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:47.461541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71368704,"num_objects":1089,"num_object_clones":0,"num_object_copies":3267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1089,"num_write_kb":69696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":489,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033399+0000","last_change":"2023-09-23T22:20:18.464810+0000","last_active":"2023-09-23T22:20:28.033399+0000","last_peered":"2023-09-23T22:20:28.033399+0000","last_clean":"2023-09-23T22:20:28.033399+0000","last_became_active":"2023-09-23T22:20:18.464483+0000","last_became_peered":"2023-09-23T22:20:18.464483+0000","last_unstale":"2023-09-23T22:20:28.033399+0000","last_undegraded":"2023-09-23T22:20:28.033399+0000","last_fullsized":"2023-09-23T22:20:28.033399+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":550,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.031614+0000","last_change":"2023-09-23T22:15:51.379227+0000","last_active":"2023-09-23T22:20:28.031614+0000","last_peered":"2023-09-23T22:20:28.031614+0000","last_clean":"2023-09-23T22:20:28.031614+0000","last_became_active":"2023-09-23T22:15:43.928977+0000","last_became_peered":"2023-09-23T22:15:43.928977+0000","last_unstale":"2023-09-23T22:20:28.031614+0000","last_undegraded":"2023-09-23T22:20:28.031614+0000","last_fullsized":"2023-09-23T22:20:28.031614+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:04.398533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":157,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.031781+0000","last_change":"2023-09-23T22:16:49.251412+0000","last_active":"2023-09-23T22:20:28.031781+0000","last_peered":"2023-09-23T22:20:28.031781+0000","last_clean":"2023-09-23T22:20:28.031781+0000","last_became_active":"2023-09-23T22:16:49.251097+0000","last_became_peered":"2023-09-23T22:16:49.251097+0000","last_unstale":"2023-09-23T22:20:28.031781+0000","last_undegraded":"2023-09-23T22:20:28.031781+0000","last_fullsized":"2023-09-23T22:20:28.031781+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:41.601435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":213,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.032768+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:20:28.032768+0000","last_peered":"2023-09-23T22:20:28.032768+0000","last_clean":"2023-09-23T22:20:28.032768+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:20:28.032768+0000","last_undegraded":"2023-09-23T22:20:28.032768+0000","last_fullsized":"2023-09-23T22:20:28.032768+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"207'585","reported_seq":597,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:35.149240+0000","last_change":"2023-09-23T22:20:19.470871+0000","last_active":"2023-09-23T22:20:35.149240+0000","last_peered":"2023-09-23T22:20:35.149240+0000","last_clean":"2023-09-23T22:20:35.149240+0000","last_became_active":"2023-09-23T22:20:19.470597+0000","last_became_peered":"2023-09-23T22:20:19.470597+0000","last_unstale":"2023-09-23T22:20:35.149240+0000","last_undegraded":"2023-09-23T22:20:35.149240+0000","last_fullsized":"2023-09-23T22:20:35.149240+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":585,"log_dups_size":0,"ondisk_log_size":585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:32.388985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38338560,"num_objects":585,"num_object_clones":0,"num_object_copies":1755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":585,"num_write_kb":37440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":792,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150802+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:20:25.150802+0000","last_peered":"2023-09-23T22:20:25.150802+0000","last_clean":"2023-09-23T22:20:25.150802+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:20:25.150802+0000","last_undegraded":"2023-09-23T22:20:25.150802+0000","last_fullsized":"2023-09-23T22:20:25.150802+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":472,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033049+0000","last_change":"2023-09-23T22:18:57.743602+0000","last_active":"2023-09-23T22:20:28.033049+0000","last_peered":"2023-09-23T22:20:28.033049+0000","last_clean":"2023-09-23T22:20:28.033049+0000","last_became_active":"2023-09-23T22:18:57.743004+0000","last_became_peered":"2023-09-23T22:18:57.743004+0000","last_unstale":"2023-09-23T22:20:28.033049+0000","last_undegraded":"2023-09-23T22:20:28.033049+0000","last_fullsized":"2023-09-23T22:20:28.033049+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":133,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.774767+0000","last_change":"2023-09-23T22:16:49.155156+0000","last_active":"2023-09-23T22:20:27.774767+0000","last_peered":"2023-09-23T22:20:27.774767+0000","last_clean":"2023-09-23T22:20:27.774767+0000","last_became_active":"2023-09-23T22:16:49.154923+0000","last_became_peered":"2023-09-23T22:16:49.154923+0000","last_unstale":"2023-09-23T22:20:27.774767+0000","last_undegraded":"2023-09-23T22:20:27.774767+0000","last_fullsized":"2023-09-23T22:20:27.774767+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:33.532238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":94,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.975074+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:20:27.975074+0000","last_peered":"2023-09-23T22:20:27.975074+0000","last_clean":"2023-09-23T22:20:27.975074+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:20:27.975074+0000","last_undegraded":"2023-09-23T22:20:27.975074+0000","last_fullsized":"2023-09-23T22:20:27.975074+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"208'1015","reported_seq":1029,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:38.416825+0000","last_change":"2023-09-23T22:20:19.471072+0000","last_active":"2023-09-23T22:20:38.416825+0000","last_peered":"2023-09-23T22:20:38.416825+0000","last_clean":"2023-09-23T22:20:38.416825+0000","last_became_active":"2023-09-23T22:20:19.470818+0000","last_became_peered":"2023-09-23T22:20:19.470818+0000","last_unstale":"2023-09-23T22:20:38.416825+0000","last_undegraded":"2023-09-23T22:20:38.416825+0000","last_fullsized":"2023-09-23T22:20:38.416825+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1015,"log_dups_size":0,"ondisk_log_size":1015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:27.930606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66519040,"num_objects":1015,"num_object_clones":0,"num_object_copies":3045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1015,"num_write_kb":64960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":526,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.032408+0000","last_change":"2023-09-23T22:20:18.464120+0000","last_active":"2023-09-23T22:20:28.032408+0000","last_peered":"2023-09-23T22:20:28.032408+0000","last_clean":"2023-09-23T22:20:28.032408+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:20:28.032408+0000","last_undegraded":"2023-09-23T22:20:28.032408+0000","last_fullsized":"2023-09-23T22:20:28.032408+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":556,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.031924+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:20:28.031924+0000","last_peered":"2023-09-23T22:20:28.031924+0000","last_clean":"2023-09-23T22:20:28.031924+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:20:28.031924+0000","last_undegraded":"2023-09-23T22:20:28.031924+0000","last_fullsized":"2023-09-23T22:20:28.031924+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":70,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033183+0000","last_change":"2023-09-23T22:18:31.795915+0000","last_active":"2023-09-23T22:20:28.033183+0000","last_peered":"2023-09-23T22:20:28.033183+0000","last_clean":"2023-09-23T22:20:28.033183+0000","last_became_active":"2023-09-23T22:18:31.795608+0000","last_became_peered":"2023-09-23T22:18:31.795608+0000","last_unstale":"2023-09-23T22:20:28.033183+0000","last_undegraded":"2023-09-23T22:20:28.033183+0000","last_fullsized":"2023-09-23T22:20:28.033183+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:49.789927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":36,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033505+0000","last_change":"2023-09-23T22:18:57.721257+0000","last_active":"2023-09-23T22:20:28.033505+0000","last_peered":"2023-09-23T22:20:28.033505+0000","last_clean":"2023-09-23T22:20:28.033505+0000","last_became_active":"2023-09-23T22:18:57.720548+0000","last_became_peered":"2023-09-23T22:18:57.720548+0000","last_unstale":"2023-09-23T22:20:28.033505+0000","last_undegraded":"2023-09-23T22:20:28.033505+0000","last_fullsized":"2023-09-23T22:20:28.033505+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:10.709024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"208'1016","reported_seq":1030,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:38.409034+0000","last_change":"2023-09-23T22:20:19.470432+0000","last_active":"2023-09-23T22:20:38.409034+0000","last_peered":"2023-09-23T22:20:38.409034+0000","last_clean":"2023-09-23T22:20:38.409034+0000","last_became_active":"2023-09-23T22:20:19.470153+0000","last_became_peered":"2023-09-23T22:20:19.470153+0000","last_unstale":"2023-09-23T22:20:38.409034+0000","last_undegraded":"2023-09-23T22:20:38.409034+0000","last_fullsized":"2023-09-23T22:20:38.409034+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1016,"log_dups_size":0,"ondisk_log_size":1016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:00.110369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66584576,"num_objects":1016,"num_object_clones":0,"num_object_copies":3048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1016,"num_write_kb":65024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":489,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033772+0000","last_change":"2023-09-23T22:20:18.469124+0000","last_active":"2023-09-23T22:20:28.033772+0000","last_peered":"2023-09-23T22:20:28.033772+0000","last_clean":"2023-09-23T22:20:28.033772+0000","last_became_active":"2023-09-23T22:20:18.468819+0000","last_became_peered":"2023-09-23T22:20:18.468819+0000","last_unstale":"2023-09-23T22:20:28.033772+0000","last_undegraded":"2023-09-23T22:20:28.033772+0000","last_fullsized":"2023-09-23T22:20:28.033772+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":97,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033478+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:20:28.033478+0000","last_peered":"2023-09-23T22:20:28.033478+0000","last_clean":"2023-09-23T22:20:28.033478+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:20:28.033478+0000","last_undegraded":"2023-09-23T22:20:28.033478+0000","last_fullsized":"2023-09-23T22:20:28.033478+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":148,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.032834+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:20:28.032834+0000","last_peered":"2023-09-23T22:20:28.032834+0000","last_clean":"2023-09-23T22:20:28.032834+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:20:28.032834+0000","last_undegraded":"2023-09-23T22:20:28.032834+0000","last_fullsized":"2023-09-23T22:20:28.032834+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":61,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033353+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:20:28.033353+0000","last_peered":"2023-09-23T22:20:28.033353+0000","last_clean":"2023-09-23T22:20:28.033353+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:20:28.033353+0000","last_undegraded":"2023-09-23T22:20:28.033353+0000","last_fullsized":"2023-09-23T22:20:28.033353+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"208'953","reported_seq":966,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:37.455717+0000","last_change":"2023-09-23T22:20:19.469545+0000","last_active":"2023-09-23T22:20:37.455717+0000","last_peered":"2023-09-23T22:20:37.455717+0000","last_clean":"2023-09-23T22:20:37.455717+0000","last_became_active":"2023-09-23T22:20:19.469306+0000","last_became_peered":"2023-09-23T22:20:19.469306+0000","last_unstale":"2023-09-23T22:20:37.455717+0000","last_undegraded":"2023-09-23T22:20:37.455717+0000","last_fullsized":"2023-09-23T22:20:37.455717+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":953,"log_dups_size":0,"ondisk_log_size":953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:48.392898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62455808,"num_objects":953,"num_object_clones":0,"num_object_copies":2859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":953,"num_write_kb":60992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":482,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974844+0000","last_change":"2023-09-23T22:20:18.472331+0000","last_active":"2023-09-23T22:20:27.974844+0000","last_peered":"2023-09-23T22:20:27.974844+0000","last_clean":"2023-09-23T22:20:27.974844+0000","last_became_active":"2023-09-23T22:20:18.471519+0000","last_became_peered":"2023-09-23T22:20:18.471519+0000","last_unstale":"2023-09-23T22:20:27.974844+0000","last_undegraded":"2023-09-23T22:20:27.974844+0000","last_fullsized":"2023-09-23T22:20:27.974844+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":220,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.911489+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:20:28.911489+0000","last_peered":"2023-09-23T22:20:28.911489+0000","last_clean":"2023-09-23T22:20:28.911489+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:20:28.911489+0000","last_undegraded":"2023-09-23T22:20:28.911489+0000","last_fullsized":"2023-09-23T22:20:28.911489+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":848,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033240+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:20:28.033240+0000","last_peered":"2023-09-23T22:20:28.033240+0000","last_clean":"2023-09-23T22:20:28.033240+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:20:28.033240+0000","last_undegraded":"2023-09-23T22:20:28.033240+0000","last_fullsized":"2023-09-23T22:20:28.033240+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":205,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.774421+0000","last_change":"2023-09-23T22:18:57.722771+0000","last_active":"2023-09-23T22:20:27.774421+0000","last_peered":"2023-09-23T22:20:27.774421+0000","last_clean":"2023-09-23T22:20:27.774421+0000","last_became_active":"2023-09-23T22:18:57.722258+0000","last_became_peered":"2023-09-23T22:18:57.722258+0000","last_unstale":"2023-09-23T22:20:27.774421+0000","last_undegraded":"2023-09-23T22:20:27.774421+0000","last_fullsized":"2023-09-23T22:20:27.774421+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:18.267091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"208'729","reported_seq":743,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T22:20:36.269167+0000","last_change":"2023-09-23T22:20:19.465240+0000","last_active":"2023-09-23T22:20:36.269167+0000","last_peered":"2023-09-23T22:20:36.269167+0000","last_clean":"2023-09-23T22:20:36.269167+0000","last_became_active":"2023-09-23T22:20:19.464955+0000","last_became_peered":"2023-09-23T22:20:19.464955+0000","last_unstale":"2023-09-23T22:20:36.269167+0000","last_undegraded":"2023-09-23T22:20:36.269167+0000","last_fullsized":"2023-09-23T22:20:36.269167+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":729,"log_dups_size":0,"ondisk_log_size":729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:21.927943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47775744,"num_objects":729,"num_object_clones":0,"num_object_copies":2187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":729,"num_write_kb":46656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":486,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.775352+0000","last_change":"2023-09-23T22:20:18.468192+0000","last_active":"2023-09-23T22:20:27.775352+0000","last_peered":"2023-09-23T22:20:27.775352+0000","last_clean":"2023-09-23T22:20:27.775352+0000","last_became_active":"2023-09-23T22:20:18.467795+0000","last_became_peered":"2023-09-23T22:20:18.467795+0000","last_unstale":"2023-09-23T22:20:27.775352+0000","last_undegraded":"2023-09-23T22:20:27.775352+0000","last_fullsized":"2023-09-23T22:20:27.775352+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:01:25.160012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":816370,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":272,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.031992+0000","last_change":"2023-09-23T22:16:49.270015+0000","last_active":"2023-09-23T22:20:28.031992+0000","last_peered":"2023-09-23T22:20:28.031992+0000","last_clean":"2023-09-23T22:20:28.031992+0000","last_became_active":"2023-09-23T22:16:49.252726+0000","last_became_peered":"2023-09-23T22:16:49.252726+0000","last_unstale":"2023-09-23T22:20:28.031992+0000","last_undegraded":"2023-09-23T22:20:28.031992+0000","last_fullsized":"2023-09-23T22:20:28.031992+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:18.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":40,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033671+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:20:28.033671+0000","last_peered":"2023-09-23T22:20:28.033671+0000","last_clean":"2023-09-23T22:20:28.033671+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:20:28.033671+0000","last_undegraded":"2023-09-23T22:20:28.033671+0000","last_fullsized":"2023-09-23T22:20:28.033671+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974528+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:20:27.974528+0000","last_peered":"2023-09-23T22:20:27.974528+0000","last_clean":"2023-09-23T22:20:27.974528+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:20:27.974528+0000","last_undegraded":"2023-09-23T22:20:27.974528+0000","last_fullsized":"2023-09-23T22:20:27.974528+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":36,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033261+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:20:28.033261+0000","last_peered":"2023-09-23T22:20:28.033261+0000","last_clean":"2023-09-23T22:20:28.033261+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:20:28.033261+0000","last_undegraded":"2023-09-23T22:20:28.033261+0000","last_fullsized":"2023-09-23T22:20:28.033261+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":192,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.031994+0000","last_change":"2023-09-23T22:14:00.786689+0000","last_active":"2023-09-23T22:20:28.031994+0000","last_peered":"2023-09-23T22:20:28.031994+0000","last_clean":"2023-09-23T22:20:28.031994+0000","last_became_active":"2023-09-23T22:14:00.786354+0000","last_became_peered":"2023-09-23T22:14:00.786354+0000","last_unstale":"2023-09-23T22:20:28.031994+0000","last_undegraded":"2023-09-23T22:20:28.031994+0000","last_fullsized":"2023-09-23T22:20:28.031994+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:56:05.644584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":169,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150729+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:20:25.150729+0000","last_peered":"2023-09-23T22:20:25.150729+0000","last_clean":"2023-09-23T22:20:25.150729+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:20:25.150729+0000","last_undegraded":"2023-09-23T22:20:25.150729+0000","last_fullsized":"2023-09-23T22:20:25.150729+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":183,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974812+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:20:27.974812+0000","last_peered":"2023-09-23T22:20:27.974812+0000","last_clean":"2023-09-23T22:20:27.974812+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:20:27.974812+0000","last_undegraded":"2023-09-23T22:20:27.974812+0000","last_fullsized":"2023-09-23T22:20:27.974812+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":166,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T22:20:25.150896+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:20:25.150896+0000","last_peered":"2023-09-23T22:20:25.150896+0000","last_clean":"2023-09-23T22:20:25.150896+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:20:25.150896+0000","last_undegraded":"2023-09-23T22:20:25.150896+0000","last_fullsized":"2023-09-23T22:20:25.150896+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":42,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974704+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:20:27.974704+0000","last_peered":"2023-09-23T22:20:27.974704+0000","last_clean":"2023-09-23T22:20:27.974704+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:20:27.974704+0000","last_undegraded":"2023-09-23T22:20:27.974704+0000","last_fullsized":"2023-09-23T22:20:27.974704+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":189,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.775250+0000","last_change":"2023-09-23T22:16:49.145417+0000","last_active":"2023-09-23T22:20:27.775250+0000","last_peered":"2023-09-23T22:20:27.775250+0000","last_clean":"2023-09-23T22:20:27.775250+0000","last_became_active":"2023-09-23T22:16:49.145017+0000","last_became_peered":"2023-09-23T22:16:49.145017+0000","last_unstale":"2023-09-23T22:20:27.775250+0000","last_undegraded":"2023-09-23T22:20:27.775250+0000","last_fullsized":"2023-09-23T22:20:27.775250+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:47.762149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":230,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.774480+0000","last_change":"2023-09-23T22:17:15.721123+0000","last_active":"2023-09-23T22:20:27.774480+0000","last_peered":"2023-09-23T22:20:27.774480+0000","last_clean":"2023-09-23T22:20:27.774480+0000","last_became_active":"2023-09-23T22:17:15.720812+0000","last_became_peered":"2023-09-23T22:17:15.720812+0000","last_unstale":"2023-09-23T22:20:27.774480+0000","last_undegraded":"2023-09-23T22:20:27.774480+0000","last_fullsized":"2023-09-23T22:20:27.774480+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.176278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":80,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:28.033934+0000","last_change":"2023-09-23T22:18:57.729455+0000","last_active":"2023-09-23T22:20:28.033934+0000","last_peered":"2023-09-23T22:20:28.033934+0000","last_clean":"2023-09-23T22:20:28.033934+0000","last_became_active":"2023-09-23T22:18:57.729088+0000","last_became_peered":"2023-09-23T22:18:57.729088+0000","last_unstale":"2023-09-23T22:20:28.033934+0000","last_undegraded":"2023-09-23T22:20:28.033934+0000","last_fullsized":"2023-09-23T22:20:28.033934+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:17.861334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":176,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T22:20:27.974354+0000","last_change":"2023-09-23T22:16:49.164638+0000","last_active":"2023-09-23T22:20:27.974354+0000","last_peered":"2023-09-23T22:20:27.974354+0000","last_clean":"2023-09-23T22:20:27.974354+0000","last_became_active":"2023-09-23T22:16:49.164349+0000","last_became_peered":"2023-09-23T22:16:49.164349+0000","last_unstale":"2023-09-23T22:20:27.974354+0000","last_undegraded":"2023-09-23T22:20:27.974354+0000","last_fullsized":"2023-09-23T22:20:27.974354+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:26.536927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":911212544,"num_objects":13904,"num_object_clones":0,"num_object_copies":41712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13904,"num_write_kb":889856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2629697536,"data_stored":2629697536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13904,"ondisk_log_size":13904,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4892,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":536698,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":442,"num_read_kb":829,"num_write":217,"num_write_kb":2196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2790831,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234194,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":449196,"kb_used_data":363492,"kb_used_omap":0,"kb_used_meta":85696,"kb_avail":93295956,"statfs":{"total":95995035648,"available":95535058944,"internally_reserved":0,"allocated":372215808,"data_stored":371370897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":87752704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":2.4780000000000002,"15min":1.1970000000000001},"min":{"1min":0.23999999999999999,"5min":0.223,"15min":0.223},"max":{"1min":9.7799999999999994,"5min":109.208,"15min":109.208},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.752,"5min":2.7650000000000001,"15min":1.3149999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":7.4489999999999998,"5min":109.13800000000001,"15min":109.13800000000001},"last":0.59299999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:20:09 2023","interfaces":[{"interface":"back","average":{"1min":9.1620000000000008,"5min":5.9320000000000004,"15min":2.3570000000000002},"min":{"1min":0.25600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":101.145,"5min":123.79300000000001,"15min":123.79300000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":9.4949999999999992,"5min":6.2809999999999997,"15min":2.508},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":101.13,"5min":123.66500000000001,"15min":123.66500000000001},"last":0.40000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":13.066000000000001,"5min":8.5730000000000004,"15min":3.5289999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":105.34399999999999,"5min":125.101,"15min":125.101},"last":0.49199999999999999},{"interface":"front","average":{"1min":12.569000000000001,"5min":8.2129999999999992,"15min":3.1469999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":105.63200000000001,"5min":126.021,"15min":126.021},"last":1.0009999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:20:09 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":2.5619999999999998,"15min":1.2549999999999999},"min":{"1min":0.308,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8319999999999999,"5min":113.282,"15min":113.282},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.2370000000000001,"5min":2.931,"15min":1.4239999999999999},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.7910000000000004,"5min":104.74299999999999,"15min":104.74299999999999},"last":0.55300000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":6.8159999999999998,"5min":2.5489999999999999,"15min":1.173},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":88.543000000000006,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.52400000000000002},{"interface":"front","average":{"1min":6.3959999999999999,"5min":2.286,"15min":1.095},"min":{"1min":0.312,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":88.495999999999995,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":6.9299999999999997,"5min":2.2909999999999999,"15min":1.0860000000000001},"min":{"1min":0.27500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":93.816000000000003,"5min":93.816000000000003,"15min":93.816000000000003},"last":0.51400000000000001},{"interface":"front","average":{"1min":7.0010000000000003,"5min":2.8149999999999999,"15min":1.3129999999999999},"min":{"1min":0.34399999999999997,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":95.579999999999998,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.61499999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:19:38 2023","interfaces":[{"interface":"back","average":{"1min":0.44500000000000001,"5min":0.97399999999999998,"15min":0.67700000000000005},"min":{"1min":0.317,"5min":0.317,"15min":0.315},"max":{"1min":0.66600000000000004,"5min":9.0139999999999993,"15min":9.0139999999999993},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.871,"5min":1.3939999999999999,"15min":0.86199999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":3.7869999999999999,"5min":13.362,"15min":13.362},"last":0.57299999999999995}]}]},{"osd":6,"up_from":0,"seq":197568495830,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795103,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":559888,"kb_used_data":383816,"kb_used_omap":7,"kb_used_meta":176056,"kb_avail":93185264,"statfs":{"total":95995035648,"available":95421710336,"internally_reserved":0,"allocated":393027584,"data_stored":391923168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":180281716},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.9299999999999999,"15min":0.88},"min":{"1min":0.21199999999999999,"5min":0.20399999999999999,"15min":0.19400000000000001},"max":{"1min":5.8840000000000003,"5min":61.890000000000001,"15min":61.890000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.8140000000000001,"15min":1.2050000000000001},"min":{"1min":0.216,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":9.2669999999999995,"5min":62.689999999999998,"15min":62.689999999999998},"last":1.2150000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.605,"5min":2.4079999999999999,"15min":1.0549999999999999},"min":{"1min":0.223,"5min":0.216,"15min":0.216},"max":{"1min":5.2539999999999996,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":4.0620000000000003,"5min":3.032,"15min":1.2849999999999999},"min":{"1min":0.309,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":15.015000000000001,"5min":34.781999999999996,"15min":34.781999999999996},"last":1.304}]},{"osd":3,"last update":"Sat Sep 23 22:20:14 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.4580000000000002,"15min":1.0840000000000001},"min":{"1min":0.35899999999999999,"5min":0.252,"15min":0.252},"max":{"1min":6.4199999999999999,"5min":22.326000000000001,"15min":22.326000000000001},"last":1.329},{"interface":"front","average":{"1min":2.2229999999999999,"5min":2.3239999999999998,"15min":1.048},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.23000000000000001},"max":{"1min":11.005000000000001,"5min":22.259,"15min":22.259},"last":1.133}]},{"osd":4,"last update":"Sat Sep 23 22:20:14 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.0820000000000001,"15min":0.80300000000000005},"min":{"1min":0.29899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.4669999999999996,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.5489999999999999,"5min":1.702,"15min":1.008},"min":{"1min":0.47999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":6.0720000000000001,"5min":11.037000000000001,"15min":11.037000000000001},"last":1.2529999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.7350000000000001,"15min":0.98399999999999999},"min":{"1min":0.442,"5min":0.31,"15min":0.31},"max":{"1min":6.0410000000000004,"5min":11.125,"15min":11.125},"last":1.28},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.5980000000000001,"15min":0.95399999999999996},"min":{"1min":0.33800000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.872999999999999,"5min":10.872999999999999,"15min":10.872999999999999},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.6779999999999999,"15min":0.98099999999999998},"min":{"1min":0.373,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.971,"5min":10.989000000000001,"15min":10.989000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.234,"5min":1.3700000000000001,"15min":0.90200000000000002},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.34999999999999998},"max":{"1min":5.4189999999999996,"5min":10.811999999999999,"15min":10.811999999999999},"last":0.69199999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.804,"15min":1.0269999999999999},"min":{"1min":0.33800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.933999999999999,"5min":10.948,"15min":10.948},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.4319999999999999,"15min":0.92600000000000005},"min":{"1min":0.38600000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.4930000000000003,"5min":10.781000000000001,"15min":10.781000000000001},"last":0.55500000000000005}]}]},{"osd":0,"up_from":93,"seq":399431958637,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":361672,"kb_used_data":181504,"kb_used_omap":8,"kb_used_meta":180151,"kb_avail":93383480,"statfs":{"total":95995035648,"available":95624683520,"internally_reserved":0,"allocated":185860096,"data_stored":184755345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":184475498},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":2.5750000000000002,"15min":1.0860000000000001},"min":{"1min":0.20399999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.9710000000000001,"5min":44.776000000000003,"15min":44.776000000000003},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.5289999999999999,"5min":3.5259999999999998,"15min":1.4199999999999999},"min":{"1min":0.23799999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.641999999999999,"5min":48.838000000000001,"15min":48.838000000000001},"last":0.34499999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:19:41 2023","interfaces":[{"interface":"back","average":{"1min":3.2090000000000001,"5min":2.0630000000000002,"15min":0.95599999999999996},"min":{"1min":0.28399999999999997,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":13.625999999999999,"5min":25.893000000000001,"15min":25.893000000000001},"last":1.2809999999999999},{"interface":"front","average":{"1min":4.2910000000000004,"5min":3.5350000000000001,"15min":1.526},"min":{"1min":0.36199999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.16,"5min":66.665000000000006,"15min":66.665000000000006},"last":1.4990000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.5659999999999998,"15min":1.1519999999999999},"min":{"1min":0.27300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.1269999999999998,"5min":35.496000000000002,"15min":35.496000000000002},"last":1.8640000000000001},{"interface":"front","average":{"1min":1.722,"5min":2.9350000000000001,"15min":1.2709999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.8630000000000004,"5min":35.533000000000001,"15min":35.533000000000001},"last":0.29799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.6389999999999998,"15min":1.319},"min":{"1min":0.34200000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.032999999999999,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.274,"15min":0.86199999999999999},"min":{"1min":0.313,"5min":0.308,"15min":0.308},"max":{"1min":4.0270000000000001,"5min":22.209,"15min":22.209},"last":1.1719999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":2.609,"15min":1.3300000000000001},"min":{"1min":0.34799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9920000000000009,"5min":12.372,"15min":12.372},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.73899999999999999,"5min":1.2909999999999999,"15min":0.88500000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.0030000000000001,"5min":20.390999999999998,"15min":20.390999999999998},"last":1.208}]},{"osd":6,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.6680000000000001,"15min":1.3440000000000001},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.379},"max":{"1min":10.087,"5min":10.275,"15min":10.275},"last":0.625},{"interface":"front","average":{"1min":1.133,"5min":1.401,"15min":0.93700000000000006},"min":{"1min":0.34999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.4039999999999999,"5min":22.263000000000002,"15min":22.263000000000002},"last":1.2490000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":2.524,"15min":1.329},"min":{"1min":0.43099999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":4.1369999999999996,"5min":23.306000000000001,"15min":23.306000000000001},"last":1.4630000000000001},{"interface":"front","average":{"1min":3.0019999999999998,"5min":3.0840000000000001,"15min":1.5289999999999999},"min":{"1min":0.47199999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.061,"5min":23.265999999999998,"15min":23.265999999999998},"last":0.59699999999999998}]}]},{"osd":2,"up_from":103,"seq":442381631576,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":586484,"kb_used_data":406380,"kb_used_omap":10,"kb_used_meta":180085,"kb_avail":93158668,"statfs":{"total":95995035648,"available":95394476032,"internally_reserved":0,"allocated":416133120,"data_stored":415048825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":184407212},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":2.6320000000000001,"15min":1.1020000000000001},"min":{"1min":0.22800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.2009999999999996,"5min":19.344000000000001,"15min":19.344000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.8479999999999999,"15min":1.1919999999999999},"min":{"1min":0.20499999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":4.9199999999999999,"5min":19.149999999999999,"15min":19.149999999999999},"last":0.27700000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":4.9119999999999999,"15min":1.913},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.222},"max":{"1min":20.129999999999999,"5min":157.636,"15min":157.636},"last":0.314},{"interface":"front","average":{"1min":1.1379999999999999,"5min":4.8449999999999998,"15min":1.895},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.6040000000000001,"5min":155.63300000000001,"15min":155.63300000000001},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:20:06 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":2.5950000000000002,"15min":1.0920000000000001},"min":{"1min":0.27100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.9429999999999996,"5min":19.852,"15min":19.852},"last":0.29499999999999998},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.8730000000000002,"15min":1.216},"min":{"1min":0.28100000000000003,"5min":0.26000000000000001,"15min":0.246},"max":{"1min":20.234999999999999,"5min":20.581,"15min":20.581},"last":0.51200000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":1.835,"15min":1.0129999999999999},"min":{"1min":0.28599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.9779999999999998,"5min":11.244,"15min":11.244},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.9610000000000001,"15min":1.0700000000000001},"min":{"1min":0.40100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.4820000000000002,"5min":9.2799999999999994,"15min":9.2799999999999994},"last":0.39700000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":1.9450000000000001,"15min":1.069},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.0019999999999998,"5min":11.291,"15min":11.291},"last":0.44},{"interface":"front","average":{"1min":1.1919999999999999,"5min":2.0800000000000001,"15min":1.099},"min":{"1min":0.39500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.5659999999999998,"5min":9.4499999999999993,"15min":9.4499999999999993},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":2.0950000000000002,"15min":1.139},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.0529999999999999,"5min":17.640999999999998,"15min":17.640999999999998},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.385,"5min":1.9510000000000001,"15min":1.167},"min":{"1min":0.46200000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.5389999999999997,"5min":11.875,"15min":11.875},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":2.29,"15min":1.258},"min":{"1min":0.52600000000000002,"5min":0.377,"15min":0.377},"max":{"1min":20.088000000000001,"5min":20.088000000000001,"15min":20.088000000000001},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.0800000000000001,"5min":2.3620000000000001,"15min":1.2110000000000001},"min":{"1min":0.38500000000000001,"5min":0.38100000000000001,"15min":0.377},"max":{"1min":5.0259999999999998,"5min":17.738,"15min":17.738},"last":0.496}]}]},{"osd":3,"up_from":108,"seq":463856468050,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":817688,"kb_used_data":727824,"kb_used_omap":4,"kb_used_meta":89787,"kb_avail":92927464,"statfs":{"total":95995035648,"available":95157723136,"internally_reserved":0,"allocated":745291776,"data_stored":744176970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":91942445},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":4.8399999999999999,"5min":3.6309999999999998,"15min":1.462},"min":{"1min":0.23000000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":48.676000000000002,"5min":77.834999999999994,"15min":77.834999999999994},"last":1.49},{"interface":"front","average":{"1min":4.8970000000000002,"5min":4.0090000000000003,"15min":1.577},"min":{"1min":0.26600000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":48.774000000000001,"5min":75.558000000000007,"15min":75.558000000000007},"last":1.363}]},{"osd":1,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6779999999999999,"5min":4.2030000000000003,"15min":1.6359999999999999},"min":{"1min":0.24099999999999999,"5min":0.218,"15min":0.218},"max":{"1min":12.445,"5min":82.951999999999998,"15min":82.951999999999998},"last":1.159},{"interface":"front","average":{"1min":2.4119999999999999,"5min":4.0019999999999998,"15min":1.6060000000000001},"min":{"1min":0.22800000000000001,"5min":0.189,"15min":0.189},"max":{"1min":11.970000000000001,"5min":86.001000000000005,"15min":86.001000000000005},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":8.2799999999999994,"15min":3.0219999999999998},"min":{"1min":0.27700000000000002,"5min":0.223,"15min":0.223},"max":{"1min":11.708,"5min":99.962999999999994,"15min":99.962999999999994},"last":1.077},{"interface":"front","average":{"1min":3.238,"5min":7.7300000000000004,"15min":2.8599999999999999},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":11.994,"5min":100.429,"15min":100.429},"last":1.4470000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:20:14 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":1.6020000000000001,"15min":0.95099999999999996},"min":{"1min":0.29699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.911,"5min":11.911,"15min":11.911},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.881,"5min":1.708,"15min":1.006},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.720000000000001,"5min":12.036,"15min":12.036},"last":0.34599999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":4.4249999999999998,"5min":2.157,"15min":1.135},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":49.695999999999998,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":4.7279999999999998,"5min":2.3500000000000001,"15min":1.1899999999999999},"min":{"1min":0.41599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":49.668999999999997,"5min":49.668999999999997,"15min":49.668999999999997},"last":1.1339999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.841,"15min":1.0489999999999999},"min":{"1min":0.441,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.760999999999999,"5min":12.068,"15min":12.068},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.8140000000000001,"15min":1.028},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":1.1060000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":1.9039999999999999,"15min":1.0549999999999999},"min":{"1min":0.39600000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":11.596,"5min":11.624000000000001,"15min":11.624000000000001},"last":1.1890000000000001},{"interface":"front","average":{"1min":2.419,"5min":1.752,"15min":1.04},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":11.945,"5min":12.382,"15min":12.382},"last":0.58299999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019101,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":383156,"kb_used_data":259436,"kb_used_omap":0,"kb_used_meta":123712,"kb_avail":93361996,"statfs":{"total":95995035648,"available":95602683904,"internally_reserved":0,"allocated":265662464,"data_stored":264817716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":126681088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":2.488,"15min":1.246},"min":{"1min":0.30099999999999999,"5min":0.218,"15min":0.218},"max":{"1min":27.942,"5min":66.355000000000004,"15min":66.355000000000004},"last":0.44},{"interface":"front","average":{"1min":3.1339999999999999,"5min":2.7210000000000001,"15min":1.3360000000000001},"min":{"1min":0.315,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":27.956,"5min":66.332999999999998,"15min":66.332999999999998},"last":0.65700000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7570000000000001,"5min":2.1819999999999999,"15min":1.0940000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":19.053000000000001,"5min":38.878,"15min":38.878},"last":0.36099999999999999},{"interface":"front","average":{"1min":3.024,"5min":2.9100000000000001,"15min":1.3540000000000001},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":19.170999999999999,"5min":38.853999999999999,"15min":38.853999999999999},"last":0.58699999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:19:41 2023","interfaces":[{"interface":"back","average":{"1min":9.6750000000000007,"5min":4.7050000000000001,"15min":1.929},"min":{"1min":0.32400000000000001,"5min":0.255,"15min":0.255},"max":{"1min":138.06100000000001,"5min":138.06100000000001,"15min":138.06100000000001},"last":1.883},{"interface":"front","average":{"1min":10.814,"5min":5.5119999999999996,"15min":2.1779999999999999},"min":{"1min":0.309,"5min":0.308,"15min":0.308},"max":{"1min":138.15199999999999,"5min":138.15199999999999,"15min":138.15199999999999},"last":0.752}]},{"osd":3,"last update":"Sat Sep 23 22:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.7989999999999999,"15min":0.995},"min":{"1min":0.33700000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6340000000000003,"5min":8.6340000000000003,"15min":8.6340000000000003},"last":0.80400000000000005},{"interface":"front","average":{"1min":1.476,"5min":1.7869999999999999,"15min":0.97899999999999998},"min":{"1min":0.39500000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.1529999999999996,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":0.60899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.077,"15min":0.68200000000000005},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":6.6989999999999998,"5min":8.9499999999999993,"15min":8.9499999999999993},"last":0.627},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.92100000000000004,"15min":0.66300000000000003},"min":{"1min":0.313,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.6179999999999999,"5min":7.8789999999999996,"15min":7.8789999999999996},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:20:27 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.91300000000000003,"15min":0.65100000000000002},"min":{"1min":0.316,"5min":0.251,"15min":0.251},"max":{"1min":7.3079999999999998,"5min":7.3079999999999998,"15min":7.3079999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0169999999999999,"15min":0.70799999999999996},"min":{"1min":0.374,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":3.9239999999999999,"5min":8.5210000000000008,"15min":8.5210000000000008},"last":0.68500000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.228,"15min":0.79200000000000004},"min":{"1min":0.36299999999999999,"5min":0.307,"15min":0.307},"max":{"1min":6.7480000000000002,"5min":8.577,"15min":8.577},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.95999999999999996,"15min":0.71699999999999997},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2679999999999998,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.55400000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757467,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":454500,"kb_used_data":317596,"kb_used_omap":0,"kb_used_meta":136832,"kb_avail":93290652,"statfs":{"total":95995035648,"available":95529627648,"internally_reserved":0,"allocated":325218304,"data_stored":324378145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":140115968},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8439999999999999,"5min":3.7749999999999999,"15min":1.6319999999999999},"min":{"1min":0.23000000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":28.741,"5min":133.22499999999999,"15min":133.22499999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":3.3719999999999999,"5min":3.9609999999999999,"15min":1.7350000000000001},"min":{"1min":0.27100000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":28.873999999999999,"5min":132.78200000000001,"15min":132.78200000000001},"last":0.79300000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":4.8230000000000004,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.8120000000000003,"5min":83.759,"15min":83.759},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.3560000000000001,"5min":4.7699999999999996,"15min":1.9710000000000001},"min":{"1min":0.28399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":4.5629999999999997,"5min":83.741,"15min":83.741},"last":0.621}]},{"osd":2,"last update":"Sat Sep 23 22:19:40 2023","interfaces":[{"interface":"back","average":{"1min":8.5419999999999998,"5min":7.6210000000000004,"15min":2.915},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":78.137,"5min":104.254,"15min":104.254},"last":0.46000000000000002},{"interface":"front","average":{"1min":10.172000000000001,"5min":8.5760000000000005,"15min":3.2530000000000001},"min":{"1min":0.34000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":78.061000000000007,"5min":104.217,"15min":104.217},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":2.964,"15min":1.3740000000000001},"min":{"1min":0.37,"5min":0.246,"15min":0.246},"max":{"1min":9.9459999999999997,"5min":88.106999999999999,"15min":88.106999999999999},"last":1.611},{"interface":"front","average":{"1min":2.149,"5min":3.173,"15min":1.45},"min":{"1min":0.40699999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.7350000000000003,"5min":87.887,"15min":87.887},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.645,"15min":0.87},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.3399999999999999,"5min":82.530000000000001,"15min":82.530000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.78,"15min":0.94799999999999995},"min":{"1min":0.29399999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.4580000000000002,"5min":85.441000000000003,"15min":85.441000000000003},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:20:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":2.1560000000000001,"15min":1.0780000000000001},"min":{"1min":0.33200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.073,"5min":114.83499999999999,"15min":114.83499999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.505,"5min":1.95,"15min":1.02},"min":{"1min":0.33800000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.73499999999999999,"5min":110.905,"15min":110.905},"last":0.746}]},{"osd":7,"last update":"Sat Sep 23 22:19:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.2969999999999999,"15min":0.81200000000000006},"min":{"1min":0.34999999999999998,"5min":0.33500000000000002,"15min":0.32600000000000001},"max":{"1min":4.3760000000000003,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.97199999999999998,"15min":0.70199999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33200000000000002},"max":{"1min":2.379,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":0.71499999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":202113024,"data_stored":202113024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":412286976,"data_stored":412286976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":503775232,"data_stored":503775232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":712704000,"data_stored":712704000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":309657600,"data_stored":309657600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":190906368,"data_stored":190906368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":298254336,"data_stored":298254336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:20:43.090 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:42 smithi026 ceph-mon[223014]: pgmap v485: 99 pgs: 1 peering, 98 active+clean; 995 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 139 MiB/s wr, 2.20k op/s 2023-09-23T22:20:43.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:42 smithi026 ceph-mon[223014]: failed to encode map e210 with expected crc 2023-09-23T22:20:43.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:42 smithi026 ceph-mon[215775]: pgmap v485: 99 pgs: 1 peering, 98 active+clean; 995 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 139 MiB/s wr, 2.20k op/s 2023-09-23T22:20:43.091 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:42 smithi026 ceph-mon[215775]: failed to encode map e210 with expected crc 2023-09-23T22:20:43.095 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:43.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:42 smithi070 ceph-mon[192652]: pgmap v485: 99 pgs: 1 peering, 98 active+clean; 995 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 139 MiB/s wr, 2.20k op/s 2023-09-23T22:20:43.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:42 smithi070 ceph-mon[192652]: failed to encode map e210 with expected crc 2023-09-23T22:20:43.217 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:43.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:43.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:43.323 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:43.355 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:43.371 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:43.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 25092 25076 130.56 106.125 0.00488587 0.00765509 2023-09-23T22:20:43.409 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:43.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:43.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:43.420 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:43.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:43.561 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:20:43.598 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:20:43.604 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:43.647 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:20:43.690 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:20:43.728 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:20:43.742 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:43.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:43.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:43.762 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:20:43.807 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:20:43.839 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:20:43.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:20:43.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:20:43.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:20:43.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:20:43.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:20:43.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:20:43.845 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:20:43.845 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:20:43.848 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:43.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:20:43.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:20:43.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:43.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[215775]: pgmap v486: 99 pgs: 1 peering, 98 active+clean; 1.2 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 132 MiB/s wr, 2.09k op/s 2023-09-23T22:20:43.853 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4122242961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:43.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:20:43.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:20:43.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:43.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[223014]: pgmap v486: 99 pgs: 1 peering, 98 active+clean; 1.2 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 132 MiB/s wr, 2.09k op/s 2023-09-23T22:20:43.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4122242961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:43.924 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:44.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:20:44.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:20:44.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:44.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:43 smithi070 ceph-mon[192652]: pgmap v486: 99 pgs: 1 peering, 98 active+clean; 1.2 GiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 132 MiB/s wr, 2.09k op/s 2023-09-23T22:20:44.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4122242961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:44.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 26672 26656 128.111 98.75 0.00760392 0.00779833 2023-09-23T22:20:44.940 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:44.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:44.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:45.050 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:45.115 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:45.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 28567 28551 127.419 118.438 0.00443764 0.00784286 2023-09-23T22:20:46.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:20:46.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:46 smithi026 ceph-mon[223014]: 1.2 scrub starts 2023-09-23T22:20:46.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:46 smithi026 ceph-mon[223014]: 1.2 scrub ok 2023-09-23T22:20:46.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:46 smithi026 ceph-mon[223014]: pgmap v487: 99 pgs: 1 peering, 98 active+clean; 1.4 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.82k op/s 2023-09-23T22:20:46.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:46 smithi026 ceph-mon[215775]: 1.2 scrub starts 2023-09-23T22:20:46.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:46 smithi026 ceph-mon[215775]: 1.2 scrub ok 2023-09-23T22:20:46.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:46 smithi026 ceph-mon[215775]: pgmap v487: 99 pgs: 1 peering, 98 active+clean; 1.4 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.82k op/s 2023-09-23T22:20:46.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 30267 30251 126.008 106.25 0.001984 0.00791563 2023-09-23T22:20:46.564 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:46.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:46.568 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:46 smithi070 ceph-mon[192652]: 1.2 scrub starts 2023-09-23T22:20:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:46 smithi070 ceph-mon[192652]: 1.2 scrub ok 2023-09-23T22:20:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:46 smithi070 ceph-mon[192652]: pgmap v487: 99 pgs: 1 peering, 98 active+clean; 1.4 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.82k op/s 2023-09-23T22:20:46.669 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:46.695 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:47.220 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:47.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:47.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:47.330 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:47.363 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:47.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 31343 31327 122.336 67.25 0.0170454 0.00816608 2023-09-23T22:20:47.413 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:47.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:47.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:47.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:47 smithi026 ceph-mon[223014]: 1.6 scrub starts 2023-09-23T22:20:47.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:47 smithi026 ceph-mon[223014]: 1.6 scrub ok 2023-09-23T22:20:47.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:47 smithi026 ceph-mon[223014]: 1.4 scrub starts 2023-09-23T22:20:47.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:47 smithi026 ceph-mon[223014]: 1.4 scrub ok 2023-09-23T22:20:47.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:47 smithi026 ceph-mon[215775]: 1.6 scrub starts 2023-09-23T22:20:47.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:47 smithi026 ceph-mon[215775]: 1.6 scrub ok 2023-09-23T22:20:47.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:47 smithi026 ceph-mon[215775]: 1.4 scrub starts 2023-09-23T22:20:47.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:47 smithi026 ceph-mon[215775]: 1.4 scrub ok 2023-09-23T22:20:47.520 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:47.576 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:47.602 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:47 smithi070 ceph-mon[192652]: 1.6 scrub starts 2023-09-23T22:20:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:47 smithi070 ceph-mon[192652]: 1.6 scrub ok 2023-09-23T22:20:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:47 smithi070 ceph-mon[192652]: 1.4 scrub starts 2023-09-23T22:20:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:47 smithi070 ceph-mon[192652]: 1.4 scrub ok 2023-09-23T22:20:47.948 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:47.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:47.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:48.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:48.086 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:48.138 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:48.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:48.247 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:48.276 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:48.286 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:20:48.286 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:20:48.326 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:48.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:48.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:48.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 32732 32716 120.246 86.8125 0.010317 0.00830949 2023-09-23T22:20:48.432 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:48.508 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:48.824 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:20:49.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:48 smithi070 ceph-mon[192652]: pgmap v488: 99 pgs: 99 active+clean; 1.7 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 104 MiB/s wr, 1.64k op/s 2023-09-23T22:20:49.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2377820809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:20:49.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:48 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:20:49.193 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:20:49.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:48 smithi026 ceph-mon[223014]: pgmap v488: 99 pgs: 99 active+clean; 1.7 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 104 MiB/s wr, 1.64k op/s 2023-09-23T22:20:49.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2377820809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:20:49.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:48 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:20:49.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:48 smithi026 ceph-mon[215775]: pgmap v488: 99 pgs: 99 active+clean; 1.7 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 104 MiB/s wr, 1.64k op/s 2023-09-23T22:20:49.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2377820809' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:20:49.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:48 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:20:49.389 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 34251 34235 118.84 94.9375 0.00230041 0.00840504 2023-09-23T22:20:49.491 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:49.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:49.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:49.602 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:49.631 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:49.677 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:49.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:49.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:49.785 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:49.803 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:20:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:20:49] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2023-09-23T22:20:49.811 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:49.853 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:49.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:49.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:49.960 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:20:49.992 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:49 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:20:49.993 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:49 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:20:49.993 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:49 smithi070 ceph-mon[192652]: failed to encode map e211 with expected crc 2023-09-23T22:20:49.993 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:49 smithi070 ceph-mon[192652]: failed to encode map e211 with expected crc 2023-09-23T22:20:49.993 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:49 smithi070 ceph-mon[192652]: failed to encode map e211 with expected crc 2023-09-23T22:20:49.993 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:49 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:20:49.993 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:49 smithi070 ceph-mon[192652]: osdmap e211: 8 total, 8 up, 7 in 2023-09-23T22:20:49.993 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:49 smithi070 ceph-mon[192652]: failed to encode map e211 with expected crc 2023-09-23T22:20:49.996 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:50.059 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:20:50.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:50.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:20:50.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:20:50.087 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[215775]: failed to encode map e211 with expected crc 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[215775]: failed to encode map e211 with expected crc 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[215775]: failed to encode map e211 with expected crc 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[215775]: osdmap e211: 8 total, 8 up, 7 in 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[215775]: failed to encode map e211 with expected crc 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[223014]: failed to encode map e211 with expected crc 2023-09-23T22:20:50.088 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[223014]: failed to encode map e211 with expected crc 2023-09-23T22:20:50.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[223014]: failed to encode map e211 with expected crc 2023-09-23T22:20:50.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:20:50.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[223014]: osdmap e211: 8 total, 8 up, 7 in 2023-09-23T22:20:50.089 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:49 smithi026 ceph-mon[223014]: failed to encode map e211 with expected crc 2023-09-23T22:20:50.165 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:50.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:50.258 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:50.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:50.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:50.364 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:50.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 35288 35272 115.996 64.8125 0.00589692 0.00859053 2023-09-23T22:20:50.392 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:50 smithi070 ceph-mon[192652]: pgmap v490: 99 pgs: 99 active+clean; 1.9 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 115 MiB/s wr, 1.82k op/s 2023-09-23T22:20:51.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:50 smithi026 ceph-mon[215775]: pgmap v490: 99 pgs: 99 active+clean; 1.9 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 115 MiB/s wr, 1.82k op/s 2023-09-23T22:20:51.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:50 smithi026 ceph-mon[223014]: pgmap v490: 99 pgs: 99 active+clean; 1.9 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 115 MiB/s wr, 1.82k op/s 2023-09-23T22:20:51.389 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:20:51.389503+0000 min lat: 0.00131411 max lat: 0.217315 avg lat: 0.00877295 2023-09-23T22:20:51.389 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:20:51.389 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 36482 36466 113.928 74.625 0.013526 0.00877295 2023-09-23T22:20:51.470 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:51.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:51.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:51.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:51.599 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:20:51.607 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:51.659 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:51.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:51.766 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:51.797 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:51.850 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:51.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:51.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:51 smithi070 ceph-mon[192652]: failed to encode map e211 with expected crc 2023-09-23T22:20:51.957 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:52.032 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:52.055 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:52.189 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:52.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:52.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:52.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:51 smithi026 ceph-mon[223014]: failed to encode map e211 with expected crc 2023-09-23T22:20:52.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:51 smithi026 ceph-mon[215775]: failed to encode map e211 with expected crc 2023-09-23T22:20:52.303 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:52.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:52.382 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:52.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:52.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:52.392 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 37718 37702 112.165 77.25 0.0110268 0.00890869 2023-09-23T22:20:52.487 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:52.522 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:52.576 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:52.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:52.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:52.602 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:20:52.626 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:52.627 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:20:52.682 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:20:52.738 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:52 smithi070 ceph-mon[192652]: pgmap v491: 99 pgs: 99 active+clean; 2.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T22:20:52.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:52 smithi026 ceph-mon[223014]: pgmap v491: 99 pgs: 99 active+clean; 2.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T22:20:52.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:52 smithi026 ceph-mon[215775]: pgmap v491: 99 pgs: 99 active+clean; 2.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T22:20:53.392 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 39126 39110 111.065 88 0.00402897 0.00899834 2023-09-23T22:20:53.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:20:54.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:53 smithi070 ceph-mon[192652]: from='client.45023 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:54.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:53 smithi026 ceph-mon[215775]: from='client.45023 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:54.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:53 smithi026 ceph-mon[223014]: from='client.45023 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:20:54.392 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 39991 39975 108.587 54.0625 0.00909828 0.00920315 2023-09-23T22:20:54.615 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:20:54.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:54.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:20:54.725 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:54.754 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:54.793 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":491,"stamp":"2023-09-23T22:20:51.156456+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2127828915,"num_objects":32423,"num_object_clones":0,"num_object_copies":97269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32423,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":33302,"num_write_kb":2086005,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":139,"num_bytes_recovered":8393538,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":35197,"ondisk_log_size":35197,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7472884,"kb_used_data":6405308,"kb_used_omap":31,"kb_used_meta":1067488,"kb_avail":648743180,"statfs":{"total":671965249536,"available":664313016320,"internally_reserved":0,"allocated":6559035392,"data_stored":6551998660,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31817,"internal_metadata":1093108663},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1084966007,"num_objects":16556,"num_object_clones":0,"num_object_copies":49668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16556,"num_whiteouts":0,"num_read":-309,"num_read_kb":-580,"num_write":16404,"num_write_kb":1058046,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-7,"num_bytes_recovered":-1782427,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003169"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":187,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837598+0000","last_change":"2023-09-23T22:13:27.664671+0000","last_active":"2023-09-23T22:20:39.837598+0000","last_peered":"2023-09-23T22:20:39.837598+0000","last_clean":"2023-09-23T22:20:39.837598+0000","last_became_active":"2023-09-23T22:13:27.664167+0000","last_became_peered":"2023-09-23T22:13:27.664167+0000","last_unstale":"2023-09-23T22:20:39.837598+0000","last_undegraded":"2023-09-23T22:20:39.837598+0000","last_fullsized":"2023-09-23T22:20:39.837598+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:05.523046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":190,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.853421+0000","last_change":"2023-09-23T22:14:00.790313+0000","last_active":"2023-09-23T22:20:40.853421+0000","last_peered":"2023-09-23T22:20:40.853421+0000","last_clean":"2023-09-23T22:20:40.853421+0000","last_became_active":"2023-09-23T22:14:00.789578+0000","last_became_peered":"2023-09-23T22:14:00.789578+0000","last_unstale":"2023-09-23T22:20:40.853421+0000","last_undegraded":"2023-09-23T22:20:40.853421+0000","last_fullsized":"2023-09-23T22:20:40.853421+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:26.894219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":150,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.832880+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:20:39.832880+0000","last_peered":"2023-09-23T22:20:39.832880+0000","last_clean":"2023-09-23T22:20:39.832880+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:20:39.832880+0000","last_undegraded":"2023-09-23T22:20:39.832880+0000","last_fullsized":"2023-09-23T22:20:39.832880+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":38,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.846558+0000","last_change":"2023-09-23T22:18:57.724501+0000","last_active":"2023-09-23T22:20:39.846558+0000","last_peered":"2023-09-23T22:20:39.846558+0000","last_clean":"2023-09-23T22:20:39.846558+0000","last_became_active":"2023-09-23T22:18:57.724203+0000","last_became_peered":"2023-09-23T22:18:57.724203+0000","last_unstale":"2023-09-23T22:20:39.846558+0000","last_undegraded":"2023-09-23T22:20:39.846558+0000","last_fullsized":"2023-09-23T22:20:39.846558+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:52.168267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":230,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.844488+0000","last_change":"2023-09-23T22:18:57.722574+0000","last_active":"2023-09-23T22:20:40.844488+0000","last_peered":"2023-09-23T22:20:40.844488+0000","last_clean":"2023-09-23T22:20:40.844488+0000","last_became_active":"2023-09-23T22:18:57.721800+0000","last_became_peered":"2023-09-23T22:18:57.721800+0000","last_unstale":"2023-09-23T22:20:40.844488+0000","last_undegraded":"2023-09-23T22:20:40.844488+0000","last_fullsized":"2023-09-23T22:20:40.844488+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:09.303838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":166,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851823+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:20:40.851823+0000","last_peered":"2023-09-23T22:20:40.851823+0000","last_clean":"2023-09-23T22:20:40.851823+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:20:40.851823+0000","last_undegraded":"2023-09-23T22:20:40.851823+0000","last_fullsized":"2023-09-23T22:20:40.851823+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":76,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.844899+0000","last_change":"2023-09-23T22:18:57.729038+0000","last_active":"2023-09-23T22:20:39.844899+0000","last_peered":"2023-09-23T22:20:39.844899+0000","last_clean":"2023-09-23T22:20:39.844899+0000","last_became_active":"2023-09-23T22:18:57.728650+0000","last_became_peered":"2023-09-23T22:18:57.728650+0000","last_unstale":"2023-09-23T22:20:39.844899+0000","last_undegraded":"2023-09-23T22:20:39.844899+0000","last_fullsized":"2023-09-23T22:20:39.844899+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:11.461883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":185,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.844707+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:20:40.844707+0000","last_peered":"2023-09-23T22:20:40.844707+0000","last_clean":"2023-09-23T22:20:40.844707+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:20:40.844707+0000","last_undegraded":"2023-09-23T22:20:40.844707+0000","last_fullsized":"2023-09-23T22:20:40.844707+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":171,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851401+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:20:40.851401+0000","last_peered":"2023-09-23T22:20:40.851401+0000","last_clean":"2023-09-23T22:20:40.851401+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:20:40.851401+0000","last_undegraded":"2023-09-23T22:20:40.851401+0000","last_fullsized":"2023-09-23T22:20:40.851401+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":135,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.852162+0000","last_change":"2023-09-23T22:16:49.154341+0000","last_active":"2023-09-23T22:20:40.852162+0000","last_peered":"2023-09-23T22:20:40.852162+0000","last_clean":"2023-09-23T22:20:40.852162+0000","last_became_active":"2023-09-23T22:16:49.154041+0000","last_became_peered":"2023-09-23T22:16:49.154041+0000","last_unstale":"2023-09-23T22:20:40.852162+0000","last_undegraded":"2023-09-23T22:20:40.852162+0000","last_fullsized":"2023-09-23T22:20:40.852162+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:05.733262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":123,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.853313+0000","last_change":"2023-09-23T22:15:14.829731+0000","last_active":"2023-09-23T22:20:40.853313+0000","last_peered":"2023-09-23T22:20:40.853313+0000","last_clean":"2023-09-23T22:20:40.853313+0000","last_became_active":"2023-09-23T22:15:14.829437+0000","last_became_peered":"2023-09-23T22:15:14.829437+0000","last_unstale":"2023-09-23T22:20:40.853313+0000","last_undegraded":"2023-09-23T22:20:40.853313+0000","last_fullsized":"2023-09-23T22:20:40.853313+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:45.456558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":177,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.834830+0000","last_change":"2023-09-23T22:18:57.723165+0000","last_active":"2023-09-23T22:20:39.834830+0000","last_peered":"2023-09-23T22:20:39.834830+0000","last_clean":"2023-09-23T22:20:39.834830+0000","last_became_active":"2023-09-23T22:18:57.722863+0000","last_became_peered":"2023-09-23T22:18:57.722863+0000","last_unstale":"2023-09-23T22:20:39.834830+0000","last_undegraded":"2023-09-23T22:20:39.834830+0000","last_fullsized":"2023-09-23T22:20:39.834830+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:19.645029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":194,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837603+0000","last_change":"2023-09-23T22:16:49.265991+0000","last_active":"2023-09-23T22:20:39.837603+0000","last_peered":"2023-09-23T22:20:39.837603+0000","last_clean":"2023-09-23T22:20:39.837603+0000","last_became_active":"2023-09-23T22:16:49.251169+0000","last_became_peered":"2023-09-23T22:16:49.251169+0000","last_unstale":"2023-09-23T22:20:39.837603+0000","last_undegraded":"2023-09-23T22:20:39.837603+0000","last_fullsized":"2023-09-23T22:20:39.837603+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:25.502218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":178,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837752+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:20:39.837752+0000","last_peered":"2023-09-23T22:20:39.837752+0000","last_clean":"2023-09-23T22:20:39.837752+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:20:39.837752+0000","last_undegraded":"2023-09-23T22:20:39.837752+0000","last_fullsized":"2023-09-23T22:20:39.837752+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":121,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837511+0000","last_change":"2023-09-23T22:15:14.826143+0000","last_active":"2023-09-23T22:20:39.837511+0000","last_peered":"2023-09-23T22:20:39.837511+0000","last_clean":"2023-09-23T22:20:39.837511+0000","last_became_active":"2023-09-23T22:15:14.825780+0000","last_became_peered":"2023-09-23T22:15:14.825780+0000","last_unstale":"2023-09-23T22:20:39.837511+0000","last_undegraded":"2023-09-23T22:20:39.837511+0000","last_fullsized":"2023-09-23T22:20:39.837511+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:22.608676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":176,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.853004+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:20:40.853004+0000","last_peered":"2023-09-23T22:20:40.853004+0000","last_clean":"2023-09-23T22:20:40.853004+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:20:40.853004+0000","last_undegraded":"2023-09-23T22:20:40.853004+0000","last_fullsized":"2023-09-23T22:20:40.853004+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":38,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.845592+0000","last_change":"2023-09-23T22:18:57.726112+0000","last_active":"2023-09-23T22:20:39.845592+0000","last_peered":"2023-09-23T22:20:39.845592+0000","last_clean":"2023-09-23T22:20:39.845592+0000","last_became_active":"2023-09-23T22:18:57.725558+0000","last_became_peered":"2023-09-23T22:18:57.725558+0000","last_unstale":"2023-09-23T22:20:39.845592+0000","last_undegraded":"2023-09-23T22:20:39.845592+0000","last_fullsized":"2023-09-23T22:20:39.845592+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:38.315664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":180,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.858793+0000","last_change":"2023-09-23T22:20:40.858603+0000","last_active":"2023-09-23T22:20:40.858793+0000","last_peered":"2023-09-23T22:20:40.858793+0000","last_clean":"2023-09-23T22:20:40.858793+0000","last_became_active":"2023-09-23T22:20:40.858216+0000","last_became_peered":"2023-09-23T22:20:40.858216+0000","last_unstale":"2023-09-23T22:20:40.858793+0000","last_undegraded":"2023-09-23T22:20:40.858793+0000","last_fullsized":"2023-09-23T22:20:40.858793+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:42.220293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":133,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837345+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:20:39.837345+0000","last_peered":"2023-09-23T22:20:39.837345+0000","last_clean":"2023-09-23T22:20:39.837345+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:20:39.837345+0000","last_undegraded":"2023-09-23T22:20:39.837345+0000","last_fullsized":"2023-09-23T22:20:39.837345+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":38,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.846836+0000","last_change":"2023-09-23T22:18:57.726691+0000","last_active":"2023-09-23T22:20:39.846836+0000","last_peered":"2023-09-23T22:20:39.846836+0000","last_clean":"2023-09-23T22:20:39.846836+0000","last_became_active":"2023-09-23T22:18:57.726386+0000","last_became_peered":"2023-09-23T22:18:57.726386+0000","last_unstale":"2023-09-23T22:20:39.846836+0000","last_undegraded":"2023-09-23T22:20:39.846836+0000","last_fullsized":"2023-09-23T22:20:39.846836+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:34.829149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":179,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.845165+0000","last_change":"2023-09-23T22:18:57.728767+0000","last_active":"2023-09-23T22:20:39.845165+0000","last_peered":"2023-09-23T22:20:39.845165+0000","last_clean":"2023-09-23T22:20:39.845165+0000","last_became_active":"2023-09-23T22:18:57.728377+0000","last_became_peered":"2023-09-23T22:18:57.728377+0000","last_unstale":"2023-09-23T22:20:39.845165+0000","last_undegraded":"2023-09-23T22:20:39.845165+0000","last_fullsized":"2023-09-23T22:20:39.845165+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:01:01.370904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":176,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.854376+0000","last_change":"2023-09-23T22:14:00.794463+0000","last_active":"2023-09-23T22:20:40.854376+0000","last_peered":"2023-09-23T22:20:40.854376+0000","last_clean":"2023-09-23T22:20:40.854376+0000","last_became_active":"2023-09-23T22:14:00.794204+0000","last_became_peered":"2023-09-23T22:14:00.794204+0000","last_unstale":"2023-09-23T22:20:40.854376+0000","last_undegraded":"2023-09-23T22:20:40.854376+0000","last_fullsized":"2023-09-23T22:20:40.854376+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.816065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"211'2200","reported_seq":2215,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T22:20:50.158212+0000","last_change":"2023-09-23T22:20:19.470064+0000","last_active":"2023-09-23T22:20:50.158212+0000","last_peered":"2023-09-23T22:20:50.158212+0000","last_clean":"2023-09-23T22:20:50.158212+0000","last_became_active":"2023-09-23T22:20:19.469807+0000","last_became_peered":"2023-09-23T22:20:19.469807+0000","last_unstale":"2023-09-23T22:20:50.158212+0000","last_undegraded":"2023-09-23T22:20:50.158212+0000","last_fullsized":"2023-09-23T22:20:50.158212+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":2200,"log_dups_size":0,"ondisk_log_size":2200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:45.351484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144179200,"num_objects":2200,"num_object_clones":0,"num_object_copies":6600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2200,"num_write_kb":140800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":177,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837793+0000","last_change":"2023-09-23T22:19:00.682889+0000","last_active":"2023-09-23T22:20:39.837793+0000","last_peered":"2023-09-23T22:20:39.837793+0000","last_clean":"2023-09-23T22:20:39.837793+0000","last_became_active":"2023-09-23T22:18:57.718002+0000","last_became_peered":"2023-09-23T22:18:57.718002+0000","last_unstale":"2023-09-23T22:20:39.837793+0000","last_undegraded":"2023-09-23T22:20:39.837793+0000","last_fullsized":"2023-09-23T22:20:39.837793+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:22.517277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":163,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.850783+0000","last_change":"2023-09-23T22:16:49.158580+0000","last_active":"2023-09-23T22:20:40.850783+0000","last_peered":"2023-09-23T22:20:40.850783+0000","last_clean":"2023-09-23T22:20:40.850783+0000","last_became_active":"2023-09-23T22:16:49.158266+0000","last_became_peered":"2023-09-23T22:16:49.158266+0000","last_unstale":"2023-09-23T22:20:40.850783+0000","last_undegraded":"2023-09-23T22:20:40.850783+0000","last_fullsized":"2023-09-23T22:20:40.850783+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:03.791658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"211'2120","reported_seq":2136,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T22:20:48.913306+0000","last_change":"2023-09-23T22:20:19.474568+0000","last_active":"2023-09-23T22:20:48.913306+0000","last_peered":"2023-09-23T22:20:48.913306+0000","last_clean":"2023-09-23T22:20:48.913306+0000","last_became_active":"2023-09-23T22:20:19.474222+0000","last_became_peered":"2023-09-23T22:20:19.474222+0000","last_unstale":"2023-09-23T22:20:48.913306+0000","last_undegraded":"2023-09-23T22:20:48.913306+0000","last_fullsized":"2023-09-23T22:20:48.913306+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":2120,"log_dups_size":0,"ondisk_log_size":2120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:25.959782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138936320,"num_objects":2120,"num_object_clones":0,"num_object_copies":6360,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2120,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2120,"num_write_kb":135680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":121,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.838198+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:20:39.838198+0000","last_peered":"2023-09-23T22:20:39.838198+0000","last_clean":"2023-09-23T22:20:39.838198+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:20:39.838198+0000","last_undegraded":"2023-09-23T22:20:39.838198+0000","last_fullsized":"2023-09-23T22:20:39.838198+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":177,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837332+0000","last_change":"2023-09-23T22:14:00.795430+0000","last_active":"2023-09-23T22:20:39.837332+0000","last_peered":"2023-09-23T22:20:39.837332+0000","last_clean":"2023-09-23T22:20:39.837332+0000","last_became_active":"2023-09-23T22:14:00.795100+0000","last_became_peered":"2023-09-23T22:14:00.795100+0000","last_unstale":"2023-09-23T22:20:39.837332+0000","last_undegraded":"2023-09-23T22:20:39.837332+0000","last_fullsized":"2023-09-23T22:20:39.837332+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:43.270833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"210'1900","reported_seq":1916,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:46.277811+0000","last_change":"2023-09-23T22:20:19.468974+0000","last_active":"2023-09-23T22:20:46.277811+0000","last_peered":"2023-09-23T22:20:46.277811+0000","last_clean":"2023-09-23T22:20:46.277811+0000","last_became_active":"2023-09-23T22:20:19.468672+0000","last_became_peered":"2023-09-23T22:20:19.468672+0000","last_unstale":"2023-09-23T22:20:46.277811+0000","last_undegraded":"2023-09-23T22:20:46.277811+0000","last_fullsized":"2023-09-23T22:20:46.277811+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1900,"log_dups_size":0,"ondisk_log_size":1900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:39.575600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124518400,"num_objects":1900,"num_object_clones":0,"num_object_copies":5700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1900,"num_write_kb":121600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":136,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851510+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:20:40.851510+0000","last_peered":"2023-09-23T22:20:40.851510+0000","last_clean":"2023-09-23T22:20:40.851510+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:20:40.851510+0000","last_undegraded":"2023-09-23T22:20:40.851510+0000","last_fullsized":"2023-09-23T22:20:40.851510+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":125,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.834914+0000","last_change":"2023-09-23T22:16:49.161676+0000","last_active":"2023-09-23T22:20:39.834914+0000","last_peered":"2023-09-23T22:20:39.834914+0000","last_clean":"2023-09-23T22:20:39.834914+0000","last_became_active":"2023-09-23T22:16:49.161247+0000","last_became_peered":"2023-09-23T22:16:49.161247+0000","last_unstale":"2023-09-23T22:20:39.834914+0000","last_undegraded":"2023-09-23T22:20:39.834914+0000","last_fullsized":"2023-09-23T22:20:39.834914+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:30.708371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"210'1933","reported_seq":1949,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:46.222550+0000","last_change":"2023-09-23T22:20:19.471789+0000","last_active":"2023-09-23T22:20:46.222550+0000","last_peered":"2023-09-23T22:20:46.222550+0000","last_clean":"2023-09-23T22:20:46.222550+0000","last_became_active":"2023-09-23T22:20:19.471481+0000","last_became_peered":"2023-09-23T22:20:19.471481+0000","last_unstale":"2023-09-23T22:20:46.222550+0000","last_undegraded":"2023-09-23T22:20:46.222550+0000","last_fullsized":"2023-09-23T22:20:46.222550+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1933,"log_dups_size":0,"ondisk_log_size":1933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:43.740017+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126681088,"num_objects":1933,"num_object_clones":0,"num_object_copies":5799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1933,"num_write_kb":123712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":181,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851238+0000","last_change":"2023-09-23T22:16:49.164833+0000","last_active":"2023-09-23T22:20:40.851238+0000","last_peered":"2023-09-23T22:20:40.851238+0000","last_clean":"2023-09-23T22:20:40.851238+0000","last_became_active":"2023-09-23T22:16:49.163833+0000","last_became_peered":"2023-09-23T22:16:49.163833+0000","last_unstale":"2023-09-23T22:20:40.851238+0000","last_undegraded":"2023-09-23T22:20:40.851238+0000","last_fullsized":"2023-09-23T22:20:40.851238+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:41.891395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.846233+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:20:39.846233+0000","last_peered":"2023-09-23T22:20:39.846233+0000","last_clean":"2023-09-23T22:20:39.846233+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:20:39.846233+0000","last_undegraded":"2023-09-23T22:20:39.846233+0000","last_fullsized":"2023-09-23T22:20:39.846233+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"210'1990","reported_seq":2005,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:47.438084+0000","last_change":"2023-09-23T22:20:19.469349+0000","last_active":"2023-09-23T22:20:47.438084+0000","last_peered":"2023-09-23T22:20:47.438084+0000","last_clean":"2023-09-23T22:20:47.438084+0000","last_became_active":"2023-09-23T22:20:19.469086+0000","last_became_peered":"2023-09-23T22:20:19.469086+0000","last_unstale":"2023-09-23T22:20:47.438084+0000","last_undegraded":"2023-09-23T22:20:47.438084+0000","last_fullsized":"2023-09-23T22:20:47.438084+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1990,"log_dups_size":0,"ondisk_log_size":1990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:28.871375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130416640,"num_objects":1990,"num_object_clones":0,"num_object_copies":5970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1990,"num_write_kb":127360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":496,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.859368+0000","last_change":"2023-09-23T22:20:40.859368+0000","last_active":"2023-09-23T22:20:40.859368+0000","last_peered":"2023-09-23T22:20:40.859368+0000","last_clean":"2023-09-23T22:20:40.859368+0000","last_became_active":"2023-09-23T22:20:18.460615+0000","last_became_peered":"2023-09-23T22:20:18.460615+0000","last_unstale":"2023-09-23T22:20:40.859368+0000","last_undegraded":"2023-09-23T22:20:40.859368+0000","last_fullsized":"2023-09-23T22:20:40.859368+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:44:01.936102+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":223,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:43.914217+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:20:43.914217+0000","last_peered":"2023-09-23T22:20:43.914217+0000","last_clean":"2023-09-23T22:20:43.914217+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:20:43.914217+0000","last_undegraded":"2023-09-23T22:20:43.914217+0000","last_fullsized":"2023-09-23T22:20:43.914217+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":178,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837515+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:20:39.837515+0000","last_peered":"2023-09-23T22:20:39.837515+0000","last_clean":"2023-09-23T22:20:39.837515+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:20:39.837515+0000","last_undegraded":"2023-09-23T22:20:39.837515+0000","last_fullsized":"2023-09-23T22:20:39.837515+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"210'1908","reported_seq":1924,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:46.244969+0000","last_change":"2023-09-23T22:20:19.470036+0000","last_active":"2023-09-23T22:20:46.244969+0000","last_peered":"2023-09-23T22:20:46.244969+0000","last_clean":"2023-09-23T22:20:46.244969+0000","last_became_active":"2023-09-23T22:20:19.468632+0000","last_became_peered":"2023-09-23T22:20:19.468632+0000","last_unstale":"2023-09-23T22:20:46.244969+0000","last_undegraded":"2023-09-23T22:20:46.244969+0000","last_fullsized":"2023-09-23T22:20:46.244969+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1908,"log_dups_size":0,"ondisk_log_size":1908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:04.458920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125042688,"num_objects":1908,"num_object_clones":0,"num_object_copies":5724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1908,"num_write_kb":122112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":499,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.850631+0000","last_change":"2023-09-23T22:20:39.551790+0000","last_active":"2023-09-23T22:20:40.850631+0000","last_peered":"2023-09-23T22:20:40.850631+0000","last_clean":"2023-09-23T22:20:40.850631+0000","last_became_active":"2023-09-23T22:20:18.471844+0000","last_became_peered":"2023-09-23T22:20:18.471844+0000","last_unstale":"2023-09-23T22:20:40.850631+0000","last_undegraded":"2023-09-23T22:20:40.850631+0000","last_fullsized":"2023-09-23T22:20:40.850631+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:13.376798+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":126,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837409+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:20:39.837409+0000","last_peered":"2023-09-23T22:20:39.837409+0000","last_clean":"2023-09-23T22:20:39.837409+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:20:39.837409+0000","last_undegraded":"2023-09-23T22:20:39.837409+0000","last_fullsized":"2023-09-23T22:20:39.837409+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":185,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837664+0000","last_change":"2023-09-23T22:13:27.663241+0000","last_active":"2023-09-23T22:20:39.837664+0000","last_peered":"2023-09-23T22:20:39.837664+0000","last_clean":"2023-09-23T22:20:39.837664+0000","last_became_active":"2023-09-23T22:13:27.662891+0000","last_became_peered":"2023-09-23T22:13:27.662891+0000","last_unstale":"2023-09-23T22:20:39.837664+0000","last_undegraded":"2023-09-23T22:20:39.837664+0000","last_fullsized":"2023-09-23T22:20:39.837664+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:34.493603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"211'2046","reported_seq":2063,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T22:20:48.910038+0000","last_change":"2023-09-23T22:20:19.469847+0000","last_active":"2023-09-23T22:20:48.910038+0000","last_peered":"2023-09-23T22:20:48.910038+0000","last_clean":"2023-09-23T22:20:48.910038+0000","last_became_active":"2023-09-23T22:20:19.469569+0000","last_became_peered":"2023-09-23T22:20:19.469569+0000","last_unstale":"2023-09-23T22:20:48.910038+0000","last_undegraded":"2023-09-23T22:20:48.910038+0000","last_fullsized":"2023-09-23T22:20:48.910038+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":2046,"log_dups_size":0,"ondisk_log_size":2046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:16.691256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134086656,"num_objects":2046,"num_object_clones":0,"num_object_copies":6138,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2046,"num_write_kb":130944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":161,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837180+0000","last_change":"2023-09-23T22:13:27.665993+0000","last_active":"2023-09-23T22:20:39.837180+0000","last_peered":"2023-09-23T22:20:39.837180+0000","last_clean":"2023-09-23T22:20:39.837180+0000","last_became_active":"2023-09-23T22:13:27.665594+0000","last_became_peered":"2023-09-23T22:13:27.665594+0000","last_unstale":"2023-09-23T22:20:39.837180+0000","last_undegraded":"2023-09-23T22:20:39.837180+0000","last_fullsized":"2023-09-23T22:20:39.837180+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:22.687112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":188,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.853041+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:20:40.853041+0000","last_peered":"2023-09-23T22:20:40.853041+0000","last_clean":"2023-09-23T22:20:40.853041+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:20:40.853041+0000","last_undegraded":"2023-09-23T22:20:40.853041+0000","last_fullsized":"2023-09-23T22:20:40.853041+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"211'2188","reported_seq":2204,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T22:20:50.163541+0000","last_change":"2023-09-23T22:20:19.473637+0000","last_active":"2023-09-23T22:20:50.163541+0000","last_peered":"2023-09-23T22:20:50.163541+0000","last_clean":"2023-09-23T22:20:50.163541+0000","last_became_active":"2023-09-23T22:20:19.473385+0000","last_became_peered":"2023-09-23T22:20:19.473385+0000","last_unstale":"2023-09-23T22:20:50.163541+0000","last_undegraded":"2023-09-23T22:20:50.163541+0000","last_fullsized":"2023-09-23T22:20:50.163541+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":2188,"log_dups_size":0,"ondisk_log_size":2188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:01.911861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143392768,"num_objects":2188,"num_object_clones":0,"num_object_copies":6564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2188,"num_write_kb":140032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":495,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.850720+0000","last_change":"2023-09-23T22:20:36.566780+0000","last_active":"2023-09-23T22:20:40.850720+0000","last_peered":"2023-09-23T22:20:40.850720+0000","last_clean":"2023-09-23T22:20:40.850720+0000","last_became_active":"2023-09-23T22:20:18.472892+0000","last_became_peered":"2023-09-23T22:20:18.472892+0000","last_unstale":"2023-09-23T22:20:40.850720+0000","last_undegraded":"2023-09-23T22:20:40.850720+0000","last_fullsized":"2023-09-23T22:20:40.850720+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:12.879736+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":178,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.844331+0000","last_change":"2023-09-23T22:14:00.795928+0000","last_active":"2023-09-23T22:20:40.844331+0000","last_peered":"2023-09-23T22:20:40.844331+0000","last_clean":"2023-09-23T22:20:40.844331+0000","last_became_active":"2023-09-23T22:14:00.795642+0000","last_became_peered":"2023-09-23T22:14:00.795642+0000","last_unstale":"2023-09-23T22:20:40.844331+0000","last_undegraded":"2023-09-23T22:20:40.844331+0000","last_fullsized":"2023-09-23T22:20:40.844331+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:18.143679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":150,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.836186+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:20:39.836186+0000","last_peered":"2023-09-23T22:20:39.836186+0000","last_clean":"2023-09-23T22:20:39.836186+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:20:39.836186+0000","last_undegraded":"2023-09-23T22:20:39.836186+0000","last_fullsized":"2023-09-23T22:20:39.836186+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"210'1923","reported_seq":1939,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:46.238895+0000","last_change":"2023-09-23T22:20:19.471690+0000","last_active":"2023-09-23T22:20:46.238895+0000","last_peered":"2023-09-23T22:20:46.238895+0000","last_clean":"2023-09-23T22:20:46.238895+0000","last_became_active":"2023-09-23T22:20:19.471393+0000","last_became_peered":"2023-09-23T22:20:19.471393+0000","last_unstale":"2023-09-23T22:20:46.238895+0000","last_undegraded":"2023-09-23T22:20:46.238895+0000","last_fullsized":"2023-09-23T22:20:46.238895+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1923,"log_dups_size":0,"ondisk_log_size":1923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:23.131932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126025728,"num_objects":1923,"num_object_clones":0,"num_object_copies":5769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1923,"num_write_kb":123072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":496,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.854261+0000","last_change":"2023-09-23T22:20:36.790562+0000","last_active":"2023-09-23T22:20:40.854261+0000","last_peered":"2023-09-23T22:20:40.854261+0000","last_clean":"2023-09-23T22:20:40.854261+0000","last_became_active":"2023-09-23T22:20:18.467701+0000","last_became_peered":"2023-09-23T22:20:18.467701+0000","last_unstale":"2023-09-23T22:20:40.854261+0000","last_undegraded":"2023-09-23T22:20:40.854261+0000","last_fullsized":"2023-09-23T22:20:40.854261+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:31.210808+0000","scrub_duration":0.0070908769999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":757,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.835246+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:20:39.835246+0000","last_peered":"2023-09-23T22:20:39.835246+0000","last_clean":"2023-09-23T22:20:39.835246+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:20:39.835246+0000","last_undegraded":"2023-09-23T22:20:39.835246+0000","last_fullsized":"2023-09-23T22:20:39.835246+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":188,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851592+0000","last_change":"2023-09-23T22:14:00.792797+0000","last_active":"2023-09-23T22:20:40.851592+0000","last_peered":"2023-09-23T22:20:40.851592+0000","last_clean":"2023-09-23T22:20:40.851592+0000","last_became_active":"2023-09-23T22:14:00.792515+0000","last_became_peered":"2023-09-23T22:14:00.792515+0000","last_unstale":"2023-09-23T22:20:40.851592+0000","last_undegraded":"2023-09-23T22:20:40.851592+0000","last_fullsized":"2023-09-23T22:20:40.851592+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:56:19.991633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":132,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.849799+0000","last_change":"2023-09-23T22:13:27.665798+0000","last_active":"2023-09-23T22:20:39.849799+0000","last_peered":"2023-09-23T22:20:39.849799+0000","last_clean":"2023-09-23T22:20:39.849799+0000","last_became_active":"2023-09-23T22:13:27.665577+0000","last_became_peered":"2023-09-23T22:13:27.665577+0000","last_unstale":"2023-09-23T22:20:39.849799+0000","last_undegraded":"2023-09-23T22:20:39.849799+0000","last_fullsized":"2023-09-23T22:20:39.849799+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:15.531831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"211'2109","reported_seq":2126,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T22:20:48.901485+0000","last_change":"2023-09-23T22:20:19.469302+0000","last_active":"2023-09-23T22:20:48.901485+0000","last_peered":"2023-09-23T22:20:48.901485+0000","last_clean":"2023-09-23T22:20:48.901485+0000","last_became_active":"2023-09-23T22:20:19.469029+0000","last_became_peered":"2023-09-23T22:20:19.469029+0000","last_unstale":"2023-09-23T22:20:48.901485+0000","last_undegraded":"2023-09-23T22:20:48.901485+0000","last_fullsized":"2023-09-23T22:20:48.901485+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":2109,"log_dups_size":0,"ondisk_log_size":2109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:08.337445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138215424,"num_objects":2109,"num_object_clones":0,"num_object_copies":6327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2109,"num_write_kb":134976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":495,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.850874+0000","last_change":"2023-09-23T22:20:38.712424+0000","last_active":"2023-09-23T22:20:40.850874+0000","last_peered":"2023-09-23T22:20:40.850874+0000","last_clean":"2023-09-23T22:20:40.850874+0000","last_became_active":"2023-09-23T22:20:18.469020+0000","last_became_peered":"2023-09-23T22:20:18.469020+0000","last_unstale":"2023-09-23T22:20:40.850874+0000","last_undegraded":"2023-09-23T22:20:40.850874+0000","last_fullsized":"2023-09-23T22:20:40.850874+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:01.792474+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":943,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.850848+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:20:40.850848+0000","last_peered":"2023-09-23T22:20:40.850848+0000","last_clean":"2023-09-23T22:20:40.850848+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:20:40.850848+0000","last_undegraded":"2023-09-23T22:20:40.850848+0000","last_fullsized":"2023-09-23T22:20:40.850848+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":219,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837403+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:20:39.837403+0000","last_peered":"2023-09-23T22:20:39.837403+0000","last_clean":"2023-09-23T22:20:39.837403+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:20:39.837403+0000","last_undegraded":"2023-09-23T22:20:39.837403+0000","last_fullsized":"2023-09-23T22:20:39.837403+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":185,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851365+0000","last_change":"2023-09-23T22:14:00.796811+0000","last_active":"2023-09-23T22:20:40.851365+0000","last_peered":"2023-09-23T22:20:40.851365+0000","last_clean":"2023-09-23T22:20:40.851365+0000","last_became_active":"2023-09-23T22:14:00.796311+0000","last_became_peered":"2023-09-23T22:14:00.796311+0000","last_unstale":"2023-09-23T22:20:40.851365+0000","last_undegraded":"2023-09-23T22:20:40.851365+0000","last_fullsized":"2023-09-23T22:20:40.851365+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:22.749914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"211'2037","reported_seq":2054,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T22:20:48.893180+0000","last_change":"2023-09-23T22:20:19.473261+0000","last_active":"2023-09-23T22:20:48.893180+0000","last_peered":"2023-09-23T22:20:48.893180+0000","last_clean":"2023-09-23T22:20:48.893180+0000","last_became_active":"2023-09-23T22:20:19.472983+0000","last_became_peered":"2023-09-23T22:20:19.472983+0000","last_unstale":"2023-09-23T22:20:48.893180+0000","last_undegraded":"2023-09-23T22:20:48.893180+0000","last_fullsized":"2023-09-23T22:20:48.893180+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":2037,"log_dups_size":0,"ondisk_log_size":2037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:47.461541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133496832,"num_objects":2037,"num_object_clones":0,"num_object_copies":6111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2037,"num_write_kb":130368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":503,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:44.599843+0000","last_change":"2023-09-23T22:20:44.599961+0000","last_active":"2023-09-23T22:20:44.599843+0000","last_peered":"2023-09-23T22:20:44.599843+0000","last_clean":"2023-09-23T22:20:44.599843+0000","last_became_active":"2023-09-23T22:20:18.464483+0000","last_became_peered":"2023-09-23T22:20:18.464483+0000","last_unstale":"2023-09-23T22:20:44.599843+0000","last_undegraded":"2023-09-23T22:20:44.599843+0000","last_fullsized":"2023-09-23T22:20:44.599843+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:01.545919+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":552,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.842725+0000","last_change":"2023-09-23T22:15:51.379227+0000","last_active":"2023-09-23T22:20:39.842725+0000","last_peered":"2023-09-23T22:20:39.842725+0000","last_clean":"2023-09-23T22:20:39.842725+0000","last_became_active":"2023-09-23T22:15:43.928977+0000","last_became_peered":"2023-09-23T22:15:43.928977+0000","last_unstale":"2023-09-23T22:20:39.842725+0000","last_undegraded":"2023-09-23T22:20:39.842725+0000","last_fullsized":"2023-09-23T22:20:39.842725+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:04.398533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":159,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.833442+0000","last_change":"2023-09-23T22:16:49.251412+0000","last_active":"2023-09-23T22:20:39.833442+0000","last_peered":"2023-09-23T22:20:39.833442+0000","last_clean":"2023-09-23T22:20:39.833442+0000","last_became_active":"2023-09-23T22:16:49.251097+0000","last_became_peered":"2023-09-23T22:16:49.251097+0000","last_unstale":"2023-09-23T22:20:39.833442+0000","last_undegraded":"2023-09-23T22:20:39.833442+0000","last_fullsized":"2023-09-23T22:20:39.833442+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:41.601435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":215,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.836266+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:20:39.836266+0000","last_peered":"2023-09-23T22:20:39.836266+0000","last_clean":"2023-09-23T22:20:39.836266+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:20:39.836266+0000","last_undegraded":"2023-09-23T22:20:39.836266+0000","last_fullsized":"2023-09-23T22:20:39.836266+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"211'2207","reported_seq":2221,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T22:20:50.158413+0000","last_change":"2023-09-23T22:20:19.470871+0000","last_active":"2023-09-23T22:20:50.158413+0000","last_peered":"2023-09-23T22:20:50.158413+0000","last_clean":"2023-09-23T22:20:50.158413+0000","last_became_active":"2023-09-23T22:20:19.470597+0000","last_became_peered":"2023-09-23T22:20:19.470597+0000","last_unstale":"2023-09-23T22:20:50.158413+0000","last_undegraded":"2023-09-23T22:20:50.158413+0000","last_fullsized":"2023-09-23T22:20:50.158413+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":2207,"log_dups_size":0,"ondisk_log_size":2207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:32.388985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144637952,"num_objects":2207,"num_object_clones":0,"num_object_copies":6621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2207,"num_write_kb":141248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":796,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851188+0000","last_change":"2023-09-23T22:13:27.666413+0000","last_active":"2023-09-23T22:20:40.851188+0000","last_peered":"2023-09-23T22:20:40.851188+0000","last_clean":"2023-09-23T22:20:40.851188+0000","last_became_active":"2023-09-23T22:13:27.665826+0000","last_became_peered":"2023-09-23T22:13:27.665826+0000","last_unstale":"2023-09-23T22:20:40.851188+0000","last_undegraded":"2023-09-23T22:20:40.851188+0000","last_fullsized":"2023-09-23T22:20:40.851188+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:07.664883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":486,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.844987+0000","last_change":"2023-09-23T22:20:39.627654+0000","last_active":"2023-09-23T22:20:39.844987+0000","last_peered":"2023-09-23T22:20:39.844987+0000","last_clean":"2023-09-23T22:20:39.844987+0000","last_became_active":"2023-09-23T22:18:57.743004+0000","last_became_peered":"2023-09-23T22:18:57.743004+0000","last_unstale":"2023-09-23T22:20:39.844987+0000","last_undegraded":"2023-09-23T22:20:39.844987+0000","last_fullsized":"2023-09-23T22:20:39.844987+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:43:01.140050+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":136,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851957+0000","last_change":"2023-09-23T22:16:49.155156+0000","last_active":"2023-09-23T22:20:40.851957+0000","last_peered":"2023-09-23T22:20:40.851957+0000","last_clean":"2023-09-23T22:20:40.851957+0000","last_became_active":"2023-09-23T22:16:49.154923+0000","last_became_peered":"2023-09-23T22:16:49.154923+0000","last_unstale":"2023-09-23T22:20:40.851957+0000","last_undegraded":"2023-09-23T22:20:40.851957+0000","last_fullsized":"2023-09-23T22:20:40.851957+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:33.532238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":96,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837934+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:20:39.837934+0000","last_peered":"2023-09-23T22:20:39.837934+0000","last_clean":"2023-09-23T22:20:39.837934+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:20:39.837934+0000","last_undegraded":"2023-09-23T22:20:39.837934+0000","last_fullsized":"2023-09-23T22:20:39.837934+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"210'2029","reported_seq":2045,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:48.504449+0000","last_change":"2023-09-23T22:20:19.471072+0000","last_active":"2023-09-23T22:20:48.504449+0000","last_peered":"2023-09-23T22:20:48.504449+0000","last_clean":"2023-09-23T22:20:48.504449+0000","last_became_active":"2023-09-23T22:20:19.470818+0000","last_became_peered":"2023-09-23T22:20:19.470818+0000","last_unstale":"2023-09-23T22:20:48.504449+0000","last_undegraded":"2023-09-23T22:20:48.504449+0000","last_fullsized":"2023-09-23T22:20:48.504449+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":2029,"log_dups_size":0,"ondisk_log_size":2029,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:27.930606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132972544,"num_objects":2029,"num_object_clones":0,"num_object_copies":6087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2029,"num_write_kb":129856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":540,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.833097+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:20:39.833097+0000","last_peered":"2023-09-23T22:20:39.833097+0000","last_clean":"2023-09-23T22:20:39.833097+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:20:39.833097+0000","last_undegraded":"2023-09-23T22:20:39.833097+0000","last_fullsized":"2023-09-23T22:20:39.833097+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":558,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.833576+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:20:39.833576+0000","last_peered":"2023-09-23T22:20:39.833576+0000","last_clean":"2023-09-23T22:20:39.833576+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:20:39.833576+0000","last_undegraded":"2023-09-23T22:20:39.833576+0000","last_fullsized":"2023-09-23T22:20:39.833576+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":72,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.845785+0000","last_change":"2023-09-23T22:18:31.795915+0000","last_active":"2023-09-23T22:20:39.845785+0000","last_peered":"2023-09-23T22:20:39.845785+0000","last_clean":"2023-09-23T22:20:39.845785+0000","last_became_active":"2023-09-23T22:18:31.795608+0000","last_became_peered":"2023-09-23T22:18:31.795608+0000","last_unstale":"2023-09-23T22:20:39.845785+0000","last_undegraded":"2023-09-23T22:20:39.845785+0000","last_fullsized":"2023-09-23T22:20:39.845785+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:49.789927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":38,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.846477+0000","last_change":"2023-09-23T22:18:57.721257+0000","last_active":"2023-09-23T22:20:39.846477+0000","last_peered":"2023-09-23T22:20:39.846477+0000","last_clean":"2023-09-23T22:20:39.846477+0000","last_became_active":"2023-09-23T22:18:57.720548+0000","last_became_peered":"2023-09-23T22:18:57.720548+0000","last_unstale":"2023-09-23T22:20:39.846477+0000","last_undegraded":"2023-09-23T22:20:39.846477+0000","last_fullsized":"2023-09-23T22:20:39.846477+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:10.709024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"210'1999","reported_seq":2014,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:48.488004+0000","last_change":"2023-09-23T22:20:19.470432+0000","last_active":"2023-09-23T22:20:48.488004+0000","last_peered":"2023-09-23T22:20:48.488004+0000","last_clean":"2023-09-23T22:20:48.488004+0000","last_became_active":"2023-09-23T22:20:19.470153+0000","last_became_peered":"2023-09-23T22:20:19.470153+0000","last_unstale":"2023-09-23T22:20:48.488004+0000","last_undegraded":"2023-09-23T22:20:48.488004+0000","last_fullsized":"2023-09-23T22:20:48.488004+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1999,"log_dups_size":0,"ondisk_log_size":1999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:00.110369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131006464,"num_objects":1999,"num_object_clones":0,"num_object_copies":5997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1999,"num_write_kb":127936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":503,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:45.626292+0000","last_change":"2023-09-23T22:20:45.626419+0000","last_active":"2023-09-23T22:20:45.626292+0000","last_peered":"2023-09-23T22:20:45.626292+0000","last_clean":"2023-09-23T22:20:45.626292+0000","last_became_active":"2023-09-23T22:20:18.468819+0000","last_became_peered":"2023-09-23T22:20:18.468819+0000","last_unstale":"2023-09-23T22:20:45.626292+0000","last_undegraded":"2023-09-23T22:20:45.626292+0000","last_fullsized":"2023-09-23T22:20:45.626292+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:56.951048+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":99,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.845948+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:20:39.845948+0000","last_peered":"2023-09-23T22:20:39.845948+0000","last_clean":"2023-09-23T22:20:39.845948+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:20:39.845948+0000","last_undegraded":"2023-09-23T22:20:39.845948+0000","last_fullsized":"2023-09-23T22:20:39.845948+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.836035+0000","last_change":"2023-09-23T22:13:27.662175+0000","last_active":"2023-09-23T22:20:39.836035+0000","last_peered":"2023-09-23T22:20:39.836035+0000","last_clean":"2023-09-23T22:20:39.836035+0000","last_became_active":"2023-09-23T22:13:27.661885+0000","last_became_peered":"2023-09-23T22:13:27.661885+0000","last_unstale":"2023-09-23T22:20:39.836035+0000","last_undegraded":"2023-09-23T22:20:39.836035+0000","last_fullsized":"2023-09-23T22:20:39.836035+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:24.788937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":63,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.846156+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:20:39.846156+0000","last_peered":"2023-09-23T22:20:39.846156+0000","last_clean":"2023-09-23T22:20:39.846156+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:20:39.846156+0000","last_undegraded":"2023-09-23T22:20:39.846156+0000","last_fullsized":"2023-09-23T22:20:39.846156+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"210'1961","reported_seq":1976,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:47.444008+0000","last_change":"2023-09-23T22:20:19.469545+0000","last_active":"2023-09-23T22:20:47.444008+0000","last_peered":"2023-09-23T22:20:47.444008+0000","last_clean":"2023-09-23T22:20:47.444008+0000","last_became_active":"2023-09-23T22:20:19.469306+0000","last_became_peered":"2023-09-23T22:20:19.469306+0000","last_unstale":"2023-09-23T22:20:47.444008+0000","last_undegraded":"2023-09-23T22:20:47.444008+0000","last_fullsized":"2023-09-23T22:20:47.444008+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1961,"log_dups_size":0,"ondisk_log_size":1961,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:48.392898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128516096,"num_objects":1961,"num_object_clones":0,"num_object_copies":5883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1961,"num_write_kb":125504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":494,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837498+0000","last_change":"2023-09-23T22:20:37.492583+0000","last_active":"2023-09-23T22:20:39.837498+0000","last_peered":"2023-09-23T22:20:39.837498+0000","last_clean":"2023-09-23T22:20:39.837498+0000","last_became_active":"2023-09-23T22:20:18.471519+0000","last_became_peered":"2023-09-23T22:20:18.471519+0000","last_unstale":"2023-09-23T22:20:39.837498+0000","last_undegraded":"2023-09-23T22:20:39.837498+0000","last_fullsized":"2023-09-23T22:20:39.837498+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:25.241723+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":223,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.854354+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:20:40.854354+0000","last_peered":"2023-09-23T22:20:40.854354+0000","last_clean":"2023-09-23T22:20:40.854354+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:20:40.854354+0000","last_undegraded":"2023-09-23T22:20:40.854354+0000","last_fullsized":"2023-09-23T22:20:40.854354+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":850,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.835864+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:20:39.835864+0000","last_peered":"2023-09-23T22:20:39.835864+0000","last_clean":"2023-09-23T22:20:39.835864+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:20:39.835864+0000","last_undegraded":"2023-09-23T22:20:39.835864+0000","last_fullsized":"2023-09-23T22:20:39.835864+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":208,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.844408+0000","last_change":"2023-09-23T22:18:57.722771+0000","last_active":"2023-09-23T22:20:40.844408+0000","last_peered":"2023-09-23T22:20:40.844408+0000","last_clean":"2023-09-23T22:20:40.844408+0000","last_became_active":"2023-09-23T22:18:57.722258+0000","last_became_peered":"2023-09-23T22:18:57.722258+0000","last_unstale":"2023-09-23T22:20:40.844408+0000","last_undegraded":"2023-09-23T22:20:40.844408+0000","last_fullsized":"2023-09-23T22:20:40.844408+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:18.267091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"210'1845","reported_seq":1861,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:46.290013+0000","last_change":"2023-09-23T22:20:19.465240+0000","last_active":"2023-09-23T22:20:46.290013+0000","last_peered":"2023-09-23T22:20:46.290013+0000","last_clean":"2023-09-23T22:20:46.290013+0000","last_became_active":"2023-09-23T22:20:19.464955+0000","last_became_peered":"2023-09-23T22:20:19.464955+0000","last_unstale":"2023-09-23T22:20:46.290013+0000","last_undegraded":"2023-09-23T22:20:46.290013+0000","last_fullsized":"2023-09-23T22:20:46.290013+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":1845,"log_dups_size":0,"ondisk_log_size":1845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:21.927943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120913920,"num_objects":1845,"num_object_clones":0,"num_object_copies":5535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1845,"num_write_kb":118080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":501,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:46.903205+0000","last_change":"2023-09-23T22:20:46.903205+0000","last_active":"2023-09-23T22:20:46.903205+0000","last_peered":"2023-09-23T22:20:46.903205+0000","last_clean":"2023-09-23T22:20:46.903205+0000","last_became_active":"2023-09-23T22:20:18.467795+0000","last_became_peered":"2023-09-23T22:20:18.467795+0000","last_unstale":"2023-09-23T22:20:46.903205+0000","last_undegraded":"2023-09-23T22:20:46.903205+0000","last_fullsized":"2023-09-23T22:20:46.903205+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:46.903071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:46.903071+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:58:14.711974+0000","scrub_duration":0.0040897939999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":274,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.844519+0000","last_change":"2023-09-23T22:16:49.270015+0000","last_active":"2023-09-23T22:20:39.844519+0000","last_peered":"2023-09-23T22:20:39.844519+0000","last_clean":"2023-09-23T22:20:39.844519+0000","last_became_active":"2023-09-23T22:16:49.252726+0000","last_became_peered":"2023-09-23T22:16:49.252726+0000","last_unstale":"2023-09-23T22:20:39.844519+0000","last_undegraded":"2023-09-23T22:20:39.844519+0000","last_fullsized":"2023-09-23T22:20:39.844519+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:18.526640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":42,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.846652+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:20:39.846652+0000","last_peered":"2023-09-23T22:20:39.846652+0000","last_clean":"2023-09-23T22:20:39.846652+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:20:39.846652+0000","last_undegraded":"2023-09-23T22:20:39.846652+0000","last_fullsized":"2023-09-23T22:20:39.846652+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":173,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837234+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:20:39.837234+0000","last_peered":"2023-09-23T22:20:39.837234+0000","last_clean":"2023-09-23T22:20:39.837234+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:20:39.837234+0000","last_undegraded":"2023-09-23T22:20:39.837234+0000","last_fullsized":"2023-09-23T22:20:39.837234+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":38,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.846066+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:20:39.846066+0000","last_peered":"2023-09-23T22:20:39.846066+0000","last_clean":"2023-09-23T22:20:39.846066+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:20:39.846066+0000","last_undegraded":"2023-09-23T22:20:39.846066+0000","last_fullsized":"2023-09-23T22:20:39.846066+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":203,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.852779+0000","last_change":"2023-09-23T22:20:40.852779+0000","last_active":"2023-09-23T22:20:40.852779+0000","last_peered":"2023-09-23T22:20:40.852779+0000","last_clean":"2023-09-23T22:20:40.852779+0000","last_became_active":"2023-09-23T22:20:40.852417+0000","last_became_peered":"2023-09-23T22:20:40.852417+0000","last_unstale":"2023-09-23T22:20:40.852779+0000","last_undegraded":"2023-09-23T22:20:40.852779+0000","last_fullsized":"2023-09-23T22:20:40.852779+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:28.570551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":173,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851155+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:20:40.851155+0000","last_peered":"2023-09-23T22:20:40.851155+0000","last_clean":"2023-09-23T22:20:40.851155+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:20:40.851155+0000","last_undegraded":"2023-09-23T22:20:40.851155+0000","last_fullsized":"2023-09-23T22:20:40.851155+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":185,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837597+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:20:39.837597+0000","last_peered":"2023-09-23T22:20:39.837597+0000","last_clean":"2023-09-23T22:20:39.837597+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:20:39.837597+0000","last_undegraded":"2023-09-23T22:20:39.837597+0000","last_fullsized":"2023-09-23T22:20:39.837597+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":170,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.851599+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:20:40.851599+0000","last_peered":"2023-09-23T22:20:40.851599+0000","last_clean":"2023-09-23T22:20:40.851599+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:20:40.851599+0000","last_undegraded":"2023-09-23T22:20:40.851599+0000","last_fullsized":"2023-09-23T22:20:40.851599+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":44,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.837495+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:20:39.837495+0000","last_peered":"2023-09-23T22:20:39.837495+0000","last_clean":"2023-09-23T22:20:39.837495+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:20:39.837495+0000","last_undegraded":"2023-09-23T22:20:39.837495+0000","last_fullsized":"2023-09-23T22:20:39.837495+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":192,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.853391+0000","last_change":"2023-09-23T22:16:49.145417+0000","last_active":"2023-09-23T22:20:40.853391+0000","last_peered":"2023-09-23T22:20:40.853391+0000","last_clean":"2023-09-23T22:20:40.853391+0000","last_became_active":"2023-09-23T22:16:49.145017+0000","last_became_peered":"2023-09-23T22:16:49.145017+0000","last_unstale":"2023-09-23T22:20:40.853391+0000","last_undegraded":"2023-09-23T22:20:40.853391+0000","last_fullsized":"2023-09-23T22:20:40.853391+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:47.762149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":233,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.852085+0000","last_change":"2023-09-23T22:17:15.721123+0000","last_active":"2023-09-23T22:20:40.852085+0000","last_peered":"2023-09-23T22:20:40.852085+0000","last_clean":"2023-09-23T22:20:40.852085+0000","last_became_active":"2023-09-23T22:17:15.720812+0000","last_became_peered":"2023-09-23T22:17:15.720812+0000","last_unstale":"2023-09-23T22:20:40.852085+0000","last_undegraded":"2023-09-23T22:20:40.852085+0000","last_fullsized":"2023-09-23T22:20:40.852085+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.176278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":82,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T22:20:39.844818+0000","last_change":"2023-09-23T22:18:57.729455+0000","last_active":"2023-09-23T22:20:39.844818+0000","last_peered":"2023-09-23T22:20:39.844818+0000","last_clean":"2023-09-23T22:20:39.844818+0000","last_became_active":"2023-09-23T22:18:57.729088+0000","last_became_peered":"2023-09-23T22:18:57.729088+0000","last_unstale":"2023-09-23T22:20:39.844818+0000","last_undegraded":"2023-09-23T22:20:39.844818+0000","last_fullsized":"2023-09-23T22:20:39.844818+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:17.861334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":188,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T22:20:40.853005+0000","last_change":"2023-09-23T22:20:40.853005+0000","last_active":"2023-09-23T22:20:40.853005+0000","last_peered":"2023-09-23T22:20:40.853005+0000","last_clean":"2023-09-23T22:20:40.853005+0000","last_became_active":"2023-09-23T22:20:40.852694+0000","last_became_peered":"2023-09-23T22:20:40.852694+0000","last_unstale":"2023-09-23T22:20:40.853005+0000","last_undegraded":"2023-09-23T22:20:40.853005+0000","last_fullsized":"2023-09-23T22:20:40.853005+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:51:06.984566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2123038720,"num_objects":32395,"num_object_clones":0,"num_object_copies":97185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":32395,"num_write_kb":2073280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6409682944,"data_stored":6409682944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32395,"ondisk_log_size":32395,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":4892,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234197,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1274584,"kb_used_data":1171792,"kb_used_omap":0,"kb_used_meta":102784,"kb_avail":92470568,"statfs":{"total":95995035648,"available":94689861632,"internally_reserved":0,"allocated":1199915008,"data_stored":1199057797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":105250816},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:01 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":2.4780000000000002,"15min":1.1970000000000001},"min":{"1min":0.23999999999999999,"5min":0.223,"15min":0.223},"max":{"1min":9.7799999999999994,"5min":109.208,"15min":109.208},"last":0.24199999999999999},{"interface":"front","average":{"1min":1.752,"5min":2.7650000000000001,"15min":1.3149999999999999},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":7.4489999999999998,"5min":109.13800000000001,"15min":109.13800000000001},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:20:09 2023","interfaces":[{"interface":"back","average":{"1min":9.1620000000000008,"5min":5.9320000000000004,"15min":2.3570000000000002},"min":{"1min":0.25600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":101.145,"5min":123.79300000000001,"15min":123.79300000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":9.4949999999999992,"5min":6.2809999999999997,"15min":2.508},"min":{"1min":0.26600000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":101.13,"5min":123.66500000000001,"15min":123.66500000000001},"last":1.502}]},{"osd":2,"last update":"Sat Sep 23 22:20:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1739999999999999,"5min":8.625,"15min":3.54},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.68,"5min":125.101,"15min":125.101},"last":0.436},{"interface":"front","average":{"1min":1.2190000000000001,"5min":8.3179999999999996,"15min":3.1869999999999998},"min":{"1min":0.29199999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.7169999999999996,"5min":126.021,"15min":126.021},"last":0.31}]},{"osd":3,"last update":"Sat Sep 23 22:20:09 2023","interfaces":[{"interface":"back","average":{"1min":1.056,"5min":2.5619999999999998,"15min":1.2549999999999999},"min":{"1min":0.308,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":2.8319999999999999,"5min":113.282,"15min":113.282},"last":1.3540000000000001},{"interface":"front","average":{"1min":1.2370000000000001,"5min":2.931,"15min":1.4239999999999999},"min":{"1min":0.36699999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.7910000000000004,"5min":104.74299999999999,"15min":104.74299999999999},"last":0.75700000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:20:39 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":2.5459999999999998,"15min":1.1759999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.0329999999999999,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.38},{"interface":"front","average":{"1min":0.85799999999999998,"5min":2.3530000000000002,"15min":1.1200000000000001},"min":{"1min":0.26700000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.5759999999999996,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.35799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:20:39 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":2.3130000000000002,"15min":1.097},"min":{"1min":0.28699999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.5640000000000001,"5min":93.816000000000003,"15min":93.816000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.57799999999999996,"5min":2.8069999999999999,"15min":1.3160000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29799999999999999},"max":{"1min":1.01,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:20:39 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.97599999999999998,"15min":0.68300000000000005},"min":{"1min":0.33200000000000002,"5min":0.317,"15min":0.315},"max":{"1min":1.873,"5min":9.0139999999999993,"15min":9.0139999999999993},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.3839999999999999,"15min":0.86399999999999999},"min":{"1min":0.34499999999999997,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":0.92600000000000005,"5min":13.362,"15min":13.362},"last":0.53500000000000003}]}]},{"osd":6,"up_from":0,"seq":197568495833,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795105,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":946480,"kb_used_data":766312,"kb_used_omap":7,"kb_used_meta":180152,"kb_avail":92798672,"statfs":{"total":95995035648,"available":95025840128,"internally_reserved":0,"allocated":784703488,"data_stored":783583658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":184476020},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":1.9299999999999999,"15min":0.88},"min":{"1min":0.21199999999999999,"5min":0.20399999999999999,"15min":0.19400000000000001},"max":{"1min":5.8840000000000003,"5min":61.890000000000001,"15min":61.890000000000001},"last":0.90500000000000003},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.8140000000000001,"15min":1.2050000000000001},"min":{"1min":0.216,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":9.2669999999999995,"5min":62.689999999999998,"15min":62.689999999999998},"last":1.7}]},{"osd":2,"last update":"Sat Sep 23 22:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":2.5219999999999998,"15min":1.0920000000000001},"min":{"1min":0.27800000000000002,"5min":0.223,"15min":0.216},"max":{"1min":4.5439999999999996,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.94099999999999995},{"interface":"front","average":{"1min":0.73799999999999999,"5min":3.0870000000000002,"15min":1.3069999999999999},"min":{"1min":0.253,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.0710000000000002,"5min":34.781999999999996,"15min":34.781999999999996},"last":1.9750000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:20:14 2023","interfaces":[{"interface":"back","average":{"1min":1.7350000000000001,"5min":2.4580000000000002,"15min":1.0840000000000001},"min":{"1min":0.35899999999999999,"5min":0.252,"15min":0.252},"max":{"1min":6.4199999999999999,"5min":22.326000000000001,"15min":22.326000000000001},"last":1.9019999999999999},{"interface":"front","average":{"1min":2.2229999999999999,"5min":2.3239999999999998,"15min":1.048},"min":{"1min":0.27500000000000002,"5min":0.25900000000000001,"15min":0.23000000000000001},"max":{"1min":11.005000000000001,"5min":22.259,"15min":22.259},"last":1.4610000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:20:14 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":1.0820000000000001,"15min":0.80300000000000005},"min":{"1min":0.29899999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.4669999999999996,"5min":10.845000000000001,"15min":10.845000000000001},"last":0.97599999999999998},{"interface":"front","average":{"1min":1.5489999999999999,"5min":1.702,"15min":1.008},"min":{"1min":0.47999999999999998,"5min":0.318,"15min":0.318},"max":{"1min":6.0720000000000001,"5min":11.037000000000001,"15min":11.037000000000001},"last":1.75}]},{"osd":5,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":1.7350000000000001,"15min":0.98399999999999999},"min":{"1min":0.442,"5min":0.31,"15min":0.31},"max":{"1min":6.0410000000000004,"5min":11.125,"15min":11.125},"last":1.9419999999999999},{"interface":"front","average":{"1min":2.1339999999999999,"5min":1.5980000000000001,"15min":0.95399999999999996},"min":{"1min":0.33800000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.872999999999999,"5min":10.872999999999999,"15min":10.872999999999999},"last":1.5629999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.6779999999999999,"15min":0.98099999999999998},"min":{"1min":0.373,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":10.971,"5min":10.989000000000001,"15min":10.989000000000001},"last":1.4990000000000001},{"interface":"front","average":{"1min":1.234,"5min":1.3700000000000001,"15min":0.90200000000000002},"min":{"1min":0.36199999999999999,"5min":0.35699999999999998,"15min":0.34999999999999998},"max":{"1min":5.4189999999999996,"5min":10.811999999999999,"15min":10.811999999999999},"last":1.0129999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.804,"15min":1.0269999999999999},"min":{"1min":0.33800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.933999999999999,"5min":10.948,"15min":10.948},"last":1.5329999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.4319999999999999,"15min":0.92600000000000005},"min":{"1min":0.38600000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.4930000000000003,"5min":10.781000000000001,"15min":10.781000000000001},"last":1.0449999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958639,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":699116,"kb_used_data":478244,"kb_used_omap":8,"kb_used_meta":220855,"kb_avail":93046036,"statfs":{"total":95995035648,"available":95279140864,"internally_reserved":0,"allocated":489721856,"data_stored":488605250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":226156394},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:20:13 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":2.5750000000000002,"15min":1.0860000000000001},"min":{"1min":0.20399999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":4.9710000000000001,"5min":44.776000000000003,"15min":44.776000000000003},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.5289999999999999,"5min":3.5259999999999998,"15min":1.4199999999999999},"min":{"1min":0.23799999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":12.641999999999999,"5min":48.838000000000001,"15min":48.838000000000001},"last":1.3380000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:20:43 2023","interfaces":[{"interface":"back","average":{"1min":5.2599999999999998,"5min":3.0270000000000001,"15min":1.28},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":70.316999999999993,"5min":70.316999999999993,"15min":70.316999999999993},"last":0.70199999999999996},{"interface":"front","average":{"1min":5.0949999999999998,"5min":4.4710000000000001,"15min":1.831},"min":{"1min":0.29699999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":69.944000000000003,"5min":69.944000000000003,"15min":69.944000000000003},"last":1.1779999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":2.5659999999999998,"15min":1.1519999999999999},"min":{"1min":0.27300000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.1269999999999998,"5min":35.496000000000002,"15min":35.496000000000002},"last":3.8250000000000002},{"interface":"front","average":{"1min":1.722,"5min":2.9350000000000001,"15min":1.2709999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.8630000000000004,"5min":35.533000000000001,"15min":35.533000000000001},"last":0.442}]},{"osd":4,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9180000000000001,"5min":2.6389999999999998,"15min":1.319},"min":{"1min":0.34200000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.032999999999999,"5min":10.217000000000001,"15min":10.217000000000001},"last":2.4700000000000002},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.274,"15min":0.86199999999999999},"min":{"1min":0.313,"5min":0.308,"15min":0.308},"max":{"1min":4.0270000000000001,"5min":22.209,"15min":22.209},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":2.609,"15min":1.3300000000000001},"min":{"1min":0.34799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.9920000000000009,"5min":12.372,"15min":12.372},"last":2.5499999999999998},{"interface":"front","average":{"1min":0.73899999999999999,"5min":1.2909999999999999,"15min":0.88500000000000001},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":4.0030000000000001,"5min":20.390999999999998,"15min":20.390999999999998},"last":0.40200000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":2.6680000000000001,"15min":1.3440000000000001},"min":{"1min":0.38900000000000001,"5min":0.379,"15min":0.379},"max":{"1min":10.087,"5min":10.275,"15min":10.275},"last":2.5899999999999999},{"interface":"front","average":{"1min":1.133,"5min":1.401,"15min":0.93700000000000006},"min":{"1min":0.34999999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":7.4039999999999999,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.48599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:19:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":2.524,"15min":1.329},"min":{"1min":0.43099999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":4.1369999999999996,"5min":23.306000000000001,"15min":23.306000000000001},"last":0.94199999999999995},{"interface":"front","average":{"1min":3.0019999999999998,"5min":3.0840000000000001,"15min":1.5289999999999999},"min":{"1min":0.47199999999999998,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":10.061,"5min":23.265999999999998,"15min":23.265999999999998},"last":3.121}]}]},{"osd":2,"up_from":103,"seq":442381631578,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1404444,"kb_used_data":1206292,"kb_used_omap":10,"kb_used_meta":198133,"kb_avail":92340708,"statfs":{"total":95995035648,"available":94556884992,"internally_reserved":0,"allocated":1235243008,"data_stored":1234143209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11092,"internal_metadata":202888364},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":2.6320000000000001,"15min":1.1020000000000001},"min":{"1min":0.22800000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.2009999999999996,"5min":19.344000000000001,"15min":19.344000000000001},"last":0.26700000000000002},{"interface":"front","average":{"1min":1.0189999999999999,"5min":2.8479999999999999,"15min":1.1919999999999999},"min":{"1min":0.20499999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":4.9199999999999999,"5min":19.149999999999999,"15min":19.149999999999999},"last":0.40999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":1.7529999999999999,"5min":4.9119999999999999,"15min":1.913},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.222},"max":{"1min":20.129999999999999,"5min":157.636,"15min":157.636},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.1379999999999999,"5min":4.8449999999999998,"15min":1.895},"min":{"1min":0.25600000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.6040000000000001,"5min":155.63300000000001,"15min":155.63300000000001},"last":0.92200000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:20:06 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":2.5950000000000002,"15min":1.0920000000000001},"min":{"1min":0.27100000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.9429999999999996,"5min":19.852,"15min":19.852},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.8730000000000002,"15min":1.216},"min":{"1min":0.28100000000000003,"5min":0.26000000000000001,"15min":0.246},"max":{"1min":20.234999999999999,"5min":20.581,"15min":20.581},"last":0.48699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":0.873,"5min":1.835,"15min":1.0129999999999999},"min":{"1min":0.28599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.9779999999999998,"5min":11.244,"15min":11.244},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.9610000000000001,"15min":1.0700000000000001},"min":{"1min":0.40100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.4820000000000002,"5min":9.2799999999999994,"15min":9.2799999999999994},"last":0.32400000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":1.9450000000000001,"15min":1.069},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.0019999999999998,"5min":11.291,"15min":11.291},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.1919999999999999,"5min":2.0800000000000001,"15min":1.099},"min":{"1min":0.39500000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.5659999999999998,"5min":9.4499999999999993,"15min":9.4499999999999993},"last":0.38300000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":1.071,"5min":2.0950000000000002,"15min":1.139},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":5.0529999999999999,"5min":17.640999999999998,"15min":17.640999999999998},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.385,"5min":1.9510000000000001,"15min":1.167},"min":{"1min":0.46200000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.5389999999999997,"5min":11.875,"15min":11.875},"last":1.9550000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:20:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0190000000000001,"5min":2.29,"15min":1.258},"min":{"1min":0.52600000000000002,"5min":0.377,"15min":0.377},"max":{"1min":20.088000000000001,"5min":20.088000000000001,"15min":20.088000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.0800000000000001,"5min":2.3620000000000001,"15min":1.2110000000000001},"min":{"1min":0.38500000000000001,"5min":0.38100000000000001,"15min":0.377},"max":{"1min":5.0259999999999998,"5min":17.738,"15min":17.738},"last":0.51500000000000001}]}]},{"osd":3,"up_from":108,"seq":463856468052,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1459528,"kb_used_data":1359872,"kb_used_omap":4,"kb_used_meta":99643,"kb_avail":92285624,"statfs":{"total":95995035648,"available":94500478976,"internally_reserved":0,"allocated":1392508928,"data_stored":1391370989,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":102034989},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":4.8399999999999999,"5min":3.6309999999999998,"15min":1.462},"min":{"1min":0.23000000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":48.676000000000002,"5min":77.834999999999994,"15min":77.834999999999994},"last":2.7690000000000001},{"interface":"front","average":{"1min":4.8970000000000002,"5min":4.0090000000000003,"15min":1.577},"min":{"1min":0.26600000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":48.774000000000001,"5min":75.558000000000007,"15min":75.558000000000007},"last":0.30299999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":3.6779999999999999,"5min":4.2030000000000003,"15min":1.6359999999999999},"min":{"1min":0.24099999999999999,"5min":0.218,"15min":0.218},"max":{"1min":12.445,"5min":82.951999999999998,"15min":82.951999999999998},"last":0.376},{"interface":"front","average":{"1min":2.4119999999999999,"5min":4.0019999999999998,"15min":1.6060000000000001},"min":{"1min":0.22800000000000001,"5min":0.189,"15min":0.189},"max":{"1min":11.970000000000001,"5min":86.001000000000005,"15min":86.001000000000005},"last":2.5840000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":8.2799999999999994,"15min":3.0219999999999998},"min":{"1min":0.27700000000000002,"5min":0.223,"15min":0.223},"max":{"1min":11.708,"5min":99.962999999999994,"15min":99.962999999999994},"last":1.0629999999999999},{"interface":"front","average":{"1min":3.238,"5min":7.7300000000000004,"15min":2.8599999999999999},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":11.994,"5min":100.429,"15min":100.429},"last":2.6419999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:20:14 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":1.6020000000000001,"15min":0.95099999999999996},"min":{"1min":0.29699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.911,"5min":11.911,"15min":11.911},"last":2.738},{"interface":"front","average":{"1min":1.881,"5min":1.708,"15min":1.006},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.720000000000001,"5min":12.036,"15min":12.036},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":4.4249999999999998,"5min":2.157,"15min":1.135},"min":{"1min":0.36399999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":49.695999999999998,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":4.7279999999999998,"5min":2.3500000000000001,"15min":1.1899999999999999},"min":{"1min":0.41599999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":49.668999999999997,"5min":49.668999999999997,"15min":49.668999999999997},"last":0.40799999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.841,"15min":1.0489999999999999},"min":{"1min":0.441,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":10.760999999999999,"5min":12.068,"15min":12.068},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.8140000000000001,"15min":1.028},"min":{"1min":0.45200000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":11.68,"5min":11.68,"15min":11.68},"last":0.46400000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:20:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":1.9039999999999999,"15min":1.0549999999999999},"min":{"1min":0.39600000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":11.596,"5min":11.624000000000001,"15min":11.624000000000001},"last":0.435},{"interface":"front","average":{"1min":2.419,"5min":1.752,"15min":1.04},"min":{"1min":0.435,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":11.945,"5min":12.382,"15min":12.382},"last":2.6949999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019103,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":735308,"kb_used_data":614276,"kb_used_omap":0,"kb_used_meta":121024,"kb_avail":93009844,"statfs":{"total":95995035648,"available":95242080256,"internally_reserved":0,"allocated":629018624,"data_stored":628165736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":123928576},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":3.0529999999999999,"5min":2.488,"15min":1.246},"min":{"1min":0.30099999999999999,"5min":0.218,"15min":0.218},"max":{"1min":27.942,"5min":66.355000000000004,"15min":66.355000000000004},"last":0.52400000000000002},{"interface":"front","average":{"1min":3.1339999999999999,"5min":2.7210000000000001,"15min":1.3360000000000001},"min":{"1min":0.315,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":27.956,"5min":66.332999999999998,"15min":66.332999999999998},"last":0.503}]},{"osd":1,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7570000000000001,"5min":2.1819999999999999,"15min":1.0940000000000001},"min":{"1min":0.23999999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":19.053000000000001,"5min":38.878,"15min":38.878},"last":0.80100000000000005},{"interface":"front","average":{"1min":3.024,"5min":2.9100000000000001,"15min":1.3540000000000001},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":19.170999999999999,"5min":38.853999999999999,"15min":38.853999999999999},"last":1.23}]},{"osd":2,"last update":"Sat Sep 23 22:20:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":4.8289999999999997,"15min":1.9750000000000001},"min":{"1min":0.32700000000000001,"5min":0.255,"15min":0.255},"max":{"1min":8.9920000000000009,"5min":138.06100000000001,"15min":138.06100000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.67100000000000004,"5min":5.5170000000000003,"15min":2.1890000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.8450000000000002,"5min":138.15199999999999,"15min":138.15199999999999},"last":0.45600000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":1.7989999999999999,"15min":0.995},"min":{"1min":0.33700000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":8.6340000000000003,"5min":8.6340000000000003,"15min":8.6340000000000003},"last":3.286},{"interface":"front","average":{"1min":1.476,"5min":1.7869999999999999,"15min":0.97899999999999998},"min":{"1min":0.39500000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.1529999999999996,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":1.087}]},{"osd":5,"last update":"Sat Sep 23 22:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.077,"15min":0.68200000000000005},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.26800000000000002},"max":{"1min":6.6989999999999998,"5min":8.9499999999999993,"15min":8.9499999999999993},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.92100000000000004,"15min":0.66300000000000003},"min":{"1min":0.313,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.6179999999999999,"5min":7.8789999999999996,"15min":7.8789999999999996},"last":0.30299999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:20:27 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":0.91300000000000003,"15min":0.65100000000000002},"min":{"1min":0.316,"5min":0.251,"15min":0.251},"max":{"1min":7.3079999999999998,"5min":7.3079999999999998,"15min":7.3079999999999998},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.96299999999999997,"5min":1.0169999999999999,"15min":0.70799999999999996},"min":{"1min":0.374,"5min":0.30499999999999999,"15min":0.28899999999999998},"max":{"1min":3.9239999999999999,"5min":8.5210000000000008,"15min":8.5210000000000008},"last":0.28599999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:20:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.228,"15min":0.79200000000000004},"min":{"1min":0.36299999999999999,"5min":0.307,"15min":0.307},"max":{"1min":6.7480000000000002,"5min":8.577,"15min":8.577},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":0.95999999999999996,"15min":0.71699999999999997},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.2679999999999998,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.312}]}]},{"osd":5,"up_from":38,"seq":163208757470,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":953424,"kb_used_data":808520,"kb_used_omap":0,"kb_used_meta":144896,"kb_avail":92791728,"statfs":{"total":95995035648,"available":95018729472,"internally_reserved":0,"allocated":827924480,"data_stored":827072021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":148373504},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:19:49 2023","interfaces":[{"interface":"back","average":{"1min":3.8439999999999999,"5min":3.7749999999999999,"15min":1.6319999999999999},"min":{"1min":0.23000000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":28.741,"5min":133.22499999999999,"15min":133.22499999999999},"last":1.083},{"interface":"front","average":{"1min":3.3719999999999999,"5min":3.9609999999999999,"15min":1.7350000000000001},"min":{"1min":0.27100000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":28.873999999999999,"5min":132.78200000000001,"15min":132.78200000000001},"last":0.22900000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.748,"5min":4.8230000000000004,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":7.8120000000000003,"5min":83.759,"15min":83.759},"last":0.31900000000000001},{"interface":"front","average":{"1min":1.3560000000000001,"5min":4.7699999999999996,"15min":1.9710000000000001},"min":{"1min":0.28399999999999997,"5min":0.252,"15min":0.252},"max":{"1min":4.5629999999999997,"5min":83.741,"15min":83.741},"last":0.33100000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:20:45 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":7.5780000000000003,"15min":2.9079999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.65700000000000003,"5min":104.254,"15min":104.254},"last":5.1159999999999997},{"interface":"front","average":{"1min":0.60799999999999998,"5min":8.5749999999999993,"15min":3.254},"min":{"1min":0.32200000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.819,"5min":104.217,"15min":104.217},"last":1.474}]},{"osd":3,"last update":"Sat Sep 23 22:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":2.964,"15min":1.3740000000000001},"min":{"1min":0.37,"5min":0.246,"15min":0.246},"max":{"1min":9.9459999999999997,"5min":88.106999999999999,"15min":88.106999999999999},"last":4.2549999999999999},{"interface":"front","average":{"1min":2.149,"5min":3.173,"15min":1.45},"min":{"1min":0.40699999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.7350000000000003,"5min":87.887,"15min":87.887},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 22:20:03 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.645,"15min":0.87},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.3399999999999999,"5min":82.530000000000001,"15min":82.530000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.78,"15min":0.94799999999999995},"min":{"1min":0.29399999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.4580000000000002,"5min":85.441000000000003,"15min":85.441000000000003},"last":0.41899999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:20:33 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":2.1560000000000001,"15min":1.0780000000000001},"min":{"1min":0.33200000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.073,"5min":114.83499999999999,"15min":114.83499999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.505,"5min":1.95,"15min":1.02},"min":{"1min":0.33800000000000002,"5min":0.31,"15min":0.31},"max":{"1min":0.73499999999999999,"5min":110.905,"15min":110.905},"last":0.98599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:19:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":1.2969999999999999,"15min":0.81200000000000006},"min":{"1min":0.34999999999999998,"5min":0.33500000000000002,"15min":0.32600000000000001},"max":{"1min":4.3760000000000003,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":1.3620000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.97199999999999998,"15min":0.70199999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33200000000000002},"max":{"1min":2.379,"5min":7.6609999999999996,"15min":7.6609999999999996},"last":0.495}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":500498432,"data_stored":500498432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":803012608,"data_stored":803012608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1236467712,"data_stored":1236467712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1374093312,"data_stored":1374093312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":648609792,"data_stored":648609792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":713490432,"data_stored":713490432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1133510656,"data_stored":1133510656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:20:54.800 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T22:20:54.800 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:20:54.800 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T22:20:54.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T22:20:54.811 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:54.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:54.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:54.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:54 smithi026 ceph-mon[215775]: pgmap v492: 99 pgs: 99 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 96 MiB/s wr, 1.52k op/s 2023-09-23T22:20:54.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:54.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:54.881 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:54 smithi026 ceph-mon[223014]: pgmap v492: 99 pgs: 99 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 96 MiB/s wr, 1.52k op/s 2023-09-23T22:20:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:54.882 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:54.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:20:54.948 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:55.002 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:20:55.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:54 smithi070 ceph-mon[192652]: pgmap v492: 99 pgs: 99 active+clean; 2.1 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 96 MiB/s wr, 1.52k op/s 2023-09-23T22:20:55.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:55.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:20:55.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:20:55.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:55.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:20:55.108 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:55.139 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:55.192 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:55.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:55.297 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:55.391 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:55.394 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 41165 41149 107.113 73.375 0.0100287 0.00932902 2023-09-23T22:20:56.066 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:20:56.066 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:20:56.066 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:20:56.066 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:20:56.073 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:20:56.117 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:20:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:55 smithi070 ceph-mon[192652]: pgmap v493: 99 pgs: 99 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 91 MiB/s wr, 1.44k op/s 2023-09-23T22:20:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3823550534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:56.150 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:20:56.202 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:20:56.213 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:20:56.213 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:20:56.213 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:20:56.214 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:20:56.214 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:20:56.214 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:20:56.214 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:20:56.214 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:20:56.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:55 smithi026 ceph-mon[215775]: pgmap v493: 99 pgs: 99 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 91 MiB/s wr, 1.44k op/s 2023-09-23T22:20:56.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3823550534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:55 smithi026 ceph-mon[223014]: pgmap v493: 99 pgs: 99 active+clean; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 91 MiB/s wr, 1.44k op/s 2023-09-23T22:20:56.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3823550534' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:20:56.293 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:56.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:56.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:56.394 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 42328 42312 105.736 72.6875 0.00732283 0.00945138 2023-09-23T22:20:56.402 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:56.478 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:56.584 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:56.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:56.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:56.690 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:56.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:56.756 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:56.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:56.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:56.861 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:56.921 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:56.956 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:57.222 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:57.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:57.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:57.333 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:57.396 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 43383 43367 104.199 65.9375 0.0228248 0.00959082 2023-09-23T22:20:57.417 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:57.578 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:57.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:57.687 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:20:57.737 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:20:57.760 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:57.883 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:20:57.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:57.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:20:57.996 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:20:58.021 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:58.064 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:20:58.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:58.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:20:58.138 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:20:58.138 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:20:58.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.139 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:44089 6.3 6:c85743cc:::benchmark_data_smithi026_355573_object44088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:20:58.139 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:20:57.986398+0000", 2023-09-23T22:20:58.139 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.013169449, 2023-09-23T22:20:58.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013194921, 2023-09-23T22:20:58.139 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:20:58.139 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:20:58.140 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:20:58.140 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:20:58.140 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:20:58.140 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 44089 2023-09-23T22:20:58.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.140 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:20:58.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.986398+0000", 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.986398+0000", 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:20:58.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.986398+0000", 2023-09-23T22:20:58.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:20:58.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:20:58.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.986418+0000", 2023-09-23T22:20:58.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0032000000000001e-05 2023-09-23T22:20:58.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:20:58.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.986425+0000", 2023-09-23T22:20:58.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8249999999999999e-06 2023-09-23T22:20:58.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:20:58.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.986430+0000", 2023-09-23T22:20:58.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8910000000000003e-06 2023-09-23T22:20:58.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:20:58.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.987065+0000", 2023-09-23T22:20:58.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000635135 2023-09-23T22:20:58.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:20:58.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.987200+0000", 2023-09-23T22:20:58.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013557400000000001 2023-09-23T22:20:58.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:20:58.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.987327+0000", 2023-09-23T22:20:58.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012702199999999999 2023-09-23T22:20:58.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:20:58.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.988959+0000", 2023-09-23T22:20:58.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016316060000000001 2023-09-23T22:20:58.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:20:58.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.989722+0000", 2023-09-23T22:20:58.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00076263399999999999 2023-09-23T22:20:58.148 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.148 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:20:58.148 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.148 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:44071 6.3 6:ceffd8d3:::benchmark_data_smithi026_355573_object44070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:20:58.149 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:20:57.975340+0000", 2023-09-23T22:20:58.149 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.024227617999999999, 2023-09-23T22:20:58.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024314266000000001, 2023-09-23T22:20:58.149 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:20:58.149 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:20:58.149 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:20:58.150 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:20:58.150 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:20:58.150 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 44071 2023-09-23T22:20:58.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.150 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:20:58.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:20:58.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.975340+0000", 2023-09-23T22:20:58.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:20:58.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.975340+0000", 2023-09-23T22:20:58.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:20:58.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.975340+0000", 2023-09-23T22:20:58.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:20:58.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:20:58.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.975362+0000", 2023-09-23T22:20:58.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2033000000000001e-05 2023-09-23T22:20:58.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:20:58.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.975374+0000", 2023-09-23T22:20:58.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1979e-05 2023-09-23T22:20:58.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:20:58.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.975380+0000", 2023-09-23T22:20:58.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3459999999999999e-06 2023-09-23T22:20:58.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:20:58.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.975513+0000", 2023-09-23T22:20:58.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013265999999999999 2023-09-23T22:20:58.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:20:58.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.975644+0000", 2023-09-23T22:20:58.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131149 2023-09-23T22:20:58.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:20:58.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.975783+0000", 2023-09-23T22:20:58.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013973800000000001 2023-09-23T22:20:58.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:20:58.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.977492+0000", 2023-09-23T22:20:58.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001708396 2023-09-23T22:20:58.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:20:58.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.984980+0000", 2023-09-23T22:20:58.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074883149999999997 2023-09-23T22:20:58.159 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.159 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:20:58.159 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.159 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:44104 6.e e211/203 6:7c040ecf:::benchmark_data_smithi026_355573_object44103:head v 211'2777, mlcod=211'2772)", 2023-09-23T22:20:58.159 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:20:57.999106+0000", 2023-09-23T22:20:58.159 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00046161200000000002, 2023-09-23T22:20:58.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059997000000000002, 2023-09-23T22:20:58.160 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:20:58.160 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:20:58.160 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:20:58.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:20:58.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.999106+0000", 2023-09-23T22:20:58.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:20:58.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.999106+0000", 2023-09-23T22:20:58.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:20:58.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.999105+0000", 2023-09-23T22:20:58.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:20:58.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:20:58.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.999211+0000", 2023-09-23T22:20:58.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010508799999999999 2023-09-23T22:20:58.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:20:58.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.999225+0000", 2023-09-23T22:20:58.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4139e-05 2023-09-23T22:20:58.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:20:58.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.999232+0000", 2023-09-23T22:20:58.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8580000000000002e-06 2023-09-23T22:20:58.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:20:58.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.999274+0000", 2023-09-23T22:20:58.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2219999999999999e-05 2023-09-23T22:20:58.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:20:58.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.999291+0000", 2023-09-23T22:20:58.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7438000000000001e-05 2023-09-23T22:20:58.166 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.166 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:20:58.166 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.167 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:44078 6.3 6:c8ee5456:::benchmark_data_smithi026_355573_object44077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:20:58.167 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:20:57.980999+0000", 2023-09-23T22:20:58.167 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.01856915, 2023-09-23T22:20:58.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018760833000000001, 2023-09-23T22:20:58.167 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:20:58.168 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:20:58.168 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:20:58.168 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:20:58.168 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:20:58.168 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 44078 2023-09-23T22:20:58.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.168 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:20:58.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:20:58.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.980999+0000", 2023-09-23T22:20:58.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:20:58.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.980999+0000", 2023-09-23T22:20:58.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:20:58.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.980998+0000", 2023-09-23T22:20:58.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:20:58.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:20:58.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981020+0000", 2023-09-23T22:20:58.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2359000000000001e-05 2023-09-23T22:20:58.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:20:58.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981029+0000", 2023-09-23T22:20:58.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0930000000000003e-06 2023-09-23T22:20:58.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:20:58.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981035+0000", 2023-09-23T22:20:58.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3520000000000003e-06 2023-09-23T22:20:58.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:20:58.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981486+0000", 2023-09-23T22:20:58.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045087300000000002 2023-09-23T22:20:58.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:20:58.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981650+0000", 2023-09-23T22:20:58.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000164697 2023-09-23T22:20:58.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:20:58.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981807+0000", 2023-09-23T22:20:58.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015649699999999999 2023-09-23T22:20:58.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:20:58.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.984938+0000", 2023-09-23T22:20:58.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031312060000000001 2023-09-23T22:20:58.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:20:58.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.985986+0000", 2023-09-23T22:20:58.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001047746 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:44083 6.3 6:cf47a55a:::benchmark_data_smithi026_355573_object44082:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:20:57.981094+0000", 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.018474072000000001, 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018717820999999999, 2023-09-23T22:20:58.177 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 44083 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981094+0000", 2023-09-23T22:20:58.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981094+0000", 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981093+0000", 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:20:58.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981147+0000", 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3498000000000002e-05 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981155+0000", 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4030000000000007e-06 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.981162+0000", 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0559999999999999e-06 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.983399+0000", 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022370699999999999 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.983485+0000", 2023-09-23T22:20:58.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5399e-05 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.983573+0000", 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.831e-05 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.985104+0000", 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001531438 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:20:58.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:20:57.986016+0000", 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00091152900000000001 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 5 2023-09-23T22:20:58.183 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:20:58.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:20:58.218 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:58.396 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 44343 44327 102.562 60 0.00716867 0.00973349 2023-09-23T22:20:58.772 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:20:58.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:58.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:20:58.881 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:20:58.913 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:58.961 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:20:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:58.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:20:59.066 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:59.091 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:20:58 smithi026 ceph-mon[223014]: pgmap v494: 99 pgs: 99 active+clean; 2.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T22:20:59.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:20:58 smithi026 ceph-mon[215775]: pgmap v494: 99 pgs: 99 active+clean; 2.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T22:20:59.095 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:20:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:20:58 smithi070 ceph-mon[192652]: pgmap v494: 99 pgs: 99 active+clean; 2.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T22:20:59.238 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:20:59.396 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 45403 45387 101.266 66.25 0.0146289 0.00986992 2023-09-23T22:20:59.569 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:20:59.638 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:20:59.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:20:59.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:20:59.742 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:20:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:20:59] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.43.0" 2023-09-23T22:20:59.753 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:20:59.882 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:00.062 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:00.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:00.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:00.168 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:00.199 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:00.256 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:00.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:00.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:00.362 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:00.393 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:00.396 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 46527 46511 100.196 70.25 0.0100473 0.00997472 2023-09-23T22:21:00.451 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:00.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:00.558 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:00.587 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:00.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:00 smithi026 ceph-mon[223014]: pgmap v495: 99 pgs: 99 active+clean; 2.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 84 MiB/s wr, 1.33k op/s 2023-09-23T22:21:00.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:00 smithi026 ceph-mon[215775]: pgmap v495: 99 pgs: 99 active+clean; 2.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 84 MiB/s wr, 1.33k op/s 2023-09-23T22:21:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:00 smithi070 ceph-mon[192652]: pgmap v495: 99 pgs: 99 active+clean; 2.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 84 MiB/s wr, 1.33k op/s 2023-09-23T22:21:01.210 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:01.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:01.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:01.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:01.321 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:01.344 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:01.350 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:01.396 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 47511 47495 98.9067 61.5 0.0100945 0.0101042 2023-09-23T22:21:01.405 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:01.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:01.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:01.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:01.567 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:01.831 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:21:01.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1167637341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:21:01.868 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1167637341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:21:01.905 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:01.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:01.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:02.016 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:02.095 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:02.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1167637341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:21:02.398 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 48672 48656 98.0518 72.5625 0.00760372 0.0101936 2023-09-23T22:21:02.448 INFO:teuthology.orchestra.run.smithi026.stdout:moved 0 / 297 (0%) 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 4.65548 -> 4.65548 (expected baseline 6.03053) 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 36 -> 36 pgs (0.848485 -> 0.848485 * mean) 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.3 with 50 -> 50 pgs (1.17845 -> 1.17845 * mean) 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 1 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0152 2023-09-23T22:21:02.449 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.1523 2023-09-23T22:21:02.458 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T22:21:02.659 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:02.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:02.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:02.772 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:02.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[215775]: pgmap v496: 99 pgs: 99 active+clean; 2.7 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T22:21:02.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:02.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[215775]: failed to encode map e212 with expected crc 2023-09-23T22:21:02.805 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[215775]: failed to encode map e212 with expected crc 2023-09-23T22:21:02.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1167637341' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:21:02.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[215775]: osdmap e212: 8 total, 8 up, 7 in 2023-09-23T22:21:02.844 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:03.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:02 smithi070 ceph-mon[192652]: pgmap v496: 99 pgs: 99 active+clean; 2.7 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T22:21:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:02 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:02 smithi070 ceph-mon[192652]: failed to encode map e212 with expected crc 2023-09-23T22:21:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:02 smithi070 ceph-mon[192652]: failed to encode map e212 with expected crc 2023-09-23T22:21:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1167637341' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:21:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:02 smithi070 ceph-mon[192652]: osdmap e212: 8 total, 8 up, 7 in 2023-09-23T22:21:03.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[223014]: pgmap v496: 99 pgs: 99 active+clean; 2.7 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 67 MiB/s wr, 1.07k op/s 2023-09-23T22:21:03.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:03.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[223014]: failed to encode map e212 with expected crc 2023-09-23T22:21:03.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[223014]: failed to encode map e212 with expected crc 2023-09-23T22:21:03.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1167637341' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:21:03.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:02 smithi026 ceph-mon[223014]: osdmap e212: 8 total, 8 up, 7 in 2023-09-23T22:21:03.398 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 49956 49940 97.4951 80.25 0.00891713 0.0102514 2023-09-23T22:21:03.532 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:03.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:03.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:03.638 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:03.700 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:03.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:03 smithi026 ceph-mon[215775]: from='client.54865 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:03.814 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:03 smithi026 ceph-mon[223014]: from='client.54865 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:04.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:03 smithi070 ceph-mon[192652]: from='client.54865 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:04.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 51170 51154 96.8389 75.875 0.0147731 0.0103206 2023-09-23T22:21:05.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:04 smithi070 ceph-mon[192652]: pgmap v498: 99 pgs: 99 active+clean; 2.8 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T22:21:05.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:04 smithi026 ceph-mon[215775]: pgmap v498: 99 pgs: 99 active+clean; 2.8 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T22:21:05.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:04 smithi026 ceph-mon[223014]: pgmap v498: 99 pgs: 99 active+clean; 2.8 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 70 MiB/s wr, 1.13k op/s 2023-09-23T22:21:05.335 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T22:21:05.335 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-23T22:21:05.336 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:05.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:05.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:05.402 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 52192 52176 95.8695 63.875 0.027701 0.0104254 2023-09-23T22:21:05.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:05.531 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:05.548 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:21:05.700 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:05.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:05.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:05.811 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:05.840 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:05.888 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:05.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:05.993 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:06.084 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:06.187 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:06.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:06.193 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:06.216 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:21:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:21:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:21:06.216 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:21:06.216 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:21:06.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:06.324 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:06.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:06.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:06.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:06.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 53134 53118 94.8128 58.875 0.00779567 0.0105418 2023-09-23T22:21:06.488 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:06.522 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:06.575 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:06.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:06.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:06.682 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:06.704 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:21:06.715 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:06.747 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:21:06.768 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:06.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:06.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:06.781 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:21:06.819 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:21:06.859 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:21:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:06 smithi070 ceph-mon[192652]: pgmap v499: 99 pgs: 99 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T22:21:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2261898640' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:06.876 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:06.907 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:06.910 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:21:06.946 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:21:06.964 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:06.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:06.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:06.987 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:21:06.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:21:06.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:21:06.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:21:06.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:21:06.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:21:06.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:21:06.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:21:06.992 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:21:07.069 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:07.124 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:06 smithi026 ceph-mon[223014]: pgmap v499: 99 pgs: 99 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T22:21:07.124 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2261898640' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:07.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:06 smithi026 ceph-mon[215775]: pgmap v499: 99 pgs: 99 active+clean; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 70 MiB/s wr, 1.11k op/s 2023-09-23T22:21:07.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2261898640' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:07.130 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:07.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:07.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:07.234 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:07.335 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:07.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 54271 54255 94.1521 71.0625 0.00896418 0.0106141 2023-09-23T22:21:07.407 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:07.663 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:07.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:07.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:07.766 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:07.829 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:08.236 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:08.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:21:08.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:08.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:08.329 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:08.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:08.378 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:08.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 55445 55429 93.5906 73.375 0.0177021 0.0106786 2023-09-23T22:21:08.445 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:08.729 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:08.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:08.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:08.842 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:08.925 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:08.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:08 smithi026 ceph-mon[215775]: pgmap v500: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:21:08.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:08 smithi026 ceph-mon[223014]: pgmap v500: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:21:09.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:08 smithi070 ceph-mon[192652]: pgmap v500: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:21:09.209 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:09.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:09.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:09.317 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:09.344 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:09.401 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 56441 56425 92.7631 62.25 0.0115501 0.0107762 2023-09-23T22:21:09.401 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:09.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:09.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:09.507 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:09.544 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:09.593 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:09.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:09.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:09.699 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:09.735 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:09.787 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:09.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:09.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:09.893 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:09.918 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:09.968 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:09.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:09.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:09.987 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:21:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:21:09] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-09-23T22:21:10.076 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:10.173 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:10.272 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:21:10.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:10.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:10.298 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:10 smithi026 ceph-mon[215775]: pgmap v501: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:21:10.298 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:10.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:10.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:10 smithi026 ceph-mon[223014]: pgmap v501: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:21:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:10 smithi070 ceph-mon[192652]: pgmap v501: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:21:10.401 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 57426 57410 91.9632 61.5625 0.0133683 0.0108689 2023-09-23T22:21:10.820 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:10.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:10.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:10.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:10.959 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:11.011 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:11.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:11.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:11.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:11.192 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:11.401 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:21:11.401977+0000 min lat: 0.00131411 max lat: 0.238942 avg lat: 0.0109753 2023-09-23T22:21:11.401 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:21:11.401 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 58325 58309 91.0681 56.1875 0.0162612 0.0109753 2023-09-23T22:21:11.451 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:11.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:11.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:11.561 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:11.586 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:11.644 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:11.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:11.857 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:11.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:11.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:11.967 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:12.000 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:12.051 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:12.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:12.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:12.158 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:12.246 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:21:12.246 INFO:teuthology.orchestra.run.smithi026.stdout:moved 26 / 297 (8.75421%) 2023-09-23T22:21:12.246 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:21:12.246 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 4.65548 -> 7.00728 (expected baseline 6.03053) 2023-09-23T22:21:12.246 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 36 -> 28 pgs (0.848485 -> 0.659933 * mean) 2023-09-23T22:21:12.246 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.3 with 50 -> 41 pgs (1.17845 -> 0.96633 * mean) 2023-09-23T22:21:12.247 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:21:12.247 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T22:21:12.247 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 1 2023-09-23T22:21:12.247 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:21:12.247 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0180 2023-09-23T22:21:12.247 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0198 2023-09-23T22:21:12.247 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 1.0000 -> 0.7174 2023-09-23T22:21:12.247 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 weight 1.0000 -> 0.8539 2023-09-23T22:21:12.247 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 weight 1.0000 -> 0.7242 2023-09-23T22:21:12.258 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:12.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:12 smithi070 ceph-mon[192652]: pgmap v502: 99 pgs: 99 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:21:12.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:12 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'47013','3':'47463','7':'55958'}"}]: dispatch 2023-09-23T22:21:12.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/942497357' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:12.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:12 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:12.401 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 59085 59069 90.0056 47.5 0.0034112 0.0110661 2023-09-23T22:21:12.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:12 smithi026 ceph-mon[215775]: pgmap v502: 99 pgs: 99 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:21:12.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:12 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'47013','3':'47463','7':'55958'}"}]: dispatch 2023-09-23T22:21:12.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/942497357' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:12.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:12 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:12.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:12 smithi026 ceph-mon[223014]: pgmap v502: 99 pgs: 99 active+clean; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:21:12.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:12 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'47013','3':'47463','7':'55958'}"}]: dispatch 2023-09-23T22:21:12.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/942497357' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:12.481 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:12 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:13.401 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 59085 59069 87.8632 0 - 0.0110661 2023-09-23T22:21:13.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: from='client.45050 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'47013','3':'47463','7':'55958'}"}]': finished 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: osdmap e213: 8 total, 8 up, 7 in 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: failed to encode map e213 with expected crc 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: failed to encode map e213 with expected crc 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: osdmap e214: 8 total, 8 up, 7 in 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:13 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: from='client.45050 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:13.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:13.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'47013','3':'47463','7':'55958'}"}]': finished 2023-09-23T22:21:13.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:21:13.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: osdmap e213: 8 total, 8 up, 7 in 2023-09-23T22:21:13.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: failed to encode map e213 with expected crc 2023-09-23T22:21:13.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: failed to encode map e213 with expected crc 2023-09-23T22:21:13.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: osdmap e214: 8 total, 8 up, 7 in 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: from='client.45050 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'47013','3':'47463','7':'55958'}"}]': finished 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:21:13.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: osdmap e213: 8 total, 8 up, 7 in 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: failed to encode map e213 with expected crc 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: failed to encode map e213 with expected crc 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: osdmap e214: 8 total, 8 up, 7 in 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:13 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:13.876 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:13.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:13.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:13.980 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:14.065 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:14.145 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T22:21:14.146 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T22:21:14.401 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 59407 59391 86.2883 10.0625 0.00401943 0.0115672 2023-09-23T22:21:14.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:14.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:14.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[223014]: failed to encode map e214 with expected crc 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[223014]: pgmap v505: 99 pgs: 4 peering, 95 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[223014]: failed to encode map e215 with expected crc 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[223014]: osdmap e215: 8 total, 8 up, 7 in 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[223014]: failed to encode map e215 with expected crc 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[223014]: failed to encode map e215 with expected crc 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[215775]: failed to encode map e214 with expected crc 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[215775]: pgmap v505: 99 pgs: 4 peering, 95 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T22:21:14.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[215775]: failed to encode map e215 with expected crc 2023-09-23T22:21:14.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[215775]: osdmap e215: 8 total, 8 up, 7 in 2023-09-23T22:21:14.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[215775]: failed to encode map e215 with expected crc 2023-09-23T22:21:14.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:14 smithi026 ceph-mon[215775]: failed to encode map e215 with expected crc 2023-09-23T22:21:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:14 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:14 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:14 smithi070 ceph-mon[192652]: failed to encode map e214 with expected crc 2023-09-23T22:21:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:14 smithi070 ceph-mon[192652]: pgmap v505: 99 pgs: 4 peering, 95 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T22:21:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:14 smithi070 ceph-mon[192652]: failed to encode map e215 with expected crc 2023-09-23T22:21:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:14 smithi070 ceph-mon[192652]: osdmap e215: 8 total, 8 up, 7 in 2023-09-23T22:21:14.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:14 smithi070 ceph-mon[192652]: failed to encode map e215 with expected crc 2023-09-23T22:21:14.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:14 smithi070 ceph-mon[192652]: failed to encode map e215 with expected crc 2023-09-23T22:21:14.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:14.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:14.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:14.862 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:14.896 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:15.263 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:15.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:15.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:15.469 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 60082 60066 85.2818 42.1875 0.00867864 0.0117185 2023-09-23T22:21:15.569 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:15 smithi070 ceph-mon[192652]: failed to encode map e215 with expected crc 2023-09-23T22:21:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:15 smithi070 ceph-mon[192652]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:15 smithi070 ceph-mon[192652]: osdmap e216: 8 total, 8 up, 7 in 2023-09-23T22:21:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:15 smithi070 ceph-mon[192652]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:15 smithi070 ceph-mon[192652]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:15 smithi070 ceph-mon[192652]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[215775]: failed to encode map e215 with expected crc 2023-09-23T22:21:15.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[215775]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[215775]: osdmap e216: 8 total, 8 up, 7 in 2023-09-23T22:21:15.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[215775]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[215775]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[215775]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[223014]: failed to encode map e215 with expected crc 2023-09-23T22:21:15.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[223014]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[223014]: osdmap e216: 8 total, 8 up, 7 in 2023-09-23T22:21:15.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[223014]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[223014]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.667 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:15 smithi026 ceph-mon[223014]: failed to encode map e216 with expected crc 2023-09-23T22:21:15.682 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:15.853 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:15.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:15.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:15.966 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:15.997 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:16.005 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:16.067 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:16.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:16.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:16.174 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:16.175 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:21:16.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:16.229 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:21:16.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:16 smithi070 ceph-mon[192652]: failed to encode map e216 with expected crc 2023-09-23T22:21:16.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:16 smithi070 ceph-mon[192652]: pgmap v508: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 92 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 55 MiB/s wr, 881 op/s; 3667/175575 objects degraded (2.089%); 3716/175575 objects misplaced (2.116%); 16 KiB/s, 0 objects/s recovering 2023-09-23T22:21:16.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:16 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 3667/175575 objects degraded (2.089%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:21:16.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:16 smithi070 ceph-mon[192652]: osdmap e217: 8 total, 8 up, 7 in 2023-09-23T22:21:16.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:16 smithi070 ceph-mon[192652]: failed to encode map e217 with expected crc 2023-09-23T22:21:16.255 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2250930620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:16.263 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:21:16.263 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:21:16.280 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:16.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:16.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:16.288 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:21:16.353 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:21:16.384 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:16.403 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:21:16.404 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 60813 60797 84.4019 45.6875 0.0409609 0.0118424 2023-09-23T22:21:16.419 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:16.460 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:21:16.478 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:16.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:16.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:16.495 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:21:16.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[223014]: failed to encode map e216 with expected crc 2023-09-23T22:21:16.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[223014]: pgmap v508: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 92 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 55 MiB/s wr, 881 op/s; 3667/175575 objects degraded (2.089%); 3716/175575 objects misplaced (2.116%); 16 KiB/s, 0 objects/s recovering 2023-09-23T22:21:16.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 3667/175575 objects degraded (2.089%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:21:16.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[223014]: osdmap e217: 8 total, 8 up, 7 in 2023-09-23T22:21:16.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[223014]: failed to encode map e217 with expected crc 2023-09-23T22:21:16.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2250930620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:16.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[215775]: failed to encode map e216 with expected crc 2023-09-23T22:21:16.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[215775]: pgmap v508: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 4 peering, 92 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 55 MiB/s wr, 881 op/s; 3667/175575 objects degraded (2.089%); 3716/175575 objects misplaced (2.116%); 16 KiB/s, 0 objects/s recovering 2023-09-23T22:21:16.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 3667/175575 objects degraded (2.089%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:21:16.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[215775]: osdmap e217: 8 total, 8 up, 7 in 2023-09-23T22:21:16.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[215775]: failed to encode map e217 with expected crc 2023-09-23T22:21:16.541 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2250930620' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:16.542 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:21:16.555 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:21:16.555 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:21:16.556 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:21:16.556 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:21:16.556 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:21:16.556 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:21:16.556 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:21:16.556 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:21:16.582 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:16.623 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:16.676 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:16.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:16.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:16.782 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:16.834 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:16.901 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:16.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:16.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:17.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:17.078 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:17.272 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:17 smithi070 ceph-mon[192652]: failed to encode map e217 with expected crc 2023-09-23T22:21:17.404 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 61645 61629 83.6976 52 0.0037748 0.0119377 2023-09-23T22:21:17.471 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:21:17.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:17 smithi026 ceph-mon[215775]: failed to encode map e217 with expected crc 2023-09-23T22:21:17.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:17 smithi026 ceph-mon[223014]: failed to encode map e217 with expected crc 2023-09-23T22:21:18.404 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 62327 62311 82.824 42.625 0.0192909 0.0120662 2023-09-23T22:21:18.562 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:18.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:18.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:18.669 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:18.742 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:18.969 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:18.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:18.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:18.973 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:18 smithi026 ceph-mon[223014]: pgmap v510: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 5 peering, 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 19 KiB/s rd, 35 MiB/s wr, 574 op/s; 11075/178956 objects degraded (6.189%); 3716/178956 objects misplaced (2.076%); 51 KiB/s, 1 objects/s recovering 2023-09-23T22:21:18.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:18 smithi026 ceph-mon[215775]: pgmap v510: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 5 peering, 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 19 KiB/s rd, 35 MiB/s wr, 574 op/s; 11075/178956 objects degraded (6.189%); 3716/178956 objects misplaced (2.076%); 51 KiB/s, 1 objects/s recovering 2023-09-23T22:21:19.074 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:19.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:18 smithi070 ceph-mon[192652]: pgmap v510: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 remapped+peering, 5 peering, 89 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 19 KiB/s rd, 35 MiB/s wr, 574 op/s; 11075/178956 objects degraded (6.189%); 3716/178956 objects misplaced (2.076%); 51 KiB/s, 1 objects/s recovering 2023-09-23T22:21:19.148 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:19.405 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 62966 62950 81.9308 39.9375 0.168387 0.0121848 2023-09-23T22:21:19.824 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:21:19.873 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:21:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:21:19] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-09-23T22:21:19.988 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:19.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:19.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:20.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:20.128 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:20.240 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:20.405 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 63701 63685 81.1964 45.9375 0.0520032 0.0123086 2023-09-23T22:21:20.532 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:20.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:20.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:20.660 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:20.703 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:20.771 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:20.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:20.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:20.880 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:20.915 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:20.978 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:20.982 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:20 smithi070 ceph-mon[192652]: pgmap v511: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 peering, 91 active+clean; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 14 KiB/s rd, 45 MiB/s wr, 732 op/s; 18451/184587 objects degraded (9.996%); 6973/184587 objects misplaced (3.778%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:21:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:20.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:21.084 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:21.179 INFO:teuthology.orchestra.run.smithi026.stdout:moved 0 / 297 (0%) 2023-09-23T22:21:21.179 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:21:21.179 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 7.00728 -> 7.00728 (expected baseline 6.03053) 2023-09-23T22:21:21.179 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 28 -> 28 pgs (0.659933 -> 0.659933 * mean) 2023-09-23T22:21:21.179 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.1 with 51 -> 51 pgs (1.20202 -> 1.20202 * mean) 2023-09-23T22:21:21.179 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:21:21.179 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T22:21:21.179 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 1 2023-09-23T22:21:21.179 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:21:21.180 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0198 2023-09-23T22:21:21.180 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.1983 2023-09-23T22:21:21.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:20 smithi026 ceph-mon[223014]: pgmap v511: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 peering, 91 active+clean; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 14 KiB/s rd, 45 MiB/s wr, 732 op/s; 18451/184587 objects degraded (9.996%); 6973/184587 objects misplaced (3.778%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:21:21.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:20 smithi026 ceph-mon[215775]: pgmap v511: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 peering, 91 active+clean; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 14 KiB/s rd, 45 MiB/s wr, 732 op/s; 18451/184587 objects degraded (9.996%); 6973/184587 objects misplaced (3.778%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:21:21.194 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T22:21:21.194 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:21.287 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:21.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:21.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:21.391 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:21.404 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 64441 64425 80.4976 46.25 0.0141229 0.012417 2023-09-23T22:21:21.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:21.487 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:21.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:21.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:21.596 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:21.673 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:21.912 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:21:21.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:21 smithi026 ceph-mon[215775]: from='client.54889 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:21.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:21 smithi026 ceph-mon[215775]: pgmap v512: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 11 KiB/s rd, 39 MiB/s wr, 628 op/s; 18451/186624 objects degraded (9.887%); 6973/186624 objects misplaced (3.736%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T22:21:21.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:21 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 18451/186624 objects degraded (9.887%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:21.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:21 smithi026 ceph-mon[223014]: from='client.54889 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:21.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:21 smithi026 ceph-mon[223014]: pgmap v512: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 11 KiB/s rd, 39 MiB/s wr, 628 op/s; 18451/186624 objects degraded (9.887%); 6973/186624 objects misplaced (3.736%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T22:21:21.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:21 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 18451/186624 objects degraded (9.887%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:22.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:21 smithi070 ceph-mon[192652]: from='client.54889 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:22.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:21 smithi070 ceph-mon[192652]: pgmap v512: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 11 KiB/s rd, 39 MiB/s wr, 628 op/s; 18451/186624 objects degraded (9.887%); 6973/186624 objects misplaced (3.736%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T22:21:22.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:21 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 18451/186624 objects degraded (9.887%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:22.405 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 65153 65137 79.7919 44.5 0.0420482 0.0125264 2023-09-23T22:21:22.719 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:22.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:22.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:22.823 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:22.850 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:22.898 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:22.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:22.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:23.010 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:23.111 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:23.335 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:23.357 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:23.369 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T22:21:23.370 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 1.0 2023-09-23T22:21:23.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:23.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:23.405 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 65871 65855 79.1205 44.875 0.0101642 0.0126315 2023-09-23T22:21:23.479 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:23.517 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:23.521 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:23.586 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:23.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:23.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:23.699 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:23.730 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:23.785 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:23.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:23.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:23.893 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:23.931 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:24.005 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:24.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:24.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:24.114 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:24.148 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:24.210 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:24.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:24.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:24.315 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:24.350 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:24.409 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 66548 66532 78.4206 42.3125 0.0133954 0.0127457 2023-09-23T22:21:24.414 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:24.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:24.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:24.614 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:24 smithi026 ceph-mon[215775]: pgmap v513: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 9.9 KiB/s rd, 37 MiB/s wr, 606 op/s; 18451/190620 objects degraded (9.679%); 6571/190620 objects misplaced (3.447%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T22:21:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/269565278' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:21:24.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:24 smithi026 ceph-mon[223014]: pgmap v513: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 9.9 KiB/s rd, 37 MiB/s wr, 606 op/s; 18451/190620 objects degraded (9.679%); 6571/190620 objects misplaced (3.447%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T22:21:24.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:24.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:24.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/269565278' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:21:25.005 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:25.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:25.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:25.079 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:21:25.120 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:24 smithi070 ceph-mon[192652]: pgmap v513: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 9.9 KiB/s rd, 37 MiB/s wr, 606 op/s; 18451/190620 objects degraded (9.679%); 6571/190620 objects misplaced (3.447%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T22:21:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/269565278' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:21:25.151 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:25.409 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 67093 67077 77.5992 34.0625 0.0915281 0.0128667 2023-09-23T22:21:25.938 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:25.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:25.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:26.049 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:26.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:25 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:26.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:25 smithi070 ceph-mon[192652]: failed to encode map e218 with expected crc 2023-09-23T22:21:26.078 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/269565278' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:21:26.079 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:25 smithi070 ceph-mon[192652]: osdmap e218: 8 total, 8 up, 7 in 2023-09-23T22:21:26.083 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:26.141 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:26.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:26.145 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:26.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:26.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:25 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:26.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:25 smithi026 ceph-mon[215775]: failed to encode map e218 with expected crc 2023-09-23T22:21:26.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/269565278' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:21:26.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:25 smithi026 ceph-mon[215775]: osdmap e218: 8 total, 8 up, 7 in 2023-09-23T22:21:26.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:25 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:26.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:25 smithi026 ceph-mon[223014]: failed to encode map e218 with expected crc 2023-09-23T22:21:26.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/269565278' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:21:26.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:25 smithi026 ceph-mon[223014]: osdmap e218: 8 total, 8 up, 7 in 2023-09-23T22:21:26.320 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:26.409 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 15 67905 67890 77.1123 50.8125 0.00873122 0.0129615 2023-09-23T22:21:26.439 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:26.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:26.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:26.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:26.570 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:26.632 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:26.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:26.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:26.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:26.829 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:26 smithi070 ceph-mon[192652]: pgmap v515: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s; 18451/197283 objects degraded (9.353%); 6321/197283 objects misplaced (3.204%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T22:21:26.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:26 smithi026 ceph-mon[223014]: pgmap v515: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s; 18451/197283 objects degraded (9.353%); 6321/197283 objects misplaced (3.204%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T22:21:26.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:26 smithi026 ceph-mon[215775]: pgmap v515: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.0 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s; 18451/197283 objects degraded (9.353%); 6321/197283 objects misplaced (3.204%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T22:21:27.409 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 68623 68607 76.5355 44.8125 0.0212807 0.0130597 2023-09-23T22:21:27.618 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:27.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:27.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:27.722 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:27.755 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:27.829 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:27.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:27.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:27.858 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:27.940 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:27.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:27 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 18451/197283 objects degraded (9.353%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:27.962 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:27 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 18451/197283 objects degraded (9.353%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:27.974 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:27.980 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:21:27.980 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:21:27.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.980 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:69007 6.b 6:db9d6b00:::benchmark_data_smithi026_355573_object69006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T22:21:27.980 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:27.974176+0000", 2023-09-23T22:21:27.980 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0051308109999999999, 2023-09-23T22:21:27.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005174077, 2023-09-23T22:21:27.980 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:27.980 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 69007 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974176+0000", 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974176+0000", 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974175+0000", 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974245+0000", 2023-09-23T22:21:27.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0479000000000001e-05 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974266+0000", 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1080000000000001e-05 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974278+0000", 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1770000000000001e-05 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974416+0000", 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013807199999999999 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974587+0000", 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000170853 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974780+0000", 2023-09-23T22:21:27.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019348799999999999 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:69008 6.b 6:d15921cb:::benchmark_data_smithi026_355573_object69007:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:27.974293+0000", 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0050137139999999998, 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051528390000000002, 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:21:27.985 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 69008 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974293+0000", 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974293+0000", 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974293+0000", 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974316+0000", 2023-09-23T22:21:27.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3646e-05 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974324+0000", 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9840000000000001e-06 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.974330+0000", 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4049999999999999e-06 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.977557+0000", 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032272659999999999 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.977734+0000", 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017653999999999999 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.977908+0000", 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017446900000000001 2023-09-23T22:21:27.989 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:69005 6.7 e218/203 6:ef339acf:::benchmark_data_smithi026_355573_object69004:head v 218'4366, mlcod=218'4364)", 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:27.976884+0000", 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0024230860000000001, 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026491819999999999, 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:27.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.976884+0000", 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.976884+0000", 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.976884+0000", 2023-09-23T22:21:27.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.976933+0000", 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9259999999999999e-05 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.976944+0000", 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0869e-05 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.976951+0000", 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9140000000000002e-06 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.977023+0000", 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1984999999999996e-05 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.977046+0000", 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3756000000000001e-05 2023-09-23T22:21:27.993 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:69001 6.5 6:a3806c9b:::benchmark_data_smithi026_355573_object69000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:27.969552+0000", 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0097554770000000002, 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010056054, 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:21:27.994 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 69001 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.969552+0000", 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.969552+0000", 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.969550+0000", 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.969599+0000", 2023-09-23T22:21:27.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9450999999999998e-05 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.969617+0000", 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7286e-05 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.969626+0000", 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8619999999999998e-06 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.969677+0000", 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0779000000000003e-05 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.969853+0000", 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017580499999999999 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:21:27.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.970888+0000", 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001034742 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.976853+0000", 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059655150000000002 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:27.976908+0000", 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4744999999999998e-05 2023-09-23T22:21:27.999 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:28.000 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:28.000 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:28.000 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:28.000 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:21:28.000 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 4 2023-09-23T22:21:28.000 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:21:28.047 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:28.056 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:27 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 18451/197283 objects degraded (9.353%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:28.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:28.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:28.157 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:28.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:28.265 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:28.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:28.371 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:28.411 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 69333 69317 75.9692 44.375 0.0265992 0.0131563 2023-09-23T22:21:28.462 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:21:28.473 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:28.500 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:21:28.549 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:21:28.587 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:21:28.625 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:21:28.664 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:21:28.701 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:21:28.746 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:21:28.752 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:21:28.752 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:21:28.752 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:21:28.752 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:21:28.752 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:21:28.752 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:21:28.752 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:21:28.752 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:21:29.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:28 smithi070 ceph-mon[192652]: pgmap v516: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 697 op/s; 18451/199884 objects degraded (9.231%); 6321/199884 objects misplaced (3.162%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:21:29.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:28 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3335880590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:29.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:28 smithi026 ceph-mon[223014]: pgmap v516: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 697 op/s; 18451/199884 objects degraded (9.231%); 6321/199884 objects misplaced (3.162%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:21:29.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:28 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3335880590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:29.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:28 smithi026 ceph-mon[215775]: pgmap v516: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 697 op/s; 18451/199884 objects degraded (9.231%); 6321/199884 objects misplaced (3.162%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:21:29.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:28 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3335880590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:29.411 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 70146 70130 75.5355 50.8125 0.0071614 0.0132335 2023-09-23T22:21:29.418 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:21:29.989 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:21:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:21:29] "GET /metrics HTTP/1.1" 200 36442 "" "Prometheus/2.43.0" 2023-09-23T22:21:30.032 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:30.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:30.080 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:21:30.149 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:30.177 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:30.245 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:30.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:30.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:30.349 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:30.411 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 70881 70865 75.034 45.9375 0.0530535 0.0133202 2023-09-23T22:21:30.422 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:30.531 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:30.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:30.646 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:30.695 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:30.762 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:30.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:30.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:30.873 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:30.910 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:30.960 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:30.970 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:30 smithi070 ceph-mon[192652]: pgmap v517: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s; 18451/205911 objects degraded (8.961%); 5620/205911 objects misplaced (2.729%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T22:21:30.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:30.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:30.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:30 smithi026 ceph-mon[215775]: pgmap v517: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s; 18451/205911 objects degraded (8.961%); 5620/205911 objects misplaced (2.729%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T22:21:30.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:30 smithi026 ceph-mon[223014]: pgmap v517: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 710 op/s; 18451/205911 objects degraded (8.961%); 5620/205911 objects misplaced (2.729%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T22:21:31.072 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:31.103 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:31.155 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:31.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:31.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:31.265 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:31.347 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:31.411 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:21:31.412033+0000 min lat: 0.00131411 max lat: 1.79188 avg lat: 0.0134098 2023-09-23T22:21:31.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:21:31.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 71609 71593 74.5419 45.5 0.0336306 0.0134098 2023-09-23T22:21:31.738 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:31.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:31.822 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:31.848 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:31.881 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:31.954 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:31.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:31.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:32.060 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:32.107 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:32.172 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:32.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:32.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:32.256 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:32.281 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:32.316 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:32.372 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:32.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:32.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:32.411 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 72341 72325 74.0699 45.75 0.00679154 0.0134947 2023-09-23T22:21:32.479 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:32.516 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:32.578 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:32.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:32.684 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:32.718 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:32.774 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:32.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:32.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:32.866 INFO:teuthology.orchestra.run.smithi026.stdout:moved 35 / 297 (11.7845%) 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 7.00728 -> 13.7618 (expected baseline 6.03053) 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 28 -> 26 pgs (0.659933 -> 0.612795 * mean) 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.1 with 51 -> 59 pgs (1.20202 -> 1.39057 * mean) 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 1 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0224 2023-09-23T22:21:32.867 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0246 2023-09-23T22:21:32.868 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 weight 0.7242 -> 0.5238 2023-09-23T22:21:32.868 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 weight 0.8539 -> 0.7293 2023-09-23T22:21:32.868 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 0.7174 -> 0.5412 2023-09-23T22:21:32.883 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:32.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:32 smithi026 ceph-mon[223014]: pgmap v518: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 18451/207894 objects degraded (8.875%); 5620/207894 objects misplaced (2.703%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T22:21:32.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:32 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'35469','3':'34325','7':'47793'}"}]: dispatch 2023-09-23T22:21:32.929 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:32 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 18451/207894 objects degraded (8.875%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:32.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:32 smithi026 ceph-mon[215775]: pgmap v518: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 18451/207894 objects degraded (8.875%); 5620/207894 objects misplaced (2.703%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T22:21:32.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:32 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'35469','3':'34325','7':'47793'}"}]: dispatch 2023-09-23T22:21:32.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:32 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 18451/207894 objects degraded (8.875%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:32.935 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:32 smithi070 ceph-mon[192652]: pgmap v518: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 44 MiB/s wr, 708 op/s; 18451/207894 objects degraded (8.875%); 5620/207894 objects misplaced (2.703%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T22:21:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:32 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'35469','3':'34325','7':'47793'}"}]: dispatch 2023-09-23T22:21:33.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:32 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 18451/207894 objects degraded (8.875%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:33.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 72676 72660 73.2132 20.9375 0.00643219 0.0135387 2023-09-23T22:21:33.645 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:21:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[223014]: from='client.35121 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'35469','3':'34325','7':'47793'}"}]': finished 2023-09-23T22:21:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[223014]: osdmap e219: 8 total, 8 up, 7 in 2023-09-23T22:21:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[223014]: failed to encode map e219 with expected crc 2023-09-23T22:21:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[223014]: failed to encode map e219 with expected crc 2023-09-23T22:21:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[223014]: pgmap v520: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 682 op/s; 18451/211995 objects degraded (8.704%); 5162/211995 objects misplaced (2.435%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T22:21:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[215775]: from='client.35121 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:33.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'35469','3':'34325','7':'47793'}"}]': finished 2023-09-23T22:21:33.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[215775]: osdmap e219: 8 total, 8 up, 7 in 2023-09-23T22:21:33.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[215775]: failed to encode map e219 with expected crc 2023-09-23T22:21:33.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[215775]: failed to encode map e219 with expected crc 2023-09-23T22:21:33.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:33 smithi026 ceph-mon[215775]: pgmap v520: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 682 op/s; 18451/211995 objects degraded (8.704%); 5162/211995 objects misplaced (2.435%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T22:21:34.090 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:34.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:34.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:33 smithi070 ceph-mon[192652]: from='client.35121 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:33 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'35469','3':'34325','7':'47793'}"}]': finished 2023-09-23T22:21:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:33 smithi070 ceph-mon[192652]: osdmap e219: 8 total, 8 up, 7 in 2023-09-23T22:21:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:33 smithi070 ceph-mon[192652]: failed to encode map e219 with expected crc 2023-09-23T22:21:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:33 smithi070 ceph-mon[192652]: failed to encode map e219 with expected crc 2023-09-23T22:21:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:33 smithi070 ceph-mon[192652]: pgmap v520: 99 pgs: 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 92 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 43 MiB/s wr, 682 op/s; 18451/211995 objects degraded (8.704%); 5162/211995 objects misplaced (2.435%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T22:21:34.194 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:34.222 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:34.414 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 72676 72660 72.0515 0 - 0.0135387 2023-09-23T22:21:35.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[223014]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[223014]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[223014]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[223014]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[223014]: osdmap e220: 8 total, 8 up, 7 in 2023-09-23T22:21:35.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[215775]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[215775]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[215775]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[215775]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:34 smithi026 ceph-mon[215775]: osdmap e220: 8 total, 8 up, 7 in 2023-09-23T22:21:35.301 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:35.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:35.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:34 smithi070 ceph-mon[192652]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:34 smithi070 ceph-mon[192652]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:34 smithi070 ceph-mon[192652]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:34 smithi070 ceph-mon[192652]: failed to encode map e220 with expected crc 2023-09-23T22:21:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:34 smithi070 ceph-mon[192652]: osdmap e220: 8 total, 8 up, 7 in 2023-09-23T22:21:35.407 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:35.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 72761 72745 71.0091 2.65625 0.00524488 0.0138852 2023-09-23T22:21:35.437 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:36.044 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:36.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:36.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:36.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:36.180 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: osdmap e221: 8 total, 8 up, 7 in 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[223014]: pgmap v523: 99 pgs: 2 remapped+peering, 5 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 29 MiB/s wr, 468 op/s; 17382/214347 objects degraded (8.109%); 2198/214347 objects misplaced (1.025%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T22:21:36.181 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: osdmap e221: 8 total, 8 up, 7 in 2023-09-23T22:21:36.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.182 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:35 smithi026 ceph-mon[215775]: pgmap v523: 99 pgs: 2 remapped+peering, 5 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 29 MiB/s wr, 468 op/s; 17382/214347 objects degraded (8.109%); 2198/214347 objects misplaced (1.025%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T22:21:36.183 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T22:21:36.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T22:21:36.188 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:36.251 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:36.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:36.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:36.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:36.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: osdmap e221: 8 total, 8 up, 7 in 2023-09-23T22:21:36.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: failed to encode map e221 with expected crc 2023-09-23T22:21:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:35 smithi070 ceph-mon[192652]: pgmap v523: 99 pgs: 2 remapped+peering, 5 peering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 29 MiB/s wr, 468 op/s; 17382/214347 objects degraded (8.109%); 2198/214347 objects misplaced (1.025%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T22:21:36.388 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:36.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 72761 72745 69.917 0 - 0.0138852 2023-09-23T22:21:36.501 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:36.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:36.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:36.571 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:36.605 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:36.650 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:36.717 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:21:36.717 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:21:36.722 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:36.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:36.826 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:36.851 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:36.898 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:36.906 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:21:36.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:36.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:21:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[223014]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[223014]: osdmap e222: 8 total, 8 up, 7 in 2023-09-23T22:21:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[223014]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[223014]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[223014]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[223014]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1981686086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:36.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:21:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[215775]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[215775]: osdmap e222: 8 total, 8 up, 7 in 2023-09-23T22:21:36.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[215775]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[215775]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[215775]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[215775]: failed to encode map e222 with expected crc 2023-09-23T22:21:36.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1981686086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:36.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:37.007 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:37.032 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:37.077 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:37.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:37.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:37.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:36 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:21:37.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:36 smithi070 ceph-mon[192652]: failed to encode map e222 with expected crc 2023-09-23T22:21:37.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:36 smithi070 ceph-mon[192652]: osdmap e222: 8 total, 8 up, 7 in 2023-09-23T22:21:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:36 smithi070 ceph-mon[192652]: failed to encode map e222 with expected crc 2023-09-23T22:21:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:36 smithi070 ceph-mon[192652]: failed to encode map e222 with expected crc 2023-09-23T22:21:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:36 smithi070 ceph-mon[192652]: failed to encode map e222 with expected crc 2023-09-23T22:21:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:36 smithi070 ceph-mon[192652]: failed to encode map e222 with expected crc 2023-09-23T22:21:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:36 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1981686086' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:37.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:37.214 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:37.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 72761 72745 68.858 0 - 0.0138852 2023-09-23T22:21:38.068 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:38.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:38.172 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:38.196 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:21:38.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:21:38.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: osdmap e223: 8 total, 8 up, 7 in 2023-09-23T22:21:38.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: pgmap v526: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 2 remapped+peering, 6 peering, 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 79 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 21 KiB/s rd, 26 MiB/s wr, 433 op/s; 27404/216996 objects degraded (12.629%); 5729/216996 objects misplaced (2.640%); 9.5 MiB/s, 149 objects/s recovering 2023-09-23T22:21:38.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:37 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17382/214347 objects degraded (8.109%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:38.199 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:38.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:21:38.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:21:38.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: osdmap e223: 8 total, 8 up, 7 in 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: pgmap v526: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 2 remapped+peering, 6 peering, 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 79 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 21 KiB/s rd, 26 MiB/s wr, 433 op/s; 27404/216996 objects degraded (12.629%); 5729/216996 objects misplaced (2.640%); 9.5 MiB/s, 149 objects/s recovering 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17382/214347 objects degraded (8.109%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:21:38.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: osdmap e223: 8 total, 8 up, 7 in 2023-09-23T22:21:38.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: failed to encode map e223 with expected crc 2023-09-23T22:21:38.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: pgmap v526: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 2 remapped+peering, 6 peering, 1 active+undersized+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 79 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 21 KiB/s rd, 26 MiB/s wr, 433 op/s; 27404/216996 objects degraded (12.629%); 5729/216996 objects misplaced (2.640%); 9.5 MiB/s, 149 objects/s recovering 2023-09-23T22:21:38.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:37 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17382/214347 objects degraded (8.109%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:38.241 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:38.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:38.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:38.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:38.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:38.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 72761 72745 67.8306 0 - 0.0138852 2023-09-23T22:21:38.422 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:38.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:38.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:38.528 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:38.553 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:38.560 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:38.598 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:38.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:38.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:38.703 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:38.733 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:38.883 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:38.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:38.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:38.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:39.015 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:39.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 72761 72745 66.8334 0 - 0.0138852 2023-09-23T22:21:39.775 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:39.776 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:21:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:21:39] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.43.0" 2023-09-23T22:21:39.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:39.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:39.878 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:39.906 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:39.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:21:40.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:40.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:40.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:40.185 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:40.211 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:40.253 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:40.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:40.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:40.360 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:40.405 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:40.405 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:40.405 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:40 smithi070 ceph-mon[192652]: pgmap v527: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 1 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 251 op/s; 40060/217044 objects degraded (18.457%); 5729/217044 objects misplaced (2.640%); 7.2 MiB/s, 112 objects/s recovering 2023-09-23T22:21:40.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 72761 72745 65.8651 0 - 0.0138852 2023-09-23T22:21:40.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:40.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:40.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:40 smithi026 ceph-mon[223014]: pgmap v527: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 1 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 251 op/s; 40060/217044 objects degraded (18.457%); 5729/217044 objects misplaced (2.640%); 7.2 MiB/s, 112 objects/s recovering 2023-09-23T22:21:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:40 smithi026 ceph-mon[215775]: pgmap v527: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 1 peering, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 84 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 16 KiB/s rd, 15 MiB/s wr, 251 op/s; 40060/217044 objects degraded (18.457%); 5729/217044 objects misplaced (2.640%); 7.2 MiB/s, 112 objects/s recovering 2023-09-23T22:21:40.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:41.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:41 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T22:21:41.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 72761 72745 64.9245 0 - 0.0138852 2023-09-23T22:21:41.431 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:41.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:41 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T22:21:41.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:41 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T22:21:41.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:41.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:41.535 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:41.541 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:21:41.572 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:41.685 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:41.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:41.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:41.788 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:41.791 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:41.809 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:21:41.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:41.847 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:21:41.860 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:41.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:41.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:41.875 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:21:41.907 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:21:41.968 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:41.993 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:42.039 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:42.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:42.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:42 smithi070 ceph-mon[192652]: pgmap v528: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 156 op/s; 40060/217116 objects degraded (18.451%); 5729/217116 objects misplaced (2.639%); 2.4 MiB/s, 36 objects/s recovering 2023-09-23T22:21:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2946564212' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:42.145 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:42.190 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:42.247 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:42.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:42.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:42 smithi026 ceph-mon[215775]: pgmap v528: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 156 op/s; 40060/217116 objects degraded (18.451%); 5729/217116 objects misplaced (2.639%); 2.4 MiB/s, 36 objects/s recovering 2023-09-23T22:21:42.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2946564212' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:42.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:42 smithi026 ceph-mon[223014]: pgmap v528: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 13 KiB/s rd, 9.2 MiB/s wr, 156 op/s; 40060/217116 objects degraded (18.451%); 5729/217116 objects misplaced (2.639%); 2.4 MiB/s, 36 objects/s recovering 2023-09-23T22:21:42.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2946564212' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:42.342 INFO:teuthology.orchestra.run.smithi026.stdout:moved 15 / 297 (5.05051%) 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 13.7618 -> 17.6462 (expected baseline 6.03053) 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.3 with 25 -> 19 pgs (0.589226 -> 0.447811 * mean) 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.1 with 59 -> 63 pgs (1.39057 -> 1.48485 * mean) 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 1 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0237 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0260 2023-09-23T22:21:42.343 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 weight 0.5238 -> 0.3758 2023-09-23T22:21:42.344 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7 weight 0.7293 -> 0.6286 2023-09-23T22:21:42.344 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 0.5412 -> 0.4165 2023-09-23T22:21:42.348 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T22:21:42.353 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:42.393 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:42.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 72761 72745 64.0104 0 - 0.0138852 2023-09-23T22:21:42.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:21:42 smithi026 conmon[202047]: 2023-09-23T22:21:42.341+0000 7f47954a6700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T22:21:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:43 smithi070 ceph-mon[192652]: from='client.45080 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 40060/217116 objects degraded (18.451%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:43.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 72761 72745 63.1216 0 - 0.0138852 2023-09-23T22:21:43.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:43 smithi026 ceph-mon[215775]: from='client.45080 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:43.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 40060/217116 objects degraded (18.451%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:43 smithi026 ceph-mon[223014]: from='client.45080 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:21:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 40060/217116 objects degraded (18.451%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:44 smithi070 ceph-mon[192652]: pgmap v529: 99 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 123 op/s; 40060/217116 objects degraded (18.451%); 5136/217116 objects misplaced (2.366%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:21:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:44 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:21:44.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 72761 72745 62.2572 0 - 0.0138852 2023-09-23T22:21:44.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:44 smithi026 ceph-mon[215775]: pgmap v529: 99 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 123 op/s; 40060/217116 objects degraded (18.451%); 5136/217116 objects misplaced (2.366%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:21:44.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:44 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:21:44.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:44 smithi026 ceph-mon[223014]: pgmap v529: 99 pgs: 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 10 KiB/s rd, 7.2 MiB/s wr, 123 op/s; 40060/217116 objects degraded (18.451%); 5136/217116 objects misplaced (2.366%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:21:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:44 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:21:44.501 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:44.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:44.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:44.605 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:44.618 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:21:44.637 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:44.648 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:21:44.648 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:44.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:21:44.721 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:21:44.728 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:44.729 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:21:44.729 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:21:44.729 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:21:44.729 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:21:44.730 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:21:44.730 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:21:44.730 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:21:44.730 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:21:44.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:44.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:44.839 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:44.872 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:44.956 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:44.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:44.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:45.079 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:45.176 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:45.293 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:45.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:45.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:45.404 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:45.415 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 73135 73119 61.73 2.3375 0.0180459 0.0161935 2023-09-23T22:21:45.451 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:45.598 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:45.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:45.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:45.709 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:45.812 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:46.105 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:46.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:46.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:46.221 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:46.295 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:46.298 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:46723 6.c 6:3f8a6929:::benchmark_data_smithi026_355573_object46722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.530280+0000", 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.765758757, 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23437707499999999, 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.299 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 46723 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530280+0000", 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530280+0000", 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530280+0000", 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530303+0000", 2023-09-23T22:21:46.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2925000000000002e-05 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530310+0000", 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7170000000000008e-06 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530316+0000", 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3959999999999997e-06 2023-09-23T22:21:46.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530360+0000", 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3822000000000002e-05 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530481+0000", 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121314 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530608+0000", 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126649 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.535533+0000", 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049258699999999997 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.540271+0000", 2023-09-23T22:21:46.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047371790000000002 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.540297+0000", 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6217999999999999e-05 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764603+0000", 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22430586599999999 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764657+0000", 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4645999999999998e-05 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:46726 6.5 6:a9fd2bee:::benchmark_data_smithi026_355573_object46725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.530782+0000", 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.765256790999999, 2023-09-23T22:21:46.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23415875999999999, 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 46726 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530782+0000", 2023-09-23T22:21:46.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530782+0000", 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530781+0000", 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530801+0000", 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9981999999999999e-05 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530812+0000", 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0988000000000001e-05 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530819+0000", 2023-09-23T22:21:46.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0520000000000004e-06 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532474+0000", 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001654881 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532583+0000", 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010846200000000001 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532703+0000", 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012066999999999999 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539642+0000", 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069380580000000004 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.540199+0000", 2023-09-23T22:21:46.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000557641 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764852+0000", 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22465268599999999 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764897+0000", 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5550000000000003e-05 2023-09-23T22:21:46.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764941+0000", 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3569999999999998e-05 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:46728 6.5 6:a79a6956:::benchmark_data_smithi026_355573_object46727:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.530877+0000", 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.765162445000001, 2023-09-23T22:21:46.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23419050899999999, 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 46728 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530877+0000", 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530877+0000", 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530876+0000", 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:21:46.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530896+0000", 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0049000000000001e-05 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530903+0000", 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3840000000000002e-06 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530908+0000", 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3190000000000002e-06 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534268+0000", 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003360287 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534422+0000", 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153504 2023-09-23T22:21:46.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534553+0000", 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130889 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539647+0000", 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050944359999999999 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.540227+0000", 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000580413 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764990+0000", 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22476283899999999 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.765030+0000", 2023-09-23T22:21:46.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9477999999999997e-05 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.765067+0000", 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7382000000000002e-05 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:46729 6.c 6:31849389:::benchmark_data_smithi026_355573_object46728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.530913+0000", 2023-09-23T22:21:46.320 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.765126445, 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.233938744, 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 46729 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530913+0000", 2023-09-23T22:21:46.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530913+0000", 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530912+0000", 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530956+0000", 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4209999999999999e-05 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530963+0000", 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.708e-06 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.530968+0000", 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.967e-06 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532344+0000", 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013759519999999999 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532459+0000", 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115141 2023-09-23T22:21:46.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532589+0000", 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012945900000000001 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764682+0000", 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.232093841 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764734+0000", 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1969999999999999e-05 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764772+0000", 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7174e-05 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764814+0000", 2023-09-23T22:21:46.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1946999999999997e-05 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764851+0000", 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7790999999999998e-05 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:46725 6.4 e211/203 6:2ab3e0e3:::benchmark_data_smithi026_355573_object46724:head v 211'2844, mlcod=211'2842)", 2023-09-23T22:21:46.327 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.532345+0000", 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.763694022999999, 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23238627000000001, 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532345+0000", 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532345+0000", 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532344+0000", 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532369+0000", 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4295999999999999e-05 2023-09-23T22:21:46.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532378+0000", 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5990000000000008e-06 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.532384+0000", 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2729999999999999e-06 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534116+0000", 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017317490000000001 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534135+0000", 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8987e-05 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764686+0000", 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23055102799999999 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764731+0000", 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5065000000000003e-05 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.332 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:46731 6.5 6:ada99428:::benchmark_data_smithi026_355573_object46730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.533888+0000", 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.762151400999997, 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23130003299999999, 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 46731 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.533888+0000", 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.533888+0000", 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.533886+0000", 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.533910+0000", 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3472000000000002e-05 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.533920+0000", 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0546e-05 2023-09-23T22:21:46.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.533928+0000", 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3339999999999996e-06 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.536029+0000", 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021014219999999999 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.536255+0000", 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022574199999999999 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.536443+0000", 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000187639 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764592+0000", 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.228149147 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764715+0000", 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012366999999999999 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.765125+0000", 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040969200000000002 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.765154+0000", 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8445e-05 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.765188+0000", 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4125999999999998e-05 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.339 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:46735 6.6 6:6d689b66:::benchmark_data_smithi026_355573_object46734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.534894+0000", 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.761145493999997, 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.229854594, 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 46735 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534894+0000", 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534894+0000", 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534893+0000", 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:21:46.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534918+0000", 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5457e-05 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534931+0000", 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2845e-05 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534937+0000", 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7660000000000004e-06 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.534975+0000", 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8479999999999997e-05 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.535052+0000", 2023-09-23T22:21:46.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6897999999999995e-05 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.535134+0000", 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1812999999999997e-05 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537226+0000", 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020921199999999998 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539996+0000", 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027702120000000002 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764647+0000", 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22465094599999999 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764699+0000", 2023-09-23T22:21:46.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2266999999999999e-05 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764748+0000", 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8662000000000003e-05 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:46736 6.c 6:3b16b609:::benchmark_data_smithi026_355573_object46735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e211)", 2023-09-23T22:21:46.346 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.537060+0000", 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.758979167, 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22824656600000001, 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 46736 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537060+0000", 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537060+0000", 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537058+0000", 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:21:46.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537085+0000", 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6611999999999999e-05 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537098+0000", 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3023999999999999e-05 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537105+0000", 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9439999999999999e-06 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537177+0000", 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2050000000000003e-05 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537292+0000", 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114765 2023-09-23T22:21:46.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537418+0000", 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012666899999999999 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764870+0000", 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.227451499 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764936+0000", 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5724999999999998e-05 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.765228+0000", 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029227000000000002 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.765275+0000", 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6961e-05 2023-09-23T22:21:46.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.765306+0000", 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1584000000000001e-05 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:46733 6.4 e211/203 6:2fe19310:::benchmark_data_smithi026_355573_object46732:head v 211'2845, mlcod=211'2842)", 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.537759+0000", 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.758279659999999, 2023-09-23T22:21:46.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.227139546, 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537759+0000", 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537759+0000", 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537759+0000", 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537783+0000", 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4734000000000001e-05 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537794+0000", 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0355e-05 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.537801+0000", 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8870000000000003e-06 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539087+0000", 2023-09-23T22:21:46.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001286817 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539105+0000", 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7946999999999999e-05 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764872+0000", 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.225766826 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764899+0000", 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6781e-05 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:46737 6.4 e211/203 6:278ea1f9:::benchmark_data_smithi026_355573_object46736:head v 211'2846, mlcod=211'2842)", 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.539437+0000", 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.756601777999997, 2023-09-23T22:21:46.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.231040313, 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539437+0000", 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539437+0000", 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539437+0000", 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539457+0000", 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0656999999999999e-05 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539465+0000", 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.452e-06 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539471+0000", 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0820000000000004e-06 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764958+0000", 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22548721199999999 2023-09-23T22:21:46.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764981+0000", 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2614999999999999e-05 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.770442+0000", 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054611119999999997 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.770478+0000", 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5599000000000003e-05 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.45002.0:46729 6.c e211/203 ondisk, result = 0)", 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:00.539959+0000", 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.756080392000001, 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.224943171, 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.363 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539959+0000", 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539959+0000", 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539958+0000", 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539960+0000", 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8560000000000001e-06 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539965+0000", 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7809999999999996e-06 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.539969+0000", 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5650000000000002e-06 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764757+0000", 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22478859800000001 2023-09-23T22:21:46.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764766+0000", 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9959999999999999e-06 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:00.764902+0000", 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013553400000000001 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.367 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:61617 6.5 6:ab940e82:::benchmark_data_smithi026_355573_object61616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:17.365530+0000", 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.930509422, 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.27139324799999998, 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.368 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 61617 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.365530+0000", 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.365530+0000", 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.365528+0000", 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.365556+0000", 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7920999999999999e-05 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.365570+0000", 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4158000000000001e-05 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.365579+0000", 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8030000000000003e-06 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.365909+0000", 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032995600000000001 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.366055+0000", 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014653600000000001 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.366207+0000", 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151719 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.369177+0000", 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002969766 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.369185+0000", 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8930000000000005e-06 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636178+0000", 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26699286700000002 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636884+0000", 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00070655100000000003 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636923+0000", 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8720999999999997e-05 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:61621 6.6 6:6f6811b8:::benchmark_data_smithi026_355573_object61620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:17.370523+0000", 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.925515967999999, 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26563256699999999, 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 61621 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.375 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370523+0000", 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370523+0000", 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.376 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370521+0000", 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370545+0000", 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3985000000000002e-05 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370556+0000", 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0784999999999999e-05 2023-09-23T22:21:46.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370563+0000", 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2300000000000002e-06 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370624+0000", 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0949000000000002e-05 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370785+0000", 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016093100000000001 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370945+0000", 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016003 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.378286+0000", 2023-09-23T22:21:46.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073402880000000004 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635983+0000", 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.257697448 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636051+0000", 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8192e-05 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636107+0000", 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6041000000000001e-05 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636156+0000", 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8248999999999998e-05 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:61629 6.6 6:63b9d7cd:::benchmark_data_smithi026_355573_object61628:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:17.370772+0000", 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.925266979, 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.27020333099999999, 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 61629 2023-09-23T22:21:46.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370772+0000", 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370772+0000", 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370771+0000", 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370826+0000", 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4403000000000001e-05 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370836+0000", 2023-09-23T22:21:46.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0382e-05 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.370843+0000", 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9709999999999998e-06 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.373389+0000", 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025456340000000002 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.373521+0000", 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013256499999999999 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,7", 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.373673+0000", 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151588 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.378294+0000", 2023-09-23T22:21:46.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046214130000000004 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636020+0000", 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.257726177 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.640886+0000", 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048658060000000003 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.640940+0000", 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3711999999999997e-05 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.640975+0000", 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5474e-05 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:61623 6.7 e217/203 6:e1663d1d:::benchmark_data_smithi026_355573_object61622:head v 217'3904, mlcod=217'3902)", 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:17.371085+0000", 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.924954317000001, 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26651580800000002, 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.371085+0000", 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.371085+0000", 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.371084+0000", 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.371137+0000", 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2602999999999998e-05 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.371147+0000", 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0644e-05 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.371154+0000", 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9290000000000001e-06 2023-09-23T22:21:46.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.375911+0000", 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047569480000000004 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.375932+0000", 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0421e-05 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.637530+0000", 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.261598632 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.637601+0000", 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0234999999999994e-05 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.393 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:61630 6.4 e217/203 6:2dcd571c:::benchmark_data_smithi026_355573_object61629:head v 217'3769, mlcod=217'3767)", 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:17.378047+0000", 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.917992113, 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26251838500000002, 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.378047+0000", 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.378047+0000", 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.378046+0000", 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.378073+0000", 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7033999999999999e-05 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.378088+0000", 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4878000000000001e-05 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.378097+0000", 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9420000000000001e-06 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.637016+0000", 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25891955799999999 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.637040+0000", 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3547999999999999e-05 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.640522+0000", 2023-09-23T22:21:46.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034820900000000002 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.640565+0000", 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3538999999999998e-05 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:61632 6.0 e217/203 6:072ea117:::benchmark_data_smithi026_355573_object61631:head v 217'3922, mlcod=217'3920)", 2023-09-23T22:21:46.398 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:17.379001+0000", 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.917038087000002, 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26880630300000002, 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.379001+0000", 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.379001+0000", 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.379000+0000", 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.379024+0000", 2023-09-23T22:21:46.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3394999999999999e-05 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.379036+0000", 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2121e-05 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.379044+0000", 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8029999999999997e-06 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.379086+0000", 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2153000000000001e-05 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.379106+0000", 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0482000000000001e-05 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.647765+0000", 2023-09-23T22:21:46.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26865927099999998 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.647807+0000", 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1761000000000002e-05 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.45002.0:61621 6.6 e217/203 ondisk, result = 0)", 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:17.381167+0000", 2023-09-23T22:21:46.403 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.914871947000002, 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25482102299999998, 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.381167+0000", 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.381167+0000", 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.381166+0000", 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.381173+0000", 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4729999999999997e-06 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635658+0000", 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25448547900000001 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635664+0000", 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.485e-06 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635966+0000", 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030234000000000001 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635976+0000", 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8190000000000003e-06 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635988+0000", 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2347000000000001e-05 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.407 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.45002.0:61617 6.5 e217/203 ondisk, result = 0)", 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:17.381758+0000", 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.914281251999999, 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25517441499999999, 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.381758+0000", 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.381758+0000", 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.381757+0000", 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:21:46.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.381763+0000", 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6740000000000002e-06 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635531+0000", 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25376799 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635554+0000", 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2665999999999999e-05 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636163+0000", 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00060904399999999999 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636171+0000", 2023-09-23T22:21:46.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4810000000000006e-06 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.636932+0000", 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00076112499999999997 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:61641 6.9 e217/214 6:980cd376:::benchmark_data_smithi026_355573_object61640:head v 217'3932, mlcod=0'0)", 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:21:17.387948+0000", 2023-09-23T22:21:46.412 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 28.908090745999999, 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25264489600000001, 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.387948+0000", 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.387948+0000", 2023-09-23T22:21:46.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.387947+0000", 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.387973+0000", 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.58e-05 2023-09-23T22:21:46.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635545+0000", 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.24757236199999999 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635557+0000", 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1949e-05 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:21:46.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635719+0000", 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016203600000000001 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.635760+0000", 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0849000000000001e-05 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.640551+0000", 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047908300000000003 2023-09-23T22:21:46.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:21:17.640593+0000", 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2379000000000002e-05 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:21:46.417 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:21:46.418 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:46 smithi026 ceph-mon[223014]: pgmap v530: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 5.1 MiB/s wr, 80 op/s; 47413/218328 objects degraded (21.716%); 6539/218328 objects misplaced (2.995%); 7.8 MiB/s, 123 objects/s recovering 2023-09-23T22:21:46.418 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:46 smithi026 ceph-mon[215775]: pgmap v530: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 5.1 MiB/s wr, 80 op/s; 47413/218328 objects degraded (21.716%); 6539/218328 objects misplaced (2.995%); 7.8 MiB/s, 123 objects/s recovering 2023-09-23T22:21:46.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 73803 73787 61.4588 41.75 0.0251238 0.0162648 2023-09-23T22:21:46.466 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:46.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:46.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:46.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:46 smithi070 ceph-mon[192652]: pgmap v530: 99 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 5.1 MiB/s wr, 80 op/s; 47413/218328 objects degraded (21.716%); 6539/218328 objects misplaced (2.995%); 7.8 MiB/s, 123 objects/s recovering 2023-09-23T22:21:46.661 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:46.736 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:46.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:46.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:46.846 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:46.923 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:47.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 74445 74429 61.1782 40.125 0.0173678 0.0163399 2023-09-23T22:21:47.695 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:47.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:47.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:47.799 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:47.874 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:47.947 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:21:47.955 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:47.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:47.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:48.065 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:48.087 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:21:48.155 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:48.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 75054 75038 60.8781 38.0625 0.0171207 0.0164204 2023-09-23T22:21:48.437 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:48.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:48.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:48.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:48 smithi026 ceph-mon[223014]: pgmap v531: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 4.5 MiB/s wr, 71 op/s; 47413/219195 objects degraded (21.631%); 5986/219195 objects misplaced (2.731%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T22:21:48.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 47413/219195 objects degraded (21.631%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:48.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:48 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:48.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:48 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:48.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:48 smithi026 ceph-mon[215775]: pgmap v531: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 4.5 MiB/s wr, 71 op/s; 47413/219195 objects degraded (21.631%); 5986/219195 objects misplaced (2.731%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T22:21:48.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 47413/219195 objects degraded (21.631%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:48.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:48.540 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:48.550 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:21:48.557 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:48.604 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:48 smithi070 ceph-mon[192652]: pgmap v531: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 4.5 MiB/s wr, 71 op/s; 47413/219195 objects degraded (21.631%); 5986/219195 objects misplaced (2.731%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T22:21:48.604 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 47413/219195 objects degraded (21.631%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:48.604 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:48 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:48.604 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:48 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:48.630 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:48.883 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:48.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:48.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:48.990 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:49.037 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:49.097 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:49.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:49.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:49.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:49.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 75622 75606 60.5528 35.5 0.0135529 0.0165084 2023-09-23T22:21:49.454 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:49.546 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:49.619 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:21:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:21:49] "GET /metrics HTTP/1.1" 200 36465 "" "Prometheus/2.43.0" 2023-09-23T22:21:49.667 INFO:tasks.thrashosds.thrasher:in_osds: [1, 2, 3, 4, 5, 7, 0] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:21:49.667 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:21:49.667 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 2, 3, 4, 5, 7, 0] 2023-09-23T22:21:49.668 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-23T22:21:49.769 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:49.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:49.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:49.882 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:49.941 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:50.030 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:50.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:50.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:50.194 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:50.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:50 smithi070 ceph-mon[192652]: pgmap v532: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.0 MiB/s wr, 111 op/s; 47413/221022 objects degraded (21.452%); 5986/221022 objects misplaced (2.708%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:21:50.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/626940133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:21:50.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:50 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:21:50.278 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:21:50.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 76169 76153 60.2191 34.1875 0.0119853 0.0165627 2023-09-23T22:21:50.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:50 smithi026 ceph-mon[223014]: pgmap v532: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.0 MiB/s wr, 111 op/s; 47413/221022 objects degraded (21.452%); 5986/221022 objects misplaced (2.708%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:21:50.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/626940133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:21:50.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:50 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:21:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:50 smithi026 ceph-mon[215775]: pgmap v532: 99 pgs: 1 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 7.0 MiB/s wr, 111 op/s; 47413/221022 objects degraded (21.452%); 5986/221022 objects misplaced (2.708%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:21:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/626940133' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:21:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:50 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:21:50.893 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:50.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:50.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:51.006 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:51.059 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:51.130 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:51.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:51.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:51.233 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:21:51.285 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:51.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:51 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:51 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:21:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:51 smithi070 ceph-mon[192652]: osdmap e224: 8 total, 8 up, 7 in 2023-09-23T22:21:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:51 smithi070 ceph-mon[192652]: failed to encode map e224 with expected crc 2023-09-23T22:21:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:51 smithi070 ceph-mon[192652]: failed to encode map e224 with expected crc 2023-09-23T22:21:51.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:21:51.421978+0000 min lat: 0.00131411 max lat: 12.143 avg lat: 0.0166754 2023-09-23T22:21:51.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:21:51.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 76786 76770 59.9485 38.5625 0.0211726 0.0166754 2023-09-23T22:21:51.615 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:21:51.624 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:51.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:21:51.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[223014]: osdmap e224: 8 total, 8 up, 7 in 2023-09-23T22:21:51.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[223014]: failed to encode map e224 with expected crc 2023-09-23T22:21:51.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[223014]: failed to encode map e224 with expected crc 2023-09-23T22:21:51.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:21:51.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:21:51.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[215775]: osdmap e224: 8 total, 8 up, 7 in 2023-09-23T22:21:51.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[215775]: failed to encode map e224 with expected crc 2023-09-23T22:21:51.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:51 smithi026 ceph-mon[215775]: failed to encode map e224 with expected crc 2023-09-23T22:21:51.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:51.626 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:21:51.727 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:51.789 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:51.932 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:51.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:51.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:52.044 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:52.086 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:52.125 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:52.160 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:52.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:52.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:52.265 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:52.323 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:21:52.341 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:52 smithi070 ceph-mon[192652]: failed to encode map e224 with expected crc 2023-09-23T22:21:52.341 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:52 smithi070 ceph-mon[192652]: failed to encode map e224 with expected crc 2023-09-23T22:21:52.341 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:52 smithi070 ceph-mon[192652]: pgmap v534: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 17 MiB/s wr, 279 op/s; 47403/225510 objects degraded (21.020%); 5779/225510 objects misplaced (2.563%); 11 MiB/s, 168 objects/s recovering 2023-09-23T22:21:52.341 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1919894200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:52.347 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:52.368 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:21:52.421 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:21:52.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 15 77426 77411 59.703 40.0625 0.0272288 0.0167432 2023-09-23T22:21:52.465 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:21:52.488 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:52.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:52.514 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:21:52.551 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:21:52.593 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:21:52.595 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:52.607 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:21:52.607 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:21:52.654 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:21:52.661 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:52.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:21:52.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:21:52.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:21:52.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:21:52.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:21:52.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:21:52.665 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:21:52.665 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:21:52.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:52 smithi026 ceph-mon[223014]: failed to encode map e224 with expected crc 2023-09-23T22:21:52.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:52 smithi026 ceph-mon[223014]: failed to encode map e224 with expected crc 2023-09-23T22:21:52.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:52 smithi026 ceph-mon[223014]: pgmap v534: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 17 MiB/s wr, 279 op/s; 47403/225510 objects degraded (21.020%); 5779/225510 objects misplaced (2.563%); 11 MiB/s, 168 objects/s recovering 2023-09-23T22:21:52.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1919894200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:52.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:52 smithi026 ceph-mon[215775]: failed to encode map e224 with expected crc 2023-09-23T22:21:52.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:52 smithi026 ceph-mon[215775]: failed to encode map e224 with expected crc 2023-09-23T22:21:52.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:52 smithi026 ceph-mon[215775]: pgmap v534: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 17 MiB/s wr, 279 op/s; 47403/225510 objects degraded (21.020%); 5779/225510 objects misplaced (2.563%); 11 MiB/s, 168 objects/s recovering 2023-09-23T22:21:52.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1919894200' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:21:52.911 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:52.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:52.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:53.015 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:53.089 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:53.195 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:53.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:53.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:53.297 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 47403/225510 objects degraded (21.020%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:53.301 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:53.401 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:53.421 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 78116 78100 59.5001 43.0625 0.00805936 0.0168011 2023-09-23T22:21:53.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 47403/225510 objects degraded (21.020%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:53.725 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:21:53.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 47403/225510 objects degraded (21.020%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:53.823 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:21:54.411 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:21:54.422 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 78682 78666 59.2095 35.375 0.027316 0.0168824 2023-09-23T22:21:54.902 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:54.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:54.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:55.011 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:55.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:54 smithi070 ceph-mon[192652]: pgmap v535: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 23 MiB/s wr, 362 op/s; 47403/227985 objects degraded (20.792%); 5550/227985 objects misplaced (2.434%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T22:21:55.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:55.065 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:55.066 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:55.157 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:55.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:55.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:55.218 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:55.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:54 smithi026 ceph-mon[215775]: pgmap v535: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 23 MiB/s wr, 362 op/s; 47403/227985 objects degraded (20.792%); 5550/227985 objects misplaced (2.434%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T22:21:55.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:55.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:55.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:54 smithi026 ceph-mon[223014]: pgmap v535: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.6 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 23 MiB/s wr, 362 op/s; 47403/227985 objects degraded (20.792%); 5550/227985 objects misplaced (2.434%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T22:21:55.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:55.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:21:55.264 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:55.360 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:55.422 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 79356 79340 59.0062 42.125 0.0382332 0.0169419 2023-09-23T22:21:55.754 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:21:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:55.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:21:55.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:55.915 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:55.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:55 smithi026 conmon[215751]: 2023-09-23T22:21:55.980+0000 7f4580f04700 -1 mon.a@0(leader).osd e224 definitely_dead 0 2023-09-23T22:21:56.118 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:56.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:56.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:56.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:56.290 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:56.422 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 79875 79859 58.6937 32.4375 0.0255978 0.0170316 2023-09-23T22:21:56.550 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:56.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:56.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:56.666 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:21:56.696 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:56.800 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:21:56.800 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:21:56.831 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:21:56.838 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.2. 2023-09-23T22:21:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:56 smithi026 ceph-mon[215775]: pgmap v536: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 30 MiB/s wr, 472 op/s; 47403/232515 objects degraded (20.387%); 5302/232515 objects misplaced (2.280%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T22:21:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2207765326' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:21:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:56 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:21:56.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:56 smithi026 ceph-mon[215775]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T22:21:56.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:56 smithi026 ceph-mon[223014]: pgmap v536: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 30 MiB/s wr, 472 op/s; 47403/232515 objects degraded (20.387%); 5302/232515 objects misplaced (2.280%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T22:21:56.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2207765326' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:21:56.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:56 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:21:56.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:56 smithi026 ceph-mon[223014]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T22:21:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:56 smithi070 ceph-mon[192652]: pgmap v536: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 30 MiB/s wr, 472 op/s; 47403/232515 objects degraded (20.387%); 5302/232515 objects misplaced (2.280%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T22:21:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2207765326' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:21:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:56 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:21:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:56 smithi070 ceph-mon[192652]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T22:21:57.422 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 80132 80116 58.1981 16.0625 0.00694431 0.0170865 2023-09-23T22:21:57.497 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:21:57.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:57.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:21:57.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:57.631 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:57.677 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:57.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:57.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:57.780 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:21:57.808 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:57.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:57 smithi070 ceph-mon[192652]: failed to encode map e225 with expected crc 2023-09-23T22:21:57.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T22:21:57.952 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:57 smithi070 ceph-mon[192652]: osdmap e225: 8 total, 8 up, 6 in 2023-09-23T22:21:57.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:57 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:57.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:57 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:57.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:57 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 47403/232515 objects degraded (20.387%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:57.953 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:57 smithi070 ceph-mon[192652]: osdmap e226: 8 total, 8 up, 6 in 2023-09-23T22:21:57.968 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:21:57.972 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:21:57.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:57.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:21:58.077 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:58.103 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:58.147 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:58.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:58.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:58.187 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[223014]: failed to encode map e225 with expected crc 2023-09-23T22:21:58.187 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T22:21:58.187 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[223014]: osdmap e225: 8 total, 8 up, 6 in 2023-09-23T22:21:58.187 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:58.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:58.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 47403/232515 objects degraded (20.387%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:58.188 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[223014]: osdmap e226: 8 total, 8 up, 6 in 2023-09-23T22:21:58.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[215775]: failed to encode map e225 with expected crc 2023-09-23T22:21:58.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T22:21:58.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[215775]: osdmap e225: 8 total, 8 up, 6 in 2023-09-23T22:21:58.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:58.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:58.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 47403/232515 objects degraded (20.387%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T22:21:58.189 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:57 smithi026 ceph-mon[215775]: osdmap e226: 8 total, 8 up, 6 in 2023-09-23T22:21:58.254 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:21:58.292 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:58.332 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:21:58.422 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 80132 80116 57.5294 0 - 0.0170865 2023-09-23T22:21:58.518 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:21:58.559 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:21:58.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:58.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:21:58.666 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:58.694 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:58.891 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:58.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:58.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:58.997 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:21:59.026 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:59.098 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:21:59.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:59.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:21:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: pgmap v538: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 42 MiB/s wr, 667 op/s; 47403/237048 objects degraded (19.997%); 5008/237048 objects misplaced (2.113%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:21:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:21:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:58 smithi070 ceph-mon[192652]: osdmap e227: 8 total, 8 up, 6 in 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: pgmap v538: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 42 MiB/s wr, 667 op/s; 47403/237048 objects degraded (19.997%); 5008/237048 objects misplaced (2.113%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:21:59.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[215775]: osdmap e227: 8 total, 8 up, 6 in 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: pgmap v538: 99 pgs: 1 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 85 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 42 MiB/s wr, 667 op/s; 47403/237048 objects degraded (19.997%); 5008/237048 objects misplaced (2.113%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: failed to encode map e226 with expected crc 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:21:59.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:58 smithi026 ceph-mon[223014]: osdmap e227: 8 total, 8 up, 6 in 2023-09-23T22:21:59.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:21:59.237 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:59.287 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:21:59.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:59.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:21:59.390 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:21:59.419 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:21:59.422 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 80258 80242 56.9651 3.9375 0.0075949 0.0175363 2023-09-23T22:21:59.716 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:21:59.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:21:59.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:21:59.820 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:21:59.850 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:21:59.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[215775]: failed to encode map e227 with expected crc 2023-09-23T22:21:59.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[215775]: failed to encode map e227 with expected crc 2023-09-23T22:21:59.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1557607814' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:21:59.880 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:21:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:21:59] "GET /metrics HTTP/1.1" 200 36466 "" "Prometheus/2.43.0" 2023-09-23T22:21:59.949 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:00.058 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:00.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:00.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:59 smithi070 ceph-mon[192652]: failed to encode map e227 with expected crc 2023-09-23T22:22:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:59 smithi070 ceph-mon[192652]: failed to encode map e227 with expected crc 2023-09-23T22:22:00.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:59 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1557607814' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:00.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:59 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:00.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:59 smithi070 ceph-mon[192652]: failed to encode map e227 with expected crc 2023-09-23T22:22:00.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:59 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:00.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:59 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:22:00.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:21:59 smithi070 ceph-mon[192652]: osdmap e228: 8 total, 8 up, 6 in 2023-09-23T22:22:00.163 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:00.227 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:00.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[223014]: failed to encode map e227 with expected crc 2023-09-23T22:22:00.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[223014]: failed to encode map e227 with expected crc 2023-09-23T22:22:00.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1557607814' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:00.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:00.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[223014]: failed to encode map e227 with expected crc 2023-09-23T22:22:00.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:00.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:22:00.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[223014]: osdmap e228: 8 total, 8 up, 6 in 2023-09-23T22:22:00.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:00.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[215775]: failed to encode map e227 with expected crc 2023-09-23T22:22:00.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:00.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:22:00.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:21:59 smithi026 ceph-mon[215775]: osdmap e228: 8 total, 8 up, 6 in 2023-09-23T22:22:00.376 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:00.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:00.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:00.426 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 80899 80883 56.773 40.0625 0.0276086 0.017606 2023-09-23T22:22:00.445 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:22:00.445 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:22:00.481 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:00.557 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:01.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[215775]: pgmap v541: 99 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 82 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 37 MiB/s wr, 588 op/s; 54852/238587 objects degraded (22.990%); 7477/238587 objects misplaced (3.134%); 7.0 MiB/s, 110 objects/s recovering 2023-09-23T22:22:01.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[215775]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[215775]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[215775]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[215775]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[215775]: osdmap e229: 8 total, 8 up, 6 in 2023-09-23T22:22:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[223014]: pgmap v541: 99 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 82 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 37 MiB/s wr, 588 op/s; 54852/238587 objects degraded (22.990%); 7477/238587 objects misplaced (3.134%); 7.0 MiB/s, 110 objects/s recovering 2023-09-23T22:22:01.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[223014]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[223014]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[223014]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[223014]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:00 smithi026 ceph-mon[223014]: osdmap e229: 8 total, 8 up, 6 in 2023-09-23T22:22:01.270 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:00 smithi070 ceph-mon[192652]: pgmap v541: 99 pgs: 1 activating+undersized+degraded+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 3 active+recovering+undersized+remapped, 82 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 37 MiB/s wr, 588 op/s; 54852/238587 objects degraded (22.990%); 7477/238587 objects misplaced (3.134%); 7.0 MiB/s, 110 objects/s recovering 2023-09-23T22:22:01.270 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:00 smithi070 ceph-mon[192652]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.270 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:00 smithi070 ceph-mon[192652]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.271 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:00 smithi070 ceph-mon[192652]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.271 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:00 smithi070 ceph-mon[192652]: failed to encode map e228 with expected crc 2023-09-23T22:22:01.271 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:00 smithi070 ceph-mon[192652]: osdmap e229: 8 total, 8 up, 6 in 2023-09-23T22:22:01.295 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:22:01.426 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.0055 2023-09-23T22:22:01.426 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 81347 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 56.4875 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 41.4401 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 187.5 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 903 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 663.055 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 3000 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.0176988 2023-09-23T22:22:01.427 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.154691 2023-09-23T22:22:01.428 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 12.143 2023-09-23T22:22:01.428 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00131411 2023-09-23T22:22:01.430 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:22:01.785 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:01.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:01.808 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:01.831 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:22:01.909 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:01.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[215775]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[215775]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[215775]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[215775]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[215775]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.981 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[215775]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[215775]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[215775]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[215775]: pgmap v544: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 21 MiB/s wr, 328 op/s; 62882/240993 objects degraded (26.093%); 4331/240993 objects misplaced (1.797%); 5.2 MiB/s, 81 objects/s recovering 2023-09-23T22:22:01.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[223014]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[223014]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[223014]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[223014]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[223014]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[223014]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[223014]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[223014]: failed to encode map e229 with expected crc 2023-09-23T22:22:01.984 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:01 smithi026 ceph-mon[223014]: pgmap v544: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 21 MiB/s wr, 328 op/s; 62882/240993 objects degraded (26.093%); 4331/240993 objects misplaced (1.797%); 5.2 MiB/s, 81 objects/s recovering 2023-09-23T22:22:01.987 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:02.185 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:02.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:02.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:02.279 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:01 smithi070 ceph-mon[192652]: failed to encode map e229 with expected crc 2023-09-23T22:22:02.279 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:01 smithi070 ceph-mon[192652]: failed to encode map e229 with expected crc 2023-09-23T22:22:02.279 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:01 smithi070 ceph-mon[192652]: failed to encode map e229 with expected crc 2023-09-23T22:22:02.279 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:01 smithi070 ceph-mon[192652]: failed to encode map e229 with expected crc 2023-09-23T22:22:02.279 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:01 smithi070 ceph-mon[192652]: failed to encode map e229 with expected crc 2023-09-23T22:22:02.279 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:01 smithi070 ceph-mon[192652]: failed to encode map e229 with expected crc 2023-09-23T22:22:02.280 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:01 smithi070 ceph-mon[192652]: failed to encode map e229 with expected crc 2023-09-23T22:22:02.280 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:01 smithi070 ceph-mon[192652]: failed to encode map e229 with expected crc 2023-09-23T22:22:02.280 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:01 smithi070 ceph-mon[192652]: pgmap v544: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 21 MiB/s wr, 328 op/s; 62882/240993 objects degraded (26.093%); 4331/240993 objects misplaced (1.797%); 5.2 MiB/s, 81 objects/s recovering 2023-09-23T22:22:02.290 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:02.354 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:02.503 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:22:02.542 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:22:02.576 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:22:02.586 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:02.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:02.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:02.624 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:22:02.693 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:02.719 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:02.761 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:02.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:02.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:02.855 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:22:02.864 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:02.897 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:22:02.922 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:02.938 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:22:02.989 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:22:03.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:22:03.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:22:03.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:22:03.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:22:03.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:22:03.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:22:03.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:22:03.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:22:03.193 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/268468181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:03.193 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:02 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 62882/240993 objects degraded (26.093%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:22:03.206 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:03.215 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 7, 0] out_osds: [6, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:22:03.216 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:22:03.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:03.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:03.224 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/268468181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:03.224 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:02 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 62882/240993 objects degraded (26.093%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:22:03.224 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/268468181' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:03.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:02 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 62882/240993 objects degraded (26.093%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T22:22:03.319 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:03.386 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:03.473 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:03.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:03.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:03.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:03.630 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:03.694 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:03.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:03.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:03.797 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:03.808 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:22:03.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:03.931 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:22:03.931 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:22:03.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:03.931 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45002.0:82522 6.6 e229/220 6:6d7498c5:::benchmark_data_smithi026_355573_object1172:head v 229'5187, mlcod=0'0)", 2023-09-23T22:22:03.932 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:22:03.930598+0000", 2023-09-23T22:22:03.932 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00076599399999999998, 2023-09-23T22:22:03.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00079076700000000005, 2023-09-23T22:22:03.932 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:03.932 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:22:03.932 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:03.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:03.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:03.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:03.930598+0000", 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:03.930598+0000", 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:03.930598+0000", 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:03.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:03.930605+0000", 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1450000000000002e-06 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:03.930616+0000", 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1598e-05 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:03.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:03.930622+0000", 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.037e-06 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:03.930664+0000", 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1650999999999998e-05 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:03.930689+0000", 2023-09-23T22:22:03.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5391000000000001e-05 2023-09-23T22:22:03.936 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:03.936 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:03.936 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:03.936 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:03.936 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T22:22:03.936 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 1 2023-09-23T22:22:03.936 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:22:04.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:03 smithi026 ceph-mon[223014]: pgmap v545: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 17 MiB/s wr, 276 op/s; 62882/241518 objects degraded (26.036%); 3840/241518 objects misplaced (1.590%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T22:22:04.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:03 smithi026 ceph-mon[215775]: pgmap v545: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 17 MiB/s wr, 276 op/s; 62882/241518 objects degraded (26.036%); 3840/241518 objects misplaced (1.590%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T22:22:04.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:03 smithi070 ceph-mon[192652]: pgmap v545: 99 pgs: 2 peering, 1 activating+undersized+degraded+remapped, 1 remapped+peering, 4 active+recovery_wait+degraded, 13 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 17 MiB/s wr, 276 op/s; 62882/241518 objects degraded (26.036%); 3840/241518 objects misplaced (1.590%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T22:22:04.599 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:04.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:04.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:04.706 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:04.777 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:04.884 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:22:04.916 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:04.917 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:04.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:04.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:04.982 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:05.032 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:05.090 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:05.206 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:05.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:05.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:05.312 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:05.395 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:05.452 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:22:05.514 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:05.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:05.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:05.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:05.717 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:06.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:06 smithi070 ceph-mon[192652]: pgmap v546: 99 pgs: 2 peering, 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 19 MiB/s wr, 422 op/s; 67333/241377 objects degraded (27.895%); 3549/241377 objects misplaced (1.470%); 13 MiB/s, 210 objects/s recovering 2023-09-23T22:22:06.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:06 smithi026 ceph-mon[223014]: pgmap v546: 99 pgs: 2 peering, 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 19 MiB/s wr, 422 op/s; 67333/241377 objects degraded (27.895%); 3549/241377 objects misplaced (1.470%); 13 MiB/s, 210 objects/s recovering 2023-09-23T22:22:06.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:06 smithi026 ceph-mon[215775]: pgmap v546: 99 pgs: 2 peering, 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 19 MiB/s wr, 422 op/s; 67333/241377 objects degraded (27.895%); 3549/241377 objects misplaced (1.470%); 13 MiB/s, 210 objects/s recovering 2023-09-23T22:22:06.713 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:22:07.089 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:07.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:07.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:07.193 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:07.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:07.326 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:22:07.326 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:22:07.327 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:22:07.327 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:22:07.327 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:22:07.327 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:07.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:07.430 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:07.505 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:08.061 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:08.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:08.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:08.165 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:08.257 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:08.297 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:22:08.365 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:08.392 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:08.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:08 smithi026 ceph-mon[223014]: pgmap v547: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 414 op/s; 67333/239700 objects degraded (28.091%); 3195/239700 objects misplaced (1.333%); 12 MiB/s, 196 objects/s recovering 2023-09-23T22:22:08.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 67333/239700 objects degraded (28.091%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T22:22:08.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:08 smithi026 ceph-mon[215775]: pgmap v547: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 414 op/s; 67333/239700 objects degraded (28.091%); 3195/239700 objects misplaced (1.333%); 12 MiB/s, 196 objects/s recovering 2023-09-23T22:22:08.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 67333/239700 objects degraded (28.091%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T22:22:08.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:08.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:08.497 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:08.596 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:08.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:08 smithi070 ceph-mon[192652]: pgmap v547: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 127 B/s rd, 14 MiB/s wr, 414 op/s; 67333/239700 objects degraded (28.091%); 3195/239700 objects misplaced (1.333%); 12 MiB/s, 196 objects/s recovering 2023-09-23T22:22:08.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 67333/239700 objects degraded (28.091%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T22:22:08.801 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:22:08.877 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:08.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:08.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:08.993 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:09.045 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:09.128 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:09.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:09.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:09.241 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:09.312 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:09.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:09.527 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:09.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:09.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:09.535 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:09.535 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:09.634 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:09.666 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:09.709 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:09.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:09.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:09.813 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:09.903 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:22:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:22:09] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2023-09-23T22:22:09.924 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:10.049 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:10.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:10.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:10.163 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:10.192 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:10.238 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:10.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:10.304 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:22:10.305 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:22:10.343 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:10.376 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:10.403 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:10 smithi070 ceph-mon[192652]: pgmap v548: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 109 B/s rd, 9.4 MiB/s wr, 399 op/s; 66765/237141 objects degraded (28.154%); 2850/237141 objects misplaced (1.202%); 13 MiB/s, 213 objects/s recovering 2023-09-23T22:22:10.403 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3933113080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:10.403 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:10.403 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2921305402' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:22:10.429 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:10.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:10.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:10.533 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:10.597 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:10.665 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:22:10.665 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:22:10.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:22:10.665 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:22:10.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.665 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19259 5.7 5:e3ec1517:::benchmark_data_smithi026_286012_object19258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:22:10.665 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.612249+0000", 2023-09-23T22:22:10.665 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 402.04871145800001, 2023-09-23T22:22:10.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0598800580000001, 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19259 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612249+0000", 2023-09-23T22:22:10.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612249+0000", 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612247+0000", 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612300+0000", 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2468e-05 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612319+0000", 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9431000000000001e-05 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612325+0000", 2023-09-23T22:22:10.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4820000000000002e-06 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612377+0000", 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1932999999999999e-05 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612500+0000", 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122898 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612640+0000", 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000140168 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.613682+0000", 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010420690000000001 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.617494+0000", 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038125429999999998 2023-09-23T22:22:10.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.617503+0000", 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3459999999999994e-06 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627652+0000", 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.010149754 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627667+0000", 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4209000000000001e-05 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672028+0000", 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0443611349999999 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672129+0000", 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010098 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.672 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19268 5.7 5:e4a7ba5f:::benchmark_data_smithi026_286012_object19267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.635055+0000", 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 402.025905534, 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.042290714, 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.673 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19268 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635055+0000", 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635055+0000", 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635053+0000", 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635096+0000", 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2938999999999997e-05 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635110+0000", 2023-09-23T22:22:10.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3417e-05 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635116+0000", 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.139e-06 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635172+0000", 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6567999999999998e-05 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635186+0000", 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3509999999999999e-05 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627702+0000", 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99251634700000002 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627708+0000", 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3789999999999997e-06 2023-09-23T22:22:10.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672153+0000", 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.044445216 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672284+0000", 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013079899999999999 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:22:10.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672410+0000", 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012686899999999999 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.673809+0000", 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013983579999999999 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.674260+0000", 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045079799999999998 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.676705+0000", 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024456840000000001 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.676749+0000", 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4087e-05 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.677345+0000", 2023-09-23T22:22:10.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059599999999999996 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:36982 5.7 5:ee40b8fd:::benchmark_data_smithi026_286012_object36981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.905294+0000", 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 387.75566662199998, 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0459705229999998, 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.681 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36982 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905294+0000", 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905294+0000", 2023-09-23T22:22:10.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905292+0000", 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905348+0000", 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5515e-05 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905359+0000", 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1467e-05 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905366+0000", 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.229e-06 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905414+0000", 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8909000000000002e-05 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905562+0000", 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014794 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905720+0000", 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015746799999999999 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.907741+0000", 2023-09-23T22:22:10.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020212910000000001 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.912362+0000", 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046209500000000004 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.918820+0000", 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064581580000000003 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.918832+0000", 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1997000000000001e-05 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915724+0000", 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99689173499999995 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915741+0000", 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7453999999999999e-05 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951176+0000", 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0354348980000001 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951264+0000", 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7725000000000005e-05 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.688 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:36996 5.7 5:e8cdce36:::benchmark_data_smithi026_286012_object36995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.923128+0000", 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 387.73783207000002, 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0387094659999998, 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36996 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923128+0000", 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923128+0000", 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923127+0000", 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923158+0000", 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1696999999999998e-05 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923168+0000", 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8609999999999993e-06 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923175+0000", 2023-09-23T22:22:10.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3940000000000001e-06 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923656+0000", 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048170500000000002 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923667+0000", 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0723999999999999e-05 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915782+0000", 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99211468700000005 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915789+0000", 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7309999999999993e-06 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951301+0000", 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.035511493 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951393+0000", 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2347000000000002e-05 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951500+0000", 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010628600000000001 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956154+0000", 2023-09-23T22:22:10.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046545290000000001 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961149+0000", 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049946900000000004 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961196+0000", 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6995e-05 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961241+0000", 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5599000000000002e-05 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961838+0000", 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059618699999999998 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.696 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37004 5.7 5:eeedc2c2:::benchmark_data_smithi026_286012_object37003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.940630+0000", 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 387.72033057800002, 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0219609119999999, 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37004 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940630+0000", 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940630+0000", 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940622+0000", 2023-09-23T22:22:10.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999924 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940657+0000", 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5079000000000001e-05 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940672+0000", 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5036e-05 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940678+0000", 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2940000000000002e-06 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940726+0000", 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8319000000000003e-05 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940734+0000", 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1199999999999996e-06 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915819+0000", 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97508563500000001 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915826+0000", 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4749999999999998e-06 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.952797+0000", 2023-09-23T22:22:10.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0369709229999999 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.952925+0000", 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128796 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.953102+0000", 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000177011 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.958840+0000", 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057373930000000004 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961876+0000", 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003036029 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962106+0000", 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022989700000000001 2023-09-23T22:22:10.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962141+0000", 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5138999999999997e-05 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962590+0000", 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044962099999999998 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37005 5.7 5:ebf3df01:::benchmark_data_smithi026_286012_object37004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.942342+0000", 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 387.71861854299999, 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0208824060000001, 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37005 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.705 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942342+0000", 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942342+0000", 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942340+0000", 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942383+0000", 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2715e-05 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942409+0000", 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6746000000000001e-05 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942421+0000", 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1542999999999999e-05 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942572+0000", 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015078199999999999 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942607+0000", 2023-09-23T22:22:10.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5651999999999997e-05 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915850+0000", 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97324232600000005 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915855+0000", 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8389999999999997e-06 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.954081+0000", 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.038225908 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.954178+0000", 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7398000000000005e-05 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.954284+0000", 2023-09-23T22:22:10.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106477 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.958847+0000", 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045624619999999998 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961899+0000", 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030523680000000002 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962660+0000", 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00076089199999999999 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962727+0000", 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7230000000000005e-05 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963224+0000", 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049668100000000001 2023-09-23T22:22:10.712 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37018 5.7 5:e9bfc706:::benchmark_data_smithi026_286012_object37017:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.022925+0000", 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 386.63803557599999, 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94092104499999996, 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.713 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37018 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022925+0000", 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022925+0000", 2023-09-23T22:22:10.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022923+0000", 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022979+0000", 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5467000000000003e-05 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022992+0000", 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3006999999999999e-05 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022999+0000", 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0439999999999998e-06 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.023072+0000", 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3577000000000004e-05 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.023087+0000", 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4313e-05 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.955164+0000", 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93207778100000005 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.955263+0000", 2023-09-23T22:22:10.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8320999999999996e-05 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.955401+0000", 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013850399999999999 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.959959+0000", 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045578490000000001 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961930+0000", 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001971048 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963292+0000", 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013615999999999999 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963355+0000", 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3847e-05 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963846+0000", 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049014800000000004 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37021 5.7 5:e7df05ef:::benchmark_data_smithi026_286012_object37020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.024001+0000", 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 386.63695924500001, 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94048630600000005, 2023-09-23T22:22:10.720 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37021 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024001+0000", 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024001+0000", 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024000+0000", 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024054+0000", 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4146e-05 2023-09-23T22:22:10.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024071+0000", 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6731999999999998e-05 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024078+0000", 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2099999999999996e-06 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024135+0000", 2023-09-23T22:22:10.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7099999999999999e-05 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024146+0000", 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.084e-05 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956202+0000", 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93205636999999997 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956349+0000", 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147093 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956522+0000", 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017295900000000001 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.959972+0000", 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034492390000000002 2023-09-23T22:22:10.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961978+0000", 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020065759999999999 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963918+0000", 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001940034 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963991+0000", 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2616999999999999e-05 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964487+0000", 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049637900000000003 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37045 5.7 5:e066772a:::benchmark_data_smithi026_286012_object37044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:22:10.727 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.042677+0000", 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 386.61828267599998, 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92228575899999998, 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37045 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042677+0000", 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042677+0000", 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042676+0000", 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042737+0000", 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0795999999999999e-05 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042751+0000", 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4453e-05 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042759+0000", 2023-09-23T22:22:10.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1729999999999998e-06 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042834+0000", 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5841999999999997e-05 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042847+0000", 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.255e-05 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957481+0000", 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91463394099999995 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957625+0000", 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000144027 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957796+0000", 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017132399999999999 2023-09-23T22:22:10.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961086+0000", 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003289882 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962006+0000", 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00091965299999999999 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964538+0000", 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025323949999999998 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964578+0000", 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9359e-05 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964963+0000", 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038571299999999998 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.734 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59924 5.7 5:e199b184:::benchmark_data_smithi026_286012_object59923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.616936+0000", 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 363.04402401499999, 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0339116370000001, 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.735 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59924 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616936+0000", 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616936+0000", 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616935+0000", 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616983+0000", 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8554999999999998e-05 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616996+0000", 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3098000000000001e-05 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.617003+0000", 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8950000000000001e-06 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.618902+0000", 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001899022 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.618913+0000", 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0487e-05 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.629818+0000", 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.01090558 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.629940+0000", 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121614 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.630050+0000", 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110517 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636053+0000", 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060029760000000001 2023-09-23T22:22:10.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650747+0000", 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014693632999999999 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650777+0000", 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9669999999999999e-05 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650811+0000", 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4023999999999998e-05 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650848+0000", 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7058999999999999e-05 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.742 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59932 5.7 5:eebeefdc:::benchmark_data_smithi026_286012_object59931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.623830+0000", 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 363.03713043200003, 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027128673, 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59932 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623830+0000", 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623830+0000", 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623829+0000", 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623852+0000", 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3107000000000001e-05 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623863+0000", 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1671999999999999e-05 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623868+0000", 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1880000000000001e-06 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623915+0000", 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6591999999999997e-05 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623926+0000", 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0712e-05 2023-09-23T22:22:10.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.632001+0000", 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0080748070000001 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.632141+0000", 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014057300000000001 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.632325+0000", 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000183525 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638341+0000", 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060166400000000002 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650882+0000", 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012540689000000001 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650908+0000", 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5919e-05 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650934+0000", 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6418999999999999e-05 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650958+0000", 2023-09-23T22:22:10.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4048000000000001e-05 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59934 5.7 5:e79a8a57:::benchmark_data_smithi026_286012_object59933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.625976+0000", 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 363.03498371900002, 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.025169207, 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.750 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59934 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.625976+0000", 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.625976+0000", 2023-09-23T22:22:10.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.625975+0000", 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626005+0000", 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0679999999999998e-05 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626023+0000", 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7546000000000001e-05 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626031+0000", 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9349999999999994e-06 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626091+0000", 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9787999999999997e-05 2023-09-23T22:22:10.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626100+0000", 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6150000000000003e-06 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.633572+0000", 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0074717230000001 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.633653+0000", 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1277000000000001e-05 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.633745+0000", 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1607999999999994e-05 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638348+0000", 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046025989999999998 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650976+0000", 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012628758 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651066+0000", 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9389999999999996e-05 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651119+0000", 2023-09-23T22:22:10.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3155999999999997e-05 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651146+0000", 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6772999999999999e-05 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59943 5.7 5:e02a7111:::benchmark_data_smithi026_286012_object59942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.635287+0000", 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 363.02567325799998, 2023-09-23T22:22:10.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0159653, 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59943 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635287+0000", 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635287+0000", 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635285+0000", 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635349+0000", 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3595000000000004e-05 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635365+0000", 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6218e-05 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635371+0000", 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1619999999999996e-06 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635435+0000", 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3384000000000002e-05 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635446+0000", 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0754000000000001e-05 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.634774+0000", 2023-09-23T22:22:10.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99932857799999997 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.634895+0000", 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012116999999999999 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.635052+0000", 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000156162 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640622+0000", 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055706200000000001 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651186+0000", 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010564146999999999 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651210+0000", 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3373000000000001e-05 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651235+0000", 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5253999999999999e-05 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651252+0000", 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7295999999999998e-05 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.764 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59963 5.7 5:ec56653b:::benchmark_data_smithi026_286012_object59962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.657440+0000", 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 363.00352055000002, 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99390371200000005, 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59963 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657440+0000", 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657440+0000", 2023-09-23T22:22:10.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657438+0000", 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657481+0000", 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2404999999999998e-05 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657495+0000", 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4557e-05 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657502+0000", 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6989999999999997e-06 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657564+0000", 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1988000000000003e-05 2023-09-23T22:22:10.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657573+0000", 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4689999999999997e-06 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636095+0000", 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97852269199999997 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636182+0000", 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6561e-05 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636276+0000", 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4087999999999998e-05 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640638+0000", 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043621459999999999 2023-09-23T22:22:10.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651280+0000", 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010641651 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651303+0000", 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2943999999999999e-05 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651326+0000", 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3354999999999999e-05 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651343+0000", 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7377999999999999e-05 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59987 5.7 5:ea604450:::benchmark_data_smithi026_286012_object59986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:22:10.772 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.678386+0000", 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 362.98257447100002, 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97304345299999995, 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59987 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678386+0000", 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678386+0000", 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678384+0000", 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678451+0000", 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6938000000000002e-05 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678463+0000", 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.146e-05 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678470+0000", 2023-09-23T22:22:10.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5079999999999999e-06 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678519+0000", 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8720000000000001e-05 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678530+0000", 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0726e-05 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.637191+0000", 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95866113100000006 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.637310+0000", 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119374 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.637455+0000", 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014488599999999999 2023-09-23T22:22:10.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.643133+0000", 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056781310000000003 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651368+0000", 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082348999999999999 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651390+0000", 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2087999999999999e-05 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651413+0000", 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2354e-05 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651429+0000", 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6606999999999999e-05 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.779 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:60017 5.7 5:e7911ea8:::benchmark_data_smithi026_286012_object60016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.703829+0000", 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 362.95713163400001, 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94768691699999996, 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.780 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60017 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703829+0000", 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703829+0000", 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703827+0000", 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703929+0000", 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102562 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703949+0000", 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9378999999999999e-05 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703957+0000", 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1659999999999994e-06 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.704024+0000", 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7266000000000003e-05 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.704036+0000", 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1711e-05 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638386+0000", 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93434993 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638486+0000", 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9834000000000007e-05 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638579+0000", 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3580999999999997e-05 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.643145+0000", 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045657620000000001 2023-09-23T22:22:10.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651454+0000", 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083090219999999992 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651477+0000", 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2541999999999999e-05 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651499+0000", 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2277999999999999e-05 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651515+0000", 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6566e-05 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.787 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:60025 5.7 5:ee51d15d:::benchmark_data_smithi026_286012_object60024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.711079+0000", 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 362.94988108699999, 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94052496900000004, 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60025 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711079+0000", 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711079+0000", 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711078+0000", 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711136+0000", 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7855000000000001e-05 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711150+0000", 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4219000000000001e-05 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711156+0000", 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5379999999999999e-06 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711209+0000", 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2714999999999999e-05 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711221+0000", 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1578e-05 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.639471+0000", 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92825038299999996 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.639589+0000", 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011770200000000001 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.639740+0000", 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015135499999999999 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.645463+0000", 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057232120000000001 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651540+0000", 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060769939999999996 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651563+0000", 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3059000000000001e-05 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651586+0000", 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2154000000000002e-05 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651604+0000", 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8445999999999999e-05 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:60041 5.7 5:e6579469:::benchmark_data_smithi026_286012_object60040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.728814+0000", 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 362.932146359, 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92288071299999996, 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.795 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60041 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728814+0000", 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728814+0000", 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728813+0000", 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728906+0000", 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3043999999999999e-05 2023-09-23T22:22:10.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728918+0000", 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2201000000000001e-05 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728923+0000", 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1590000000000001e-06 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728988+0000", 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5283000000000005e-05 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728999+0000", 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0393e-05 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640683+0000", 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.911684351 2023-09-23T22:22:10.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640810+0000", 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012665500000000001 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640977+0000", 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016692500000000001 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.645469+0000", 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044926890000000002 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651632+0000", 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061627959999999999 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651654+0000", 2023-09-23T22:22:10.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2266e-05 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651677+0000", 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2198999999999999e-05 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651695+0000", 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7923000000000001e-05 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.802 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:93693 5.f 5:f82a8543:::benchmark_data_smithi026_286012_object93692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:48.289235+0000", 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 322.371724825, 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0721697720000001, 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.803 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 93693 2023-09-23T22:22:10.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.804 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289235+0000", 2023-09-23T22:22:10.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289235+0000", 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289234+0000", 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289280+0000", 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6313000000000002e-05 2023-09-23T22:22:10.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289294+0000", 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3896999999999999e-05 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289299+0000", 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4210000000000003e-06 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289365+0000", 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6038999999999996e-05 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289378+0000", 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2371000000000001e-05 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152914+0000", 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.863536533 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152926+0000", 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1677e-05 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.346430+0000", 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1935035489999999 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.346618+0000", 2023-09-23T22:22:10.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000188378 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.346836+0000", 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021775200000000001 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358797+0000", 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011961090000000001 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358858+0000", 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1081999999999996e-05 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358963+0000", 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105529 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.359038+0000", 2023-09-23T22:22:10.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4793e-05 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.361405+0000", 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002366908 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:93720 5.f 5:f1e7c9aa:::benchmark_data_smithi026_286012_object93719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:22:10.812 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:48.318037+0000", 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 322.34292329200002, 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0455209810000001, 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 93720 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318037+0000", 2023-09-23T22:22:10.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318037+0000", 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318035+0000", 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:10.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318080+0000", 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4404999999999999e-05 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318094+0000", 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4014e-05 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:10.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318100+0000", 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6380000000000001e-06 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318152+0000", 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2349000000000003e-05 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318161+0000", 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9260000000000006e-06 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152958+0000", 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.834797067 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152965+0000", 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3849999999999996e-06 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.350144+0000", 2023-09-23T22:22:10.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.197178563 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.350310+0000", 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000166584 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.350530+0000", 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000219235 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358894+0000", 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083640529999999998 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358901+0000", 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1829999999999997e-06 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.361482+0000", 2023-09-23T22:22:10.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002581046 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.361552+0000", 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9646000000000004e-05 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.363558+0000", 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020062909999999999 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.820 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.821 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:10.821 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:10.821 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:22:10.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:10 smithi026 ceph-mon[215775]: pgmap v548: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 109 B/s rd, 9.4 MiB/s wr, 399 op/s; 66765/237141 objects degraded (28.154%); 2850/237141 objects misplaced (1.202%); 13 MiB/s, 213 objects/s recovering 2023-09-23T22:22:10.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3933113080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:10.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:10.822 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2921305402' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:22:10.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:10 smithi026 ceph-mon[223014]: pgmap v548: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.8 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 109 B/s rd, 9.4 MiB/s wr, 399 op/s; 66765/237141 objects degraded (28.154%); 2850/237141 objects misplaced (1.202%); 13 MiB/s, 213 objects/s recovering 2023-09-23T22:22:10.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3933113080' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:10.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:10.822 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2921305402' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:22:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:10.823 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:10.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:10.923 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:10.971 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:11.056 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:11.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:11.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:11.159 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:11.201 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:11.269 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:11.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:11.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:11.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:11.463 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:11.558 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:11 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:11.558 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:22:11.558 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2921305402' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:22:11.558 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:11 smithi070 ceph-mon[192652]: osdmap e230: 8 total, 8 up, 6 in 2023-09-23T22:22:11.558 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:11 smithi070 ceph-mon[192652]: failed to encode map e230 with expected crc 2023-09-23T22:22:11.558 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:11 smithi070 ceph-mon[192652]: failed to encode map e230 with expected crc 2023-09-23T22:22:11.575 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:22:11.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:11.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:22:11.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2921305402' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:22:11.584 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[215775]: osdmap e230: 8 total, 8 up, 6 in 2023-09-23T22:22:11.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[215775]: failed to encode map e230 with expected crc 2023-09-23T22:22:11.585 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[215775]: failed to encode map e230 with expected crc 2023-09-23T22:22:11.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:11.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:22:11.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2921305402' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:22:11.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[223014]: osdmap e230: 8 total, 8 up, 6 in 2023-09-23T22:22:11.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[223014]: failed to encode map e230 with expected crc 2023-09-23T22:22:11.585 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:11 smithi026 ceph-mon[223014]: failed to encode map e230 with expected crc 2023-09-23T22:22:12.111 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:22:12.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T22:22:12.246 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:12.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:12.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:12.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:12.448 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:12.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:12 smithi026 ceph-mon[223014]: pgmap v550: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 6.5 MiB/s wr, 453 op/s; 66436/233631 objects degraded (28.436%); 2850/233631 objects misplaced (1.220%); 13 MiB/s, 216 objects/s recovering 2023-09-23T22:22:12.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:12 smithi026 ceph-mon[215775]: pgmap v550: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 6.5 MiB/s wr, 453 op/s; 66436/233631 objects degraded (28.436%); 2850/233631 objects misplaced (1.220%); 13 MiB/s, 216 objects/s recovering 2023-09-23T22:22:12.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:12 smithi070 ceph-mon[192652]: pgmap v550: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 6.5 MiB/s wr, 453 op/s; 66436/233631 objects degraded (28.436%); 2850/233631 objects misplaced (1.220%); 13 MiB/s, 216 objects/s recovering 2023-09-23T22:22:12.638 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:12.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:12.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:12.746 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:12.833 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:13.063 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:13.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:13.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:13.184 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:22:13.188 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:13.209 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:13.228 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:22:13.262 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:13.297 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:22:13.348 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:22:13.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:22:13.392 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:13.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:13.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 66436/233631 objects degraded (28.436%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T22:22:13.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2508514116' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:13.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:13.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 66436/233631 objects degraded (28.436%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T22:22:13.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2508514116' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:13.430 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:22:13.469 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:22:13.504 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:13.520 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:22:13.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:22:13.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:22:13.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:22:13.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:22:13.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:22:13.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:22:13.534 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:22:13.535 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:22:13.555 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:13.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 66436/233631 objects degraded (28.436%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T22:22:13.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2508514116' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:13.642 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:13.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:13.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:13.762 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:13.912 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:14.164 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:22:14.165 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:22:14.325 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:14.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:14.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:14.432 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:14.465 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:14.504 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:22:15.069 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:22:15.080 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:14 smithi026 ceph-mon[215775]: pgmap v551: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 5.4 MiB/s wr, 463 op/s; 66436/232800 objects degraded (28.538%); 2410/232800 objects misplaced (1.035%); 13 MiB/s, 210 objects/s recovering 2023-09-23T22:22:15.084 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:14 smithi026 ceph-mon[223014]: pgmap v551: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 5.4 MiB/s wr, 463 op/s; 66436/232800 objects degraded (28.538%); 2410/232800 objects misplaced (1.035%); 13 MiB/s, 210 objects/s recovering 2023-09-23T22:22:15.100 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:15.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:15.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:15.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:14 smithi070 ceph-mon[192652]: pgmap v551: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 5.4 MiB/s wr, 463 op/s; 66436/232800 objects degraded (28.538%); 2410/232800 objects misplaced (1.035%); 13 MiB/s, 210 objects/s recovering 2023-09-23T22:22:15.208 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:15.264 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:15.321 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:15.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:15.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:15.430 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:15.504 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:15.661 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:22:15.825 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:15.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:15.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:15.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:16.003 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:16.029 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:16.233 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:16.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:16.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:16.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:16.384 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:16.454 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:16.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:16.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:16.559 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:16.654 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:16.846 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:16.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:16.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:16.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:16 smithi026 ceph-mon[215775]: pgmap v552: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 927 KiB/s wr, 444 op/s; 65689/228924 objects degraded (28.695%); 2055/228924 objects misplaced (0.898%); 14 MiB/s, 231 objects/s recovering 2023-09-23T22:22:16.994 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:16 smithi026 ceph-mon[223014]: pgmap v552: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 927 KiB/s wr, 444 op/s; 65689/228924 objects degraded (28.695%); 2055/228924 objects misplaced (0.898%); 14 MiB/s, 231 objects/s recovering 2023-09-23T22:22:17.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:17.110 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:17.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:17.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:16 smithi070 ceph-mon[192652]: pgmap v552: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 102 B/s rd, 927 KiB/s wr, 444 op/s; 65689/228924 objects degraded (28.695%); 2055/228924 objects misplaced (0.898%); 14 MiB/s, 231 objects/s recovering 2023-09-23T22:22:17.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:17.221 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:17.262 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:17.398 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:17.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:17.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:17.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:17.616 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:17.706 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:22:17.706 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:22:17.746 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:17.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:17.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:17.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:22:17.855 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:17.900 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T22:22:18.004 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:17 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/190566337' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:22:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:17 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:22:18.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:17 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 65689/228924 objects degraded (28.695%), 19 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:18.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:17 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/190566337' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:22:18.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:17 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:22:18.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:17 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 65689/228924 objects degraded (28.695%), 19 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:18.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:17 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/190566337' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:22:18.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:17 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:22:18.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:17 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 65689/228924 objects degraded (28.695%), 19 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:18.184 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:18.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:18.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:18.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:18.315 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:18.361 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:18.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:18.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:18.464 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:18.495 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:18.691 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:22:18.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[215775]: pgmap v553: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 444 op/s; 65689/226362 objects degraded (29.019%); 1711/226362 objects misplaced (0.756%); 14 MiB/s, 230 objects/s recovering 2023-09-23T22:22:18.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:22:18.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[215775]: osdmap e231: 8 total, 8 up, 6 in 2023-09-23T22:22:18.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[215775]: failed to encode map e231 with expected crc 2023-09-23T22:22:18.867 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[215775]: failed to encode map e231 with expected crc 2023-09-23T22:22:18.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[223014]: pgmap v553: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 444 op/s; 65689/226362 objects degraded (29.019%); 1711/226362 objects misplaced (0.756%); 14 MiB/s, 230 objects/s recovering 2023-09-23T22:22:18.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:22:18.867 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[223014]: osdmap e231: 8 total, 8 up, 6 in 2023-09-23T22:22:18.868 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[223014]: failed to encode map e231 with expected crc 2023-09-23T22:22:18.868 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:18 smithi026 ceph-mon[223014]: failed to encode map e231 with expected crc 2023-09-23T22:22:19.047 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:19.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:19.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:19.122 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:22:19.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:18 smithi070 ceph-mon[192652]: pgmap v553: 99 pgs: 4 active+recovery_wait+degraded, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 444 op/s; 65689/226362 objects degraded (29.019%); 1711/226362 objects misplaced (0.756%); 14 MiB/s, 230 objects/s recovering 2023-09-23T22:22:19.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:18 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:22:19.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:18 smithi070 ceph-mon[192652]: osdmap e231: 8 total, 8 up, 6 in 2023-09-23T22:22:19.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:18 smithi070 ceph-mon[192652]: failed to encode map e231 with expected crc 2023-09-23T22:22:19.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:18 smithi070 ceph-mon[192652]: failed to encode map e231 with expected crc 2023-09-23T22:22:19.155 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:19.189 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:19.197 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T22:22:19.268 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:19.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:19.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:19.373 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:19.420 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:19.450 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:19.522 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:19.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:19.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:19.628 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:19.662 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:19.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:22:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:22:19] "GET /metrics HTTP/1.1" 200 36451 "" "Prometheus/2.43.0" 2023-09-23T22:22:19.811 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:19.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:19.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:19.949 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:20.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:19 smithi070 ceph-mon[192652]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:19 smithi070 ceph-mon[192652]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:19 smithi070 ceph-mon[192652]: osdmap e232: 8 total, 8 up, 6 in 2023-09-23T22:22:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:19 smithi070 ceph-mon[192652]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:19 smithi070 ceph-mon[192652]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.145 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:20.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[223014]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[223014]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[223014]: osdmap e232: 8 total, 8 up, 6 in 2023-09-23T22:22:20.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[223014]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[223014]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[215775]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[215775]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[215775]: osdmap e232: 8 total, 8 up, 6 in 2023-09-23T22:22:20.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[215775]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:19 smithi026 ceph-mon[215775]: failed to encode map e232 with expected crc 2023-09-23T22:22:20.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:20.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:20.261 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:20.292 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:20.355 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:20.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:20.459 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:20.495 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:20.649 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:20.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:20.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:20.763 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:20.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:20.898 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:22:21.097 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:21.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:21.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:21.173 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:22:21.174 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:22:21.200 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:20 smithi070 ceph-mon[192652]: pgmap v556: 99 pgs: 1 remapped+peering, 3 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 53408/224136 objects degraded (23.828%); 8480/224136 objects misplaced (3.783%); 18 MiB/s, 287 objects/s recovering 2023-09-23T22:22:21.200 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:20 smithi070 ceph-mon[192652]: failed to encode map e232 with expected crc 2023-09-23T22:22:21.200 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:20 smithi070 ceph-mon[192652]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.200 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:20 smithi070 ceph-mon[192652]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.200 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/854520271' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:21.200 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:20 smithi070 ceph-mon[192652]: osdmap e233: 8 total, 8 up, 6 in 2023-09-23T22:22:21.200 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:20 smithi070 ceph-mon[192652]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.200 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:20 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:21.200 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:20 smithi070 ceph-mon[192652]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.205 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:21.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[223014]: pgmap v556: 99 pgs: 1 remapped+peering, 3 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 53408/224136 objects degraded (23.828%); 8480/224136 objects misplaced (3.783%); 18 MiB/s, 287 objects/s recovering 2023-09-23T22:22:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[223014]: failed to encode map e232 with expected crc 2023-09-23T22:22:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[223014]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[223014]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/854520271' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[223014]: osdmap e233: 8 total, 8 up, 6 in 2023-09-23T22:22:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[223014]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:21.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[223014]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[215775]: pgmap v556: 99 pgs: 1 remapped+peering, 3 peering, 4 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 76 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 392 op/s; 53408/224136 objects degraded (23.828%); 8480/224136 objects misplaced (3.783%); 18 MiB/s, 287 objects/s recovering 2023-09-23T22:22:21.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[215775]: failed to encode map e232 with expected crc 2023-09-23T22:22:21.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[215775]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[215775]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/854520271' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:21.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[215775]: osdmap e233: 8 total, 8 up, 6 in 2023-09-23T22:22:21.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[215775]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:21.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:20 smithi026 ceph-mon[215775]: failed to encode map e233 with expected crc 2023-09-23T22:22:21.239 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:21.417 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:21.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:21.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:21.522 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:21.561 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:21.813 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:21.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:21.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:21.924 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:22.013 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:22.033 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:22:22.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:22:22.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[223014]: failed to encode map e234 with expected crc 2023-09-23T22:22:22.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:22:22.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[223014]: osdmap e234: 8 total, 8 up, 6 in 2023-09-23T22:22:22.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[223014]: failed to encode map e234 with expected crc 2023-09-23T22:22:22.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:22:22.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[215775]: failed to encode map e234 with expected crc 2023-09-23T22:22:22.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:22:22.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[215775]: osdmap e234: 8 total, 8 up, 6 in 2023-09-23T22:22:22.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:21 smithi026 ceph-mon[215775]: failed to encode map e234 with expected crc 2023-09-23T22:22:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:21 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:22:22.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:21 smithi070 ceph-mon[192652]: failed to encode map e234 with expected crc 2023-09-23T22:22:22.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:21 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:22:22.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:21 smithi070 ceph-mon[192652]: osdmap e234: 8 total, 8 up, 6 in 2023-09-23T22:22:22.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:21 smithi070 ceph-mon[192652]: failed to encode map e234 with expected crc 2023-09-23T22:22:22.463 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:22:22.654 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:22.700 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:22.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:22.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:22.816 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:22.852 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:22.900 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:22 smithi026 ceph-mon[215775]: pgmap v559: 99 pgs: 2 remapped+peering, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 393 op/s; 35615/221580 objects degraded (16.073%); 8480/221580 objects misplaced (3.827%); 14 MiB/s, 2 keys/s, 224 objects/s recovering 2023-09-23T22:22:22.900 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:22 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 35615/221580 objects degraded (16.073%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:23.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:22 smithi070 ceph-mon[192652]: pgmap v559: 99 pgs: 2 remapped+peering, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 393 op/s; 35615/221580 objects degraded (16.073%); 8480/221580 objects misplaced (3.827%); 14 MiB/s, 2 keys/s, 224 objects/s recovering 2023-09-23T22:22:23.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:22 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 35615/221580 objects degraded (16.073%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:23.114 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:23.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:23.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:23.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:22 smithi026 ceph-mon[223014]: pgmap v559: 99 pgs: 2 remapped+peering, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 78 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 393 op/s; 35615/221580 objects degraded (16.073%); 8480/221580 objects misplaced (3.827%); 14 MiB/s, 2 keys/s, 224 objects/s recovering 2023-09-23T22:22:23.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:22 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 35615/221580 objects degraded (16.073%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:23.218 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:23.247 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:23.289 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:23.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:23.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:23.513 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:23.849 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:22:23.849 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:22:23.849 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:22:23.849 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:22:23.850 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:22:23.850 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:22:23.850 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:22:23.850 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:22:23.867 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:22:23.868 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:22:23.868 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:22:23.868 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:22:23.868 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:22:23.868 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:22:23.868 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:22:23.868 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:22:23.902 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:23.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:23.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:24.010 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:24.044 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:23 smithi070 ceph-mon[192652]: pgmap v560: 99 pgs: 2 remapped+peering, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 78 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 35653/220914 objects degraded (16.139%); 8247/220914 objects misplaced (3.733%); 13 MiB/s, 1 keys/s, 204 objects/s recovering 2023-09-23T22:22:24.044 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1524328246' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:24.080 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:24.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:23 smithi026 ceph-mon[215775]: pgmap v560: 99 pgs: 2 remapped+peering, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 78 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 35653/220914 objects degraded (16.139%); 8247/220914 objects misplaced (3.733%); 13 MiB/s, 1 keys/s, 204 objects/s recovering 2023-09-23T22:22:24.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1524328246' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:24.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:23 smithi026 ceph-mon[223014]: pgmap v560: 99 pgs: 2 remapped+peering, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 78 active+clean; 4.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 35653/220914 objects degraded (16.139%); 8247/220914 objects misplaced (3.733%); 13 MiB/s, 1 keys/s, 204 objects/s recovering 2023-09-23T22:22:24.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1524328246' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:24.313 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:22:24.313 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:22:24.313 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:22:24.313 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:22:24.680 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:24.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:24.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:24.685 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:22:24.784 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:24.837 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:25.089 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:22:25.179 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:25.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:25.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:25.285 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:25.326 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:22:25.327 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:25.330 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:25.393 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:25.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:25.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:22:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:22:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:25.500 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:25.544 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:22:25.619 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:25.774 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:25.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:25.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:25.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:25.906 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:25.997 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:26.092 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:26 smithi070 ceph-mon[192652]: pgmap v561: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 439 op/s; 40580/216813 objects degraded (18.717%); 7444/216813 objects misplaced (3.433%); 17 MiB/s, 1 keys/s, 274 objects/s recovering 2023-09-23T22:22:26.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:26.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:26 smithi026 ceph-mon[223014]: pgmap v561: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 439 op/s; 40580/216813 objects degraded (18.717%); 7444/216813 objects misplaced (3.433%); 17 MiB/s, 1 keys/s, 274 objects/s recovering 2023-09-23T22:22:26.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:26 smithi026 ceph-mon[215775]: pgmap v561: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 439 op/s; 40580/216813 objects degraded (18.717%); 7444/216813 objects misplaced (3.433%); 17 MiB/s, 1 keys/s, 274 objects/s recovering 2023-09-23T22:22:26.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:26.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:26.341 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:26.378 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:26.426 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:26.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:26.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:26.529 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:26.630 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:26.758 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:26.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:26.771 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:26.872 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:26.954 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:27.090 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:27.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:27.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:27.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2102082668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:22:27.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:27 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:22:27.135 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T22:22:27.200 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:27.255 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:27.364 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:27.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:27.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:27.428 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:22:27.428 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:22:27.428 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:22:27.428 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:22:27.429 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:22:27.468 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:27.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2102082668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:22:27.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:27 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:22:27.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2102082668' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:22:27.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:27 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:22:27.535 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:27.751 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:22:28.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[215775]: failed to encode map e235 with expected crc 2023-09-23T22:22:28.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:22:28.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[215775]: osdmap e235: 8 total, 8 up, 6 in 2023-09-23T22:22:28.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[215775]: pgmap v563: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 399 op/s; 40580/214155 objects degraded (18.949%); 7444/214155 objects misplaced (3.476%); 8.6 MiB/s, 136 objects/s recovering 2023-09-23T22:22:28.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 40580/216813 objects degraded (18.717%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:28.330 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:22:28.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[223014]: failed to encode map e235 with expected crc 2023-09-23T22:22:28.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:22:28.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[223014]: osdmap e235: 8 total, 8 up, 6 in 2023-09-23T22:22:28.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[223014]: pgmap v563: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 399 op/s; 40580/214155 objects degraded (18.949%); 7444/214155 objects misplaced (3.476%); 8.6 MiB/s, 136 objects/s recovering 2023-09-23T22:22:28.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 40580/216813 objects degraded (18.717%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:28.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:28 smithi070 ceph-mon[192652]: failed to encode map e235 with expected crc 2023-09-23T22:22:28.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:28 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:22:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:28 smithi070 ceph-mon[192652]: osdmap e235: 8 total, 8 up, 6 in 2023-09-23T22:22:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:28 smithi070 ceph-mon[192652]: pgmap v563: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 399 op/s; 40580/214155 objects degraded (18.949%); 7444/214155 objects misplaced (3.476%); 8.6 MiB/s, 136 objects/s recovering 2023-09-23T22:22:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 40580/216813 objects degraded (18.717%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:28.803 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:22:28.849 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:28.850 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:22:28.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:28.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:28.952 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:28.985 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:29.099 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:29.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:29.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:29.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:29 smithi026 ceph-mon[215775]: failed to encode map e235 with expected crc 2023-09-23T22:22:29.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:29 smithi026 ceph-mon[215775]: failed to encode map e235 with expected crc 2023-09-23T22:22:29.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:29.239 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:29 smithi070 ceph-mon[192652]: failed to encode map e235 with expected crc 2023-09-23T22:22:29.239 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:29 smithi070 ceph-mon[192652]: failed to encode map e235 with expected crc 2023-09-23T22:22:29.240 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:29.293 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:29.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:29.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:29.399 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:29.462 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:29.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:29 smithi026 ceph-mon[223014]: failed to encode map e235 with expected crc 2023-09-23T22:22:29.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:29 smithi026 ceph-mon[223014]: failed to encode map e235 with expected crc 2023-09-23T22:22:29.480 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:29.616 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:29.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:29.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:29.718 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:29.722 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:29.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:22:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:22:29] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.43.0" 2023-09-23T22:22:29.755 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:29.802 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:29.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:29.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:29.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:29.934 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:29.979 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:29.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:29.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:30.082 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:30.109 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:30.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[215775]: failed to encode map e236 with expected crc 2023-09-23T22:22:30.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[215775]: osdmap e236: 8 total, 8 up, 6 in 2023-09-23T22:22:30.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[215775]: failed to encode map e236 with expected crc 2023-09-23T22:22:30.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[215775]: pgmap v565: 99 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 433 op/s; 40272/211365 objects degraded (19.053%); 6779/211365 objects misplaced (3.207%); 13 MiB/s, 213 objects/s recovering 2023-09-23T22:22:30.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3345182735' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:22:30.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:22:30.160 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:22:30.263 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:30.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:30.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:30.371 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:30.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:30 smithi070 ceph-mon[192652]: failed to encode map e236 with expected crc 2023-09-23T22:22:30.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:30 smithi070 ceph-mon[192652]: osdmap e236: 8 total, 8 up, 6 in 2023-09-23T22:22:30.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:30 smithi070 ceph-mon[192652]: failed to encode map e236 with expected crc 2023-09-23T22:22:30.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:30 smithi070 ceph-mon[192652]: pgmap v565: 99 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 433 op/s; 40272/211365 objects degraded (19.053%); 6779/211365 objects misplaced (3.207%); 13 MiB/s, 213 objects/s recovering 2023-09-23T22:22:30.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3345182735' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:22:30.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:30 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:22:30.419 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:30.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[223014]: failed to encode map e236 with expected crc 2023-09-23T22:22:30.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[223014]: osdmap e236: 8 total, 8 up, 6 in 2023-09-23T22:22:30.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[223014]: failed to encode map e236 with expected crc 2023-09-23T22:22:30.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[223014]: pgmap v565: 99 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 433 op/s; 40272/211365 objects degraded (19.053%); 6779/211365 objects misplaced (3.207%); 13 MiB/s, 213 objects/s recovering 2023-09-23T22:22:30.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3345182735' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:22:30.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:30 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:22:30.602 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:30.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:30.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:30.706 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:30.791 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:30.853 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:22:30.853 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:22:30.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:22:30.853 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:22:30.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:14264 5.1 e115/111 5:85e28b51:::benchmark_data_smithi026_286012_object14263:head v 115'855, mlcod=115'853)", 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.815928+0000", 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.03481588900001, 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16326769999999999, 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815928+0000", 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815928+0000", 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815928+0000", 2023-09-23T22:22:30.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815954+0000", 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6536000000000001e-05 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815971+0000", 2023-09-23T22:22:30.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6943999999999999e-05 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815977+0000", 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5149999999999997e-06 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816016+0000", 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9079000000000003e-05 2023-09-23T22:22:30.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977199+0000", 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.161183399 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979161+0000", 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001961646 2023-09-23T22:22:30.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979196+0000", 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5267e-05 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.859 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:14265 5.1 e115/111 5:89dc4bcb:::benchmark_data_smithi026_286012_object14264:head v 115'856, mlcod=115'853)", 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.816637+0000", 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.03410756300002, 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16258351400000001, 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816637+0000", 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816637+0000", 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816636+0000", 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816659+0000", 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2614999999999999e-05 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977207+0000", 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16054861300000001 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977227+0000", 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9551999999999998e-05 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977927+0000", 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00069972700000000003 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977953+0000", 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6245999999999999e-05 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979199+0000", 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012457709999999999 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979220+0000", 2023-09-23T22:22:30.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.158e-05 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:99732 5.5 e160/152 5:a0b53350:::benchmark_data_smithi026_286012_object5924:head v 160'6376, mlcod=0'0)", 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:18.766981+0000", 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 312.083763519, 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036776971999999998, 2023-09-23T22:22:30.865 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.766981+0000", 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.766981+0000", 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.766979+0000", 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.766993+0000", 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3393e-05 2023-09-23T22:22:30.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.767012+0000", 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9236e-05 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.767020+0000", 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2060000000000007e-06 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.767077+0000", 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7348e-05 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.767104+0000", 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6933000000000001e-05 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.803716+0000", 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036611363000000001 2023-09-23T22:22:30.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.803758+0000", 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1959000000000003e-05 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:99743 5.5 e160/152 5:ae85e855:::benchmark_data_smithi026_286012_object5935:head v 160'6378, mlcod=0'0)", 2023-09-23T22:22:30.870 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:18.772875+0000", 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 312.07786922299999, 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037407917999999998, 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.772875+0000", 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.772875+0000", 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.772874+0000", 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.772896+0000", 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.234e-05 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.772918+0000", 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1977e-05 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.772927+0000", 2023-09-23T22:22:30.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4740000000000005e-06 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.780509+0000", 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075827050000000003 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.780534+0000", 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4705000000000001e-05 2023-09-23T22:22:30.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.810240+0000", 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.02970623 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:18.810283+0000", 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2630999999999999e-05 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.875 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.876 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:100632 5.5 e161/152 5:a9460d07:::benchmark_data_smithi026_286012_object6824:head v 161'6435, mlcod=0'0)", 2023-09-23T22:22:30.876 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:19.911998+0000", 2023-09-23T22:22:30.876 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.938746288, 2023-09-23T22:22:30.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037518451000000001, 2023-09-23T22:22:30.876 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.876 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.876 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.911998+0000", 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.911998+0000", 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.911997+0000", 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.912006+0000", 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4839999999999993e-06 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.912020+0000", 2023-09-23T22:22:30.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3339999999999999e-05 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.912026+0000", 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5300000000000002e-06 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.915823+0000", 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037968310000000001 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.915842+0000", 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9417999999999999e-05 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.949475+0000", 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033632452 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:19.949516+0000", 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1536999999999999e-05 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101023 5.5 e161/152 5:a887f58c:::benchmark_data_smithi026_286012_object7215:head v 161'6460, mlcod=0'0)", 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.654324+0000", 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.19642032899998, 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.053302833000000001, 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.881 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654324+0000", 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654324+0000", 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654323+0000", 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654333+0000", 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8530000000000004e-06 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654346+0000", 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3842000000000001e-05 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654353+0000", 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.489e-06 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.664025+0000", 2023-09-23T22:22:30.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096725869999999999 2023-09-23T22:22:30.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.664052+0000", 2023-09-23T22:22:30.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6594999999999999e-05 2023-09-23T22:22:30.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707599+0000", 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.043547359000000001 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707627+0000", 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7282000000000001e-05 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.887 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101021 5.d e161/111 5:ba53bdac:::benchmark_data_smithi026_286012_object7213:head v 161'6371, mlcod=161'6368)", 2023-09-23T22:22:30.887 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.656838+0000", 2023-09-23T22:22:30.887 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.19390611400001, 2023-09-23T22:22:30.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050757414000000001, 2023-09-23T22:22:30.887 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.888 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.888 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656838+0000", 2023-09-23T22:22:30.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656838+0000", 2023-09-23T22:22:30.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656836+0000", 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656849+0000", 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2418e-05 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656861+0000", 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2558000000000001e-05 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656869+0000", 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3440000000000003e-06 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659763+0000", 2023-09-23T22:22:30.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002893825 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659783+0000", 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0205999999999999e-05 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707552+0000", 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047768928000000002 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707595+0000", 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3696000000000003e-05 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101026 5.d e161/111 5:bd4efaeb:::benchmark_data_smithi026_286012_object7218:head v 161'6372, mlcod=161'6368)", 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.657139+0000", 2023-09-23T22:22:30.893 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.19360533600002, 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050510059000000003, 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657139+0000", 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657139+0000", 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657138+0000", 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657144+0000", 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9499999999999998e-06 2023-09-23T22:22:30.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657156+0000", 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2381999999999999e-05 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657162+0000", 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5629999999999999e-06 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.662814+0000", 2023-09-23T22:22:30.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005652341 2023-09-23T22:22:30.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.662834+0000", 2023-09-23T22:22:30.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9296999999999998e-05 2023-09-23T22:22:30.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707629+0000", 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.044795436000000001 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707649+0000", 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9789999999999999e-05 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.898 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101024 5.5 e161/152 5:adf8d131:::benchmark_data_smithi026_286012_object7216:head v 161'6461, mlcod=0'0)", 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.659691+0000", 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.19105336899997, 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048001597999999999, 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659691+0000", 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659691+0000", 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659689+0000", 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659701+0000", 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2172e-05 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659726+0000", 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5106e-05 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659733+0000", 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1729999999999998e-06 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.673568+0000", 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013834377 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.673593+0000", 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4805999999999999e-05 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707672+0000", 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034079600000000002 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707692+0000", 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0287000000000001e-05 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.904 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101025 5.5 e161/152 5:ac250153:::benchmark_data_smithi026_286012_object7217:head v 161'6462, mlcod=0'0)", 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.660139+0000", 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.19060553999998, 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047578966, 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660139+0000", 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660139+0000", 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660138+0000", 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660144+0000", 2023-09-23T22:22:30.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8930000000000002e-06 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660153+0000", 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8540000000000003e-06 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660158+0000", 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8180000000000003e-06 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.680575+0000", 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020417219 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.680606+0000", 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0701000000000003e-05 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707696+0000", 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.02709056 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707718+0000", 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1460000000000001e-05 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101029 5.d e161/111 5:b06f3af1:::benchmark_data_smithi026_286012_object7221:head v 161'6373, mlcod=161'6368)", 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.668426+0000", 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.18231841199997, 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039244399999999999, 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668426+0000", 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668426+0000", 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668424+0000", 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668437+0000", 2023-09-23T22:22:30.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2584e-05 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668456+0000", 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9378e-05 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668465+0000", 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3520000000000007e-06 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670857+0000", 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023919100000000001 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670874+0000", 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7413000000000001e-05 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707651+0000", 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036776674000000002 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707670+0000", 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.95e-05 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101033 5.5 e161/152 5:ae2bad64:::benchmark_data_smithi026_286012_object7225:head v 161'6463, mlcod=0'0)", 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.670148+0000", 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 310.180596267, 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037583591, 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670148+0000", 2023-09-23T22:22:30.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670148+0000", 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670147+0000", 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:30.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670155+0000", 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4349999999999997e-06 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670167+0000", 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1858000000000001e-05 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670173+0000", 2023-09-23T22:22:30.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9220000000000002e-06 2023-09-23T22:22:30.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.687178+0000", 2023-09-23T22:22:30.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017004251000000001 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.687206+0000", 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8778000000000001e-05 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707719+0000", 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020513053999999999 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707731+0000", 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2147000000000001e-05 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101956 5.d e161/111 5:b91a2013:::benchmark_data_smithi026_286012_object8148:head v 161'6423, mlcod=161'6418)", 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:22:30.920 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 308.877125971, 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.075111800000000006, 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973635+0000", 2023-09-23T22:22:30.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6799e-05 2023-09-23T22:22:30.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973642+0000", 2023-09-23T22:22:30.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9199999999999998e-06 2023-09-23T22:22:30.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973646+0000", 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1540000000000004e-06 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.979441+0000", 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057947739999999999 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.979459+0000", 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7938999999999999e-05 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048696+0000", 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.069237189000000005 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048730+0000", 2023-09-23T22:22:30.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4269e-05 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101969 5.4 e161/111 5:2c36eb9f:::benchmark_data_smithi026_286012_object8161:head v 161'6383, mlcod=161'6381)", 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:21.982992+0000", 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 308.86775265, 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.071878993000000002, 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.926 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.927 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982992+0000", 2023-09-23T22:22:30.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982992+0000", 2023-09-23T22:22:30.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982991+0000", 2023-09-23T22:22:30.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:30.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982997+0000", 2023-09-23T22:22:30.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1310000000000003e-06 2023-09-23T22:22:30.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048677+0000", 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.065680429999999998 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048692+0000", 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4908e-05 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.930 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048737+0000", 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4496000000000002e-05 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048761+0000", 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4675999999999999e-05 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.054828+0000", 2023-09-23T22:22:30.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006067037 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.054871+0000", 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2276e-05 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:107855 5.4 e162/111 5:272601b4:::benchmark_data_smithi026_286012_object14047:head v 162'6737, mlcod=162'6734)", 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:31.719396+0000", 2023-09-23T22:22:30.932 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 299.13134838600001, 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036590755000000003, 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719396+0000", 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719396+0000", 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719395+0000", 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719406+0000", 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0997e-05 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719413+0000", 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7440000000000001e-06 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719417+0000", 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0749999999999999e-06 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719455+0000", 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8173000000000002e-05 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719473+0000", 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7504999999999999e-05 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755942+0000", 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036469517999999999 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755987+0000", 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.405e-05 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:107856 5.4 e162/111 5:2b1debf3:::benchmark_data_smithi026_286012_object14048:head v 162'6738, mlcod=162'6734)", 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:31.719455+0000", 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 299.13128957399999, 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036556289999999998, 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719455+0000", 2023-09-23T22:22:30.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719455+0000", 2023-09-23T22:22:30.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719454+0000", 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967296 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719458+0000", 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.247e-06 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719463+0000", 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1830000000000002e-06 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.719466+0000", 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4350000000000001e-06 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722187+0000", 2023-09-23T22:22:30.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027202200000000002 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722208+0000", 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1756e-05 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755989+0000", 2023-09-23T22:22:30.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033781071000000003 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.756011+0000", 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1546999999999999e-05 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.943 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:107865 5.d e162/111 5:b3790187:::benchmark_data_smithi026_286012_object14057:head v 162'6782, mlcod=162'6780)", 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:31.722221+0000", 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 299.12852303300002, 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045419582, 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722221+0000", 2023-09-23T22:22:30.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722221+0000", 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722220+0000", 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722227+0000", 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0509999999999999e-06 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722236+0000", 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.149e-06 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722242+0000", 2023-09-23T22:22:30.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2900000000000002e-06 2023-09-23T22:22:30.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722299+0000", 2023-09-23T22:22:30.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7377000000000003e-05 2023-09-23T22:22:30.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722316+0000", 2023-09-23T22:22:30.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6634999999999999e-05 2023-09-23T22:22:30.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.767615+0000", 2023-09-23T22:22:30.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045299530999999997 2023-09-23T22:22:30.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.767641+0000", 2023-09-23T22:22:30.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5423999999999999e-05 2023-09-23T22:22:30.949 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.949 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.950 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.950 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:107867 5.b e162/155 5:df3eb44e:::benchmark_data_smithi026_286012_object14059:head v 162'6625, mlcod=162'6622)", 2023-09-23T22:22:30.950 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:31.730105+0000", 2023-09-23T22:22:30.950 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 299.120638977, 2023-09-23T22:22:30.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037833218000000002, 2023-09-23T22:22:30.950 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.950 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.730105+0000", 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.730105+0000", 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.730104+0000", 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.730115+0000", 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1071e-05 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.952 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755382+0000", 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025266956 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755388+0000", 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7529999999999998e-06 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755412+0000", 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4536e-05 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755427+0000", 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4117999999999999e-05 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.767913+0000", 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012486667999999999 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.767938+0000", 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.514e-05 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.955 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:109357 5.5 e163/152 5:a53d5020:::benchmark_data_smithi026_286012_object15549:head v 163'6993, mlcod=0'0)", 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:34.041874+0000", 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.80887008799999, 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.038666124000000003, 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041874+0000", 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041874+0000", 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041873+0000", 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041884+0000", 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.119e-05 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041899+0000", 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.446e-05 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041905+0000", 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6900000000000003e-06 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.052998+0000", 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01109245 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.053028+0000", 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0366e-05 2023-09-23T22:22:30.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.080486+0000", 2023-09-23T22:22:30.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027457314 2023-09-23T22:22:30.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.080540+0000", 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4583000000000002e-05 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:149531 5.5 e176/152 5:a016ed63:::benchmark_data_smithi026_286012_object55723:head v 176'9549, mlcod=0'0)", 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:18:37.874547+0000", 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 232.97619707999999, 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036934320999999999, 2023-09-23T22:22:30.961 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.874547+0000", 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.874547+0000", 2023-09-23T22:22:30.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.874546+0000", 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.874557+0000", 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.114e-05 2023-09-23T22:22:30.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.874569+0000", 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1789e-05 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.874575+0000", 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4139999999999999e-06 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.891992+0000", 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01741707 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.892014+0000", 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1365e-05 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:22:30.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.911443+0000", 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019428754999999999 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:18:37.911481+0000", 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8829999999999999e-05 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.966 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:22:30.967 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:22:30.967 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:22:31.169 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:31 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:31.170 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:31 smithi070 ceph-mon[192652]: failed to encode map e237 with expected crc 2023-09-23T22:22:31.170 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:31 smithi070 ceph-mon[192652]: failed to encode map e237 with expected crc 2023-09-23T22:22:31.170 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:31 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:22:31.170 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:31 smithi070 ceph-mon[192652]: osdmap e237: 8 total, 8 up, 6 in 2023-09-23T22:22:31.170 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:31 smithi070 ceph-mon[192652]: failed to encode map e237 with expected crc 2023-09-23T22:22:31.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:31.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[215775]: failed to encode map e237 with expected crc 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[215775]: failed to encode map e237 with expected crc 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[215775]: osdmap e237: 8 total, 8 up, 6 in 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[215775]: failed to encode map e237 with expected crc 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[223014]: failed to encode map e237 with expected crc 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[223014]: failed to encode map e237 with expected crc 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[223014]: osdmap e237: 8 total, 8 up, 6 in 2023-09-23T22:22:31.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:31 smithi026 ceph-mon[223014]: failed to encode map e237 with expected crc 2023-09-23T22:22:31.722 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:31.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:31.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:31.830 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:31.872 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:31.995 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:22:32.309 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:32.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:32 smithi070 ceph-mon[192652]: pgmap v567: 99 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 500 op/s; 37397/207807 objects degraded (17.996%); 9557/207807 objects misplaced (4.599%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:22:32.423 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:32.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:32 smithi026 ceph-mon[215775]: pgmap v567: 99 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 500 op/s; 37397/207807 objects degraded (17.996%); 9557/207807 objects misplaced (4.599%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:22:32.463 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:32 smithi026 ceph-mon[223014]: pgmap v567: 99 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 500 op/s; 37397/207807 objects degraded (17.996%); 9557/207807 objects misplaced (4.599%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:22:32.470 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:32.997 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:33.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:33.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:33.112 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:33.150 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:33.209 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:33.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:33.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:33.293 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 37397/207807 objects degraded (17.996%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:33.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 37397/207807 objects degraded (17.996%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:33.314 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:33.344 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:33.396 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:33.399 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 37397/207807 objects degraded (17.996%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:33.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:33.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:33.502 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:33.638 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:33.650 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:22:33.704 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:22:33.735 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:22:33.778 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:22:33.823 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:22:33.841 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:33.866 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:22:33.888 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:33.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:33.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:33.939 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:22:33.998 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:34.003 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:22:34.008 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:22:34.008 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:22:34.008 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:22:34.008 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:22:34.008 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:22:34.008 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:22:34.009 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:22:34.009 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:22:34.030 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:34.093 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:34.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:34.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:34.203 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:34.296 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:34.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:34 smithi070 ceph-mon[192652]: pgmap v568: 99 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 394 op/s; 37397/206928 objects degraded (18.072%); 9557/206928 objects misplaced (4.619%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T22:22:34.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:34 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4183373557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:34.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:34 smithi026 ceph-mon[215775]: pgmap v568: 99 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 394 op/s; 37397/206928 objects degraded (18.072%); 9557/206928 objects misplaced (4.619%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T22:22:34.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:34 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4183373557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:34.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:34 smithi026 ceph-mon[223014]: pgmap v568: 99 pgs: 2 active+recovery_wait+degraded, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 394 op/s; 37397/206928 objects degraded (18.072%); 9557/206928 objects misplaced (4.619%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T22:22:34.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:34 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4183373557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:35.141 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:22:35.270 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:22:35.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1845000225' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:22:35.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:35 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:22:35.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1845000225' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:22:35.461 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:35 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:22:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1845000225' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:22:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:35 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:22:35.707 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:35.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:35.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:35.814 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:35.897 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:36.492 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[215775]: pgmap v569: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 37105/202518 objects degraded (18.322%); 9297/202518 objects misplaced (4.591%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T22:22:36.492 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[215775]: failed to encode map e238 with expected crc 2023-09-23T22:22:36.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[215775]: failed to encode map e238 with expected crc 2023-09-23T22:22:36.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:22:36.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[215775]: osdmap e238: 8 total, 8 up, 6 in 2023-09-23T22:22:36.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[223014]: pgmap v569: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 37105/202518 objects degraded (18.322%); 9297/202518 objects misplaced (4.591%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T22:22:36.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[223014]: failed to encode map e238 with expected crc 2023-09-23T22:22:36.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[223014]: failed to encode map e238 with expected crc 2023-09-23T22:22:36.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:22:36.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:36 smithi026 ceph-mon[223014]: osdmap e238: 8 total, 8 up, 6 in 2023-09-23T22:22:36.504 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:36.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:36.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:36.612 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:36.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:36 smithi070 ceph-mon[192652]: pgmap v569: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 484 op/s; 37105/202518 objects degraded (18.322%); 9297/202518 objects misplaced (4.591%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T22:22:36.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:36 smithi070 ceph-mon[192652]: failed to encode map e238 with expected crc 2023-09-23T22:22:36.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:36 smithi070 ceph-mon[192652]: failed to encode map e238 with expected crc 2023-09-23T22:22:36.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:22:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:36 smithi070 ceph-mon[192652]: osdmap e238: 8 total, 8 up, 6 in 2023-09-23T22:22:36.722 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:36.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T22:22:37.153 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:37.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:37.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:37.272 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:37.390 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:37.420 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:22:37.444 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:37.550 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:37.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:37.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:37.657 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:37.684 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:37.731 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:37.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:37.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:37.835 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:37.865 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:37.938 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:37.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:37.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:38.044 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:38.078 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:38.095 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:38.132 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:38.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:38.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:38.242 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:38.267 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:22:38.268 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:22:38.327 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:38 smithi026 ceph-mon[215775]: pgmap v571: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 37105/199752 objects degraded (18.576%); 9038/199752 objects misplaced (4.525%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:22:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 37105/199752 objects degraded (18.576%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:38.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:38 smithi026 ceph-mon[223014]: pgmap v571: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 37105/199752 objects degraded (18.576%); 9038/199752 objects misplaced (4.525%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:22:38.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 37105/199752 objects degraded (18.576%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:38.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:38 smithi070 ceph-mon[192652]: pgmap v571: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 483 op/s; 37105/199752 objects degraded (18.576%); 9038/199752 objects misplaced (4.525%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:22:38.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 37105/199752 objects degraded (18.576%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:38.812 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:38.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:38.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:38.922 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:38.989 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:39.054 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:39.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:39.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:39.158 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:39.197 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:39.242 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:39.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:39.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:39.387 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:39.444 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:39.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:39.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:39.549 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:39.575 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:39.616 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:39.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:39.724 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:39.758 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:39.822 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:39.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:39.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:39.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:39.991 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:22:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:22:39] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.43.0" 2023-09-23T22:22:40.024 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:40.272 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:40.281 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:22:40.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:22:40.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:40.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:40 smithi026 ceph-mon[215775]: pgmap v572: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 36765/197040 objects degraded (18.659%); 8720/197040 objects misplaced (4.425%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T22:22:40.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:22:40.283 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:40.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:40 smithi026 ceph-mon[223014]: pgmap v572: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 36765/197040 objects degraded (18.659%); 8720/197040 objects misplaced (4.425%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T22:22:40.295 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:40.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:40.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:22:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:40 smithi070 ceph-mon[192652]: pgmap v572: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 4.0 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 459 op/s; 36765/197040 objects degraded (18.659%); 8720/197040 objects misplaced (4.425%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T22:22:40.401 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:40.554 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:41.691 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:41.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:41.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:41.807 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:41.922 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:42.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:42 smithi070 ceph-mon[192652]: pgmap v573: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 36765/193512 objects degraded (18.999%); 8522/193512 objects misplaced (4.404%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T22:22:42.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:42 smithi026 ceph-mon[215775]: pgmap v573: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 36765/193512 objects degraded (18.999%); 8522/193512 objects misplaced (4.404%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T22:22:42.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:42 smithi026 ceph-mon[223014]: pgmap v573: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 36765/193512 objects degraded (18.999%); 8522/193512 objects misplaced (4.404%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T22:22:43.253 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:43.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:43.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:43.344 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:22:43.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:43.373 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T22:22:43.413 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:43.457 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:43.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/764846784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:43.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:43.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 36765/193512 objects degraded (18.999%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:43.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2379710343' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:22:43.575 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:22:43.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/764846784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:43.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:43.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 36765/193512 objects degraded (18.999%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:43.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2379710343' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:22:43.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:43 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:22:43.597 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:43.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:43.604 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/764846784' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:43 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:22:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 36765/193512 objects degraded (18.999%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2379710343' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:22:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:43 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:22:43.705 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:43.738 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:44.295 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:22:43 smithi070 conmon[172396]: logger=sqlstore.transactions t=2023-09-23T22:22:43.98376566Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T22:22:44.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:44 smithi070 ceph-mon[192652]: pgmap v574: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 36765/192624 objects degraded (19.086%); 8522/192624 objects misplaced (4.424%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T22:22:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:44 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:44 smithi070 ceph-mon[192652]: failed to encode map e239 with expected crc 2023-09-23T22:22:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:44 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:22:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:44 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:22:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:44 smithi070 ceph-mon[192652]: osdmap e239: 8 total, 8 up, 6 in 2023-09-23T22:22:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:44 smithi070 ceph-mon[192652]: failed to encode map e239 with expected crc 2023-09-23T22:22:44.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[215775]: pgmap v574: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 36765/192624 objects degraded (19.086%); 8522/192624 objects misplaced (4.424%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T22:22:44.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:44.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[215775]: failed to encode map e239 with expected crc 2023-09-23T22:22:44.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:22:44.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:22:44.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[215775]: osdmap e239: 8 total, 8 up, 6 in 2023-09-23T22:22:44.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[215775]: failed to encode map e239 with expected crc 2023-09-23T22:22:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[223014]: pgmap v574: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 3.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 476 op/s; 36765/192624 objects degraded (19.086%); 8522/192624 objects misplaced (4.424%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T22:22:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[223014]: failed to encode map e239 with expected crc 2023-09-23T22:22:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:22:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:22:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[223014]: osdmap e239: 8 total, 8 up, 6 in 2023-09-23T22:22:44.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:44 smithi026 ceph-mon[223014]: failed to encode map e239 with expected crc 2023-09-23T22:22:45.420 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T22:22:45.471 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:45.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:45.594 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:45 smithi070 ceph-mon[192652]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:45 smithi070 ceph-mon[192652]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:45 smithi070 ceph-mon[192652]: osdmap e240: 8 total, 8 up, 6 in 2023-09-23T22:22:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:45 smithi070 ceph-mon[192652]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:45 smithi070 ceph-mon[192652]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:45 smithi070 ceph-mon[192652]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.622 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:45.671 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:45.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:45.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:45.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[215775]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[215775]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[215775]: osdmap e240: 8 total, 8 up, 6 in 2023-09-23T22:22:45.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[215775]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[215775]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[215775]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[223014]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[223014]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[223014]: osdmap e240: 8 total, 8 up, 6 in 2023-09-23T22:22:45.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[223014]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[223014]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:45 smithi026 ceph-mon[223014]: failed to encode map e240 with expected crc 2023-09-23T22:22:45.778 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:45.811 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:45.869 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:45.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:45.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:45.974 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:46.078 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:46.192 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:22:46.237 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:22:46.267 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:22:46.325 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:22:46.358 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:22:46.399 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:22:46.425 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:46.434 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:22:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[223014]: pgmap v577: 99 pgs: 2 remapped+peering, 8 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 454 op/s; 25426/188829 objects degraded (13.465%); 6064/188829 objects misplaced (3.211%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:22:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[223014]: failed to encode map e241 with expected crc 2023-09-23T22:22:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[223014]: osdmap e241: 8 total, 8 up, 6 in 2023-09-23T22:22:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[223014]: failed to encode map e241 with expected crc 2023-09-23T22:22:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[223014]: 6.b starting backfill to osd.7 from (0'0,212'3717] MIN to 238'6268 2023-09-23T22:22:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[223014]: failed to encode map e241 with expected crc 2023-09-23T22:22:46.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2252721080' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:46.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:46.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:46.444 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[215775]: pgmap v577: 99 pgs: 2 remapped+peering, 8 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 454 op/s; 25426/188829 objects degraded (13.465%); 6064/188829 objects misplaced (3.211%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:22:46.444 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[215775]: failed to encode map e241 with expected crc 2023-09-23T22:22:46.445 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[215775]: osdmap e241: 8 total, 8 up, 6 in 2023-09-23T22:22:46.445 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[215775]: failed to encode map e241 with expected crc 2023-09-23T22:22:46.445 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[215775]: 6.b starting backfill to osd.7 from (0'0,212'3717] MIN to 238'6268 2023-09-23T22:22:46.445 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[215775]: failed to encode map e241 with expected crc 2023-09-23T22:22:46.445 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2252721080' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:46.472 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:22:46.482 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:22:46.482 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:22:46.482 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:22:46.482 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:22:46.482 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:22:46.482 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:22:46.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:22:46.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:22:46.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:46.579 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:46 smithi070 ceph-mon[192652]: pgmap v577: 99 pgs: 2 remapped+peering, 8 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 454 op/s; 25426/188829 objects degraded (13.465%); 6064/188829 objects misplaced (3.211%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:22:46.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:46 smithi070 ceph-mon[192652]: failed to encode map e241 with expected crc 2023-09-23T22:22:46.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:46 smithi070 ceph-mon[192652]: osdmap e241: 8 total, 8 up, 6 in 2023-09-23T22:22:46.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:46 smithi070 ceph-mon[192652]: failed to encode map e241 with expected crc 2023-09-23T22:22:46.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:46 smithi070 ceph-mon[192652]: 6.b starting backfill to osd.7 from (0'0,212'3717] MIN to 238'6268 2023-09-23T22:22:46.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:46 smithi070 ceph-mon[192652]: failed to encode map e241 with expected crc 2023-09-23T22:22:46.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2252721080' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:46.582 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:46.672 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:46.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:46.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:46.792 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:46.825 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:46.910 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:46.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:46.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:47.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:47.128 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:47.337 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:47 smithi070 ceph-mon[192652]: osdmap e242: 8 total, 8 up, 6 in 2023-09-23T22:22:47.337 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:47 smithi070 ceph-mon[192652]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.337 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:47 smithi070 ceph-mon[192652]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.337 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:47 smithi070 ceph-mon[192652]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.337 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:47 smithi070 ceph-mon[192652]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.337 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:47 smithi070 ceph-mon[192652]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.337 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:47 smithi070 ceph-mon[192652]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[215775]: osdmap e242: 8 total, 8 up, 6 in 2023-09-23T22:22:47.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[215775]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[215775]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[215775]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[215775]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[215775]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[215775]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[223014]: osdmap e242: 8 total, 8 up, 6 in 2023-09-23T22:22:47.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[223014]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[223014]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[223014]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[223014]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[223014]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:47 smithi026 ceph-mon[223014]: failed to encode map e242 with expected crc 2023-09-23T22:22:47.494 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:22:47.993 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:48.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:48.106 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:48.135 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:48.583 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:48.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[223014]: pgmap v580: 99 pgs: 2 remapped+peering, 10 peering, 1 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 22033/187062 objects degraded (11.778%); 8912/187062 objects misplaced (4.764%); 13 MiB/s, 206 objects/s recovering 2023-09-23T22:22:48.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[223014]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[223014]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[223014]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[223014]: osdmap e243: 8 total, 8 up, 6 in 2023-09-23T22:22:48.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[223014]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[223014]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 22033/187062 objects degraded (11.778%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:48.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:48.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:48.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[215775]: pgmap v580: 99 pgs: 2 remapped+peering, 10 peering, 1 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 22033/187062 objects degraded (11.778%); 8912/187062 objects misplaced (4.764%); 13 MiB/s, 206 objects/s recovering 2023-09-23T22:22:48.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[215775]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[215775]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[215775]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[215775]: osdmap e243: 8 total, 8 up, 6 in 2023-09-23T22:22:48.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[215775]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[215775]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 22033/187062 objects degraded (11.778%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:48.628 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:48 smithi070 ceph-mon[192652]: pgmap v580: 99 pgs: 2 remapped+peering, 10 peering, 1 active+recovery_wait+degraded, 3 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 73 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 22033/187062 objects degraded (11.778%); 8912/187062 objects misplaced (4.764%); 13 MiB/s, 206 objects/s recovering 2023-09-23T22:22:48.628 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:48 smithi070 ceph-mon[192652]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.628 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:48 smithi070 ceph-mon[192652]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.628 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:48 smithi070 ceph-mon[192652]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:48 smithi070 ceph-mon[192652]: osdmap e243: 8 total, 8 up, 6 in 2023-09-23T22:22:48.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:48 smithi070 ceph-mon[192652]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:48 smithi070 ceph-mon[192652]: failed to encode map e243 with expected crc 2023-09-23T22:22:48.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 22033/187062 objects degraded (11.778%), 14 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:48.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:48.731 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:48.776 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:48.791 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:48.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:48.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:48.899 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:48.949 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:49.042 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:49.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:49.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:49.151 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:49.199 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:49.263 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:49.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:49.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:49.381 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:49.396 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:49.418 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:49.488 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:49.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:49.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:49.597 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:49.613 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:22:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:22:49] "GET /metrics HTTP/1.1" 200 36429 "" "Prometheus/2.43.0" 2023-09-23T22:22:49.636 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:49.709 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:49.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:49.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:49.822 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:49.885 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:49.953 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:49.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:49.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:50.063 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:50.146 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:50.229 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:22:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:22:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:22:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:22:50.230 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:22:50.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:50 smithi026 ceph-mon[215775]: pgmap v582: 99 pgs: 2 active+recovery_wait+degraded, 2 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 375 op/s; 35175/184761 objects degraded (19.038%); 8918/184761 objects misplaced (4.827%); 7.5 MiB/s, 121 objects/s recovering 2023-09-23T22:22:50.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:50 smithi026 ceph-mon[223014]: pgmap v582: 99 pgs: 2 active+recovery_wait+degraded, 2 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 375 op/s; 35175/184761 objects degraded (19.038%); 8918/184761 objects misplaced (4.827%); 7.5 MiB/s, 121 objects/s recovering 2023-09-23T22:22:50.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:50 smithi070 ceph-mon[192652]: pgmap v582: 99 pgs: 2 active+recovery_wait+degraded, 2 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 375 op/s; 35175/184761 objects degraded (19.038%); 8918/184761 objects misplaced (4.827%); 7.5 MiB/s, 121 objects/s recovering 2023-09-23T22:22:50.725 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:50.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:50.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:50.808 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:22:50.835 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:50.996 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:51.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:51.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:51.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:51.185 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:51.259 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:51.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:51.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:51.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:51.802 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:51.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:22:51.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:52.186 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:52.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:52.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:52.297 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:52.344 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:52.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:52 smithi026 ceph-mon[215775]: pgmap v583: 99 pgs: 2 active+recovery_wait+degraded, 2 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 35175/180477 objects degraded (19.490%); 8650/180477 objects misplaced (4.793%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T22:22:52.361 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:52 smithi026 ceph-mon[223014]: pgmap v583: 99 pgs: 2 active+recovery_wait+degraded, 2 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 35175/180477 objects degraded (19.490%); 8650/180477 objects misplaced (4.793%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T22:22:52.405 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:52.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:52.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:52.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:52.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:52 smithi070 ceph-mon[192652]: pgmap v583: 99 pgs: 2 active+recovery_wait+degraded, 2 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 3.7 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 35175/180477 objects degraded (19.490%); 8650/180477 objects misplaced (4.793%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T22:22:52.632 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:52.821 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:52.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:52.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:52.925 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:22:52.965 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:53.066 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:22:53.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:53.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:22:53.176 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:53.204 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:53.253 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:53.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:53.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:53.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:53.426 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 35175/180477 objects degraded (19.490%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:53.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:53 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3362871043' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:22:53.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 35175/180477 objects degraded (19.490%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:53 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3362871043' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:22:53.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 35175/180477 objects degraded (19.490%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:53.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:53 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3362871043' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:22:53.657 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:53.658 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:22:53.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:53.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:22:53.771 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:53.832 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:53.891 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:53.930 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T22:22:54.032 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:54.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:54.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:54.137 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:54.177 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:54.252 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:54.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:54.364 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:54.410 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:54.490 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:54.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:54.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:54.598 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:54.638 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:54.723 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:54.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:54.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:54.827 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:54.873 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[215775]: pgmap v584: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 349 op/s; 35175/179268 objects degraded (19.621%); 8467/179268 objects misplaced (4.723%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:22:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3362871043' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:22:54.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[215775]: osdmap e244: 8 total, 8 up, 6 in 2023-09-23T22:22:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[215775]: failed to encode map e244 with expected crc 2023-09-23T22:22:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[223014]: pgmap v584: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 349 op/s; 35175/179268 objects degraded (19.621%); 8467/179268 objects misplaced (4.723%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:22:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3362871043' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:22:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[223014]: osdmap e244: 8 total, 8 up, 6 in 2023-09-23T22:22:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[223014]: failed to encode map e244 with expected crc 2023-09-23T22:22:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:54 smithi070 ceph-mon[192652]: pgmap v584: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.6 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 349 op/s; 35175/179268 objects degraded (19.621%); 8467/179268 objects misplaced (4.723%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T22:22:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3362871043' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:22:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:54 smithi070 ceph-mon[192652]: osdmap e244: 8 total, 8 up, 6 in 2023-09-23T22:22:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:54 smithi070 ceph-mon[192652]: failed to encode map e244 with expected crc 2023-09-23T22:22:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:22:55.859 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:22:55.859 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:22:56.316 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:56.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:56.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:56.436 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:56.581 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:56.765 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:56.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:56.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:56.880 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:22:56.980 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:57.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:56 smithi026 ceph-mon[215775]: pgmap v586: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 35175/173664 objects degraded (20.255%); 8282/173664 objects misplaced (4.769%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:22:57.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:56 smithi026 ceph-mon[223014]: pgmap v586: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 35175/173664 objects degraded (20.255%); 8282/173664 objects misplaced (4.769%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:22:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:56 smithi070 ceph-mon[192652]: pgmap v586: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 557 op/s; 35175/173664 objects degraded (20.255%); 8282/173664 objects misplaced (4.769%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:22:57.168 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:22:57.181 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:22:57.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:57.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:22:57.223 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:22:57.263 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:22:57.283 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:22:57.301 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:22:57.328 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:57.342 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:22:57.386 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:22:57.410 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:22:57.418 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:22:57.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:57.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:22:57.461 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:22:57.520 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:22:57.548 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:57.593 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:22:57.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:57.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:22:57.645 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:22:57.645 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:22:57.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:22:57.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:22:57.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:22:57.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:22:57.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:22:57.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:22:57.697 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:22:57.727 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:57.797 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:22:57.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:57.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:22:57.897 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2590696563' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:57.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:57.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2638928275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:57.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:57 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 35175/173664 objects degraded (20.255%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:57.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2590696563' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:57.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:57.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2638928275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:57.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:57 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 35175/173664 objects degraded (20.255%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:57.901 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:22:57.922 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:22:57.934 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:58.004 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:22:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:58.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:22:58.016 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2590696563' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:58.016 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:22:58.016 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2638928275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:22:58.016 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:57 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 35175/173664 objects degraded (20.255%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:22:58.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:22:58.199 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:22:59.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[223014]: pgmap v587: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 35175/170235 objects degraded (20.663%); 7847/170235 objects misplaced (4.610%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:22:59.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[223014]: failed to encode map e245 with expected crc 2023-09-23T22:22:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[223014]: failed to encode map e245 with expected crc 2023-09-23T22:22:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:22:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[223014]: osdmap e245: 8 total, 8 up, 6 in 2023-09-23T22:22:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:22:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[215775]: pgmap v587: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 35175/170235 objects degraded (20.663%); 7847/170235 objects misplaced (4.610%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:22:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[215775]: failed to encode map e245 with expected crc 2023-09-23T22:22:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[215775]: failed to encode map e245 with expected crc 2023-09-23T22:22:59.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:22:59.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[215775]: osdmap e245: 8 total, 8 up, 6 in 2023-09-23T22:22:59.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:22:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:22:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:58 smithi070 ceph-mon[192652]: pgmap v587: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.5 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 566 op/s; 35175/170235 objects degraded (20.663%); 7847/170235 objects misplaced (4.610%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:22:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:58 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:22:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:58 smithi070 ceph-mon[192652]: failed to encode map e245 with expected crc 2023-09-23T22:22:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:58 smithi070 ceph-mon[192652]: failed to encode map e245 with expected crc 2023-09-23T22:22:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:22:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:58 smithi070 ceph-mon[192652]: osdmap e245: 8 total, 8 up, 6 in 2023-09-23T22:22:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:22:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:22:59.681 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:22:59.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:22:59.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:22:59.777 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:22:59.787 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:22:59.879 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:22:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:22:59] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T22:22:59.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:00.006 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:00.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:00.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:00.119 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:00.231 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:00.324 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:23:00.324 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:23:00.338 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:00.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:00.449 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:00.554 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:00.986 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:23:01.180 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:01.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:01.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:01.210 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:00 smithi026 ceph-mon[215775]: pgmap v589: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 35175/166842 objects degraded (21.083%); 7653/166842 objects misplaced (4.587%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:23:01.210 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:00 smithi026 ceph-mon[223014]: pgmap v589: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 35175/166842 objects degraded (21.083%); 7653/166842 objects misplaced (4.587%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:23:01.307 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:01.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:00 smithi070 ceph-mon[192652]: pgmap v589: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 567 op/s; 35175/166842 objects degraded (21.083%); 7653/166842 objects misplaced (4.587%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T22:23:01.339 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:01.481 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:01.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:01.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:01.597 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:01.652 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:01.777 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:01.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:01.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:01.822 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:23:01.898 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:01.975 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:02.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:01 smithi026 ceph-mon[223014]: pgmap v590: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 708 op/s; 35175/162243 objects degraded (21.680%); 7391/162243 objects misplaced (4.556%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T22:23:02.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:01 smithi026 ceph-mon[215775]: pgmap v590: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 708 op/s; 35175/162243 objects degraded (21.680%); 7391/162243 objects misplaced (4.556%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T22:23:02.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:01 smithi070 ceph-mon[192652]: pgmap v590: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 708 op/s; 35175/162243 objects degraded (21.680%); 7391/162243 objects misplaced (4.556%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T22:23:02.450 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:23:02.762 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:02.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:02.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:02.867 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:02.907 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:02.987 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:02.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:02.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:03.051 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:03.099 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:03.231 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:03.280 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:03.585 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:03.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:03.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:03.693 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:03.738 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:03.808 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:03.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:03.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:03.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:03.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:03.886 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 35175/162243 objects degraded (21.680%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:03.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:03.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:03.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 35175/162243 objects degraded (21.680%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:03.916 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:04.023 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:04.041 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:04.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:04.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:04.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 35175/162243 objects degraded (21.680%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:04.346 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:23:04.355 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:23:04.569 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:04.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:04.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:04.677 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:04.718 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:04.802 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:04.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:04.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:04.859 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:04 smithi026 ceph-mon[223014]: pgmap v591: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 35175/161034 objects degraded (21.843%); 7226/161034 objects misplaced (4.487%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:04.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:04 smithi026 ceph-mon[215775]: pgmap v591: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 35175/161034 objects degraded (21.843%); 7226/161034 objects misplaced (4.487%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:04.914 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:04.970 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:05.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:04 smithi070 ceph-mon[192652]: pgmap v591: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.3 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 35175/161034 objects degraded (21.843%); 7226/161034 objects misplaced (4.487%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:05.815 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:05.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:05.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:05.920 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:05.968 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:05.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:05 smithi026 ceph-mon[215775]: from='client.45176 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:05.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:05 smithi026 ceph-mon[223014]: from='client.45176 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:06.048 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:06.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:06.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:06.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:05 smithi070 ceph-mon[192652]: from='client.45176 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:06.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:06.226 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:06.442 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:06.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:06.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:06.551 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:06.601 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:06.879 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:06 smithi026 ceph-mon[215775]: pgmap v592: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 598 op/s; 35016/155685 objects degraded (22.492%); 7053/155685 objects misplaced (4.530%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T22:23:07.149 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:06 smithi026 ceph-mon[223014]: pgmap v592: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 598 op/s; 35016/155685 objects degraded (22.492%); 7053/155685 objects misplaced (4.530%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T22:23:07.187 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":591,"stamp":"2023-09-23T22:23:03.234113+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3520576279,"num_objects":53678,"num_object_clones":0,"num_object_copies":161034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35175,"num_objects_misplaced":7226,"num_objects_unfound":0,"num_objects_dirty":53678,"num_whiteouts":0,"num_read":6071,"num_read_kb":12351,"num_write":109996,"num_write_kb":5219374,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15441,"num_bytes_recovered":1011225295,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86222,"ondisk_log_size":86222,"up":297,"acting":282,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10696820,"kb_used_data":9435396,"kb_used_omap":20,"kb_used_meta":1261227,"kb_avail":551774092,"statfs":{"total":575970213888,"available":565016670208,"internally_reserved":0,"allocated":9661845504,"data_stored":9654858127,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1291497227},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2,8,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-360579072,"num_objects":-5502,"num_object_clones":0,"num_object_copies":-16506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1241,"num_objects_unfound":0,"num_objects_dirty":-5502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5502,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":862,"num_bytes_recovered":56492032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.363540"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":232,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.870997+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:22:53.870997+0000","last_peered":"2023-09-23T22:22:53.870997+0000","last_clean":"2023-09-23T22:22:53.870997+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:22:53.870997+0000","last_undegraded":"2023-09-23T22:22:53.870997+0000","last_fullsized":"2023-09-23T22:22:53.870997+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":223,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878755+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:22:53.878755+0000","last_peered":"2023-09-23T22:22:53.878755+0000","last_clean":"2023-09-23T22:22:53.878755+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:22:53.878755+0000","last_undegraded":"2023-09-23T22:22:53.878755+0000","last_fullsized":"2023-09-23T22:22:53.878755+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":186,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.916328+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:22:53.916328+0000","last_peered":"2023-09-23T22:22:53.916328+0000","last_clean":"2023-09-23T22:22:53.916328+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:22:53.916328+0000","last_undegraded":"2023-09-23T22:22:53.916328+0000","last_fullsized":"2023-09-23T22:22:53.916328+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":102,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.900765+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:22:57.900765+0000","last_peered":"2023-09-23T22:22:57.900765+0000","last_clean":"2023-09-23T22:22:57.900765+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:22:57.900765+0000","last_undegraded":"2023-09-23T22:22:57.900765+0000","last_fullsized":"2023-09-23T22:22:57.900765+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":263,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.884051+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:22:53.884051+0000","last_peered":"2023-09-23T22:22:53.884051+0000","last_clean":"2023-09-23T22:22:53.884051+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:22:53.884051+0000","last_undegraded":"2023-09-23T22:22:53.884051+0000","last_fullsized":"2023-09-23T22:22:53.884051+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":200,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.879336+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:22:53.879336+0000","last_peered":"2023-09-23T22:22:53.879336+0000","last_clean":"2023-09-23T22:22:53.879336+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:22:53.879336+0000","last_undegraded":"2023-09-23T22:22:53.879336+0000","last_fullsized":"2023-09-23T22:22:53.879336+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":130,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.879986+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:22:53.879986+0000","last_peered":"2023-09-23T22:22:53.879986+0000","last_clean":"2023-09-23T22:22:53.879986+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:22:53.879986+0000","last_undegraded":"2023-09-23T22:22:53.879986+0000","last_fullsized":"2023-09-23T22:22:53.879986+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":220,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.884122+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:22:53.884122+0000","last_peered":"2023-09-23T22:22:53.884122+0000","last_clean":"2023-09-23T22:22:53.884122+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:22:53.884122+0000","last_undegraded":"2023-09-23T22:22:53.884122+0000","last_fullsized":"2023-09-23T22:22:53.884122+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":205,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.876191+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:22:53.876191+0000","last_peered":"2023-09-23T22:22:53.876191+0000","last_clean":"2023-09-23T22:22:53.876191+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:22:53.876191+0000","last_undegraded":"2023-09-23T22:22:53.876191+0000","last_fullsized":"2023-09-23T22:22:53.876191+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":188,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878660+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:22:53.878660+0000","last_peered":"2023-09-23T22:22:53.878660+0000","last_clean":"2023-09-23T22:22:53.878660+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:22:53.878660+0000","last_undegraded":"2023-09-23T22:22:53.878660+0000","last_fullsized":"2023-09-23T22:22:53.878660+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":188,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878665+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:22:53.878665+0000","last_peered":"2023-09-23T22:22:53.878665+0000","last_clean":"2023-09-23T22:22:53.878665+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:22:53.878665+0000","last_undegraded":"2023-09-23T22:22:53.878665+0000","last_fullsized":"2023-09-23T22:22:53.878665+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":222,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888947+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:22:53.888947+0000","last_peered":"2023-09-23T22:22:53.888947+0000","last_clean":"2023-09-23T22:22:53.888947+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:22:53.888947+0000","last_undegraded":"2023-09-23T22:22:53.888947+0000","last_fullsized":"2023-09-23T22:22:53.888947+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":249,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869884+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:22:53.869884+0000","last_peered":"2023-09-23T22:22:53.869884+0000","last_clean":"2023-09-23T22:22:53.869884+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:22:53.869884+0000","last_undegraded":"2023-09-23T22:22:53.869884+0000","last_fullsized":"2023-09-23T22:22:53.869884+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869796+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:22:53.869796+0000","last_peered":"2023-09-23T22:22:53.869796+0000","last_clean":"2023-09-23T22:22:53.869796+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:22:53.869796+0000","last_undegraded":"2023-09-23T22:22:53.869796+0000","last_fullsized":"2023-09-23T22:22:53.869796+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":172,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869813+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:22:53.869813+0000","last_peered":"2023-09-23T22:22:53.869813+0000","last_clean":"2023-09-23T22:22:53.869813+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:22:53.869813+0000","last_undegraded":"2023-09-23T22:22:53.869813+0000","last_fullsized":"2023-09-23T22:22:53.869813+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":211,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.879296+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:22:53.879296+0000","last_peered":"2023-09-23T22:22:53.879296+0000","last_clean":"2023-09-23T22:22:53.879296+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:22:53.879296+0000","last_undegraded":"2023-09-23T22:22:53.879296+0000","last_fullsized":"2023-09-23T22:22:53.879296+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":119,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.879756+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:22:53.879756+0000","last_peered":"2023-09-23T22:22:53.879756+0000","last_clean":"2023-09-23T22:22:53.879756+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:22:53.879756+0000","last_undegraded":"2023-09-23T22:22:53.879756+0000","last_fullsized":"2023-09-23T22:22:53.879756+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":224,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.873523+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:22:53.873523+0000","last_peered":"2023-09-23T22:22:53.873523+0000","last_clean":"2023-09-23T22:22:53.873523+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:22:53.873523+0000","last_undegraded":"2023-09-23T22:22:53.873523+0000","last_fullsized":"2023-09-23T22:22:53.873523+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":168,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869711+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:22:53.869711+0000","last_peered":"2023-09-23T22:22:53.869711+0000","last_clean":"2023-09-23T22:22:53.869711+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:22:53.869711+0000","last_undegraded":"2023-09-23T22:22:53.869711+0000","last_fullsized":"2023-09-23T22:22:53.869711+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":101,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878793+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:22:53.878793+0000","last_peered":"2023-09-23T22:22:53.878793+0000","last_clean":"2023-09-23T22:22:53.878793+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:22:53.878793+0000","last_undegraded":"2023-09-23T22:22:53.878793+0000","last_fullsized":"2023-09-23T22:22:53.878793+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":262,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.886449+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:22:53.886449+0000","last_peered":"2023-09-23T22:22:53.886449+0000","last_clean":"2023-09-23T22:22:53.886449+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:22:53.886449+0000","last_undegraded":"2023-09-23T22:22:53.886449+0000","last_fullsized":"2023-09-23T22:22:53.886449+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":33,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888671+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:22:53.888671+0000","last_peered":"2023-09-23T22:22:53.888671+0000","last_clean":"2023-09-23T22:22:53.888671+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:22:53.888671+0000","last_undegraded":"2023-09-23T22:22:53.888671+0000","last_fullsized":"2023-09-23T22:22:53.888671+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"245'6824","reported_seq":10295,"reported_epoch":245,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:00.267550+0000","last_change":"2023-09-23T22:22:28.455882+0000","last_active":"2023-09-23T22:23:00.267550+0000","last_peered":"2023-09-23T22:23:00.267550+0000","last_clean":"2023-09-23T22:21:12.198976+0000","last_became_active":"2023-09-23T22:21:14.005858+0000","last_became_peered":"2023-09-23T22:21:14.005858+0000","last_unstale":"2023-09-23T22:23:00.267550+0000","last_undegraded":"2023-09-23T22:21:33.932520+0000","last_fullsized":"2023-09-23T22:21:13.797435+0000","mapping_epoch":214,"log_start":"209'1300","ondisk_log_start":"209'1300","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5524,"log_dups_size":0,"ondisk_log_size":5524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:26.966631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211419136,"num_objects":3226,"num_object_clones":0,"num_object_copies":9678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":766,"num_objects_misplaced":1172,"num_objects_unfound":0,"num_objects_dirty":3226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6824,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1704,"num_bytes_recovered":111673344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":1172},{"shards":"0,5","objects":1288}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":240,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.871537+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:22:53.871537+0000","last_peered":"2023-09-23T22:22:53.871537+0000","last_clean":"2023-09-23T22:22:53.871537+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:22:53.871537+0000","last_undegraded":"2023-09-23T22:22:53.871537+0000","last_fullsized":"2023-09-23T22:22:53.871537+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":227,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.873141+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:22:53.873141+0000","last_peered":"2023-09-23T22:22:53.873141+0000","last_clean":"2023-09-23T22:22:53.873141+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:22:53.873141+0000","last_undegraded":"2023-09-23T22:22:53.873141+0000","last_fullsized":"2023-09-23T22:22:53.873141+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"245'6792","reported_seq":14309,"reported_epoch":245,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:22:59.176139+0000","last_change":"2023-09-23T22:22:45.430551+0000","last_active":"2023-09-23T22:22:59.176139+0000","last_peered":"2023-09-23T22:22:59.176139+0000","last_clean":"2023-09-23T22:22:43.315002+0000","last_became_active":"2023-09-23T22:22:45.409613+0000","last_became_peered":"2023-09-23T22:22:45.409613+0000","last_unstale":"2023-09-23T22:22:59.176139+0000","last_undegraded":"2023-09-23T22:22:59.176139+0000","last_fullsized":"2023-09-23T22:22:59.176139+0000","mapping_epoch":240,"log_start":"210'1391","ondisk_log_start":"210'1391","created":203,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:03.767060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220856320,"num_objects":3370,"num_object_clones":0,"num_object_copies":10110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3370,"num_objects_unfound":0,"num_objects_dirty":3370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6792,"num_write_kb":325184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3717,"num_bytes_recovered":243597312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":3370}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":154,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.875590+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:22:53.875590+0000","last_peered":"2023-09-23T22:22:53.875590+0000","last_clean":"2023-09-23T22:22:53.875590+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:22:53.875590+0000","last_undegraded":"2023-09-23T22:22:53.875590+0000","last_fullsized":"2023-09-23T22:22:53.875590+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":232,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869624+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:22:53.869624+0000","last_peered":"2023-09-23T22:22:53.869624+0000","last_clean":"2023-09-23T22:22:53.869624+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:22:53.869624+0000","last_undegraded":"2023-09-23T22:22:53.869624+0000","last_fullsized":"2023-09-23T22:22:53.869624+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"245'6958","reported_seq":10674,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:59.195807+0000","last_change":"2023-09-23T22:22:45.550371+0000","last_active":"2023-09-23T22:22:59.195807+0000","last_peered":"2023-09-23T22:22:59.195807+0000","last_clean":"2023-09-23T22:21:32.851371+0000","last_became_active":"2023-09-23T22:22:45.472694+0000","last_became_peered":"2023-09-23T22:22:45.472694+0000","last_unstale":"2023-09-23T22:22:59.195807+0000","last_undegraded":"2023-09-23T22:22:45.456856+0000","last_fullsized":"2023-09-23T22:22:45.317726+0000","mapping_epoch":240,"log_start":"210'1533","ondisk_log_start":"210'1533","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5425,"log_dups_size":0,"ondisk_log_size":5425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.540906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227409920,"num_objects":3470,"num_object_clones":0,"num_object_copies":10410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5210,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6958,"num_write_kb":333696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1215,"num_bytes_recovered":79495168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"0,1,3","objects":88},{"shards":"0,2,3","objects":495},{"shards":"2,3","objects":1},{"shards":"2,3,7","objects":2886},{"shards":"3,7","objects":-529}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":170,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.873598+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:22:53.873598+0000","last_peered":"2023-09-23T22:22:53.873598+0000","last_clean":"2023-09-23T22:22:53.873598+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:22:53.873598+0000","last_undegraded":"2023-09-23T22:22:53.873598+0000","last_fullsized":"2023-09-23T22:22:53.873598+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":179,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.889019+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:22:53.889019+0000","last_peered":"2023-09-23T22:22:53.889019+0000","last_clean":"2023-09-23T22:22:53.889019+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:22:53.889019+0000","last_undegraded":"2023-09-23T22:22:53.889019+0000","last_fullsized":"2023-09-23T22:22:53.889019+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"245'6938","reported_seq":14455,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:01.440841+0000","last_change":"2023-09-23T22:22:17.938724+0000","last_active":"2023-09-23T22:23:01.440841+0000","last_peered":"2023-09-23T22:23:01.440841+0000","last_clean":"2023-09-23T22:21:12.226652+0000","last_became_active":"2023-09-23T22:21:13.982783+0000","last_became_peered":"2023-09-23T22:21:13.982783+0000","last_unstale":"2023-09-23T22:23:01.440841+0000","last_undegraded":"2023-09-23T22:22:17.930337+0000","last_fullsized":"2023-09-23T22:21:13.795059+0000","mapping_epoch":214,"log_start":"210'2097","ondisk_log_start":"210'2097","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4841,"log_dups_size":0,"ondisk_log_size":4841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.489307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214564864,"num_objects":3274,"num_object_clones":0,"num_object_copies":9822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3274,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6938,"num_write_kb":326784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3728,"num_bytes_recovered":244318208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":38},{"shards":"0,7","objects":3236}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":235,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.874444+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:22:53.874444+0000","last_peered":"2023-09-23T22:22:53.874444+0000","last_clean":"2023-09-23T22:22:53.874444+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:22:53.874444+0000","last_undegraded":"2023-09-23T22:22:53.874444+0000","last_fullsized":"2023-09-23T22:22:53.874444+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":99,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.881441+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:22:53.881441+0000","last_peered":"2023-09-23T22:22:53.881441+0000","last_clean":"2023-09-23T22:22:53.881441+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:22:53.881441+0000","last_undegraded":"2023-09-23T22:22:53.881441+0000","last_fullsized":"2023-09-23T22:22:53.881441+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"245'6855","reported_seq":8451,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:59.862582+0000","last_change":"2023-09-23T22:21:57.837966+0000","last_active":"2023-09-23T22:22:59.862582+0000","last_peered":"2023-09-23T22:22:59.862582+0000","last_clean":"2023-09-23T22:21:12.225315+0000","last_became_active":"2023-09-23T22:21:13.940088+0000","last_became_peered":"2023-09-23T22:21:13.940088+0000","last_unstale":"2023-09-23T22:22:59.862582+0000","last_undegraded":"2023-09-23T22:21:57.835407+0000","last_fullsized":"2023-09-23T22:21:13.800025+0000","mapping_epoch":214,"log_start":"210'1500","ondisk_log_start":"210'1500","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5355,"log_dups_size":0,"ondisk_log_size":5355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:26.245766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219873280,"num_objects":3355,"num_object_clones":0,"num_object_copies":10065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6855,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":768,"num_bytes_recovered":50331648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1958},{"shards":"1,4","objects":415}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":548,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878493+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:22:53.878493+0000","last_peered":"2023-09-23T22:22:53.878493+0000","last_clean":"2023-09-23T22:22:53.878493+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:22:53.878493+0000","last_undegraded":"2023-09-23T22:22:53.878493+0000","last_fullsized":"2023-09-23T22:22:53.878493+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":258,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.901247+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:22:57.901247+0000","last_peered":"2023-09-23T22:22:57.901247+0000","last_clean":"2023-09-23T22:22:57.901247+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:22:57.901247+0000","last_undegraded":"2023-09-23T22:22:57.901247+0000","last_fullsized":"2023-09-23T22:22:57.901247+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869705+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:22:53.869705+0000","last_peered":"2023-09-23T22:22:53.869705+0000","last_clean":"2023-09-23T22:22:53.869705+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:22:53.869705+0000","last_undegraded":"2023-09-23T22:22:53.869705+0000","last_fullsized":"2023-09-23T22:22:53.869705+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"245'6754","reported_seq":8286,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:59.853933+0000","last_change":"2023-09-23T22:22:20.034548+0000","last_active":"2023-09-23T22:22:59.853933+0000","last_peered":"2023-09-23T22:22:59.853933+0000","last_clean":"2023-09-23T22:21:12.194696+0000","last_became_active":"2023-09-23T22:22:19.958601+0000","last_became_peered":"2023-09-23T22:22:19.958601+0000","last_unstale":"2023-09-23T22:22:59.853933+0000","last_undegraded":"2023-09-23T22:22:19.951596+0000","last_fullsized":"2023-09-23T22:22:19.874428+0000","mapping_epoch":232,"log_start":"210'1400","ondisk_log_start":"210'1400","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5354,"log_dups_size":0,"ondisk_log_size":5354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:10.906601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211550208,"num_objects":3228,"num_object_clones":0,"num_object_copies":9684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4172,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6754,"num_write_kb":319424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":486,"num_bytes_recovered":31850496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[4,3],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"1,4,5","objects":506},{"shards":"2,3,4","objects":1904},{"shards":"3,4","objects":-458}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":543,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:53.873062+0000","last_change":"2023-09-23T22:21:59.026062+0000","last_active":"2023-09-23T22:22:53.873062+0000","last_peered":"2023-09-23T22:22:53.873062+0000","last_clean":"2023-09-23T22:21:50.447236+0000","last_became_active":"2023-09-23T22:21:59.021009+0000","last_became_peered":"2023-09-23T22:21:59.021009+0000","last_unstale":"2023-09-23T22:22:53.873062+0000","last_undegraded":"2023-09-23T22:21:58.818399+0000","last_fullsized":"2023-09-23T22:21:58.817464+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:52.228490+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":161,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.870843+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:22:53.870843+0000","last_peered":"2023-09-23T22:22:53.870843+0000","last_clean":"2023-09-23T22:22:53.870843+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:22:53.870843+0000","last_undegraded":"2023-09-23T22:22:53.870843+0000","last_fullsized":"2023-09-23T22:22:53.870843+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":230,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869780+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:22:53.869780+0000","last_peered":"2023-09-23T22:22:53.869780+0000","last_clean":"2023-09-23T22:22:53.869780+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:22:53.869780+0000","last_undegraded":"2023-09-23T22:22:53.869780+0000","last_fullsized":"2023-09-23T22:22:53.869780+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"245'6721","reported_seq":6790,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:00.159152+0000","last_change":"2023-09-23T22:22:18.973077+0000","last_active":"2023-09-23T22:23:00.159152+0000","last_peered":"2023-09-23T22:23:00.159152+0000","last_clean":"2023-09-23T22:21:32.779211+0000","last_became_active":"2023-09-23T22:22:18.949524+0000","last_became_peered":"2023-09-23T22:22:18.949524+0000","last_unstale":"2023-09-23T22:23:00.159152+0000","last_undegraded":"2023-09-23T22:22:18.870000+0000","last_fullsized":"2023-09-23T22:22:18.859180+0000","mapping_epoch":231,"log_start":"209'1200","ondisk_log_start":"209'1200","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.194549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217251840,"num_objects":3315,"num_object_clones":0,"num_object_copies":9945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":914,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6721,"num_write_kb":321152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"1,5,7","objects":487},{"shards":"5,7","objects":2828}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":206,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869495+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:22:53.869495+0000","last_peered":"2023-09-23T22:22:53.869495+0000","last_clean":"2023-09-23T22:22:53.869495+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:22:53.869495+0000","last_undegraded":"2023-09-23T22:22:53.869495+0000","last_fullsized":"2023-09-23T22:22:53.869495+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":221,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.875461+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:22:53.875461+0000","last_peered":"2023-09-23T22:22:53.875461+0000","last_clean":"2023-09-23T22:22:53.875461+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:22:53.875461+0000","last_undegraded":"2023-09-23T22:22:53.875461+0000","last_fullsized":"2023-09-23T22:22:53.875461+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"245'6829","reported_seq":6880,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:23:00.267617+0000","last_change":"2023-09-23T22:20:19.473637+0000","last_active":"2023-09-23T22:23:00.267617+0000","last_peered":"2023-09-23T22:23:00.267617+0000","last_clean":"2023-09-23T22:23:00.267617+0000","last_became_active":"2023-09-23T22:20:19.473385+0000","last_became_peered":"2023-09-23T22:20:19.473385+0000","last_unstale":"2023-09-23T22:23:00.267617+0000","last_undegraded":"2023-09-23T22:23:00.267617+0000","last_fullsized":"2023-09-23T22:23:00.267617+0000","mapping_epoch":203,"log_start":"209'1300","ondisk_log_start":"209'1300","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5529,"log_dups_size":0,"ondisk_log_size":5529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:01.911861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219545624,"num_objects":3351,"num_object_clones":0,"num_object_copies":10053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3351,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6829,"num_write_kb":325697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":549,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.875036+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:22:53.875036+0000","last_peered":"2023-09-23T22:22:53.875036+0000","last_clean":"2023-09-23T22:22:53.875036+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:22:53.875036+0000","last_undegraded":"2023-09-23T22:22:53.875036+0000","last_fullsized":"2023-09-23T22:22:53.875036+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878586+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:22:53.878586+0000","last_peered":"2023-09-23T22:22:53.878586+0000","last_clean":"2023-09-23T22:22:53.878586+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:22:53.878586+0000","last_undegraded":"2023-09-23T22:22:53.878586+0000","last_fullsized":"2023-09-23T22:22:53.878586+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":186,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.923854+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:22:53.923854+0000","last_peered":"2023-09-23T22:22:53.923854+0000","last_clean":"2023-09-23T22:22:53.923854+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:22:53.923854+0000","last_undegraded":"2023-09-23T22:22:53.923854+0000","last_fullsized":"2023-09-23T22:22:53.923854+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"245'6976","reported_seq":8454,"reported_epoch":245,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:01.474029+0000","last_change":"2023-09-23T22:22:43.348452+0000","last_active":"2023-09-23T22:23:01.474029+0000","last_peered":"2023-09-23T22:23:01.474029+0000","last_clean":"2023-09-23T22:21:32.822849+0000","last_became_active":"2023-09-23T22:21:35.128019+0000","last_became_peered":"2023-09-23T22:21:35.128019+0000","last_unstale":"2023-09-23T22:23:01.474029+0000","last_undegraded":"2023-09-23T22:21:34.912513+0000","last_fullsized":"2023-09-23T22:21:34.880959+0000","mapping_epoch":220,"log_start":"210'2047","ondisk_log_start":"210'2047","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4929,"log_dups_size":0,"ondisk_log_size":4929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:58.687801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218365952,"num_objects":3332,"num_object_clones":0,"num_object_copies":9996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1209,"num_objects_misplaced":2684,"num_objects_unfound":0,"num_objects_dirty":3332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6976,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":709,"num_bytes_recovered":46465024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2684},{"shards":"0,3","objects":-561}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":540,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.890476+0000","last_change":"2023-09-23T22:22:47.349660+0000","last_active":"2023-09-23T22:22:53.890476+0000","last_peered":"2023-09-23T22:22:53.890476+0000","last_clean":"2023-09-23T22:22:53.890476+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:22:53.890476+0000","last_undegraded":"2023-09-23T22:22:53.890476+0000","last_fullsized":"2023-09-23T22:22:53.890476+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":793,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.898301+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:22:53.898301+0000","last_peered":"2023-09-23T22:22:53.898301+0000","last_clean":"2023-09-23T22:22:53.898301+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:22:53.898301+0000","last_undegraded":"2023-09-23T22:22:53.898301+0000","last_fullsized":"2023-09-23T22:22:53.898301+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":262,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.873385+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:22:53.873385+0000","last_peered":"2023-09-23T22:22:53.873385+0000","last_clean":"2023-09-23T22:22:53.873385+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:22:53.873385+0000","last_undegraded":"2023-09-23T22:22:53.873385+0000","last_fullsized":"2023-09-23T22:22:53.873385+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":30,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888316+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:22:53.888316+0000","last_peered":"2023-09-23T22:22:53.888316+0000","last_clean":"2023-09-23T22:22:53.888316+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:22:53.888316+0000","last_undegraded":"2023-09-23T22:22:53.888316+0000","last_fullsized":"2023-09-23T22:22:53.888316+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"245'6857","reported_seq":6939,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:59.802532+0000","last_change":"2023-09-23T22:22:18.955534+0000","last_active":"2023-09-23T22:22:59.802532+0000","last_peered":"2023-09-23T22:22:59.802532+0000","last_clean":"2023-09-23T22:21:32.860473+0000","last_became_active":"2023-09-23T22:22:18.941549+0000","last_became_peered":"2023-09-23T22:22:18.941549+0000","last_unstale":"2023-09-23T22:22:59.802532+0000","last_undegraded":"2023-09-23T22:22:18.875275+0000","last_fullsized":"2023-09-23T22:22:18.863381+0000","mapping_epoch":231,"log_start":"210'1500","ondisk_log_start":"210'1500","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5357,"log_dups_size":0,"ondisk_log_size":5357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:27.412277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221184000,"num_objects":3375,"num_object_clones":0,"num_object_copies":10125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":988,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6857,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,5,7","objects":527},{"shards":"4,7","objects":2848}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":549,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.873325+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:22:53.873325+0000","last_peered":"2023-09-23T22:22:53.873325+0000","last_clean":"2023-09-23T22:22:53.873325+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:22:53.873325+0000","last_undegraded":"2023-09-23T22:22:53.873325+0000","last_fullsized":"2023-09-23T22:22:53.873325+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":977,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.875210+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:22:53.875210+0000","last_peered":"2023-09-23T22:22:53.875210+0000","last_clean":"2023-09-23T22:22:53.875210+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:22:53.875210+0000","last_undegraded":"2023-09-23T22:22:53.875210+0000","last_fullsized":"2023-09-23T22:22:53.875210+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":254,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869708+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:22:53.869708+0000","last_peered":"2023-09-23T22:22:53.869708+0000","last_clean":"2023-09-23T22:22:53.869708+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:22:53.869708+0000","last_undegraded":"2023-09-23T22:22:53.869708+0000","last_fullsized":"2023-09-23T22:22:53.869708+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":209,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.874536+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:22:53.874536+0000","last_peered":"2023-09-23T22:22:53.874536+0000","last_clean":"2023-09-23T22:22:53.874536+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:22:53.874536+0000","last_undegraded":"2023-09-23T22:22:53.874536+0000","last_fullsized":"2023-09-23T22:22:53.874536+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"245'6653","reported_seq":6712,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:59.189622+0000","last_change":"2023-09-23T22:21:59.013998+0000","last_active":"2023-09-23T22:22:59.189622+0000","last_peered":"2023-09-23T22:22:59.189622+0000","last_clean":"2023-09-23T22:21:56.814967+0000","last_became_active":"2023-09-23T22:21:59.002385+0000","last_became_peered":"2023-09-23T22:21:59.002385+0000","last_unstale":"2023-09-23T22:22:59.189622+0000","last_undegraded":"2023-09-23T22:21:58.865154+0000","last_fullsized":"2023-09-23T22:21:58.825566+0000","mapping_epoch":226,"log_start":"209'1220","ondisk_log_start":"209'1220","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5433,"log_dups_size":0,"ondisk_log_size":5433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:00.273421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217645056,"num_objects":3321,"num_object_clones":0,"num_object_copies":9963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4918,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6653,"num_write_kb":319168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":3252},{"shards":"3,7","objects":-1597}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":573,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.879581+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:22:53.879581+0000","last_peered":"2023-09-23T22:22:53.879581+0000","last_clean":"2023-09-23T22:22:53.879581+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:22:53.879581+0000","last_undegraded":"2023-09-23T22:22:53.879581+0000","last_fullsized":"2023-09-23T22:22:53.879581+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":351,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:22:53.877299+0000","last_change":"2023-09-23T22:22:18.917671+0000","last_active":"2023-09-23T22:22:53.877299+0000","last_peered":"2023-09-23T22:22:53.877299+0000","last_clean":"2023-09-23T22:15:39.913392+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:22:53.877299+0000","last_undegraded":"2023-09-23T22:22:18.874220+0000","last_fullsized":"2023-09-23T22:22:53.877299+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":205,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.887635+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:22:53.887635+0000","last_peered":"2023-09-23T22:22:53.887635+0000","last_clean":"2023-09-23T22:22:53.887635+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:22:53.887635+0000","last_undegraded":"2023-09-23T22:22:53.887635+0000","last_fullsized":"2023-09-23T22:22:53.887635+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":251,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.929946+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:22:53.929946+0000","last_peered":"2023-09-23T22:22:53.929946+0000","last_clean":"2023-09-23T22:22:53.929946+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:22:53.929946+0000","last_undegraded":"2023-09-23T22:22:53.929946+0000","last_fullsized":"2023-09-23T22:22:53.929946+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"245'6931","reported_seq":6980,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:23:00.248330+0000","last_change":"2023-09-23T22:20:19.470871+0000","last_active":"2023-09-23T22:23:00.248330+0000","last_peered":"2023-09-23T22:23:00.248330+0000","last_clean":"2023-09-23T22:23:00.248330+0000","last_became_active":"2023-09-23T22:20:19.470597+0000","last_became_peered":"2023-09-23T22:20:19.470597+0000","last_unstale":"2023-09-23T22:23:00.248330+0000","last_undegraded":"2023-09-23T22:23:00.248330+0000","last_fullsized":"2023-09-23T22:23:00.248330+0000","mapping_epoch":203,"log_start":"210'1400","ondisk_log_start":"210'1400","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5531,"log_dups_size":0,"ondisk_log_size":5531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:32.388985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222887936,"num_objects":3401,"num_object_clones":0,"num_object_copies":10203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6931,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":840,"reported_epoch":244,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:53.873426+0000","last_change":"2023-09-23T22:21:58.853256+0000","last_active":"2023-09-23T22:22:53.873426+0000","last_peered":"2023-09-23T22:22:53.873426+0000","last_clean":"2023-09-23T22:21:50.447508+0000","last_became_active":"2023-09-23T22:21:58.852815+0000","last_became_peered":"2023-09-23T22:21:58.852815+0000","last_unstale":"2023-09-23T22:22:53.873426+0000","last_undegraded":"2023-09-23T22:21:58.818640+0000","last_fullsized":"2023-09-23T22:21:58.817793+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:25.619025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":557,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.914820+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:22:57.914820+0000","last_peered":"2023-09-23T22:22:57.914820+0000","last_clean":"2023-09-23T22:22:57.914820+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:22:57.914820+0000","last_undegraded":"2023-09-23T22:22:57.914820+0000","last_fullsized":"2023-09-23T22:22:57.914820+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":169,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878980+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:22:53.878980+0000","last_peered":"2023-09-23T22:22:53.878980+0000","last_clean":"2023-09-23T22:22:53.878980+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:22:53.878980+0000","last_undegraded":"2023-09-23T22:22:53.878980+0000","last_fullsized":"2023-09-23T22:22:53.878980+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":131,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869961+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:22:53.869961+0000","last_peered":"2023-09-23T22:22:53.869961+0000","last_clean":"2023-09-23T22:22:53.869961+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:22:53.869961+0000","last_undegraded":"2023-09-23T22:22:53.869961+0000","last_fullsized":"2023-09-23T22:22:53.869961+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"245'6693","reported_seq":6766,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:58.708773+0000","last_change":"2023-09-23T22:22:18.954521+0000","last_active":"2023-09-23T22:22:58.708773+0000","last_peered":"2023-09-23T22:22:58.708773+0000","last_clean":"2023-09-23T22:21:32.869170+0000","last_became_active":"2023-09-23T22:22:18.945835+0000","last_became_peered":"2023-09-23T22:22:18.945835+0000","last_unstale":"2023-09-23T22:22:58.708773+0000","last_undegraded":"2023-09-23T22:22:18.869474+0000","last_fullsized":"2023-09-23T22:22:18.861404+0000","mapping_epoch":231,"log_start":"210'2005","ondisk_log_start":"210'2005","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4688,"log_dups_size":0,"ondisk_log_size":4688,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:02.907936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220004352,"num_objects":3357,"num_object_clones":0,"num_object_copies":10071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6693,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,4,5","objects":591},{"shards":"1,5","objects":2766}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":576,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888953+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:22:53.888953+0000","last_peered":"2023-09-23T22:22:53.888953+0000","last_clean":"2023-09-23T22:22:53.888953+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:22:53.888953+0000","last_undegraded":"2023-09-23T22:22:53.888953+0000","last_fullsized":"2023-09-23T22:22:53.888953+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":594,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.887851+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:22:53.887851+0000","last_peered":"2023-09-23T22:22:53.887851+0000","last_clean":"2023-09-23T22:22:53.887851+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:22:53.887851+0000","last_undegraded":"2023-09-23T22:22:53.887851+0000","last_fullsized":"2023-09-23T22:22:53.887851+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":126,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.877980+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:22:53.877980+0000","last_peered":"2023-09-23T22:22:53.877980+0000","last_clean":"2023-09-23T22:22:53.877980+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:22:53.877980+0000","last_undegraded":"2023-09-23T22:22:53.877980+0000","last_fullsized":"2023-09-23T22:22:53.877980+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":83,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878445+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:22:53.878445+0000","last_peered":"2023-09-23T22:22:53.878445+0000","last_clean":"2023-09-23T22:22:53.878445+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:22:53.878445+0000","last_undegraded":"2023-09-23T22:22:53.878445+0000","last_fullsized":"2023-09-23T22:22:53.878445+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"245'6750","reported_seq":10287,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:58.700849+0000","last_change":"2023-09-23T22:22:45.524797+0000","last_active":"2023-09-23T22:22:58.700849+0000","last_peered":"2023-09-23T22:22:58.700849+0000","last_clean":"2023-09-23T22:21:12.245233+0000","last_became_active":"2023-09-23T22:22:45.454018+0000","last_became_peered":"2023-09-23T22:22:45.454018+0000","last_unstale":"2023-09-23T22:22:58.700849+0000","last_undegraded":"2023-09-23T22:22:45.437539+0000","last_fullsized":"2023-09-23T22:22:45.322462+0000","mapping_epoch":240,"log_start":"211'2130","ondisk_log_start":"211'2130","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4620,"log_dups_size":0,"ondisk_log_size":4620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:10.659044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228589568,"num_objects":3488,"num_object_clones":0,"num_object_copies":10464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6750,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1153,"num_bytes_recovered":75563008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1","objects":1},{"shards":"0,1,5","objects":565},{"shards":"1,2,7","objects":2037},{"shards":"1,7","objects":407}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":565,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878689+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:22:53.878689+0000","last_peered":"2023-09-23T22:22:53.878689+0000","last_clean":"2023-09-23T22:22:53.878689+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:22:53.878689+0000","last_undegraded":"2023-09-23T22:22:53.878689+0000","last_fullsized":"2023-09-23T22:22:53.878689+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":135,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.881797+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:22:53.881797+0000","last_peered":"2023-09-23T22:22:53.881797+0000","last_clean":"2023-09-23T22:22:53.881797+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:22:53.881797+0000","last_undegraded":"2023-09-23T22:22:53.881797+0000","last_fullsized":"2023-09-23T22:22:53.881797+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":195,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.898559+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:22:53.898559+0000","last_peered":"2023-09-23T22:22:53.898559+0000","last_clean":"2023-09-23T22:22:53.898559+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:22:53.898559+0000","last_undegraded":"2023-09-23T22:22:53.898559+0000","last_fullsized":"2023-09-23T22:22:53.898559+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":99,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.881361+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:22:53.881361+0000","last_peered":"2023-09-23T22:22:53.881361+0000","last_clean":"2023-09-23T22:22:53.881361+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:22:53.881361+0000","last_undegraded":"2023-09-23T22:22:53.881361+0000","last_fullsized":"2023-09-23T22:22:53.881361+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"245'6876","reported_seq":10444,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:59.871310+0000","last_change":"2023-09-23T22:22:19.043755+0000","last_active":"2023-09-23T22:22:59.871310+0000","last_peered":"2023-09-23T22:22:59.871310+0000","last_clean":"2023-09-23T22:21:12.214088+0000","last_became_active":"2023-09-23T22:22:18.986181+0000","last_became_peered":"2023-09-23T22:22:18.986181+0000","last_unstale":"2023-09-23T22:22:59.871310+0000","last_undegraded":"2023-09-23T22:22:18.975289+0000","last_fullsized":"2023-09-23T22:22:18.862148+0000","mapping_epoch":231,"log_start":"210'1600","ondisk_log_start":"210'1600","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5276,"log_dups_size":0,"ondisk_log_size":5276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:34.422545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225181696,"num_objects":3436,"num_object_clones":0,"num_object_copies":10308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3863,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6876,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1743,"num_bytes_recovered":114229248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,3,4","objects":1981},{"shards":"2,4,7","objects":1349},{"shards":"4,5,7","objects":83},{"shards":"4,7","objects":23}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":549,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869647+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:22:53.869647+0000","last_peered":"2023-09-23T22:22:53.869647+0000","last_clean":"2023-09-23T22:22:53.869647+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:22:53.869647+0000","last_undegraded":"2023-09-23T22:22:53.869647+0000","last_fullsized":"2023-09-23T22:22:53.869647+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":258,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.890722+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:22:53.890722+0000","last_peered":"2023-09-23T22:22:53.890722+0000","last_clean":"2023-09-23T22:22:53.890722+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:22:53.890722+0000","last_undegraded":"2023-09-23T22:22:53.890722+0000","last_fullsized":"2023-09-23T22:22:53.890722+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":886,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.893835+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:22:53.893835+0000","last_peered":"2023-09-23T22:22:53.893835+0000","last_clean":"2023-09-23T22:22:53.893835+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:22:53.893835+0000","last_undegraded":"2023-09-23T22:22:53.893835+0000","last_fullsized":"2023-09-23T22:22:53.893835+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":33,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878351+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:22:53.878351+0000","last_peered":"2023-09-23T22:22:53.878351+0000","last_clean":"2023-09-23T22:22:53.878351+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:22:53.878351+0000","last_undegraded":"2023-09-23T22:22:53.878351+0000","last_fullsized":"2023-09-23T22:22:53.878351+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"245'6638","reported_seq":6831,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:58.658161+0000","last_change":"2023-09-23T22:22:44.336083+0000","last_active":"2023-09-23T22:22:58.658161+0000","last_peered":"2023-09-23T22:22:58.658161+0000","last_clean":"2023-09-23T22:21:56.808651+0000","last_became_active":"2023-09-23T22:21:59.059105+0000","last_became_peered":"2023-09-23T22:21:59.059105+0000","last_unstale":"2023-09-23T22:22:58.658161+0000","last_undegraded":"2023-09-23T22:21:58.989636+0000","last_fullsized":"2023-09-23T22:21:58.812134+0000","mapping_epoch":226,"log_start":"210'2002","ondisk_log_start":"210'2002","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4636,"log_dups_size":0,"ondisk_log_size":4636,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:51.370288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219676672,"num_objects":3352,"num_object_clones":0,"num_object_copies":10056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6638,"num_write_kb":319680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":4259840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":3273},{"shards":"1,7","objects":-1500}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":532,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.879517+0000","last_change":"2023-09-23T22:22:19.883431+0000","last_active":"2023-09-23T22:22:53.879517+0000","last_peered":"2023-09-23T22:22:53.879517+0000","last_clean":"2023-09-23T22:22:53.879517+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:22:53.879517+0000","last_undegraded":"2023-09-23T22:22:53.879517+0000","last_fullsized":"2023-09-23T22:22:53.879517+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:46.903071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:46.903071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":226,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:22:53.884248+0000","last_change":"2023-09-23T22:22:44.342884+0000","last_active":"2023-09-23T22:22:53.884248+0000","last_peered":"2023-09-23T22:22:53.884248+0000","last_clean":"2023-09-23T22:06:25.449250+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:22:53.884248+0000","last_undegraded":"2023-09-23T22:22:44.329336+0000","last_fullsized":"2023-09-23T22:22:53.884248+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":78,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878603+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:22:53.878603+0000","last_peered":"2023-09-23T22:22:53.878603+0000","last_clean":"2023-09-23T22:22:53.878603+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:22:53.878603+0000","last_undegraded":"2023-09-23T22:22:53.878603+0000","last_fullsized":"2023-09-23T22:22:53.878603+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":208,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869539+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:22:53.869539+0000","last_peered":"2023-09-23T22:22:53.869539+0000","last_clean":"2023-09-23T22:22:53.869539+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:22:53.869539+0000","last_undegraded":"2023-09-23T22:22:53.869539+0000","last_fullsized":"2023-09-23T22:22:53.869539+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":74,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.881277+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:22:53.881277+0000","last_peered":"2023-09-23T22:22:53.881277+0000","last_clean":"2023-09-23T22:22:53.881277+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:22:53.881277+0000","last_undegraded":"2023-09-23T22:22:53.881277+0000","last_fullsized":"2023-09-23T22:22:53.881277+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":255,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.898484+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:22:53.898484+0000","last_peered":"2023-09-23T22:22:53.898484+0000","last_clean":"2023-09-23T22:22:53.898484+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:22:53.898484+0000","last_undegraded":"2023-09-23T22:22:53.898484+0000","last_fullsized":"2023-09-23T22:22:53.898484+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":207,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.874442+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:22:53.874442+0000","last_peered":"2023-09-23T22:22:53.874442+0000","last_clean":"2023-09-23T22:22:53.874442+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:22:53.874442+0000","last_undegraded":"2023-09-23T22:22:53.874442+0000","last_fullsized":"2023-09-23T22:22:53.874442+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":220,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869869+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:22:53.869869+0000","last_peered":"2023-09-23T22:22:53.869869+0000","last_clean":"2023-09-23T22:22:53.869869+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:22:53.869869+0000","last_undegraded":"2023-09-23T22:22:53.869869+0000","last_fullsized":"2023-09-23T22:22:53.869869+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":204,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.873458+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:22:53.873458+0000","last_peered":"2023-09-23T22:22:53.873458+0000","last_clean":"2023-09-23T22:22:53.873458+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:22:53.873458+0000","last_undegraded":"2023-09-23T22:22:53.873458+0000","last_fullsized":"2023-09-23T22:22:53.873458+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":79,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.870919+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:22:53.870919+0000","last_peered":"2023-09-23T22:22:53.870919+0000","last_clean":"2023-09-23T22:22:53.870919+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:22:53.870919+0000","last_undegraded":"2023-09-23T22:22:53.870919+0000","last_fullsized":"2023-09-23T22:22:53.870919+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":225,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888462+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:22:53.888462+0000","last_peered":"2023-09-23T22:22:53.888462+0000","last_clean":"2023-09-23T22:22:53.888462+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:22:53.888462+0000","last_undegraded":"2023-09-23T22:22:53.888462+0000","last_fullsized":"2023-09-23T22:22:53.888462+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":266,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878578+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:22:53.878578+0000","last_peered":"2023-09-23T22:22:53.878578+0000","last_clean":"2023-09-23T22:22:53.878578+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:22:53.878578+0000","last_undegraded":"2023-09-23T22:22:53.878578+0000","last_fullsized":"2023-09-23T22:22:53.878578+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":133,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.879940+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:22:53.879940+0000","last_peered":"2023-09-23T22:22:53.879940+0000","last_clean":"2023-09-23T22:22:53.879940+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:22:53.879940+0000","last_undegraded":"2023-09-23T22:22:53.879940+0000","last_fullsized":"2023-09-23T22:22:53.879940+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":253,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869421+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:22:53.869421+0000","last_peered":"2023-09-23T22:22:53.869421+0000","last_clean":"2023-09-23T22:22:53.869421+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:22:53.869421+0000","last_undegraded":"2023-09-23T22:22:53.869421+0000","last_fullsized":"2023-09-23T22:22:53.869421+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3516006424,"num_objects":53651,"num_object_clones":0,"num_object_copies":160953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35148,"num_objects_misplaced":7226,"num_objects_unfound":0,"num_objects_dirty":53651,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":109045,"num_write_kb":5206209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15294,"num_bytes_recovered":1002176512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12394770432,"data_stored":12394758216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83420,"ondisk_log_size":83420,"up":48,"acting":35,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":370028,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":88,"num_read_kb":166,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234223,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2048304,"kb_used_data":1887848,"kb_used_omap":0,"kb_used_meta":160448,"kb_avail":91696848,"statfs":{"total":95995035648,"available":93897572352,"internally_reserved":0,"allocated":1933156352,"data_stored":1932170906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":164298752},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:01 2023","interfaces":[{"interface":"back","average":{"1min":7.0620000000000003,"5min":2.8879999999999999,"15min":1.7010000000000001},"min":{"1min":0.28999999999999998,"5min":0.22,"15min":0.22},"max":{"1min":94.468000000000004,"5min":94.468000000000004,"15min":109.208},"last":9.9000000000000004},{"interface":"front","average":{"1min":7.5259999999999998,"5min":3.0249999999999999,"15min":1.877},"min":{"1min":0.28899999999999998,"5min":0.217,"15min":0.217},"max":{"1min":94.638000000000005,"5min":94.638000000000005,"15min":109.13800000000001},"last":9.9450000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:22:10 2023","interfaces":[{"interface":"back","average":{"1min":7.3230000000000004,"5min":5.9000000000000004,"15min":2.8290000000000002},"min":{"1min":0.29799999999999999,"5min":0.254,"15min":0.246},"max":{"1min":99.385999999999996,"5min":122.461,"15min":123.79300000000001},"last":10.127000000000001},{"interface":"front","average":{"1min":7.8239999999999998,"5min":6.1680000000000001,"15min":2.9980000000000002},"min":{"1min":0.28299999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":95.441999999999993,"5min":122.268,"15min":123.66500000000001},"last":10.92}]},{"osd":2,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":4.8609999999999998,"15min":3.6600000000000001},"min":{"1min":0.35299999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.5119999999999996,"5min":105.34399999999999,"15min":125.101},"last":0.44},{"interface":"front","average":{"1min":1.7470000000000001,"5min":4.6589999999999998,"15min":3.375},"min":{"1min":0.34699999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.0549999999999997,"5min":105.63200000000001,"15min":126.021},"last":0.35299999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:22:10 2023","interfaces":[{"interface":"back","average":{"1min":4.3239999999999998,"5min":2.0979999999999999,"15min":1.5489999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":39.073999999999998,"5min":39.073999999999998,"15min":113.282},"last":13.057},{"interface":"front","average":{"1min":4.6589999999999998,"5min":2.452,"15min":1.7250000000000001},"min":{"1min":0.39800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":38.781999999999996,"5min":38.781999999999996,"15min":104.74299999999999},"last":10.061}]},{"osd":4,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":2.3980000000000001,"15min":1.2969999999999999},"min":{"1min":0.29699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":27.922000000000001,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.377},{"interface":"front","average":{"1min":2.738,"5min":2.3719999999999999,"15min":1.2669999999999999},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":29.920000000000002,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.45200000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":2.4279999999999999,"15min":1.226},"min":{"1min":0.32700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.965,"5min":93.816000000000003,"15min":93.816000000000003},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.4710000000000001,"5min":2.5230000000000001,"15min":1.4419999999999999},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.29799999999999999},"max":{"1min":24.712,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.80200000000000005,"15min":0.71399999999999997},"min":{"1min":0.34200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.343,"5min":7.9459999999999997,"15min":9.0139999999999993},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.153,"15min":0.91500000000000004},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":6.3879999999999999,"5min":10.347,"15min":13.362},"last":0.47999999999999998}]}]},{"osd":6,"up_from":0,"seq":197568495859,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795131,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1846816,"kb_used_data":1567832,"kb_used_omap":7,"kb_used_meta":278968,"kb_avail":91898336,"statfs":{"total":95995035648,"available":94103896064,"internally_reserved":0,"allocated":1605459968,"data_stored":1604137718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":285663604},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,0,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4570000000000001,"5min":1.4690000000000001,"15min":1.014},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":11.138,"5min":11.138,"15min":61.890000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.5819999999999999,"5min":2.6139999999999999,"15min":1.4119999999999999},"min":{"1min":0.23200000000000001,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":15.768000000000001,"5min":15.768000000000001,"15min":62.689999999999998},"last":0.48999999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":2.0179999999999998,"15min":1.1850000000000001},"min":{"1min":0.252,"5min":0.223,"15min":0.216},"max":{"1min":11.179,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.1480000000000001,"5min":3.2370000000000001,"15min":1.5049999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.2670000000000003,"5min":34.781999999999996,"15min":34.781999999999996},"last":7.1449999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:22:20 2023","interfaces":[{"interface":"back","average":{"1min":6.1100000000000003,"5min":3.3300000000000001,"15min":1.5169999999999999},"min":{"1min":0.32700000000000001,"5min":0.252,"15min":0.252},"max":{"1min":44.902999999999999,"5min":44.902999999999999,"15min":44.902999999999999},"last":7.0990000000000002},{"interface":"front","average":{"1min":5.7279999999999998,"5min":3.0529999999999999,"15min":1.4770000000000001},"min":{"1min":0.28199999999999997,"5min":0.223,"15min":0.223},"max":{"1min":45.725000000000001,"5min":45.725000000000001,"15min":45.725000000000001},"last":4.0810000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:22:20 2023","interfaces":[{"interface":"back","average":{"1min":11.180999999999999,"5min":3.1230000000000002,"15min":1.526},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":234.79900000000001,"5min":234.79900000000001,"15min":234.79900000000001},"last":7.306},{"interface":"front","average":{"1min":12.012,"5min":3.9830000000000001,"15min":1.7869999999999999},"min":{"1min":0.35799999999999998,"5min":0.33200000000000002,"15min":0.318},"max":{"1min":234.81700000000001,"5min":234.81700000000001,"15min":234.81700000000001},"last":2.2610000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":10.313000000000001,"5min":3.6800000000000002,"15min":1.659},"min":{"1min":0.33100000000000002,"5min":0.31,"15min":0.31},"max":{"1min":196.79900000000001,"5min":196.79900000000001,"15min":196.79900000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":10.337,"5min":3.4889999999999999,"15min":1.645},"min":{"1min":0.41899999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":196.68299999999999,"5min":196.68299999999999,"15min":196.68299999999999},"last":4.0339999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.841,"15min":1.099},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":5.5629999999999997,"5min":10.989000000000001,"15min":10.989000000000001},"last":3.8479999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.3,"15min":0.95399999999999996},"min":{"1min":0.40999999999999998,"5min":0.35699999999999998,"15min":0.34999999999999998},"max":{"1min":11.385999999999999,"5min":11.385999999999999,"15min":11.385999999999999},"last":7.2649999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.9610000000000001,"15min":1.1519999999999999},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.5970000000000004,"5min":10.948,"15min":10.948},"last":4.1390000000000002},{"interface":"front","average":{"1min":1.159,"5min":1.389,"15min":0.98399999999999999},"min":{"1min":0.38300000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":11.204000000000001,"5min":11.204000000000001,"15min":11.204000000000001},"last":0.61499999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958666,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263620,"kb_used_data":988540,"kb_used_omap":8,"kb_used_meta":274999,"kb_avail":92481532,"statfs":{"total":95995035648,"available":94701088768,"internally_reserved":0,"allocated":1012264960,"data_stored":1010912720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":281599850},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":12.625999999999999,"5min":4.423,"15min":1.95},"min":{"1min":0.22,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":169.77199999999999,"5min":169.77199999999999,"15min":169.77199999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":12.736000000000001,"5min":5.3659999999999997,"15min":2.3439999999999999},"min":{"1min":0.29199999999999998,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":169.52099999999999,"5min":169.52099999999999,"15min":169.52099999999999},"last":0.56499999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:22:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":6.5620000000000003,"15min":2.5870000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":10.891,"5min":206.495,"15min":206.495},"last":2.8450000000000002},{"interface":"front","average":{"1min":1.8759999999999999,"5min":7.71,"15min":3.1890000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.22800000000000001},"max":{"1min":15.186,"5min":207.34399999999999,"15min":207.34399999999999},"last":1.282}]},{"osd":3,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":2.746,"5min":3.9969999999999999,"15min":1.869},"min":{"1min":0.28000000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.848000000000001,"5min":131.078,"15min":131.078},"last":2.8769999999999998},{"interface":"front","average":{"1min":1.655,"5min":4.008,"15min":1.9319999999999999},"min":{"1min":0.247,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.8289999999999997,"5min":133.083,"15min":133.083},"last":9.6549999999999994}]},{"osd":4,"last update":"Sat Sep 23 22:22:00 2023","interfaces":[{"interface":"back","average":{"1min":6.7309999999999999,"5min":3.468,"15min":1.7270000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":82.825000000000003,"5min":82.825000000000003,"15min":82.825000000000003},"last":1.3819999999999999},{"interface":"front","average":{"1min":6.101,"5min":2.3279999999999998,"15min":1.228},"min":{"1min":0.35499999999999998,"5min":0.313,"15min":0.308},"max":{"1min":75.930999999999997,"5min":75.930999999999997,"15min":75.930999999999997},"last":2.7919999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:22:00 2023","interfaces":[{"interface":"back","average":{"1min":5.8769999999999998,"5min":3.3260000000000001,"15min":1.6779999999999999},"min":{"1min":0.36899999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":76.203000000000003,"5min":76.203000000000003,"15min":76.203000000000003},"last":1.411},{"interface":"front","average":{"1min":5.9779999999999998,"5min":2.2669999999999999,"15min":1.238},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":72.576999999999998,"5min":72.576999999999998,"15min":72.576999999999998},"last":8.7899999999999991}]},{"osd":6,"last update":"Sat Sep 23 22:22:00 2023","interfaces":[{"interface":"back","average":{"1min":1.708,"5min":2.464,"15min":1.417},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":6.0839999999999996,"5min":10.275,"15min":10.275},"last":2.1040000000000001},{"interface":"front","average":{"1min":1.7330000000000001,"5min":1.536,"15min":1.004},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":7.0819999999999999,"5min":22.263000000000002,"15min":22.263000000000002},"last":6.6639999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:22:00 2023","interfaces":[{"interface":"back","average":{"1min":1.915,"5min":2.1200000000000001,"15min":1.421},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":11.259,"5min":20.491,"15min":23.306000000000001},"last":0.5},{"interface":"front","average":{"1min":1.7090000000000001,"5min":2.4670000000000001,"15min":1.597},"min":{"1min":0.42999999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":6.141,"5min":10.257999999999999,"15min":23.265999999999998},"last":1.4670000000000001}]}]},{"osd":2,"up_from":0,"seq":442381631605,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468078,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2499264,"kb_used_data":2327352,"kb_used_omap":4,"kb_used_meta":171835,"kb_avail":91245888,"statfs":{"total":95995035648,"available":93435789312,"internally_reserved":0,"allocated":2383208448,"data_stored":2381854508,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":175959597},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":3.3370000000000002,"5min":5.6219999999999999,"15min":2.5230000000000001},"min":{"1min":0.28599999999999998,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":18.292000000000002,"5min":225.86699999999999,"15min":225.86699999999999},"last":1.4299999999999999},{"interface":"front","average":{"1min":3.298,"5min":6.0110000000000001,"15min":2.6600000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":16.117999999999999,"5min":225.84800000000001,"15min":225.84800000000001},"last":1.3859999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":5.343,"5min":4.2670000000000003,"15min":2.0449999999999999},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.218},"max":{"1min":37.002000000000002,"5min":82.951999999999998,"15min":82.951999999999998},"last":1.417},{"interface":"front","average":{"1min":4.04,"5min":3.7949999999999999,"15min":1.9279999999999999},"min":{"1min":0.373,"5min":0.22600000000000001,"15min":0.189},"max":{"1min":33.119999999999997,"5min":86.001000000000005,"15min":86.001000000000005},"last":1.796}]},{"osd":2,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":5.8849999999999998,"15min":3.2959999999999998},"min":{"1min":0.26500000000000001,"5min":0.252,"15min":0.223},"max":{"1min":10.23,"5min":99.962999999999994,"15min":99.962999999999994},"last":1.3400000000000001},{"interface":"front","average":{"1min":2.387,"5min":5.8840000000000003,"15min":3.089},"min":{"1min":0.34300000000000003,"5min":0.251,"15min":0.251},"max":{"1min":8.2080000000000002,"5min":100.429,"15min":100.429},"last":1.8600000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:22:17 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":1.96,"15min":1.105},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":7.5620000000000003,"5min":11.911,"15min":11.911},"last":2.734},{"interface":"front","average":{"1min":1.8100000000000001,"5min":1.9510000000000001,"15min":1.1220000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.3579999999999997,"5min":12.036,"15min":12.036},"last":1.512}]},{"osd":5,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":2.3620000000000001,"15min":1.248},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":7.1719999999999997,"5min":49.695999999999998,"15min":49.695999999999998},"last":1.5509999999999999},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.601,"15min":1.3120000000000001},"min":{"1min":0.39800000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.031000000000001,"5min":49.668999999999997,"15min":49.668999999999997},"last":1.698}]},{"osd":6,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":2.0680000000000001,"15min":1.2090000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.1780000000000008,"5min":12.068,"15min":12.068},"last":1.7609999999999999},{"interface":"front","average":{"1min":1.881,"5min":2.0019999999999998,"15min":1.157},"min":{"1min":0.41799999999999998,"5min":0.35599999999999998,"15min":0.35399999999999998},"max":{"1min":9.2870000000000008,"5min":11.68,"15min":11.68},"last":1.732}]},{"osd":7,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.2869999999999999,"15min":1.2450000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":8.218,"5min":16.849,"15min":16.849},"last":1.6359999999999999},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.9590000000000001,"15min":1.1619999999999999},"min":{"1min":0.433,"5min":0.39300000000000002,"15min":0.36699999999999999},"max":{"1min":8.1400000000000006,"5min":12.382,"15min":12.382},"last":1.833}]}]},{"osd":4,"up_from":30,"seq":128849019128,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518452,"kb_used_data":1343788,"kb_used_omap":0,"kb_used_meta":174656,"kb_avail":92226700,"statfs":{"total":95995035648,"available":94440140800,"internally_reserved":0,"allocated":1376038912,"data_stored":1375050177,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":178847744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.99,"5min":2.5670000000000002,"15min":1.53},"min":{"1min":0.27900000000000003,"5min":0.23000000000000001,"15min":0.218},"max":{"1min":9.4049999999999994,"5min":46.18,"15min":66.355000000000004},"last":9.2579999999999991},{"interface":"front","average":{"1min":2.028,"5min":2.7290000000000001,"15min":1.6200000000000001},"min":{"1min":0.373,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":9.4280000000000008,"5min":46.207000000000001,"15min":66.332999999999998},"last":9.2390000000000008}]},{"osd":1,"last update":"Sat Sep 23 22:22:08 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":2.1949999999999998,"15min":1.2829999999999999},"min":{"1min":0.23699999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.9500000000000002,"5min":35.409999999999997,"15min":38.878},"last":0.28299999999999997},{"interface":"front","average":{"1min":1.4510000000000001,"5min":2.621,"15min":1.581},"min":{"1min":0.34599999999999997,"5min":0.255,"15min":0.255},"max":{"1min":5.3949999999999996,"5min":35.999000000000002,"15min":38.853999999999999},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9159999999999999,"5min":4.5730000000000004,"15min":2.1589999999999998},"min":{"1min":0.372,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":6.1020000000000003,"5min":138.06100000000001,"15min":138.06100000000001},"last":10.051},{"interface":"front","average":{"1min":1.8120000000000001,"5min":4.6079999999999997,"15min":2.327},"min":{"1min":0.35899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9580000000000002,"5min":138.15199999999999,"15min":138.15199999999999},"last":0.48099999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.78,"15min":1.073},"min":{"1min":0.34899999999999998,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":3.4009999999999998,"5min":8.6340000000000003,"15min":8.6340000000000003},"last":4.1580000000000004},{"interface":"front","average":{"1min":1.968,"5min":1.8200000000000001,"15min":1.0940000000000001},"min":{"1min":0.34699999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.5970000000000004,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":3.742}]},{"osd":5,"last update":"Sat Sep 23 22:22:20 2023","interfaces":[{"interface":"back","average":{"1min":2.5249999999999999,"5min":1.3100000000000001,"15min":0.82499999999999996},"min":{"1min":0.25800000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":28.399000000000001,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.9350000000000001,"5min":1.3120000000000001,"15min":0.83099999999999996},"min":{"1min":0.32600000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":28.350999999999999,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:22:33 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.82699999999999996,"15min":0.67000000000000004},"min":{"1min":0.29299999999999998,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":3.3519999999999999,"5min":7.3079999999999998,"15min":7.3079999999999998},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.86899999999999999,"15min":0.72499999999999998},"min":{"1min":0.32200000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0840000000000001,"5min":6.7279999999999998,"15min":8.5210000000000008},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:22:08 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.0780000000000001,"15min":0.81100000000000005},"min":{"1min":0.34100000000000003,"5min":0.32400000000000001,"15min":0.307},"max":{"1min":5.2590000000000003,"5min":8.577,"15min":8.577},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.0429999999999999,"5min":0.89400000000000002,"15min":0.745},"min":{"1min":0.38500000000000001,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":5.8949999999999996,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.44500000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757496,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1520364,"kb_used_data":1320036,"kb_used_omap":0,"kb_used_meta":200320,"kb_avail":92224788,"statfs":{"total":95995035648,"available":94438182912,"internally_reserved":0,"allocated":1351716864,"data_stored":1350732098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":205127680},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:55 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":2.1150000000000002,"15min":1.859},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":12.661,"5min":28.741,"15min":133.22499999999999},"last":64.344999999999999},{"interface":"front","average":{"1min":3.198,"5min":1.8959999999999999,"15min":1.8979999999999999},"min":{"1min":0.26100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.324999999999999,"5min":28.873999999999999,"15min":132.78200000000001},"last":64.022000000000006}]},{"osd":1,"last update":"Sat Sep 23 22:22:16 2023","interfaces":[{"interface":"back","average":{"1min":4.3019999999999996,"5min":5.319,"15min":2.569},"min":{"1min":0.29899999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":50.960999999999999,"5min":93.981999999999999,"15min":93.981999999999999},"last":13.385999999999999},{"interface":"front","average":{"1min":4.5069999999999997,"5min":5.3079999999999998,"15min":2.5619999999999998},"min":{"1min":0.35299999999999998,"5min":0.252,"15min":0.252},"max":{"1min":50.979999999999997,"5min":93.909999999999997,"15min":93.909999999999997},"last":13.457000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:22:49 2023","interfaces":[{"interface":"back","average":{"1min":2.972,"5min":6.0650000000000004,"15min":3.4329999999999998},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.534000000000001,"5min":104.254,"15min":104.254},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.661,"5min":6.3380000000000001,"15min":3.7040000000000002},"min":{"1min":0.35199999999999998,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":5.7380000000000004,"5min":104.217,"15min":104.217},"last":0.53300000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:22:16 2023","interfaces":[{"interface":"back","average":{"1min":4.3099999999999996,"5min":3.3140000000000001,"15min":1.992},"min":{"1min":0.314,"5min":0.314,"15min":0.246},"max":{"1min":50.942999999999998,"5min":93.739999999999995,"15min":93.739999999999995},"last":0.91700000000000004},{"interface":"front","average":{"1min":4.3090000000000002,"5min":3.3380000000000001,"15min":2.036},"min":{"1min":0.29399999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":51.235999999999997,"5min":93.197999999999993,"15min":93.197999999999993},"last":0.501}]},{"osd":4,"last update":"Sat Sep 23 22:22:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.073,"15min":0.95999999999999996},"min":{"1min":0.28299999999999997,"5min":0.252,"15min":0.252},"max":{"1min":10.163,"5min":10.163,"15min":82.530000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.2669999999999999,"5min":0.997,"15min":0.98999999999999999},"min":{"1min":0.30099999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.806,"5min":6.577,"15min":85.441000000000003},"last":1.0109999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:22:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.1399999999999999,"15min":1.123},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.26400000000000001},"max":{"1min":4.056,"5min":9.6519999999999992,"15min":114.83499999999999},"last":0.88600000000000001},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1,"15min":1.0469999999999999},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.8690000000000002,"5min":7.4640000000000004,"15min":110.905},"last":0.47599999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.159,"15min":0.84299999999999997},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.9409999999999998,"5min":8.2159999999999993,"15min":8.8030000000000008},"last":0.442},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.82499999999999996,"15min":0.745},"min":{"1min":0.39100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5299999999999998,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":0.86799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":997527552,"data_stored":997523480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1593446400,"data_stored":1593442328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2823749632,"data_stored":2823749632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2353790976,"data_stored":2353790976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1359151104,"data_stored":1359151104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1337200640,"data_stored":1337196568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1929904128,"data_stored":1929904128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:23:07.195 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:23:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:06 smithi070 ceph-mon[192652]: pgmap v592: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.2 GiB data, 9.9 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 598 op/s; 35016/155685 objects degraded (22.492%); 7053/155685 objects misplaced (4.530%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T22:23:07.532 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:07.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:07.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:07.641 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:07.703 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:08.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:07 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 35016/155685 objects degraded (22.492%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:08.107 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:07 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 35016/155685 objects degraded (22.492%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:08.183 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:08.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:08.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:08.296 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:08.329 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:07 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 35016/155685 objects degraded (22.492%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:08.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:08.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:08.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:08.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:08.513 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:08.547 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:08.619 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:08.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:08.728 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:08.762 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:08.814 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:08.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:08.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:08.873 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:23:08.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:08.931 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:23:08.959 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:08.984 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:23:08.984 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:23:08.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.984 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:114431 6.a e245/214 6:50117929:::benchmark_data_smithi026_355573_object33081:head v 245'7110, mlcod=211'2965)", 2023-09-23T22:23:08.984 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.959991+0000", 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.021012401, 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021046306000000001, 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959991+0000", 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959991+0000", 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959991+0000", 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959996+0000", 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.685e-06 2023-09-23T22:23:08.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.960002+0000", 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4099999999999996e-06 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.960007+0000", 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8579999999999999e-06 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969483+0000", 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094761970000000004 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969519+0000", 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5685999999999997e-05 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.988 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:114436 6.a e245/214)", 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.960011+0000", 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.020992328000000001, 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021096785999999999, 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.960011+0000", 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.960011+0000", 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.960011+0000", 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.960015+0000", 2023-09-23T22:23:08.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.569e-06 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.960020+0000", 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9679999999999997e-06 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.960024+0000", 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7139999999999999e-06 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:08.991 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.45002.0:114435 6.4 e245/231)", 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.969013+0000", 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.011991297999999999, 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012137139999999999, 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969013+0000", 2023-09-23T22:23:08.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969013+0000", 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969011+0000", 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:08.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969019+0000", 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2710000000000003e-06 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969041+0000", 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2923999999999999e-05 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969054+0000", 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.22e-05 2023-09-23T22:23:08.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:08.995 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:08.995 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:08.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.995 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.45002.0:114437 6.4 e245/231)", 2023-09-23T22:23:08.995 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.969070+0000", 2023-09-23T22:23:08.995 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.011934091000000001, 2023-09-23T22:23:08.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012127840000000001, 2023-09-23T22:23:08.995 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969070+0000", 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969070+0000", 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:08.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969069+0000", 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969072+0000", 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9579999999999999e-06 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:08.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969079+0000", 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4189999999999999e-06 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969084+0000", 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2310000000000004e-06 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.998 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.45002.0:114435 6.4 e245/231)", 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.969365+0000", 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.011638435000000001, 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011881904, 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969365+0000", 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:08.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969365+0000", 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969365+0000", 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969371+0000", 2023-09-23T22:23:09.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8199999999999999e-06 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969385+0000", 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3298999999999999e-05 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969393+0000", 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.49e-06 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:09.001 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.45002.0:114437 6.4 e245/231)", 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.969401+0000", 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.011603275, 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011896825, 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:09.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969401+0000", 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969401+0000", 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969400+0000", 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969403+0000", 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9830000000000001e-06 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969410+0000", 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.754e-06 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:09.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969415+0000", 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8130000000000003e-06 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:114439 6.4 6.3c9cc404 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.973938+0000", 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0070663080000000003, 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074129139999999996, 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:23:09.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114439 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.973938+0000", 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.973938+0000", 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.973935+0000", 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:09.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.973946+0000", 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.119e-05 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.973971+0000", 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4786999999999998e-05 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:09.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.973984+0000", 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2306999999999999e-05 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.45002.0:114434 6.1 e245/240)", 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.978820+0000", 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0021842369999999999, 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025884139999999998, 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:09.009 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.978820+0000", 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.978820+0000", 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.978818+0000", 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.978829+0000", 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0614999999999999e-05 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.978852+0000", 2023-09-23T22:23:09.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3659000000000002e-05 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.978866+0000", 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3346000000000001e-05 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:114434 6.1 6:898ca4f6:::benchmark_data_smithi026_355573_object33084:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.949941+0000", 2023-09-23T22:23:09.012 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.031062915999999999, 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031523822999999999, 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114434 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.949941+0000", 2023-09-23T22:23:09.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.949941+0000", 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.949938+0000", 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999967 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.949957+0000", 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9284e-05 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.949984+0000", 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7614e-05 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.949999+0000", 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4620999999999999e-05 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950270+0000", 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027100600000000001 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950447+0000", 2023-09-23T22:23:09.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017654300000000001 2023-09-23T22:23:09.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:23:09.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950620+0000", 2023-09-23T22:23:09.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017391800000000001 2023-09-23T22:23:09.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:09.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.956177+0000", 2023-09-23T22:23:09.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055562160000000001 2023-09-23T22:23:09.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:09.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.969391+0000", 2023-09-23T22:23:09.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013213971 2023-09-23T22:23:09.020 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.020 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:09.020 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.020 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:114435 6.4 6:2e1bf477:::benchmark_data_smithi026_355573_object33085:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T22:23:09.020 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.950018+0000", 2023-09-23T22:23:09.020 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.030985707000000001, 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031540371999999997, 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114435 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:09.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950018+0000", 2023-09-23T22:23:09.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950018+0000", 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950017+0000", 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:09.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950022+0000", 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.977e-06 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950031+0000", 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2830000000000001e-06 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950037+0000", 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6560000000000003e-06 2023-09-23T22:23:09.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950101+0000", 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3313999999999997e-05 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950300+0000", 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019894699999999999 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950472+0000", 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000171955 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.957015+0000", 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065435729999999996 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:114437 6.4 6:26b1688b:::benchmark_data_smithi026_355573_object33087:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e245)", 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.950052+0000", 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.030951523000000002, 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031591227, 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114437 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.027 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950052+0000", 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950052+0000", 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950052+0000", 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950055+0000", 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3189999999999999e-06 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950063+0000", 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.391e-06 2023-09-23T22:23:09.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.950068+0000", 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7320000000000003e-06 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.953287+0000", 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00321858 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.953443+0000", 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015596800000000001 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5", 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.953599+0000", 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015620200000000001 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.962144+0000", 2023-09-23T22:23:09.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085446380000000002 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(6.a 245/214 [PushOp(6:53a5edec:::benchmark_data_smithi026_355573_object47359:head, version: 211'2978, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:53a5edec:::benchmark_data_smithi026_355573_object47359:head@211'2978, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:08.959872+0000", 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.021131687, 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021899374999999999, 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:09.032 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959872+0000", 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959872+0000", 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959871+0000", 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959935+0000", 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4142999999999999e-05 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959954+0000", 2023-09-23T22:23:09.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8700999999999999e-05 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:08.959970+0000", 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5834000000000001e-05 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 12 2023-09-23T22:23:09.035 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:23:09.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:09.036 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:23:09.037 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:09.037 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:09.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:09.078 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:23:09.138 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:09.148 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:23:09.172 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:08 smithi070 ceph-mon[192652]: pgmap v593: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 591 op/s; 35016/152466 objects degraded (22.966%); 6593/152466 objects misplaced (4.324%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T22:23:09.172 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:08 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2461113006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:09.181 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:09.190 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:23:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:08 smithi026 ceph-mon[215775]: pgmap v593: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 591 op/s; 35016/152466 objects degraded (22.966%); 6593/152466 objects misplaced (4.324%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T22:23:09.202 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:08 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2461113006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:09.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:08 smithi026 ceph-mon[223014]: pgmap v593: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.1 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 591 op/s; 35016/152466 objects degraded (22.966%); 6593/152466 objects misplaced (4.324%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T22:23:09.207 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:08 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2461113006' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:09.238 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:09.243 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:23:09.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:09.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:09.275 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:23:09.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:23:09.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:23:09.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:23:09.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:23:09.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:23:09.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:23:09.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:23:09.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:23:09.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:09.414 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:09.906 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:23:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:23:09] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T22:23:09.916 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:23:10.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:10.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:09 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2253411731' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:10.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:09 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:09 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2253411731' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:10.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:09 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:10.321 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:10.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:10.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:09 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2253411731' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:10.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:09 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:10.428 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:10.499 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:11.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[223014]: pgmap v594: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 623 op/s; 35016/149001 objects degraded (23.501%); 6376/149001 objects misplaced (4.279%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:11.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:23:11.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[223014]: failed to encode map e246 with expected crc 2023-09-23T22:23:11.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[223014]: failed to encode map e246 with expected crc 2023-09-23T22:23:11.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:23:11.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[223014]: osdmap e246: 8 total, 8 up, 6 in 2023-09-23T22:23:11.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[215775]: pgmap v594: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 623 op/s; 35016/149001 objects degraded (23.501%); 6376/149001 objects misplaced (4.279%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:11.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:23:11.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[215775]: failed to encode map e246 with expected crc 2023-09-23T22:23:11.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[215775]: failed to encode map e246 with expected crc 2023-09-23T22:23:11.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:23:11.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:10 smithi026 ceph-mon[215775]: osdmap e246: 8 total, 8 up, 6 in 2023-09-23T22:23:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:10 smithi070 ceph-mon[192652]: pgmap v594: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 3.0 GiB data, 9.6 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 623 op/s; 35016/149001 objects degraded (23.501%); 6376/149001 objects misplaced (4.279%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:10 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:23:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:10 smithi070 ceph-mon[192652]: failed to encode map e246 with expected crc 2023-09-23T22:23:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:10 smithi070 ceph-mon[192652]: failed to encode map e246 with expected crc 2023-09-23T22:23:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:23:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:10 smithi070 ceph-mon[192652]: osdmap e246: 8 total, 8 up, 6 in 2023-09-23T22:23:11.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:23:12.231 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:12.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:12.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:12.346 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:12.454 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:12.579 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:12.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:12.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:12.697 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:12.808 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:13.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:12 smithi026 ceph-mon[215775]: pgmap v596: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 34810/144438 objects degraded (24.100%); 6096/144438 objects misplaced (4.220%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T22:23:13.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:12 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 34810/144438 objects degraded (24.100%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:13.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:12 smithi026 ceph-mon[223014]: pgmap v596: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 34810/144438 objects degraded (24.100%); 6096/144438 objects misplaced (4.220%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T22:23:13.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:12 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 34810/144438 objects degraded (24.100%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:13.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:12 smithi070 ceph-mon[192652]: pgmap v596: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 34810/144438 objects degraded (24.100%); 6096/144438 objects misplaced (4.220%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T22:23:13.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:12 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 34810/144438 objects degraded (24.100%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:13.681 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:13.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:13.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:13.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:23:13.786 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:13.791 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:13.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:14.007 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:14.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:14.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:14.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:14.228 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:14.262 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:23:14.567 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:14.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:14.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:14.679 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:14.787 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:14 smithi026 ceph-mon[215775]: pgmap v597: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 34810/143364 objects degraded (24.281%); 5899/143364 objects misplaced (4.115%); 8.1 MiB/s, 130 objects/s recovering 2023-09-23T22:23:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:14 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:14.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:14 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:14 smithi026 ceph-mon[223014]: pgmap v597: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 34810/143364 objects degraded (24.281%); 5899/143364 objects misplaced (4.115%); 8.1 MiB/s, 130 objects/s recovering 2023-09-23T22:23:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:14 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:14.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:14 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:15.122 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:15.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:15.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:15.211 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:23:15.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:15.238 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:23:15.308 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:15.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:14 smithi070 ceph-mon[192652]: pgmap v597: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.9 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 588 op/s; 34810/143364 objects degraded (24.281%); 5899/143364 objects misplaced (4.115%); 8.1 MiB/s, 130 objects/s recovering 2023-09-23T22:23:15.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:14 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:14 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:15 smithi026 ceph-mon[223014]: from='client.35247 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:15 smithi026 ceph-mon[223014]: pgmap v598: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 34627/137877 objects degraded (25.114%); 5724/137877 objects misplaced (4.152%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T22:23:16.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:15 smithi026 ceph-mon[215775]: from='client.35247 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:16.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:15 smithi026 ceph-mon[215775]: pgmap v598: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 34627/137877 objects degraded (25.114%); 5724/137877 objects misplaced (4.152%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T22:23:16.300 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:16.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:16.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:15 smithi070 ceph-mon[192652]: from='client.35247 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:16.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:15 smithi070 ceph-mon[192652]: pgmap v598: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.8 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 34627/137877 objects degraded (25.114%); 5724/137877 objects misplaced (4.152%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T22:23:16.421 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:16.497 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:16.887 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:16.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:23:16.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:23:16.968 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:17.097 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:17.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:17.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:17.179 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":597,"stamp":"2023-09-23T22:23:13.237958+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3134569239,"num_objects":47788,"num_object_clones":0,"num_object_copies":143364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34810,"num_objects_misplaced":5899,"num_objects_unfound":0,"num_objects_dirty":47788,"num_whiteouts":0,"num_read":6071,"num_read_kb":12351,"num_write":115886,"num_write_kb":5219374,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16743,"num_bytes_recovered":1096553167,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86012,"ondisk_log_size":86012,"up":297,"acting":282,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9781196,"kb_used_data":8508188,"kb_used_omap":20,"kb_used_meta":1272747,"kb_avail":552689716,"statfs":{"total":575970213888,"available":565954269184,"internally_reserved":0,"allocated":8712384512,"data_stored":8705389641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1303293707},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-386007040,"num_objects":-5890,"num_object_clones":0,"num_object_copies":-17670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-365,"num_objects_misplaced":-1327,"num_objects_unfound":0,"num_objects_dirty":-5890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5890,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1302,"num_bytes_recovered":85327872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003844"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":232,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.870997+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:22:53.870997+0000","last_peered":"2023-09-23T22:22:53.870997+0000","last_clean":"2023-09-23T22:22:53.870997+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:22:53.870997+0000","last_undegraded":"2023-09-23T22:22:53.870997+0000","last_fullsized":"2023-09-23T22:22:53.870997+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":223,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878755+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:22:53.878755+0000","last_peered":"2023-09-23T22:22:53.878755+0000","last_clean":"2023-09-23T22:22:53.878755+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:22:53.878755+0000","last_undegraded":"2023-09-23T22:22:53.878755+0000","last_fullsized":"2023-09-23T22:22:53.878755+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":186,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.916328+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:22:53.916328+0000","last_peered":"2023-09-23T22:22:53.916328+0000","last_clean":"2023-09-23T22:22:53.916328+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:22:53.916328+0000","last_undegraded":"2023-09-23T22:22:53.916328+0000","last_fullsized":"2023-09-23T22:22:53.916328+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":103,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:23:01.485776+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:23:01.485776+0000","last_peered":"2023-09-23T22:23:01.485776+0000","last_clean":"2023-09-23T22:23:01.485776+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:23:01.485776+0000","last_undegraded":"2023-09-23T22:23:01.485776+0000","last_fullsized":"2023-09-23T22:23:01.485776+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":263,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.884051+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:22:53.884051+0000","last_peered":"2023-09-23T22:22:53.884051+0000","last_clean":"2023-09-23T22:22:53.884051+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:22:53.884051+0000","last_undegraded":"2023-09-23T22:22:53.884051+0000","last_fullsized":"2023-09-23T22:22:53.884051+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":201,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.891042+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:22:57.891042+0000","last_peered":"2023-09-23T22:22:57.891042+0000","last_clean":"2023-09-23T22:22:57.891042+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:22:57.891042+0000","last_undegraded":"2023-09-23T22:22:57.891042+0000","last_fullsized":"2023-09-23T22:22:57.891042+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":131,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.901426+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:22:57.901426+0000","last_peered":"2023-09-23T22:22:57.901426+0000","last_clean":"2023-09-23T22:22:57.901426+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:22:57.901426+0000","last_undegraded":"2023-09-23T22:22:57.901426+0000","last_fullsized":"2023-09-23T22:22:57.901426+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":220,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.884122+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:22:53.884122+0000","last_peered":"2023-09-23T22:22:53.884122+0000","last_clean":"2023-09-23T22:22:53.884122+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:22:53.884122+0000","last_undegraded":"2023-09-23T22:22:53.884122+0000","last_fullsized":"2023-09-23T22:22:53.884122+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":206,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.889329+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:22:57.889329+0000","last_peered":"2023-09-23T22:22:57.889329+0000","last_clean":"2023-09-23T22:22:57.889329+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:22:57.889329+0000","last_undegraded":"2023-09-23T22:22:57.889329+0000","last_fullsized":"2023-09-23T22:22:57.889329+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":188,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878660+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:22:53.878660+0000","last_peered":"2023-09-23T22:22:53.878660+0000","last_clean":"2023-09-23T22:22:53.878660+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:22:53.878660+0000","last_undegraded":"2023-09-23T22:22:53.878660+0000","last_fullsized":"2023-09-23T22:22:53.878660+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":188,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878665+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:22:53.878665+0000","last_peered":"2023-09-23T22:22:53.878665+0000","last_clean":"2023-09-23T22:22:53.878665+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:22:53.878665+0000","last_undegraded":"2023-09-23T22:22:53.878665+0000","last_fullsized":"2023-09-23T22:22:53.878665+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":222,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888947+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:22:53.888947+0000","last_peered":"2023-09-23T22:22:53.888947+0000","last_clean":"2023-09-23T22:22:53.888947+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:22:53.888947+0000","last_undegraded":"2023-09-23T22:22:53.888947+0000","last_fullsized":"2023-09-23T22:22:53.888947+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":249,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869884+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:22:53.869884+0000","last_peered":"2023-09-23T22:22:53.869884+0000","last_clean":"2023-09-23T22:22:53.869884+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:22:53.869884+0000","last_undegraded":"2023-09-23T22:22:53.869884+0000","last_fullsized":"2023-09-23T22:22:53.869884+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869796+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:22:53.869796+0000","last_peered":"2023-09-23T22:22:53.869796+0000","last_clean":"2023-09-23T22:22:53.869796+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:22:53.869796+0000","last_undegraded":"2023-09-23T22:22:53.869796+0000","last_fullsized":"2023-09-23T22:22:53.869796+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":172,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869813+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:22:53.869813+0000","last_peered":"2023-09-23T22:22:53.869813+0000","last_clean":"2023-09-23T22:22:53.869813+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:22:53.869813+0000","last_undegraded":"2023-09-23T22:22:53.869813+0000","last_fullsized":"2023-09-23T22:22:53.869813+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":211,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.879296+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:22:53.879296+0000","last_peered":"2023-09-23T22:22:53.879296+0000","last_clean":"2023-09-23T22:22:53.879296+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:22:53.879296+0000","last_undegraded":"2023-09-23T22:22:53.879296+0000","last_fullsized":"2023-09-23T22:22:53.879296+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":120,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.906489+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:22:57.906489+0000","last_peered":"2023-09-23T22:22:57.906489+0000","last_clean":"2023-09-23T22:22:57.906489+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:22:57.906489+0000","last_undegraded":"2023-09-23T22:22:57.906489+0000","last_fullsized":"2023-09-23T22:22:57.906489+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":225,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.888790+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:22:57.888790+0000","last_peered":"2023-09-23T22:22:57.888790+0000","last_clean":"2023-09-23T22:22:57.888790+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:22:57.888790+0000","last_undegraded":"2023-09-23T22:22:57.888790+0000","last_fullsized":"2023-09-23T22:22:57.888790+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":168,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869711+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:22:53.869711+0000","last_peered":"2023-09-23T22:22:53.869711+0000","last_clean":"2023-09-23T22:22:53.869711+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:22:53.869711+0000","last_undegraded":"2023-09-23T22:22:53.869711+0000","last_fullsized":"2023-09-23T22:22:53.869711+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":102,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.901001+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:22:57.901001+0000","last_peered":"2023-09-23T22:22:57.901001+0000","last_clean":"2023-09-23T22:22:57.901001+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:22:57.901001+0000","last_undegraded":"2023-09-23T22:22:57.901001+0000","last_fullsized":"2023-09-23T22:22:57.901001+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":263,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.914961+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:22:57.914961+0000","last_peered":"2023-09-23T22:22:57.914961+0000","last_clean":"2023-09-23T22:22:57.914961+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:22:57.914961+0000","last_undegraded":"2023-09-23T22:22:57.914961+0000","last_fullsized":"2023-09-23T22:22:57.914961+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":33,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888671+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:22:53.888671+0000","last_peered":"2023-09-23T22:22:53.888671+0000","last_clean":"2023-09-23T22:22:53.888671+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:22:53.888671+0000","last_undegraded":"2023-09-23T22:22:53.888671+0000","last_fullsized":"2023-09-23T22:22:53.888671+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"246'7170","reported_seq":11774,"reported_epoch":246,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:10.267219+0000","last_change":"2023-09-23T22:22:28.455882+0000","last_active":"2023-09-23T22:23:10.267219+0000","last_peered":"2023-09-23T22:23:10.267219+0000","last_clean":"2023-09-23T22:21:12.198976+0000","last_became_active":"2023-09-23T22:21:14.005858+0000","last_became_peered":"2023-09-23T22:21:14.005858+0000","last_unstale":"2023-09-23T22:23:10.267219+0000","last_undegraded":"2023-09-23T22:21:33.932520+0000","last_fullsized":"2023-09-23T22:21:13.797435+0000","mapping_epoch":214,"log_start":"210'1700","ondisk_log_start":"210'1700","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:26.966631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188743680,"num_objects":2880,"num_object_clones":0,"num_object_copies":8640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":766,"num_objects_misplaced":606,"num_objects_unfound":0,"num_objects_dirty":2880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7170,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2270,"num_bytes_recovered":148766720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":606},{"shards":"0,5","objects":1508}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":240,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.871537+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:22:53.871537+0000","last_peered":"2023-09-23T22:22:53.871537+0000","last_clean":"2023-09-23T22:22:53.871537+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:22:53.871537+0000","last_undegraded":"2023-09-23T22:22:53.871537+0000","last_fullsized":"2023-09-23T22:22:53.871537+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":228,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.887119+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:22:57.887119+0000","last_peered":"2023-09-23T22:22:57.887119+0000","last_clean":"2023-09-23T22:22:57.887119+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:22:57.887119+0000","last_undegraded":"2023-09-23T22:22:57.887119+0000","last_fullsized":"2023-09-23T22:22:57.887119+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"245'7182","reported_seq":14699,"reported_epoch":245,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:23:09.212302+0000","last_change":"2023-09-23T22:22:45.430551+0000","last_active":"2023-09-23T22:23:09.212302+0000","last_peered":"2023-09-23T22:23:09.212302+0000","last_clean":"2023-09-23T22:22:43.315002+0000","last_became_active":"2023-09-23T22:22:45.409613+0000","last_became_peered":"2023-09-23T22:22:45.409613+0000","last_unstale":"2023-09-23T22:23:09.212302+0000","last_undegraded":"2023-09-23T22:23:09.212302+0000","last_fullsized":"2023-09-23T22:23:09.212302+0000","mapping_epoch":240,"log_start":"210'1791","ondisk_log_start":"210'1791","created":203,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5391,"log_dups_size":0,"ondisk_log_size":5391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:03.767060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195297280,"num_objects":2980,"num_object_clones":0,"num_object_copies":8940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2980,"num_objects_unfound":0,"num_objects_dirty":2980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7182,"num_write_kb":325184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3717,"num_bytes_recovered":243597312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":2980}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":154,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.875590+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:22:53.875590+0000","last_peered":"2023-09-23T22:22:53.875590+0000","last_clean":"2023-09-23T22:22:53.875590+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:22:53.875590+0000","last_undegraded":"2023-09-23T22:22:53.875590+0000","last_fullsized":"2023-09-23T22:22:53.875590+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":232,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869624+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:22:53.869624+0000","last_peered":"2023-09-23T22:22:53.869624+0000","last_clean":"2023-09-23T22:22:53.869624+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:22:53.869624+0000","last_undegraded":"2023-09-23T22:22:53.869624+0000","last_fullsized":"2023-09-23T22:22:53.869624+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"245'7320","reported_seq":11035,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:09.147988+0000","last_change":"2023-09-23T22:22:45.550371+0000","last_active":"2023-09-23T22:23:09.147988+0000","last_peered":"2023-09-23T22:23:09.147988+0000","last_clean":"2023-09-23T22:21:32.851371+0000","last_became_active":"2023-09-23T22:22:45.472694+0000","last_became_peered":"2023-09-23T22:22:45.472694+0000","last_unstale":"2023-09-23T22:23:09.147988+0000","last_undegraded":"2023-09-23T22:22:45.456856+0000","last_fullsized":"2023-09-23T22:22:45.317726+0000","mapping_epoch":240,"log_start":"210'1933","ondisk_log_start":"210'1933","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.540906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203685888,"num_objects":3108,"num_object_clones":0,"num_object_copies":9324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5210,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7320,"num_write_kb":333696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1215,"num_bytes_recovered":79495168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"0,1,3","objects":88},{"shards":"0,2,3","objects":495},{"shards":"2,3","objects":1},{"shards":"2,3,7","objects":2524},{"shards":"3,7","objects":-891}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":171,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.888867+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:22:57.888867+0000","last_peered":"2023-09-23T22:22:57.888867+0000","last_clean":"2023-09-23T22:22:57.888867+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:22:57.888867+0000","last_undegraded":"2023-09-23T22:22:57.888867+0000","last_fullsized":"2023-09-23T22:22:57.888867+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":179,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.889019+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:22:53.889019+0000","last_peered":"2023-09-23T22:22:53.889019+0000","last_clean":"2023-09-23T22:22:53.889019+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:22:53.889019+0000","last_undegraded":"2023-09-23T22:22:53.889019+0000","last_fullsized":"2023-09-23T22:22:53.889019+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"246'7308","reported_seq":14826,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:11.318794+0000","last_change":"2023-09-23T22:22:17.938724+0000","last_active":"2023-09-23T22:23:11.318794+0000","last_peered":"2023-09-23T22:23:11.318794+0000","last_clean":"2023-09-23T22:21:12.226652+0000","last_became_active":"2023-09-23T22:21:13.982783+0000","last_became_peered":"2023-09-23T22:21:13.982783+0000","last_unstale":"2023-09-23T22:23:11.318794+0000","last_undegraded":"2023-09-23T22:22:17.930337+0000","last_fullsized":"2023-09-23T22:21:13.795059+0000","mapping_epoch":214,"log_start":"211'2397","ondisk_log_start":"211'2397","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4911,"log_dups_size":0,"ondisk_log_size":4911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.489307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190316544,"num_objects":2904,"num_object_clones":0,"num_object_copies":8712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7308,"num_write_kb":326784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3728,"num_bytes_recovered":244318208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":38},{"shards":"0,7","objects":2866}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":236,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.888784+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:22:57.888784+0000","last_peered":"2023-09-23T22:22:57.888784+0000","last_clean":"2023-09-23T22:22:57.888784+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:22:57.888784+0000","last_undegraded":"2023-09-23T22:22:57.888784+0000","last_fullsized":"2023-09-23T22:22:57.888784+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":100,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.909620+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:22:57.909620+0000","last_peered":"2023-09-23T22:22:57.909620+0000","last_clean":"2023-09-23T22:22:57.909620+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:22:57.909620+0000","last_undegraded":"2023-09-23T22:22:57.909620+0000","last_fullsized":"2023-09-23T22:22:57.909620+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"245'7211","reported_seq":8807,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:09.738870+0000","last_change":"2023-09-23T22:21:57.837966+0000","last_active":"2023-09-23T22:23:09.738870+0000","last_peered":"2023-09-23T22:23:09.738870+0000","last_clean":"2023-09-23T22:21:12.225315+0000","last_became_active":"2023-09-23T22:21:13.940088+0000","last_became_peered":"2023-09-23T22:21:13.940088+0000","last_unstale":"2023-09-23T22:23:09.738870+0000","last_undegraded":"2023-09-23T22:21:57.835407+0000","last_fullsized":"2023-09-23T22:21:13.800025+0000","mapping_epoch":214,"log_start":"210'1900","ondisk_log_start":"210'1900","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5311,"log_dups_size":0,"ondisk_log_size":5311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:26.245766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196542464,"num_objects":2999,"num_object_clones":0,"num_object_copies":8997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7211,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":768,"num_bytes_recovered":50331648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1602},{"shards":"1,4","objects":59}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":548,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878493+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:22:53.878493+0000","last_peered":"2023-09-23T22:22:53.878493+0000","last_clean":"2023-09-23T22:22:53.878493+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:22:53.878493+0000","last_undegraded":"2023-09-23T22:22:53.878493+0000","last_fullsized":"2023-09-23T22:22:53.878493+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":258,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.901247+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:22:57.901247+0000","last_peered":"2023-09-23T22:22:57.901247+0000","last_clean":"2023-09-23T22:22:57.901247+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:22:57.901247+0000","last_undegraded":"2023-09-23T22:22:57.901247+0000","last_fullsized":"2023-09-23T22:22:57.901247+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":213,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869705+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:22:53.869705+0000","last_peered":"2023-09-23T22:22:53.869705+0000","last_clean":"2023-09-23T22:22:53.869705+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:22:53.869705+0000","last_undegraded":"2023-09-23T22:22:53.869705+0000","last_fullsized":"2023-09-23T22:22:53.869705+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"245'7100","reported_seq":8632,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:09.719469+0000","last_change":"2023-09-23T22:22:20.034548+0000","last_active":"2023-09-23T22:23:09.719469+0000","last_peered":"2023-09-23T22:23:09.719469+0000","last_clean":"2023-09-23T22:21:12.194696+0000","last_became_active":"2023-09-23T22:22:19.958601+0000","last_became_peered":"2023-09-23T22:22:19.958601+0000","last_unstale":"2023-09-23T22:23:09.719469+0000","last_undegraded":"2023-09-23T22:22:19.951596+0000","last_fullsized":"2023-09-23T22:22:19.874428+0000","mapping_epoch":232,"log_start":"210'1800","ondisk_log_start":"210'1800","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5300,"log_dups_size":0,"ondisk_log_size":5300,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:10.906601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188874752,"num_objects":2882,"num_object_clones":0,"num_object_copies":8646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4172,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2882,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7100,"num_write_kb":319424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":486,"num_bytes_recovered":31850496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[4,3],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"1,4,5","objects":506},{"shards":"2,3,4","objects":1558},{"shards":"3,4","objects":-804}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":544,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:57.887078+0000","last_change":"2023-09-23T22:21:59.026062+0000","last_active":"2023-09-23T22:22:57.887078+0000","last_peered":"2023-09-23T22:22:57.887078+0000","last_clean":"2023-09-23T22:21:50.447236+0000","last_became_active":"2023-09-23T22:21:59.021009+0000","last_became_peered":"2023-09-23T22:21:59.021009+0000","last_unstale":"2023-09-23T22:22:57.887078+0000","last_undegraded":"2023-09-23T22:21:58.818399+0000","last_fullsized":"2023-09-23T22:21:58.817464+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:52.228490+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":161,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.870843+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:22:53.870843+0000","last_peered":"2023-09-23T22:22:53.870843+0000","last_clean":"2023-09-23T22:22:53.870843+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:22:53.870843+0000","last_undegraded":"2023-09-23T22:22:53.870843+0000","last_fullsized":"2023-09-23T22:22:53.870843+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":230,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869780+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:22:53.869780+0000","last_peered":"2023-09-23T22:22:53.869780+0000","last_clean":"2023-09-23T22:22:53.869780+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:22:53.869780+0000","last_undegraded":"2023-09-23T22:22:53.869780+0000","last_fullsized":"2023-09-23T22:22:53.869780+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"246'7083","reported_seq":7153,"reported_epoch":246,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:10.259518+0000","last_change":"2023-09-23T22:22:18.973077+0000","last_active":"2023-09-23T22:23:10.259518+0000","last_peered":"2023-09-23T22:23:10.259518+0000","last_clean":"2023-09-23T22:21:32.779211+0000","last_became_active":"2023-09-23T22:22:18.949524+0000","last_became_peered":"2023-09-23T22:22:18.949524+0000","last_unstale":"2023-09-23T22:23:10.259518+0000","last_undegraded":"2023-09-23T22:22:18.870000+0000","last_fullsized":"2023-09-23T22:22:18.859180+0000","mapping_epoch":231,"log_start":"210'1600","ondisk_log_start":"210'1600","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5483,"log_dups_size":0,"ondisk_log_size":5483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.194549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193527808,"num_objects":2953,"num_object_clones":0,"num_object_copies":8859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":914,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7083,"num_write_kb":321152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"1,5,7","objects":487},{"shards":"5,7","objects":2466}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":206,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869495+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:22:53.869495+0000","last_peered":"2023-09-23T22:22:53.869495+0000","last_clean":"2023-09-23T22:22:53.869495+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:22:53.869495+0000","last_undegraded":"2023-09-23T22:22:53.869495+0000","last_fullsized":"2023-09-23T22:22:53.869495+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":221,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.875461+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:22:53.875461+0000","last_peered":"2023-09-23T22:22:53.875461+0000","last_clean":"2023-09-23T22:22:53.875461+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:22:53.875461+0000","last_undegraded":"2023-09-23T22:22:53.875461+0000","last_fullsized":"2023-09-23T22:22:53.875461+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"246'7223","reported_seq":7274,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:23:10.285049+0000","last_change":"2023-09-23T22:20:19.473637+0000","last_active":"2023-09-23T22:23:10.285049+0000","last_peered":"2023-09-23T22:23:10.285049+0000","last_clean":"2023-09-23T22:23:10.285049+0000","last_became_active":"2023-09-23T22:20:19.473385+0000","last_became_peered":"2023-09-23T22:20:19.473385+0000","last_unstale":"2023-09-23T22:23:10.285049+0000","last_undegraded":"2023-09-23T22:23:10.285049+0000","last_fullsized":"2023-09-23T22:23:10.285049+0000","mapping_epoch":203,"log_start":"210'1700","ondisk_log_start":"210'1700","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5523,"log_dups_size":0,"ondisk_log_size":5523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:01.911861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193724440,"num_objects":2957,"num_object_clones":0,"num_object_copies":8871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2957,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7223,"num_write_kb":325697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":550,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.889211+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:22:57.889211+0000","last_peered":"2023-09-23T22:22:57.889211+0000","last_clean":"2023-09-23T22:22:57.889211+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:22:57.889211+0000","last_undegraded":"2023-09-23T22:22:57.889211+0000","last_fullsized":"2023-09-23T22:22:57.889211+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":24,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878586+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:22:53.878586+0000","last_peered":"2023-09-23T22:22:53.878586+0000","last_clean":"2023-09-23T22:22:53.878586+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:22:53.878586+0000","last_undegraded":"2023-09-23T22:22:53.878586+0000","last_fullsized":"2023-09-23T22:22:53.878586+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":186,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.923854+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:22:53.923854+0000","last_peered":"2023-09-23T22:22:53.923854+0000","last_clean":"2023-09-23T22:22:53.923854+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:22:53.923854+0000","last_undegraded":"2023-09-23T22:22:53.923854+0000","last_fullsized":"2023-09-23T22:22:53.923854+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"246'7354","reported_seq":9574,"reported_epoch":246,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:11.527661+0000","last_change":"2023-09-23T22:22:43.348452+0000","last_active":"2023-09-23T22:23:11.527661+0000","last_peered":"2023-09-23T22:23:11.527661+0000","last_clean":"2023-09-23T22:21:32.822849+0000","last_became_active":"2023-09-23T22:21:35.128019+0000","last_became_peered":"2023-09-23T22:21:35.128019+0000","last_unstale":"2023-09-23T22:23:11.527661+0000","last_undegraded":"2023-09-23T22:21:34.912513+0000","last_fullsized":"2023-09-23T22:21:34.880959+0000","mapping_epoch":220,"log_start":"211'2447","ondisk_log_start":"211'2447","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4907,"log_dups_size":0,"ondisk_log_size":4907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:58.687801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193593344,"num_objects":2954,"num_object_clones":0,"num_object_copies":8862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1209,"num_objects_misplaced":2313,"num_objects_unfound":0,"num_objects_dirty":2954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7354,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1080,"num_bytes_recovered":70778880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":2313},{"shards":"0,3","objects":-568}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":540,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.890476+0000","last_change":"2023-09-23T22:22:47.349660+0000","last_active":"2023-09-23T22:22:53.890476+0000","last_peered":"2023-09-23T22:22:53.890476+0000","last_clean":"2023-09-23T22:22:53.890476+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:22:53.890476+0000","last_undegraded":"2023-09-23T22:22:53.890476+0000","last_fullsized":"2023-09-23T22:22:53.890476+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":793,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.898301+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:22:53.898301+0000","last_peered":"2023-09-23T22:22:53.898301+0000","last_clean":"2023-09-23T22:22:53.898301+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:22:53.898301+0000","last_undegraded":"2023-09-23T22:22:53.898301+0000","last_fullsized":"2023-09-23T22:22:53.898301+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":263,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.888645+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:22:57.888645+0000","last_peered":"2023-09-23T22:22:57.888645+0000","last_clean":"2023-09-23T22:22:57.888645+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:22:57.888645+0000","last_undegraded":"2023-09-23T22:22:57.888645+0000","last_fullsized":"2023-09-23T22:22:57.888645+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":30,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888316+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:22:53.888316+0000","last_peered":"2023-09-23T22:22:53.888316+0000","last_clean":"2023-09-23T22:22:53.888316+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:22:53.888316+0000","last_undegraded":"2023-09-23T22:22:53.888316+0000","last_fullsized":"2023-09-23T22:22:53.888316+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"245'7227","reported_seq":7309,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:09.758371+0000","last_change":"2023-09-23T22:22:18.955534+0000","last_active":"2023-09-23T22:23:09.758371+0000","last_peered":"2023-09-23T22:23:09.758371+0000","last_clean":"2023-09-23T22:21:32.860473+0000","last_became_active":"2023-09-23T22:22:18.941549+0000","last_became_peered":"2023-09-23T22:22:18.941549+0000","last_unstale":"2023-09-23T22:23:09.758371+0000","last_undegraded":"2023-09-23T22:22:18.875275+0000","last_fullsized":"2023-09-23T22:22:18.863381+0000","mapping_epoch":231,"log_start":"210'1900","ondisk_log_start":"210'1900","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5327,"log_dups_size":0,"ondisk_log_size":5327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:27.412277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196935680,"num_objects":3005,"num_object_clones":0,"num_object_copies":9015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":988,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7227,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,5,7","objects":527},{"shards":"4,7","objects":2478}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":550,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.887361+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:22:57.887361+0000","last_peered":"2023-09-23T22:22:57.887361+0000","last_clean":"2023-09-23T22:22:57.887361+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:22:57.887361+0000","last_undegraded":"2023-09-23T22:22:57.887361+0000","last_fullsized":"2023-09-23T22:22:57.887361+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":978,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.889399+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:22:57.889399+0000","last_peered":"2023-09-23T22:22:57.889399+0000","last_clean":"2023-09-23T22:22:57.889399+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:22:57.889399+0000","last_undegraded":"2023-09-23T22:22:57.889399+0000","last_fullsized":"2023-09-23T22:22:57.889399+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":254,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869708+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:22:53.869708+0000","last_peered":"2023-09-23T22:22:53.869708+0000","last_clean":"2023-09-23T22:22:53.869708+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:22:53.869708+0000","last_undegraded":"2023-09-23T22:22:53.869708+0000","last_fullsized":"2023-09-23T22:22:53.869708+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":210,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.888905+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:22:57.888905+0000","last_peered":"2023-09-23T22:22:57.888905+0000","last_clean":"2023-09-23T22:22:57.888905+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:22:57.888905+0000","last_undegraded":"2023-09-23T22:22:57.888905+0000","last_fullsized":"2023-09-23T22:22:57.888905+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"245'7009","reported_seq":7068,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:09.204229+0000","last_change":"2023-09-23T22:21:59.013998+0000","last_active":"2023-09-23T22:23:09.204229+0000","last_peered":"2023-09-23T22:23:09.204229+0000","last_clean":"2023-09-23T22:21:56.814967+0000","last_became_active":"2023-09-23T22:21:59.002385+0000","last_became_peered":"2023-09-23T22:21:59.002385+0000","last_unstale":"2023-09-23T22:23:09.204229+0000","last_undegraded":"2023-09-23T22:21:58.865154+0000","last_fullsized":"2023-09-23T22:21:58.825566+0000","mapping_epoch":226,"log_start":"210'1620","ondisk_log_start":"210'1620","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5389,"log_dups_size":0,"ondisk_log_size":5389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:00.273421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194314240,"num_objects":2965,"num_object_clones":0,"num_object_copies":8895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4918,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7009,"num_write_kb":319168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2896},{"shards":"3,7","objects":-1953}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":575,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:23:09.942577+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:23:09.942577+0000","last_peered":"2023-09-23T22:23:09.942577+0000","last_clean":"2023-09-23T22:23:09.942577+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:23:09.942577+0000","last_undegraded":"2023-09-23T22:23:09.942577+0000","last_fullsized":"2023-09-23T22:23:09.942577+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":351,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:22:53.877299+0000","last_change":"2023-09-23T22:22:18.917671+0000","last_active":"2023-09-23T22:22:53.877299+0000","last_peered":"2023-09-23T22:22:53.877299+0000","last_clean":"2023-09-23T22:15:39.913392+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:22:53.877299+0000","last_undegraded":"2023-09-23T22:22:18.874220+0000","last_fullsized":"2023-09-23T22:22:53.877299+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":205,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.887635+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:22:53.887635+0000","last_peered":"2023-09-23T22:22:53.887635+0000","last_clean":"2023-09-23T22:22:53.887635+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:22:53.887635+0000","last_undegraded":"2023-09-23T22:22:53.887635+0000","last_fullsized":"2023-09-23T22:22:53.887635+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":251,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.929946+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:22:53.929946+0000","last_peered":"2023-09-23T22:22:53.929946+0000","last_clean":"2023-09-23T22:22:53.929946+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:22:53.929946+0000","last_undegraded":"2023-09-23T22:22:53.929946+0000","last_fullsized":"2023-09-23T22:22:53.929946+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"246'7296","reported_seq":7347,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:23:10.272355+0000","last_change":"2023-09-23T22:20:19.470871+0000","last_active":"2023-09-23T22:23:10.272355+0000","last_peered":"2023-09-23T22:23:10.272355+0000","last_clean":"2023-09-23T22:23:10.272355+0000","last_became_active":"2023-09-23T22:20:19.470597+0000","last_became_peered":"2023-09-23T22:20:19.470597+0000","last_unstale":"2023-09-23T22:23:10.272355+0000","last_undegraded":"2023-09-23T22:23:10.272355+0000","last_fullsized":"2023-09-23T22:23:10.272355+0000","mapping_epoch":203,"log_start":"210'1800","ondisk_log_start":"210'1800","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5496,"log_dups_size":0,"ondisk_log_size":5496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:32.388985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198967296,"num_objects":3036,"num_object_clones":0,"num_object_copies":9108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7296,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":841,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:22:57.887684+0000","last_change":"2023-09-23T22:21:58.853256+0000","last_active":"2023-09-23T22:22:57.887684+0000","last_peered":"2023-09-23T22:22:57.887684+0000","last_clean":"2023-09-23T22:21:50.447508+0000","last_became_active":"2023-09-23T22:21:58.852815+0000","last_became_peered":"2023-09-23T22:21:58.852815+0000","last_unstale":"2023-09-23T22:22:57.887684+0000","last_undegraded":"2023-09-23T22:21:58.818640+0000","last_fullsized":"2023-09-23T22:21:58.817793+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:25.619025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":558,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:23:01.485645+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:23:01.485645+0000","last_peered":"2023-09-23T22:23:01.485645+0000","last_clean":"2023-09-23T22:23:01.485645+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:23:01.485645+0000","last_undegraded":"2023-09-23T22:23:01.485645+0000","last_fullsized":"2023-09-23T22:23:01.485645+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":169,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878980+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:22:53.878980+0000","last_peered":"2023-09-23T22:22:53.878980+0000","last_clean":"2023-09-23T22:22:53.878980+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:22:53.878980+0000","last_undegraded":"2023-09-23T22:22:53.878980+0000","last_fullsized":"2023-09-23T22:22:53.878980+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":131,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869961+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:22:53.869961+0000","last_peered":"2023-09-23T22:22:53.869961+0000","last_clean":"2023-09-23T22:22:53.869961+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:22:53.869961+0000","last_undegraded":"2023-09-23T22:22:53.869961+0000","last_fullsized":"2023-09-23T22:22:53.869961+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"245'7056","reported_seq":7129,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:08.669321+0000","last_change":"2023-09-23T22:22:18.954521+0000","last_active":"2023-09-23T22:23:08.669321+0000","last_peered":"2023-09-23T22:23:08.669321+0000","last_clean":"2023-09-23T22:21:32.869170+0000","last_became_active":"2023-09-23T22:22:18.945835+0000","last_became_peered":"2023-09-23T22:22:18.945835+0000","last_unstale":"2023-09-23T22:23:08.669321+0000","last_undegraded":"2023-09-23T22:22:18.869474+0000","last_fullsized":"2023-09-23T22:22:18.861404+0000","mapping_epoch":231,"log_start":"211'2405","ondisk_log_start":"211'2405","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4651,"log_dups_size":0,"ondisk_log_size":4651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:02.907936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196214784,"num_objects":2994,"num_object_clones":0,"num_object_copies":8982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7056,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,4,5","objects":591},{"shards":"1,5","objects":2403}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":576,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888953+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:22:53.888953+0000","last_peered":"2023-09-23T22:22:53.888953+0000","last_clean":"2023-09-23T22:22:53.888953+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:22:53.888953+0000","last_undegraded":"2023-09-23T22:22:53.888953+0000","last_fullsized":"2023-09-23T22:22:53.888953+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":594,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.887851+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:22:53.887851+0000","last_peered":"2023-09-23T22:22:53.887851+0000","last_clean":"2023-09-23T22:22:53.887851+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:22:53.887851+0000","last_undegraded":"2023-09-23T22:22:53.887851+0000","last_fullsized":"2023-09-23T22:22:53.887851+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":127,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.906899+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:22:57.906899+0000","last_peered":"2023-09-23T22:22:57.906899+0000","last_clean":"2023-09-23T22:22:57.906899+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:22:57.906899+0000","last_undegraded":"2023-09-23T22:22:57.906899+0000","last_fullsized":"2023-09-23T22:22:57.906899+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":84,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.900689+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:22:57.900689+0000","last_peered":"2023-09-23T22:22:57.900689+0000","last_clean":"2023-09-23T22:22:57.900689+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:22:57.900689+0000","last_undegraded":"2023-09-23T22:22:57.900689+0000","last_fullsized":"2023-09-23T22:22:57.900689+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"245'7125","reported_seq":10662,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:08.710922+0000","last_change":"2023-09-23T22:22:45.524797+0000","last_active":"2023-09-23T22:23:08.710922+0000","last_peered":"2023-09-23T22:23:08.710922+0000","last_clean":"2023-09-23T22:21:12.245233+0000","last_became_active":"2023-09-23T22:22:45.454018+0000","last_became_peered":"2023-09-23T22:22:45.454018+0000","last_unstale":"2023-09-23T22:23:08.710922+0000","last_undegraded":"2023-09-23T22:22:45.437539+0000","last_fullsized":"2023-09-23T22:22:45.322462+0000","mapping_epoch":240,"log_start":"211'2430","ondisk_log_start":"211'2430","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4695,"log_dups_size":0,"ondisk_log_size":4695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:10.659044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204013568,"num_objects":3113,"num_object_clones":0,"num_object_copies":9339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7125,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1153,"num_bytes_recovered":75563008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1","objects":1},{"shards":"0,1,5","objects":565},{"shards":"1,2,7","objects":1662},{"shards":"1,7","objects":32}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":566,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.900922+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:22:57.900922+0000","last_peered":"2023-09-23T22:22:57.900922+0000","last_clean":"2023-09-23T22:22:57.900922+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:22:57.900922+0000","last_undegraded":"2023-09-23T22:22:57.900922+0000","last_fullsized":"2023-09-23T22:22:57.900922+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":137,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:23:09.942884+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:23:09.942884+0000","last_peered":"2023-09-23T22:23:09.942884+0000","last_clean":"2023-09-23T22:23:09.942884+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:23:09.942884+0000","last_undegraded":"2023-09-23T22:23:09.942884+0000","last_fullsized":"2023-09-23T22:23:09.942884+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":195,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.898559+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:22:53.898559+0000","last_peered":"2023-09-23T22:22:53.898559+0000","last_clean":"2023-09-23T22:22:53.898559+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:22:53.898559+0000","last_undegraded":"2023-09-23T22:22:53.898559+0000","last_fullsized":"2023-09-23T22:22:53.898559+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":100,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.909544+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:22:57.909544+0000","last_peered":"2023-09-23T22:22:57.909544+0000","last_clean":"2023-09-23T22:22:57.909544+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:22:57.909544+0000","last_undegraded":"2023-09-23T22:22:57.909544+0000","last_fullsized":"2023-09-23T22:22:57.909544+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"245'7264","reported_seq":11562,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:09.816621+0000","last_change":"2023-09-23T22:22:19.043755+0000","last_active":"2023-09-23T22:23:09.816621+0000","last_peered":"2023-09-23T22:23:09.816621+0000","last_clean":"2023-09-23T22:21:12.214088+0000","last_became_active":"2023-09-23T22:22:18.986181+0000","last_became_peered":"2023-09-23T22:22:18.986181+0000","last_unstale":"2023-09-23T22:23:09.816621+0000","last_undegraded":"2023-09-23T22:22:18.975289+0000","last_fullsized":"2023-09-23T22:22:18.862148+0000","mapping_epoch":231,"log_start":"210'2000","ondisk_log_start":"210'2000","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5264,"log_dups_size":0,"ondisk_log_size":5264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:34.422545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199753728,"num_objects":3048,"num_object_clones":0,"num_object_copies":9144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3048,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7264,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2108,"num_bytes_recovered":138149888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,3,4","objects":1616},{"shards":"2,4,7","objects":1349},{"shards":"4,5,7","objects":83}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":549,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869647+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:22:53.869647+0000","last_peered":"2023-09-23T22:22:53.869647+0000","last_clean":"2023-09-23T22:22:53.869647+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:22:53.869647+0000","last_undegraded":"2023-09-23T22:22:53.869647+0000","last_fullsized":"2023-09-23T22:22:53.869647+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":258,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.890722+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:22:53.890722+0000","last_peered":"2023-09-23T22:22:53.890722+0000","last_clean":"2023-09-23T22:22:53.890722+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:22:53.890722+0000","last_undegraded":"2023-09-23T22:22:53.890722+0000","last_fullsized":"2023-09-23T22:22:53.890722+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":886,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.893835+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:22:53.893835+0000","last_peered":"2023-09-23T22:22:53.893835+0000","last_clean":"2023-09-23T22:22:53.893835+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:22:53.893835+0000","last_undegraded":"2023-09-23T22:22:53.893835+0000","last_fullsized":"2023-09-23T22:22:53.893835+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":33,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878351+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:22:53.878351+0000","last_peered":"2023-09-23T22:22:53.878351+0000","last_clean":"2023-09-23T22:22:53.878351+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:22:53.878351+0000","last_undegraded":"2023-09-23T22:22:53.878351+0000","last_fullsized":"2023-09-23T22:22:53.878351+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"245'7007","reported_seq":7200,"reported_epoch":245,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:08.659511+0000","last_change":"2023-09-23T22:22:44.336083+0000","last_active":"2023-09-23T22:23:08.659511+0000","last_peered":"2023-09-23T22:23:08.659511+0000","last_clean":"2023-09-23T22:21:56.808651+0000","last_became_active":"2023-09-23T22:21:59.059105+0000","last_became_peered":"2023-09-23T22:21:59.059105+0000","last_unstale":"2023-09-23T22:23:08.659511+0000","last_undegraded":"2023-09-23T22:21:58.989636+0000","last_fullsized":"2023-09-23T22:21:58.812134+0000","mapping_epoch":226,"log_start":"211'2302","ondisk_log_start":"211'2302","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4705,"log_dups_size":0,"ondisk_log_size":4705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:51.370288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195493888,"num_objects":2983,"num_object_clones":0,"num_object_copies":8949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7007,"num_write_kb":319680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":4259840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2904},{"shards":"1,7","objects":-1869}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":532,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.879517+0000","last_change":"2023-09-23T22:22:19.883431+0000","last_active":"2023-09-23T22:22:53.879517+0000","last_peered":"2023-09-23T22:22:53.879517+0000","last_clean":"2023-09-23T22:22:53.879517+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:22:53.879517+0000","last_undegraded":"2023-09-23T22:22:53.879517+0000","last_fullsized":"2023-09-23T22:22:53.879517+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:46.903071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:46.903071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":226,"reported_epoch":244,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:22:53.884248+0000","last_change":"2023-09-23T22:22:44.342884+0000","last_active":"2023-09-23T22:22:53.884248+0000","last_peered":"2023-09-23T22:22:53.884248+0000","last_clean":"2023-09-23T22:06:25.449250+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:22:53.884248+0000","last_undegraded":"2023-09-23T22:22:44.329336+0000","last_fullsized":"2023-09-23T22:22:53.884248+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":79,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.900841+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:22:57.900841+0000","last_peered":"2023-09-23T22:22:57.900841+0000","last_clean":"2023-09-23T22:22:57.900841+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:22:57.900841+0000","last_undegraded":"2023-09-23T22:22:57.900841+0000","last_fullsized":"2023-09-23T22:22:57.900841+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":208,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869539+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:22:53.869539+0000","last_peered":"2023-09-23T22:22:53.869539+0000","last_clean":"2023-09-23T22:22:53.869539+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:22:53.869539+0000","last_undegraded":"2023-09-23T22:22:53.869539+0000","last_fullsized":"2023-09-23T22:22:53.869539+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":75,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.909463+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:22:57.909463+0000","last_peered":"2023-09-23T22:22:57.909463+0000","last_clean":"2023-09-23T22:22:57.909463+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:22:57.909463+0000","last_undegraded":"2023-09-23T22:22:57.909463+0000","last_fullsized":"2023-09-23T22:22:57.909463+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":255,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.898484+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:22:53.898484+0000","last_peered":"2023-09-23T22:22:53.898484+0000","last_clean":"2023-09-23T22:22:53.898484+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:22:53.898484+0000","last_undegraded":"2023-09-23T22:22:53.898484+0000","last_fullsized":"2023-09-23T22:22:53.898484+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":208,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.888708+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:22:57.888708+0000","last_peered":"2023-09-23T22:22:57.888708+0000","last_clean":"2023-09-23T22:22:57.888708+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:22:57.888708+0000","last_undegraded":"2023-09-23T22:22:57.888708+0000","last_fullsized":"2023-09-23T22:22:57.888708+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":220,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869869+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:22:53.869869+0000","last_peered":"2023-09-23T22:22:53.869869+0000","last_clean":"2023-09-23T22:22:53.869869+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:22:53.869869+0000","last_undegraded":"2023-09-23T22:22:53.869869+0000","last_fullsized":"2023-09-23T22:22:53.869869+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":205,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T22:22:57.888723+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:22:57.888723+0000","last_peered":"2023-09-23T22:22:57.888723+0000","last_clean":"2023-09-23T22:22:57.888723+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:22:57.888723+0000","last_undegraded":"2023-09-23T22:22:57.888723+0000","last_fullsized":"2023-09-23T22:22:57.888723+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":79,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.870919+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:22:53.870919+0000","last_peered":"2023-09-23T22:22:53.870919+0000","last_clean":"2023-09-23T22:22:53.870919+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:22:53.870919+0000","last_undegraded":"2023-09-23T22:22:53.870919+0000","last_fullsized":"2023-09-23T22:22:53.870919+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":225,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.888462+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:22:53.888462+0000","last_peered":"2023-09-23T22:22:53.888462+0000","last_clean":"2023-09-23T22:22:53.888462+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:22:53.888462+0000","last_undegraded":"2023-09-23T22:22:53.888462+0000","last_fullsized":"2023-09-23T22:22:53.888462+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":266,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.878578+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:22:53.878578+0000","last_peered":"2023-09-23T22:22:53.878578+0000","last_clean":"2023-09-23T22:22:53.878578+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:22:53.878578+0000","last_undegraded":"2023-09-23T22:22:53.878578+0000","last_fullsized":"2023-09-23T22:22:53.878578+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":135,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T22:23:09.942626+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:23:09.942626+0000","last_peered":"2023-09-23T22:23:09.942626+0000","last_clean":"2023-09-23T22:23:09.942626+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:23:09.942626+0000","last_undegraded":"2023-09-23T22:23:09.942626+0000","last_fullsized":"2023-09-23T22:23:09.942626+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":253,"reported_epoch":244,"state":"active+clean","last_fresh":"2023-09-23T22:22:53.869421+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:22:53.869421+0000","last_peered":"2023-09-23T22:22:53.869421+0000","last_clean":"2023-09-23T22:22:53.869421+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:22:53.869421+0000","last_undegraded":"2023-09-23T22:22:53.869421+0000","last_fullsized":"2023-09-23T22:22:53.869421+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3129999384,"num_objects":47761,"num_object_clones":0,"num_object_copies":143283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":34783,"num_objects_misplaced":5899,"num_objects_unfound":0,"num_objects_dirty":47761,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114935,"num_write_kb":5206209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16596,"num_bytes_recovered":1087504384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11462389760,"data_stored":11462377544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83210,"ondisk_log_size":83210,"up":48,"acting":35,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":370028,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":88,"num_read_kb":166,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234225,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1885820,"kb_used_data":1724660,"kb_used_omap":0,"kb_used_meta":161152,"kb_avail":91859332,"statfs":{"total":95995035648,"available":94063955968,"internally_reserved":0,"allocated":1766051840,"data_stored":1765062647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":165019648},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:03 2023","interfaces":[{"interface":"back","average":{"1min":9.048,"5min":4.476,"15min":2.2679999999999998},"min":{"1min":0.23000000000000001,"5min":0.22,"15min":0.22},"max":{"1min":158.042,"5min":158.042,"15min":158.042},"last":2.3719999999999999},{"interface":"front","average":{"1min":9.6690000000000005,"5min":4.7149999999999999,"15min":2.4830000000000001},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":157.952,"5min":157.952,"15min":157.952},"last":2.1709999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:22:10 2023","interfaces":[{"interface":"back","average":{"1min":7.3230000000000004,"5min":5.9000000000000004,"15min":2.8290000000000002},"min":{"1min":0.29799999999999999,"5min":0.254,"15min":0.246},"max":{"1min":99.385999999999996,"5min":122.461,"15min":123.79300000000001},"last":1.1519999999999999},{"interface":"front","average":{"1min":7.8239999999999998,"5min":6.1680000000000001,"15min":2.9980000000000002},"min":{"1min":0.28299999999999997,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":95.441999999999993,"5min":122.268,"15min":123.66500000000001},"last":2.4039999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":4.8609999999999998,"15min":3.6600000000000001},"min":{"1min":0.35299999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.5119999999999996,"5min":105.34399999999999,"15min":125.101},"last":2.2839999999999998},{"interface":"front","average":{"1min":1.7470000000000001,"5min":4.6589999999999998,"15min":3.375},"min":{"1min":0.34699999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.0549999999999997,"5min":105.63200000000001,"15min":126.021},"last":0.379}]},{"osd":3,"last update":"Sat Sep 23 22:22:10 2023","interfaces":[{"interface":"back","average":{"1min":4.3239999999999998,"5min":2.0979999999999999,"15min":1.5489999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":39.073999999999998,"5min":39.073999999999998,"15min":113.282},"last":2.637},{"interface":"front","average":{"1min":4.6589999999999998,"5min":2.452,"15min":1.7250000000000001},"min":{"1min":0.39800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":38.781999999999996,"5min":38.781999999999996,"15min":104.74299999999999},"last":2.153}]},{"osd":4,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":2.3980000000000001,"15min":1.2969999999999999},"min":{"1min":0.29699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":27.922000000000001,"5min":88.543000000000006,"15min":88.543000000000006},"last":2.0179999999999998},{"interface":"front","average":{"1min":2.738,"5min":2.3719999999999999,"15min":1.2669999999999999},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":29.920000000000002,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.34300000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":2.4279999999999999,"15min":1.226},"min":{"1min":0.32700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.965,"5min":93.816000000000003,"15min":93.816000000000003},"last":1.107},{"interface":"front","average":{"1min":2.4710000000000001,"5min":2.5230000000000001,"15min":1.4419999999999999},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.29799999999999999},"max":{"1min":24.712,"5min":95.579999999999998,"15min":95.579999999999998},"last":2.0760000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.80200000000000005,"15min":0.71399999999999997},"min":{"1min":0.34200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.343,"5min":7.9459999999999997,"15min":9.0139999999999993},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.153,"15min":0.91500000000000004},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":6.3879999999999999,"5min":10.347,"15min":13.362},"last":2.125}]}]},{"osd":6,"up_from":0,"seq":197568495861,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795133,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1729440,"kb_used_data":1438040,"kb_used_omap":7,"kb_used_meta":291256,"kb_avail":92015712,"statfs":{"total":95995035648,"available":94224089088,"internally_reserved":0,"allocated":1472552960,"data_stored":1471230710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":298246516},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4570000000000001,"5min":1.4690000000000001,"15min":1.014},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":11.138,"5min":11.138,"15min":61.890000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.5819999999999999,"5min":2.6139999999999999,"15min":1.4119999999999999},"min":{"1min":0.23200000000000001,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":15.768000000000001,"5min":15.768000000000001,"15min":62.689999999999998},"last":1.385}]},{"osd":2,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":2.0179999999999998,"15min":1.1850000000000001},"min":{"1min":0.252,"5min":0.223,"15min":0.216},"max":{"1min":11.179,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.1480000000000001,"5min":3.2370000000000001,"15min":1.5049999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.2670000000000003,"5min":34.781999999999996,"15min":34.781999999999996},"last":6.2069999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:22:20 2023","interfaces":[{"interface":"back","average":{"1min":6.1100000000000003,"5min":3.3300000000000001,"15min":1.5169999999999999},"min":{"1min":0.32700000000000001,"5min":0.252,"15min":0.252},"max":{"1min":44.902999999999999,"5min":44.902999999999999,"15min":44.902999999999999},"last":6.25},{"interface":"front","average":{"1min":5.7279999999999998,"5min":3.0529999999999999,"15min":1.4770000000000001},"min":{"1min":0.28199999999999997,"5min":0.223,"15min":0.223},"max":{"1min":45.725000000000001,"5min":45.725000000000001,"15min":45.725000000000001},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:22:20 2023","interfaces":[{"interface":"back","average":{"1min":11.180999999999999,"5min":3.1230000000000002,"15min":1.526},"min":{"1min":0.28599999999999998,"5min":0.28299999999999997,"15min":0.27000000000000002},"max":{"1min":234.79900000000001,"5min":234.79900000000001,"15min":234.79900000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":12.012,"5min":3.9830000000000001,"15min":1.7869999999999999},"min":{"1min":0.35799999999999998,"5min":0.33200000000000002,"15min":0.318},"max":{"1min":234.81700000000001,"5min":234.81700000000001,"15min":234.81700000000001},"last":1.3220000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":10.313000000000001,"5min":3.6800000000000002,"15min":1.659},"min":{"1min":0.33100000000000002,"5min":0.31,"15min":0.31},"max":{"1min":196.79900000000001,"5min":196.79900000000001,"15min":196.79900000000001},"last":1.359},{"interface":"front","average":{"1min":10.337,"5min":3.4889999999999999,"15min":1.645},"min":{"1min":0.41899999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":196.68299999999999,"5min":196.68299999999999,"15min":196.68299999999999},"last":7.8920000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.819,"5min":1.841,"15min":1.099},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":5.5629999999999997,"5min":10.989000000000001,"15min":10.989000000000001},"last":1.048},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.3,"15min":0.95399999999999996},"min":{"1min":0.40999999999999998,"5min":0.35699999999999998,"15min":0.34999999999999998},"max":{"1min":11.385999999999999,"5min":11.385999999999999,"15min":11.385999999999999},"last":0.64200000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.9610000000000001,"15min":1.1519999999999999},"min":{"1min":0.378,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":5.5970000000000004,"5min":10.948,"15min":10.948},"last":0.85999999999999999},{"interface":"front","average":{"1min":1.159,"5min":1.389,"15min":0.98399999999999999},"min":{"1min":0.38300000000000001,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":11.204000000000001,"5min":11.204000000000001,"15min":11.204000000000001},"last":0.71799999999999997}]}]},{"osd":0,"up_from":93,"seq":399431958668,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1161168,"kb_used_data":894344,"kb_used_omap":8,"kb_used_meta":266807,"kb_avail":92583984,"statfs":{"total":95995035648,"available":94805999616,"internally_reserved":0,"allocated":915808256,"data_stored":914452269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":273211242},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":12.625999999999999,"5min":4.423,"15min":1.95},"min":{"1min":0.22,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":169.77199999999999,"5min":169.77199999999999,"15min":169.77199999999999},"last":0.436},{"interface":"front","average":{"1min":12.736000000000001,"5min":5.3659999999999997,"15min":2.3439999999999999},"min":{"1min":0.29199999999999998,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":169.52099999999999,"5min":169.52099999999999,"15min":169.52099999999999},"last":8.0809999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:22:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":6.5620000000000003,"15min":2.5870000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":10.891,"5min":206.495,"15min":206.495},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.8759999999999999,"5min":7.71,"15min":3.1890000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.22800000000000001},"max":{"1min":15.186,"5min":207.34399999999999,"15min":207.34399999999999},"last":6.9809999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:22:18 2023","interfaces":[{"interface":"back","average":{"1min":2.746,"5min":3.9969999999999999,"15min":1.869},"min":{"1min":0.28000000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.848000000000001,"5min":131.078,"15min":131.078},"last":4.6349999999999998},{"interface":"front","average":{"1min":1.655,"5min":4.008,"15min":1.9319999999999999},"min":{"1min":0.247,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.8289999999999997,"5min":133.083,"15min":133.083},"last":6.6310000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.23,"15min":1.7849999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":9.673,"5min":82.825000000000003,"15min":82.825000000000003},"last":2.649},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.306,"15min":1.2509999999999999},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.308},"max":{"1min":3.129,"5min":75.930999999999997,"15min":75.930999999999997},"last":0.377}]},{"osd":5,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":8.1050000000000004,"5min":4.4669999999999996,"15min":2.173},"min":{"1min":0.39200000000000002,"5min":0.34799999999999998,"15min":0.29499999999999998},"max":{"1min":93.685000000000002,"5min":93.685000000000002,"15min":93.685000000000002},"last":2.5990000000000002},{"interface":"front","average":{"1min":7.6550000000000002,"5min":3.5459999999999998,"15min":1.7030000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":93.546000000000006,"5min":93.546000000000006,"15min":93.546000000000006},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":2.3069999999999999,"15min":1.472},"min":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":9.7140000000000004,"5min":10.275,"15min":10.275},"last":2.7149999999999999},{"interface":"front","average":{"1min":2.3769999999999998,"5min":1.764,"15min":1.115},"min":{"1min":0.43099999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":10.919,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":4.0570000000000004,"5min":2.5800000000000001,"15min":1.643},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":32.773000000000003,"5min":32.773000000000003,"15min":32.773000000000003},"last":4.851},{"interface":"front","average":{"1min":2.069,"5min":2.4249999999999998,"15min":1.6850000000000001},"min":{"1min":0.39900000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5790000000000006,"5min":10.257999999999999,"15min":23.265999999999998},"last":2.6840000000000002}]}]},{"osd":2,"up_from":0,"seq":442381631607,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468080,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2246080,"kb_used_data":2061880,"kb_used_omap":4,"kb_used_meta":184123,"kb_avail":91499072,"statfs":{"total":95995035648,"available":93695049728,"internally_reserved":0,"allocated":2111365120,"data_stored":2110011180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":188542509},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":3.3370000000000002,"5min":5.6219999999999999,"15min":2.5230000000000001},"min":{"1min":0.28599999999999998,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":18.292000000000002,"5min":225.86699999999999,"15min":225.86699999999999},"last":1.077},{"interface":"front","average":{"1min":3.298,"5min":6.0110000000000001,"15min":2.6600000000000001},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":16.117999999999999,"5min":225.84800000000001,"15min":225.84800000000001},"last":1.1699999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":5.343,"5min":4.2670000000000003,"15min":2.0449999999999999},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.218},"max":{"1min":37.002000000000002,"5min":82.951999999999998,"15min":82.951999999999998},"last":2.891},{"interface":"front","average":{"1min":4.04,"5min":3.7949999999999999,"15min":1.9279999999999999},"min":{"1min":0.373,"5min":0.22600000000000001,"15min":0.189},"max":{"1min":33.119999999999997,"5min":86.001000000000005,"15min":86.001000000000005},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":5.8849999999999998,"15min":3.2959999999999998},"min":{"1min":0.26500000000000001,"5min":0.252,"15min":0.223},"max":{"1min":10.23,"5min":99.962999999999994,"15min":99.962999999999994},"last":2.6579999999999999},{"interface":"front","average":{"1min":2.387,"5min":5.8840000000000003,"15min":3.089},"min":{"1min":0.34300000000000003,"5min":0.251,"15min":0.251},"max":{"1min":8.2080000000000002,"5min":100.429,"15min":100.429},"last":0.56200000000000006}]},{"osd":4,"last update":"Sat Sep 23 22:22:17 2023","interfaces":[{"interface":"back","average":{"1min":1.986,"5min":1.96,"15min":1.105},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":7.5620000000000003,"5min":11.911,"15min":11.911},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.8100000000000001,"5min":1.9510000000000001,"15min":1.1220000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.3579999999999997,"5min":12.036,"15min":12.036},"last":0.53100000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":1.742,"5min":2.3620000000000001,"15min":1.248},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":7.1719999999999997,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.496},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.601,"15min":1.3120000000000001},"min":{"1min":0.39800000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":10.031000000000001,"5min":49.668999999999997,"15min":49.668999999999997},"last":2.2389999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":2.2280000000000002,"5min":2.0680000000000001,"15min":1.2090000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":8.1780000000000008,"5min":12.068,"15min":12.068},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.881,"5min":2.0019999999999998,"15min":1.157},"min":{"1min":0.41799999999999998,"5min":0.35599999999999998,"15min":0.35399999999999998},"max":{"1min":9.2870000000000008,"5min":11.68,"15min":11.68},"last":2.1549999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:22:12 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":2.2869999999999999,"15min":1.2450000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":8.218,"5min":16.849,"15min":16.849},"last":2.202},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.9590000000000001,"15min":1.1619999999999999},"min":{"1min":0.433,"5min":0.39300000000000002,"15min":0.36699999999999999},"max":{"1min":8.1400000000000006,"5min":12.382,"15min":12.382},"last":0.46200000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019130,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1357940,"kb_used_data":1188588,"kb_used_omap":0,"kb_used_meta":169344,"kb_avail":92387212,"statfs":{"total":95995035648,"available":94604505088,"internally_reserved":0,"allocated":1217114112,"data_stored":1216125377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":173408256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":2.4340000000000002,"15min":1.649},"min":{"1min":0.28399999999999997,"5min":0.23000000000000001,"15min":0.218},"max":{"1min":9.2579999999999991,"5min":46.18,"15min":66.355000000000004},"last":2.9430000000000001},{"interface":"front","average":{"1min":2.379,"5min":2.5019999999999998,"15min":1.7350000000000001},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":9.2390000000000008,"5min":46.207000000000001,"15min":66.332999999999998},"last":3.988}]},{"osd":1,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":2.0470000000000002,"15min":1.3460000000000001},"min":{"1min":0.26700000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.0750000000000002,"5min":35.409999999999997,"15min":38.878},"last":1.954},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.492,"15min":1.694},"min":{"1min":0.28100000000000003,"5min":0.255,"15min":0.255},"max":{"1min":7.8620000000000001,"5min":35.999000000000002,"15min":38.853999999999999},"last":2.3980000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9159999999999999,"5min":4.5730000000000004,"15min":2.1589999999999998},"min":{"1min":0.372,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":6.1020000000000003,"5min":138.06100000000001,"15min":138.06100000000001},"last":3.5339999999999998},{"interface":"front","average":{"1min":1.8120000000000001,"5min":4.6079999999999997,"15min":2.327},"min":{"1min":0.35899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9580000000000002,"5min":138.15199999999999,"15min":138.15199999999999},"last":4.0330000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:22:20 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":1.78,"15min":1.073},"min":{"1min":0.34899999999999998,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":3.4009999999999998,"5min":8.6340000000000003,"15min":8.6340000000000003},"last":12.129},{"interface":"front","average":{"1min":1.968,"5min":1.8200000000000001,"15min":1.0940000000000001},"min":{"1min":0.34699999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.5970000000000004,"5min":9.8109999999999999,"15min":9.8109999999999999},"last":3.8929999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:22:20 2023","interfaces":[{"interface":"back","average":{"1min":2.5249999999999999,"5min":1.3100000000000001,"15min":0.82499999999999996},"min":{"1min":0.25800000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":28.399000000000001,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.438},{"interface":"front","average":{"1min":2.9350000000000001,"5min":1.3120000000000001,"15min":0.83099999999999996},"min":{"1min":0.32600000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":28.350999999999999,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.39700000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:22:33 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.82699999999999996,"15min":0.67000000000000004},"min":{"1min":0.29299999999999998,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":3.3519999999999999,"5min":7.3079999999999998,"15min":7.3079999999999998},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.86899999999999999,"15min":0.72499999999999998},"min":{"1min":0.32200000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0840000000000001,"5min":6.7279999999999998,"15min":8.5210000000000008},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.94799999999999995,"15min":0.83999999999999997},"min":{"1min":0.313,"5min":0.313,"15min":0.307},"max":{"1min":2.754,"5min":6.7480000000000002,"15min":8.577},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.143,"5min":0.94099999999999995,"15min":0.78700000000000003},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":5.1479999999999997,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.33100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757498,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1400748,"kb_used_data":1200676,"kb_used_omap":0,"kb_used_meta":200064,"kb_avail":92344404,"statfs":{"total":95995035648,"available":94560669696,"internally_reserved":0,"allocated":1229492224,"data_stored":1228507458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":204865536},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:55 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":2.1150000000000002,"15min":1.859},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":12.661,"5min":28.741,"15min":133.22499999999999},"last":2.0739999999999998},{"interface":"front","average":{"1min":3.198,"5min":1.8959999999999999,"15min":1.8979999999999999},"min":{"1min":0.26100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.324999999999999,"5min":28.873999999999999,"15min":132.78200000000001},"last":0.316}]},{"osd":1,"last update":"Sat Sep 23 22:22:16 2023","interfaces":[{"interface":"back","average":{"1min":4.3019999999999996,"5min":5.319,"15min":2.569},"min":{"1min":0.29899999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":50.960999999999999,"5min":93.981999999999999,"15min":93.981999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":4.5069999999999997,"5min":5.3079999999999998,"15min":2.5619999999999998},"min":{"1min":0.35299999999999998,"5min":0.252,"15min":0.252},"max":{"1min":50.979999999999997,"5min":93.909999999999997,"15min":93.909999999999997},"last":0.40500000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:22:49 2023","interfaces":[{"interface":"back","average":{"1min":2.972,"5min":6.0650000000000004,"15min":3.4329999999999998},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.534000000000001,"5min":104.254,"15min":104.254},"last":1.8999999999999999},{"interface":"front","average":{"1min":1.661,"5min":6.3380000000000001,"15min":3.7040000000000002},"min":{"1min":0.35199999999999998,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":5.7380000000000004,"5min":104.217,"15min":104.217},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:22:16 2023","interfaces":[{"interface":"back","average":{"1min":4.3099999999999996,"5min":3.3140000000000001,"15min":1.992},"min":{"1min":0.314,"5min":0.314,"15min":0.246},"max":{"1min":50.942999999999998,"5min":93.739999999999995,"15min":93.739999999999995},"last":18.876000000000001},{"interface":"front","average":{"1min":4.3090000000000002,"5min":3.3380000000000001,"15min":2.036},"min":{"1min":0.29399999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":51.235999999999997,"5min":93.197999999999993,"15min":93.197999999999993},"last":5.0259999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:22:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8240000000000001,"5min":1.073,"15min":0.95999999999999996},"min":{"1min":0.28299999999999997,"5min":0.252,"15min":0.252},"max":{"1min":10.163,"5min":10.163,"15min":82.530000000000001},"last":1.587},{"interface":"front","average":{"1min":1.2669999999999999,"5min":0.997,"15min":0.98999999999999999},"min":{"1min":0.30099999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.806,"5min":6.577,"15min":85.441000000000003},"last":0.45100000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:22:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.1399999999999999,"15min":1.123},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.26400000000000001},"max":{"1min":4.056,"5min":9.6519999999999992,"15min":114.83499999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1,"15min":1.0469999999999999},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.8690000000000002,"5min":7.4640000000000004,"15min":110.905},"last":0.47299999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.159,"15min":0.84299999999999997},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.9409999999999998,"5min":8.2159999999999993,"15min":8.8030000000000008},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.82499999999999996,"15min":0.745},"min":{"1min":0.39100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5299999999999998,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":1.6930000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":900665344,"data_stored":900661272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1461129216,"data_stored":1461125144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2823749632,"data_stored":2823749632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2084634624,"data_stored":2084634624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1214382080,"data_stored":1214382080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1216286720,"data_stored":1216282648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1761542144,"data_stored":1761542144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:23:17.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:23:17.212 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:17.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:23:17.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:23:17.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:17.240 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:17.247 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:23:17.247 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:23:17.247 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:23:17.251 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:17.329 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:17.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:17.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:17.437 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:17.474 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:17.534 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:17.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:17.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:17.641 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:17.681 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:17.737 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:17.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:17.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:17.801 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:23:17.801 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:23:17.801 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:23:17.801 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:23:17.801 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:23:17.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:17.942 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:18.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:17 smithi026 ceph-mon[215775]: pgmap v599: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 591 op/s; 34627/134703 objects degraded (25.706%); 5196/134703 objects misplaced (3.857%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:23:18.161 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:17 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 34627/137877 objects degraded (25.114%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:18.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:17 smithi026 ceph-mon[223014]: pgmap v599: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 591 op/s; 34627/134703 objects degraded (25.706%); 5196/134703 objects misplaced (3.857%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:23:18.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:17 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 34627/137877 objects degraded (25.114%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:18.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:17 smithi070 ceph-mon[192652]: pgmap v599: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 591 op/s; 34627/134703 objects degraded (25.706%); 5196/134703 objects misplaced (3.857%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:23:18.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:17 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 34627/137877 objects degraded (25.114%), 17 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:19.903 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:23:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:23:19] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T22:23:20.425 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:23:20.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:20 smithi070 ceph-mon[192652]: pgmap v600: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 34627/133038 objects degraded (26.028%); 5196/133038 objects misplaced (3.906%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:23:20.662 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:20 smithi026 ceph-mon[215775]: pgmap v600: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 34627/133038 objects degraded (26.028%); 5196/133038 objects misplaced (3.906%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:23:20.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:20 smithi026 ceph-mon[223014]: pgmap v600: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.7 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 531 op/s; 34627/133038 objects degraded (26.028%); 5196/133038 objects misplaced (3.906%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:23:20.710 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:20.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:20.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:20.827 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:20.835 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:20.861 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:20.933 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:20.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:20.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:21.042 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:21.079 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:21.155 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:21.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:21.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:21.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:21.308 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:21.362 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:21.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:21.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:21.467 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:21.513 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:21.518 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:23:21.575 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:21.576 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:23:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:21.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:21.637 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:23:21.682 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:21.687 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:23:21.719 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:23:21.759 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:23:21.807 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:21.820 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:23:21.868 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:23:21.878 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:21.890 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:23:21.890 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:23:21.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:23:21.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:23:21.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:23:21.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:23:21.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:23:21.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:23:21.985 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:21.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:21.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:22.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:22.201 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:22.357 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:23:22.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:22 smithi070 ceph-mon[192652]: pgmap v601: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 652 op/s; 34325/126792 objects degraded (27.072%); 4742/126792 objects misplaced (3.740%); 10 MiB/s, 174 objects/s recovering 2023-09-23T22:23:22.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4231701760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:22.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/944781991' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:23:22.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:22 smithi026 ceph-mon[215775]: pgmap v601: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 652 op/s; 34325/126792 objects degraded (27.072%); 4742/126792 objects misplaced (3.740%); 10 MiB/s, 174 objects/s recovering 2023-09-23T22:23:22.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4231701760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:22.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/944781991' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:23:22.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:22 smithi026 ceph-mon[223014]: pgmap v601: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 652 op/s; 34325/126792 objects degraded (27.072%); 4742/126792 objects misplaced (3.740%); 10 MiB/s, 174 objects/s recovering 2023-09-23T22:23:22.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4231701760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:22.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/944781991' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:23:23.404 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:23.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:23.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:23.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:23.566 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:23 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:23:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:23 smithi070 ceph-mon[192652]: failed to encode map e247 with expected crc 2023-09-23T22:23:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/944781991' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:23:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:23 smithi070 ceph-mon[192652]: osdmap e247: 8 total, 8 up, 6 in 2023-09-23T22:23:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:23 smithi070 ceph-mon[192652]: failed to encode map e247 with expected crc 2023-09-23T22:23:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 34325/126792 objects degraded (27.072%), 17 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:23.634 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:23.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:23.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:23:23.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[215775]: failed to encode map e247 with expected crc 2023-09-23T22:23:23.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/944781991' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:23:23.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[215775]: osdmap e247: 8 total, 8 up, 6 in 2023-09-23T22:23:23.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[215775]: failed to encode map e247 with expected crc 2023-09-23T22:23:23.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 34325/126792 objects degraded (27.072%), 17 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:23.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:23:23.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[223014]: failed to encode map e247 with expected crc 2023-09-23T22:23:23.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/944781991' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:23:23.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[223014]: osdmap e247: 8 total, 8 up, 6 in 2023-09-23T22:23:23.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[223014]: failed to encode map e247 with expected crc 2023-09-23T22:23:23.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 34325/126792 objects degraded (27.072%), 17 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:23.742 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:23.786 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:23.869 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:23.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:23.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:23.977 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:24.009 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:24.084 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:24.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:24.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:24.192 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:24.245 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:24.328 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:24.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:24.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:24.445 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:24.514 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:24.737 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:24.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:24.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:24.851 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:24.909 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:24.978 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:24.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:24.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:25.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:24 smithi026 ceph-mon[223014]: pgmap v603: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 34325/125730 objects degraded (27.301%); 4554/125730 objects misplaced (3.622%); 8.6 MiB/s, 148 objects/s recovering 2023-09-23T22:23:25.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:25.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:24 smithi026 ceph-mon[215775]: pgmap v603: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 34325/125730 objects degraded (27.301%); 4554/125730 objects misplaced (3.622%); 8.6 MiB/s, 148 objects/s recovering 2023-09-23T22:23:25.062 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:25.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:25.156 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:25.344 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:25.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:25.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:25.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:24 smithi070 ceph-mon[192652]: pgmap v603: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.6 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 34325/125730 objects degraded (27.301%); 4554/125730 objects misplaced (3.622%); 8.6 MiB/s, 148 objects/s recovering 2023-09-23T22:23:25.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:25.463 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:25.510 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:25.579 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:25.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:25.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:25.691 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:25.732 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:25.791 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:25.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:25.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:25.809 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:23:25.869 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:23:25.869 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 34325/125730 objects degraded (27.301%), 17 pgs degraded, 13 pgs undersized","count":30},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":789,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":247,"num_osds":8,"num_up_osds":8,"osd_up_since":1695507239,"num_in_osds":6,"osd_in_since":1695507716,"num_remapped_pgs":16},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":81},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":13},{"state_name":"active+recovering+undersized+degraded+remapped","count":2},{"state_name":"active+recovery_wait+degraded","count":2},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":99,"num_pools":5,"num_objects":41910,"data_bytes":2749348631,"bytes_used":9168023552,"bytes_avail":566802190336,"bytes_total":575970213888,"degraded_objects":34325,"degraded_total":125730,"degraded_ratio":0.27300564702139507,"misplaced_objects":4554,"misplaced_total":125730,"misplaced_ratio":0.036220472440944881,"recovering_objects_per_sec":148,"recovering_bytes_per_sec":9063289,"recovering_keys_per_sec":0,"num_objects_recovered":1488,"num_bytes_recovered":90767360,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":586},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T22:14:59.012277+0000","services":{}},"progress_events":{"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"f9b296f7-36c4-4993-8ca1-c3652b75187b":{"message":"Global Recovery Event (2m)\n [======================......] (remaining: 27s)","progress":0.81818181276321411,"add_to_ceph_s":true}}} 2023-09-23T22:23:25.899 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:25.993 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:26.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3629736546' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:23:26.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3629736546' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:23:26.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3629736546' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:23:26.695 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:23:26.704 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:26.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:26.813 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:26.997 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:26 smithi026 ceph-mon[215775]: pgmap v604: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 34110/119964 objects degraded (28.434%); 4349/119964 objects misplaced (3.625%); 8.8 MiB/s, 151 objects/s recovering 2023-09-23T22:23:26.998 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:26 smithi026 ceph-mon[223014]: pgmap v604: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 34110/119964 objects degraded (28.434%); 4349/119964 objects misplaced (3.625%); 8.8 MiB/s, 151 objects/s recovering 2023-09-23T22:23:27.009 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:27.183 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:23:27.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:23:27.332 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:27.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:27.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:27.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:26 smithi070 ceph-mon[192652]: pgmap v604: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 596 op/s; 34110/119964 objects degraded (28.434%); 4349/119964 objects misplaced (3.625%); 8.8 MiB/s, 151 objects/s recovering 2023-09-23T22:23:27.444 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:27.485 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:27.543 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:27.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:27.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:27.597 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:27.649 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:27.728 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:27.751 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:23:27.751 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:23:27.767 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:28.108 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:27 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 34110/119964 objects degraded (28.434%), 17 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:28.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:27 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 34110/119964 objects degraded (28.434%), 17 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:28.172 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:28.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:28.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:28.280 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:28.359 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:28.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:27 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 34110/119964 objects degraded (28.434%), 17 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:29.031 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:29.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:29.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:29.136 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:29.166 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:29.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:28 smithi026 ceph-mon[215775]: pgmap v605: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 33640/116406 objects degraded (28.899%); 4122/116406 objects misplaced (3.541%); 7.0 MiB/s, 168 objects/s recovering 2023-09-23T22:23:29.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:28 smithi026 ceph-mon[215775]: osdmap e248: 8 total, 8 up, 6 in 2023-09-23T22:23:29.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:28 smithi026 ceph-mon[223014]: pgmap v605: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 33640/116406 objects degraded (28.899%); 4122/116406 objects misplaced (3.541%); 7.0 MiB/s, 168 objects/s recovering 2023-09-23T22:23:29.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:28 smithi026 ceph-mon[223014]: osdmap e248: 8 total, 8 up, 6 in 2023-09-23T22:23:29.219 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:29.222 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:28 smithi070 ceph-mon[192652]: pgmap v605: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.4 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 609 op/s; 33640/116406 objects degraded (28.899%); 4122/116406 objects misplaced (3.541%); 7.0 MiB/s, 168 objects/s recovering 2023-09-23T22:23:29.222 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:28 smithi070 ceph-mon[192652]: osdmap e248: 8 total, 8 up, 6 in 2023-09-23T22:23:29.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:29.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:29.246 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:23:29.324 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:29.355 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:29.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:23:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:23:29] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-23T22:23:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:29 smithi026 ceph-mon[215775]: failed to encode map e248 with expected crc 2023-09-23T22:23:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:29 smithi026 ceph-mon[215775]: failed to encode map e248 with expected crc 2023-09-23T22:23:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:29 smithi026 ceph-mon[215775]: failed to encode map e248 with expected crc 2023-09-23T22:23:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:29 smithi026 ceph-mon[215775]: osdmap e249: 8 total, 8 up, 6 in 2023-09-23T22:23:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:29 smithi026 ceph-mon[223014]: failed to encode map e248 with expected crc 2023-09-23T22:23:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:29 smithi026 ceph-mon[223014]: failed to encode map e248 with expected crc 2023-09-23T22:23:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:29 smithi026 ceph-mon[223014]: failed to encode map e248 with expected crc 2023-09-23T22:23:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:29 smithi026 ceph-mon[223014]: osdmap e249: 8 total, 8 up, 6 in 2023-09-23T22:23:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:29 smithi070 ceph-mon[192652]: failed to encode map e248 with expected crc 2023-09-23T22:23:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:29 smithi070 ceph-mon[192652]: failed to encode map e248 with expected crc 2023-09-23T22:23:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:29 smithi070 ceph-mon[192652]: failed to encode map e248 with expected crc 2023-09-23T22:23:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:29 smithi070 ceph-mon[192652]: osdmap e249: 8 total, 8 up, 6 in 2023-09-23T22:23:30.480 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:30.597 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:30.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:30.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:30.715 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:30.748 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:30.813 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:30.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:30.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:30.920 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:30.979 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:31.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:30 smithi070 ceph-mon[192652]: pgmap v607: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 507 op/s; 33640/114597 objects degraded (29.355%); 4122/114597 objects misplaced (3.597%); 4.9 MiB/s, 137 objects/s recovering 2023-09-23T22:23:31.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:30 smithi070 ceph-mon[192652]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:30 smithi070 ceph-mon[192652]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:30 smithi070 ceph-mon[192652]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:30 smithi070 ceph-mon[192652]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:30 smithi070 ceph-mon[192652]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:30 smithi070 ceph-mon[192652]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[215775]: pgmap v607: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 507 op/s; 33640/114597 objects degraded (29.355%); 4122/114597 objects misplaced (3.597%); 4.9 MiB/s, 137 objects/s recovering 2023-09-23T22:23:31.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[215775]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[215775]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[215775]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[215775]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[215775]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[215775]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[223014]: pgmap v607: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 81 active+clean; 2.3 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 507 op/s; 33640/114597 objects degraded (29.355%); 4122/114597 objects misplaced (3.597%); 4.9 MiB/s, 137 objects/s recovering 2023-09-23T22:23:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[223014]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[223014]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[223014]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[223014]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[223014]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:30 smithi026 ceph-mon[223014]: failed to encode map e249 with expected crc 2023-09-23T22:23:31.365 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:31.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:31.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:31.472 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:31.562 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:31.675 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:31.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:31.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:31.768 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:31.783 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:31.906 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:31.962 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:23:32.005 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:23:32.039 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:23:32.082 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:23:32.134 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:23:32.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3845955847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:32.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3845955847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:32.167 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:23:32.200 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:23:32.236 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:23:32.239 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:23:32.242 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:23:32.242 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:23:32.242 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:23:32.242 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:23:32.242 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:23:32.242 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:23:32.242 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:23:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3845955847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:32.397 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:32.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:32.407 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:32.507 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:32.560 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:32.633 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:32.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:32.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:32.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:32.779 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:32.847 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:32.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:32.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:32.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:32 smithi026 ceph-mon[215775]: pgmap v609: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 33309/109098 objects degraded (30.531%); 3935/109098 objects misplaced (3.607%); 4.5 MiB/s, 155 objects/s recovering 2023-09-23T22:23:32.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:32 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 33309/109098 objects degraded (30.531%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:32.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:32 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1888295057' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:32.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:32 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:32.956 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:33.014 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:33.110 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:33.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:33.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:32 smithi070 ceph-mon[192652]: pgmap v609: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 33309/109098 objects degraded (30.531%); 3935/109098 objects misplaced (3.607%); 4.5 MiB/s, 155 objects/s recovering 2023-09-23T22:23:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:32 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 33309/109098 objects degraded (30.531%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:32 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1888295057' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:33.121 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:32 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:33.219 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:33.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:32 smithi026 ceph-mon[223014]: pgmap v609: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 692 op/s; 33309/109098 objects degraded (30.531%); 3935/109098 objects misplaced (3.607%); 4.5 MiB/s, 155 objects/s recovering 2023-09-23T22:23:33.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:32 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 33309/109098 objects degraded (30.531%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:33.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:32 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1888295057' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:33.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:32 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:33.312 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:33.911 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:23:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:33 smithi026 ceph-mon[223014]: pgmap v610: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 33309/108009 objects degraded (30.839%); 3599/108009 objects misplaced (3.332%); 5.5 MiB/s, 170 objects/s recovering 2023-09-23T22:23:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:33 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:23:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:33 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:23:34.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:33 smithi026 ceph-mon[223014]: osdmap e250: 8 total, 8 up, 6 in 2023-09-23T22:23:34.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:33 smithi026 ceph-mon[215775]: pgmap v610: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 33309/108009 objects degraded (30.839%); 3599/108009 objects misplaced (3.332%); 5.5 MiB/s, 170 objects/s recovering 2023-09-23T22:23:34.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:33 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:23:34.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:33 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:23:34.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:33 smithi026 ceph-mon[215775]: osdmap e250: 8 total, 8 up, 6 in 2023-09-23T22:23:34.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:33 smithi070 ceph-mon[192652]: pgmap v610: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.2 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 497 op/s; 33309/108009 objects degraded (30.839%); 3599/108009 objects misplaced (3.332%); 5.5 MiB/s, 170 objects/s recovering 2023-09-23T22:23:34.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:33 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:23:34.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:33 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:23:34.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:33 smithi070 ceph-mon[192652]: osdmap e250: 8 total, 8 up, 6 in 2023-09-23T22:23:34.785 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:34.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:34.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:34.896 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:34.928 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:35.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:34 smithi026 ceph-mon[215775]: failed to encode map e250 with expected crc 2023-09-23T22:23:35.147 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:34 smithi026 ceph-mon[223014]: failed to encode map e250 with expected crc 2023-09-23T22:23:35.326 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:35.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:35.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:34 smithi070 ceph-mon[192652]: failed to encode map e250 with expected crc 2023-09-23T22:23:35.434 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:35.475 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:35.554 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:35.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:35.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:35.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:35.741 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:35.974 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:35 smithi026 ceph-mon[223014]: pgmap v612: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 33089/102267 objects degraded (32.356%); 3400/102267 objects misplaced (3.325%); 6.8 MiB/s, 139 objects/s recovering 2023-09-23T22:23:35.974 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:35 smithi026 ceph-mon[215775]: pgmap v612: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 33089/102267 objects degraded (32.356%); 3400/102267 objects misplaced (3.325%); 6.8 MiB/s, 139 objects/s recovering 2023-09-23T22:23:35.986 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:35.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:35.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:36.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:36.120 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:23:36.122 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:36.123 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:23:36.165 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:36.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:36.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:36.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:36.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:35 smithi070 ceph-mon[192652]: pgmap v612: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.1 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 33089/102267 objects degraded (32.356%); 3400/102267 objects misplaced (3.325%); 6.8 MiB/s, 139 objects/s recovering 2023-09-23T22:23:36.382 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:36.514 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:23:36.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:36 smithi026 ceph-mon[215775]: from='client.55042 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:36.995 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:36.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:36.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:37.099 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:37.145 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:37.221 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:37.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:37.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:36 smithi026 ceph-mon[223014]: from='client.55042 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:37.328 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:37.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:36 smithi070 ceph-mon[192652]: from='client.55042 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:37.374 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:37.456 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:37.466 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:23:37.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:37.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:37.485 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":612,"stamp":"2023-09-23T22:23:35.260542+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2236791575,"num_objects":34089,"num_object_clones":0,"num_object_copies":102267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33089,"num_objects_misplaced":3400,"num_objects_unfound":0,"num_objects_dirty":34089,"num_whiteouts":0,"num_read":6071,"num_read_kb":12351,"num_write":129585,"num_write_kb":5219374,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20030,"num_bytes_recovered":1261769423,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85811,"ondisk_log_size":85811,"up":297,"acting":283,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7678128,"kb_used_data":6311360,"kb_used_omap":20,"kb_used_meta":1366635,"kb_avail":554792784,"statfs":{"total":575970213888,"available":568107810816,"internally_reserved":0,"allocated":6462832640,"data_stored":6455736051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1399435019},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-269352960,"num_objects":-4110,"num_object_clones":0,"num_object_copies":-12330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-551,"num_objects_misplaced":-722,"num_objects_unfound":0,"num_objects_dirty":-4110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4110,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":887,"num_bytes_recovered":45481984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.380707"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":237,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.955303+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:23:29.955303+0000","last_peered":"2023-09-23T22:23:29.955303+0000","last_clean":"2023-09-23T22:23:29.955303+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:23:29.955303+0000","last_undegraded":"2023-09-23T22:23:29.955303+0000","last_fullsized":"2023-09-23T22:23:29.955303+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":228,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.883625+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:23:29.883625+0000","last_peered":"2023-09-23T22:23:29.883625+0000","last_clean":"2023-09-23T22:23:29.883625+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:23:29.883625+0000","last_undegraded":"2023-09-23T22:23:29.883625+0000","last_fullsized":"2023-09-23T22:23:29.883625+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":190,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.892031+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:23:28.892031+0000","last_peered":"2023-09-23T22:23:28.892031+0000","last_clean":"2023-09-23T22:23:28.892031+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:23:28.892031+0000","last_undegraded":"2023-09-23T22:23:28.892031+0000","last_fullsized":"2023-09-23T22:23:28.892031+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":106,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891497+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:23:28.891497+0000","last_peered":"2023-09-23T22:23:28.891497+0000","last_clean":"2023-09-23T22:23:28.891497+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:23:28.891497+0000","last_undegraded":"2023-09-23T22:23:28.891497+0000","last_fullsized":"2023-09-23T22:23:28.891497+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":268,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.885913+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:23:29.885913+0000","last_peered":"2023-09-23T22:23:29.885913+0000","last_clean":"2023-09-23T22:23:29.885913+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:23:29.885913+0000","last_undegraded":"2023-09-23T22:23:29.885913+0000","last_fullsized":"2023-09-23T22:23:29.885913+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":204,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.885647+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:23:28.885647+0000","last_peered":"2023-09-23T22:23:28.885647+0000","last_clean":"2023-09-23T22:23:28.885647+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:23:28.885647+0000","last_undegraded":"2023-09-23T22:23:28.885647+0000","last_fullsized":"2023-09-23T22:23:28.885647+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":134,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.892134+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:23:28.892134+0000","last_peered":"2023-09-23T22:23:28.892134+0000","last_clean":"2023-09-23T22:23:28.892134+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:23:28.892134+0000","last_undegraded":"2023-09-23T22:23:28.892134+0000","last_fullsized":"2023-09-23T22:23:28.892134+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":226,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.885959+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:23:29.885959+0000","last_peered":"2023-09-23T22:23:29.885959+0000","last_clean":"2023-09-23T22:23:29.885959+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:23:29.885959+0000","last_undegraded":"2023-09-23T22:23:29.885959+0000","last_fullsized":"2023-09-23T22:23:29.885959+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":209,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.885387+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:23:28.885387+0000","last_peered":"2023-09-23T22:23:28.885387+0000","last_clean":"2023-09-23T22:23:28.885387+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:23:28.885387+0000","last_undegraded":"2023-09-23T22:23:28.885387+0000","last_fullsized":"2023-09-23T22:23:28.885387+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":194,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.883118+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:23:29.883118+0000","last_peered":"2023-09-23T22:23:29.883118+0000","last_clean":"2023-09-23T22:23:29.883118+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:23:29.883118+0000","last_undegraded":"2023-09-23T22:23:29.883118+0000","last_fullsized":"2023-09-23T22:23:29.883118+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":193,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.883547+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:23:29.883547+0000","last_peered":"2023-09-23T22:23:29.883547+0000","last_clean":"2023-09-23T22:23:29.883547+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:23:29.883547+0000","last_undegraded":"2023-09-23T22:23:29.883547+0000","last_fullsized":"2023-09-23T22:23:29.883547+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":226,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891343+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:23:28.891343+0000","last_peered":"2023-09-23T22:23:28.891343+0000","last_clean":"2023-09-23T22:23:28.891343+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:23:28.891343+0000","last_undegraded":"2023-09-23T22:23:28.891343+0000","last_fullsized":"2023-09-23T22:23:28.891343+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":254,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.952289+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:23:29.952289+0000","last_peered":"2023-09-23T22:23:29.952289+0000","last_clean":"2023-09-23T22:23:29.952289+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:23:29.952289+0000","last_undegraded":"2023-09-23T22:23:29.952289+0000","last_fullsized":"2023-09-23T22:23:29.952289+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":218,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.952675+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:23:29.952675+0000","last_peered":"2023-09-23T22:23:29.952675+0000","last_clean":"2023-09-23T22:23:29.952675+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:23:29.952675+0000","last_undegraded":"2023-09-23T22:23:29.952675+0000","last_fullsized":"2023-09-23T22:23:29.952675+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":177,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.952776+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:23:29.952776+0000","last_peered":"2023-09-23T22:23:29.952776+0000","last_clean":"2023-09-23T22:23:29.952776+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:23:29.952776+0000","last_undegraded":"2023-09-23T22:23:29.952776+0000","last_fullsized":"2023-09-23T22:23:29.952776+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":217,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.883638+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:23:29.883638+0000","last_peered":"2023-09-23T22:23:29.883638+0000","last_clean":"2023-09-23T22:23:29.883638+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:23:29.883638+0000","last_undegraded":"2023-09-23T22:23:29.883638+0000","last_fullsized":"2023-09-23T22:23:29.883638+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":123,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.893297+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:23:28.893297+0000","last_peered":"2023-09-23T22:23:28.893297+0000","last_clean":"2023-09-23T22:23:28.893297+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:23:28.893297+0000","last_undegraded":"2023-09-23T22:23:28.893297+0000","last_fullsized":"2023-09-23T22:23:28.893297+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":228,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.887303+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:23:28.887303+0000","last_peered":"2023-09-23T22:23:28.887303+0000","last_clean":"2023-09-23T22:23:28.887303+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:23:28.887303+0000","last_undegraded":"2023-09-23T22:23:28.887303+0000","last_fullsized":"2023-09-23T22:23:28.887303+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":173,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.951887+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:23:29.951887+0000","last_peered":"2023-09-23T22:23:29.951887+0000","last_clean":"2023-09-23T22:23:29.951887+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:23:29.951887+0000","last_undegraded":"2023-09-23T22:23:29.951887+0000","last_fullsized":"2023-09-23T22:23:29.951887+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":105,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891713+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:23:28.891713+0000","last_peered":"2023-09-23T22:23:28.891713+0000","last_clean":"2023-09-23T22:23:28.891713+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:23:28.891713+0000","last_undegraded":"2023-09-23T22:23:28.891713+0000","last_fullsized":"2023-09-23T22:23:28.891713+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":266,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.893469+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:23:28.893469+0000","last_peered":"2023-09-23T22:23:28.893469+0000","last_clean":"2023-09-23T22:23:28.893469+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:23:28.893469+0000","last_undegraded":"2023-09-23T22:23:28.893469+0000","last_fullsized":"2023-09-23T22:23:28.893469+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":38,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.885788+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:23:29.885788+0000","last_peered":"2023-09-23T22:23:29.885788+0000","last_clean":"2023-09-23T22:23:29.885788+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:23:29.885788+0000","last_undegraded":"2023-09-23T22:23:29.885788+0000","last_fullsized":"2023-09-23T22:23:29.885788+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"249'7892","reported_seq":15254,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:30.269696+0000","last_change":"2023-09-23T22:23:29.940235+0000","last_active":"2023-09-23T22:23:30.269696+0000","last_peered":"2023-09-23T22:23:30.269696+0000","last_clean":"2023-09-23T22:23:30.269696+0000","last_became_active":"2023-09-23T22:23:29.892586+0000","last_became_peered":"2023-09-23T22:23:29.892586+0000","last_unstale":"2023-09-23T22:23:30.269696+0000","last_undegraded":"2023-09-23T22:23:30.269696+0000","last_fullsized":"2023-09-23T22:23:30.269696+0000","mapping_epoch":248,"log_start":"211'2400","ondisk_log_start":"211'2400","created":203,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5492,"log_dups_size":0,"ondisk_log_size":5492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:50.397106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141426688,"num_objects":2158,"num_object_clones":0,"num_object_copies":6474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7892,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3642,"num_bytes_recovered":188481536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":245,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.955610+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:23:29.955610+0000","last_peered":"2023-09-23T22:23:29.955610+0000","last_clean":"2023-09-23T22:23:29.955610+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:23:29.955610+0000","last_undegraded":"2023-09-23T22:23:29.955610+0000","last_fullsized":"2023-09-23T22:23:29.955610+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":231,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.885023+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:23:28.885023+0000","last_peered":"2023-09-23T22:23:28.885023+0000","last_clean":"2023-09-23T22:23:28.885023+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:23:28.885023+0000","last_undegraded":"2023-09-23T22:23:28.885023+0000","last_fullsized":"2023-09-23T22:23:28.885023+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"250'8115","reported_seq":15637,"reported_epoch":250,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:23:34.229022+0000","last_change":"2023-09-23T22:22:45.430551+0000","last_active":"2023-09-23T22:23:34.229022+0000","last_peered":"2023-09-23T22:23:34.229022+0000","last_clean":"2023-09-23T22:22:43.315002+0000","last_became_active":"2023-09-23T22:22:45.409613+0000","last_became_peered":"2023-09-23T22:22:45.409613+0000","last_unstale":"2023-09-23T22:23:34.229022+0000","last_undegraded":"2023-09-23T22:23:34.229022+0000","last_fullsized":"2023-09-23T22:23:34.229022+0000","mapping_epoch":240,"log_start":"211'2691","ondisk_log_start":"211'2691","created":203,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5424,"log_dups_size":0,"ondisk_log_size":5424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:03.767060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134152192,"num_objects":2047,"num_object_clones":0,"num_object_copies":6141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2047,"num_objects_unfound":0,"num_objects_dirty":2047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8115,"num_write_kb":325184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3717,"num_bytes_recovered":243597312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":2047}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":159,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.956515+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:23:29.956515+0000","last_peered":"2023-09-23T22:23:29.956515+0000","last_clean":"2023-09-23T22:23:29.956515+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:23:29.956515+0000","last_undegraded":"2023-09-23T22:23:29.956515+0000","last_fullsized":"2023-09-23T22:23:29.956515+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":237,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.951798+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:23:29.951798+0000","last_peered":"2023-09-23T22:23:29.951798+0000","last_clean":"2023-09-23T22:23:29.951798+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:23:29.951798+0000","last_undegraded":"2023-09-23T22:23:29.951798+0000","last_fullsized":"2023-09-23T22:23:29.951798+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"250'8295","reported_seq":12016,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:34.195805+0000","last_change":"2023-09-23T22:22:45.550371+0000","last_active":"2023-09-23T22:23:34.195805+0000","last_peered":"2023-09-23T22:23:34.195805+0000","last_clean":"2023-09-23T22:21:32.851371+0000","last_became_active":"2023-09-23T22:22:45.472694+0000","last_became_peered":"2023-09-23T22:22:45.472694+0000","last_unstale":"2023-09-23T22:23:34.195805+0000","last_undegraded":"2023-09-23T22:22:45.456856+0000","last_fullsized":"2023-09-23T22:22:45.317726+0000","mapping_epoch":240,"log_start":"211'2933","ondisk_log_start":"211'2933","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5362,"log_dups_size":0,"ondisk_log_size":5362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.540906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139788288,"num_objects":2133,"num_object_clones":0,"num_object_copies":6399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5210,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8295,"num_write_kb":333696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1215,"num_bytes_recovered":79495168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"0,1,3","objects":88},{"shards":"0,2,3","objects":495},{"shards":"2,3","objects":1},{"shards":"2,3,7","objects":1549},{"shards":"3,7","objects":-1866}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":174,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.887374+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:23:28.887374+0000","last_peered":"2023-09-23T22:23:28.887374+0000","last_clean":"2023-09-23T22:23:28.887374+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:23:28.887374+0000","last_undegraded":"2023-09-23T22:23:28.887374+0000","last_fullsized":"2023-09-23T22:23:28.887374+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":184,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891420+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:23:28.891420+0000","last_peered":"2023-09-23T22:23:28.891420+0000","last_clean":"2023-09-23T22:23:28.891420+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:23:28.891420+0000","last_undegraded":"2023-09-23T22:23:28.891420+0000","last_fullsized":"2023-09-23T22:23:28.891420+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"249'8053","reported_seq":15574,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:31.462910+0000","last_change":"2023-09-23T22:22:17.938724+0000","last_active":"2023-09-23T22:23:31.462910+0000","last_peered":"2023-09-23T22:23:31.462910+0000","last_clean":"2023-09-23T22:21:12.226652+0000","last_became_active":"2023-09-23T22:21:13.982783+0000","last_became_peered":"2023-09-23T22:21:13.982783+0000","last_unstale":"2023-09-23T22:23:31.462910+0000","last_undegraded":"2023-09-23T22:22:17.930337+0000","last_fullsized":"2023-09-23T22:21:13.795059+0000","mapping_epoch":214,"log_start":"212'3197","ondisk_log_start":"212'3197","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4856,"log_dups_size":0,"ondisk_log_size":4856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.489307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141492224,"num_objects":2159,"num_object_clones":0,"num_object_copies":6477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8053,"num_write_kb":326784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3728,"num_bytes_recovered":244318208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":38},{"shards":"0,7","objects":2121}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":239,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.886109+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:23:28.886109+0000","last_peered":"2023-09-23T22:23:28.886109+0000","last_clean":"2023-09-23T22:23:28.886109+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:23:28.886109+0000","last_undegraded":"2023-09-23T22:23:28.886109+0000","last_fullsized":"2023-09-23T22:23:28.886109+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":103,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891370+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:23:28.891370+0000","last_peered":"2023-09-23T22:23:28.891370+0000","last_clean":"2023-09-23T22:23:28.891370+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:23:28.891370+0000","last_undegraded":"2023-09-23T22:23:28.891370+0000","last_fullsized":"2023-09-23T22:23:28.891370+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"250'8179","reported_seq":9780,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:34.853095+0000","last_change":"2023-09-23T22:21:57.837966+0000","last_active":"2023-09-23T22:23:34.853095+0000","last_peered":"2023-09-23T22:23:34.853095+0000","last_clean":"2023-09-23T22:21:12.225315+0000","last_became_active":"2023-09-23T22:21:13.940088+0000","last_became_peered":"2023-09-23T22:21:13.940088+0000","last_unstale":"2023-09-23T22:23:34.853095+0000","last_undegraded":"2023-09-23T22:21:57.835407+0000","last_fullsized":"2023-09-23T22:21:13.800025+0000","mapping_epoch":214,"log_start":"211'2900","ondisk_log_start":"211'2900","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5279,"log_dups_size":0,"ondisk_log_size":5279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:26.245766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133103616,"num_objects":2031,"num_object_clones":0,"num_object_copies":6093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8179,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":768,"num_bytes_recovered":50331648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":634},{"shards":"1,4","objects":-909}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":553,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.882964+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:23:29.882964+0000","last_peered":"2023-09-23T22:23:29.882964+0000","last_clean":"2023-09-23T22:23:29.882964+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:23:29.882964+0000","last_undegraded":"2023-09-23T22:23:29.882964+0000","last_fullsized":"2023-09-23T22:23:29.882964+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":263,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.883129+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:23:29.883129+0000","last_peered":"2023-09-23T22:23:29.883129+0000","last_clean":"2023-09-23T22:23:29.883129+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:23:29.883129+0000","last_undegraded":"2023-09-23T22:23:29.883129+0000","last_fullsized":"2023-09-23T22:23:29.883129+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":218,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.952154+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:23:29.952154+0000","last_peered":"2023-09-23T22:23:29.952154+0000","last_clean":"2023-09-23T22:23:29.952154+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:23:29.952154+0000","last_undegraded":"2023-09-23T22:23:29.952154+0000","last_fullsized":"2023-09-23T22:23:29.952154+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"250'8010","reported_seq":9547,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:34.787343+0000","last_change":"2023-09-23T22:22:20.034548+0000","last_active":"2023-09-23T22:23:34.787343+0000","last_peered":"2023-09-23T22:23:34.787343+0000","last_clean":"2023-09-23T22:21:12.194696+0000","last_became_active":"2023-09-23T22:22:19.958601+0000","last_became_peered":"2023-09-23T22:22:19.958601+0000","last_unstale":"2023-09-23T22:23:34.787343+0000","last_undegraded":"2023-09-23T22:22:19.951596+0000","last_fullsized":"2023-09-23T22:22:19.874428+0000","mapping_epoch":232,"log_start":"211'2700","ondisk_log_start":"211'2700","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5310,"log_dups_size":0,"ondisk_log_size":5310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:10.906601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129236992,"num_objects":1972,"num_object_clones":0,"num_object_copies":5916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4172,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8010,"num_write_kb":319424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":486,"num_bytes_recovered":31850496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[4,3],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"1,4,5","objects":506},{"shards":"2,3,4","objects":648},{"shards":"3,4","objects":-1714}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":547,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:28.884801+0000","last_change":"2023-09-23T22:21:59.026062+0000","last_active":"2023-09-23T22:23:28.884801+0000","last_peered":"2023-09-23T22:23:28.884801+0000","last_clean":"2023-09-23T22:21:50.447236+0000","last_became_active":"2023-09-23T22:21:59.021009+0000","last_became_peered":"2023-09-23T22:21:59.021009+0000","last_unstale":"2023-09-23T22:23:28.884801+0000","last_undegraded":"2023-09-23T22:21:58.818399+0000","last_fullsized":"2023-09-23T22:21:58.817464+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:52.228490+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":166,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.955159+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:23:29.955159+0000","last_peered":"2023-09-23T22:23:29.955159+0000","last_clean":"2023-09-23T22:23:29.955159+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:23:29.955159+0000","last_undegraded":"2023-09-23T22:23:29.955159+0000","last_fullsized":"2023-09-23T22:23:29.955159+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":235,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.953145+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:23:29.953145+0000","last_peered":"2023-09-23T22:23:29.953145+0000","last_clean":"2023-09-23T22:23:29.953145+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:23:29.953145+0000","last_undegraded":"2023-09-23T22:23:29.953145+0000","last_fullsized":"2023-09-23T22:23:29.953145+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"249'7821","reported_seq":7894,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:30.282162+0000","last_change":"2023-09-23T22:22:18.973077+0000","last_active":"2023-09-23T22:23:30.282162+0000","last_peered":"2023-09-23T22:23:30.282162+0000","last_clean":"2023-09-23T22:21:32.779211+0000","last_became_active":"2023-09-23T22:22:18.949524+0000","last_became_peered":"2023-09-23T22:22:18.949524+0000","last_unstale":"2023-09-23T22:23:30.282162+0000","last_undegraded":"2023-09-23T22:22:18.870000+0000","last_fullsized":"2023-09-23T22:22:18.859180+0000","mapping_epoch":231,"log_start":"211'2300","ondisk_log_start":"211'2300","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.194549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145162240,"num_objects":2215,"num_object_clones":0,"num_object_copies":6645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":914,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7821,"num_write_kb":321152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"1,5,7","objects":487},{"shards":"5,7","objects":1728}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":211,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.951650+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:23:29.951650+0000","last_peered":"2023-09-23T22:23:29.951650+0000","last_clean":"2023-09-23T22:23:29.951650+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:23:29.951650+0000","last_undegraded":"2023-09-23T22:23:29.951650+0000","last_fullsized":"2023-09-23T22:23:29.951650+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":226,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.956421+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:23:29.956421+0000","last_peered":"2023-09-23T22:23:29.956421+0000","last_clean":"2023-09-23T22:23:29.956421+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:23:29.956421+0000","last_undegraded":"2023-09-23T22:23:29.956421+0000","last_fullsized":"2023-09-23T22:23:29.956421+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"249'7956","reported_seq":8011,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:30.286718+0000","last_change":"2023-09-23T22:20:19.473637+0000","last_active":"2023-09-23T22:23:30.286718+0000","last_peered":"2023-09-23T22:23:30.286718+0000","last_clean":"2023-09-23T22:23:30.286718+0000","last_became_active":"2023-09-23T22:20:19.473385+0000","last_became_peered":"2023-09-23T22:20:19.473385+0000","last_unstale":"2023-09-23T22:23:30.286718+0000","last_undegraded":"2023-09-23T22:23:30.286718+0000","last_fullsized":"2023-09-23T22:23:30.286718+0000","mapping_epoch":203,"log_start":"211'2500","ondisk_log_start":"211'2500","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:01.911861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145686552,"num_objects":2224,"num_object_clones":0,"num_object_copies":6672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2224,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7956,"num_write_kb":325697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":553,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.887443+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:23:28.887443+0000","last_peered":"2023-09-23T22:23:28.887443+0000","last_clean":"2023-09-23T22:23:28.887443+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:23:28.887443+0000","last_undegraded":"2023-09-23T22:23:28.887443+0000","last_fullsized":"2023-09-23T22:23:28.887443+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":29,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.884544+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:23:29.884544+0000","last_peered":"2023-09-23T22:23:29.884544+0000","last_clean":"2023-09-23T22:23:29.884544+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:23:29.884544+0000","last_undegraded":"2023-09-23T22:23:29.884544+0000","last_fullsized":"2023-09-23T22:23:29.884544+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":190,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.892528+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:23:28.892528+0000","last_peered":"2023-09-23T22:23:28.892528+0000","last_clean":"2023-09-23T22:23:28.892528+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:23:28.892528+0000","last_undegraded":"2023-09-23T22:23:28.892528+0000","last_fullsized":"2023-09-23T22:23:28.892528+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"249'8098","reported_seq":12240,"reported_epoch":249,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:31.561414+0000","last_change":"2023-09-23T22:22:43.348452+0000","last_active":"2023-09-23T22:23:31.561414+0000","last_peered":"2023-09-23T22:23:31.561414+0000","last_clean":"2023-09-23T22:21:32.822849+0000","last_became_active":"2023-09-23T22:21:35.128019+0000","last_became_peered":"2023-09-23T22:21:35.128019+0000","last_unstale":"2023-09-23T22:23:31.561414+0000","last_undegraded":"2023-09-23T22:21:34.912513+0000","last_fullsized":"2023-09-23T22:21:34.880959+0000","mapping_epoch":220,"log_start":"212'3247","ondisk_log_start":"212'3247","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4851,"log_dups_size":0,"ondisk_log_size":4851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:58.687801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144834560,"num_objects":2210,"num_object_clones":0,"num_object_copies":6630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1209,"num_objects_misplaced":1353,"num_objects_unfound":0,"num_objects_dirty":2210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8098,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2040,"num_bytes_recovered":133693440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1353},{"shards":"0,3","objects":-352}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":545,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.885925+0000","last_change":"2023-09-23T22:22:47.349660+0000","last_active":"2023-09-23T22:23:29.885925+0000","last_peered":"2023-09-23T22:23:29.885925+0000","last_clean":"2023-09-23T22:23:29.885925+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:23:29.885925+0000","last_undegraded":"2023-09-23T22:23:29.885925+0000","last_fullsized":"2023-09-23T22:23:29.885925+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":798,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891697+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:23:28.891697+0000","last_peered":"2023-09-23T22:23:28.891697+0000","last_clean":"2023-09-23T22:23:28.891697+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:23:28.891697+0000","last_undegraded":"2023-09-23T22:23:28.891697+0000","last_fullsized":"2023-09-23T22:23:28.891697+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":266,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.887201+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:23:28.887201+0000","last_peered":"2023-09-23T22:23:28.887201+0000","last_clean":"2023-09-23T22:23:28.887201+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:23:28.887201+0000","last_undegraded":"2023-09-23T22:23:28.887201+0000","last_fullsized":"2023-09-23T22:23:28.887201+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891113+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:23:28.891113+0000","last_peered":"2023-09-23T22:23:28.891113+0000","last_clean":"2023-09-23T22:23:28.891113+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:23:28.891113+0000","last_undegraded":"2023-09-23T22:23:28.891113+0000","last_fullsized":"2023-09-23T22:23:28.891113+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"250'8177","reported_seq":8264,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:34.836286+0000","last_change":"2023-09-23T22:22:18.955534+0000","last_active":"2023-09-23T22:23:34.836286+0000","last_peered":"2023-09-23T22:23:34.836286+0000","last_clean":"2023-09-23T22:21:32.860473+0000","last_became_active":"2023-09-23T22:22:18.941549+0000","last_became_peered":"2023-09-23T22:22:18.941549+0000","last_unstale":"2023-09-23T22:23:34.836286+0000","last_undegraded":"2023-09-23T22:22:18.875275+0000","last_fullsized":"2023-09-23T22:22:18.863381+0000","mapping_epoch":231,"log_start":"211'2900","ondisk_log_start":"211'2900","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5277,"log_dups_size":0,"ondisk_log_size":5277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:27.412277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134676480,"num_objects":2055,"num_object_clones":0,"num_object_copies":6165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":988,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8177,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,5,7","objects":527},{"shards":"4,7","objects":1528}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":553,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.885134+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:23:28.885134+0000","last_peered":"2023-09-23T22:23:28.885134+0000","last_clean":"2023-09-23T22:23:28.885134+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:23:28.885134+0000","last_undegraded":"2023-09-23T22:23:28.885134+0000","last_fullsized":"2023-09-23T22:23:28.885134+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":981,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.887552+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:23:28.887552+0000","last_peered":"2023-09-23T22:23:28.887552+0000","last_clean":"2023-09-23T22:23:28.887552+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:23:28.887552+0000","last_undegraded":"2023-09-23T22:23:28.887552+0000","last_fullsized":"2023-09-23T22:23:28.887552+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":259,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.952610+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:23:29.952610+0000","last_peered":"2023-09-23T22:23:29.952610+0000","last_clean":"2023-09-23T22:23:29.952610+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:23:29.952610+0000","last_undegraded":"2023-09-23T22:23:29.952610+0000","last_fullsized":"2023-09-23T22:23:29.952610+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":213,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.886218+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:23:28.886218+0000","last_peered":"2023-09-23T22:23:28.886218+0000","last_clean":"2023-09-23T22:23:28.886218+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:23:28.886218+0000","last_undegraded":"2023-09-23T22:23:28.886218+0000","last_fullsized":"2023-09-23T22:23:28.886218+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"250'7902","reported_seq":7965,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:34.209006+0000","last_change":"2023-09-23T22:21:59.013998+0000","last_active":"2023-09-23T22:23:34.209006+0000","last_peered":"2023-09-23T22:23:34.209006+0000","last_clean":"2023-09-23T22:21:56.814967+0000","last_became_active":"2023-09-23T22:21:59.002385+0000","last_became_peered":"2023-09-23T22:21:59.002385+0000","last_unstale":"2023-09-23T22:23:34.209006+0000","last_undegraded":"2023-09-23T22:21:58.865154+0000","last_fullsized":"2023-09-23T22:21:58.825566+0000","mapping_epoch":226,"log_start":"211'2520","ondisk_log_start":"211'2520","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5382,"log_dups_size":0,"ondisk_log_size":5382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:00.273421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135790592,"num_objects":2072,"num_object_clones":0,"num_object_copies":6216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4918,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7902,"num_write_kb":319168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":2003},{"shards":"3,7","objects":-2846}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":578,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.893214+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:23:28.893214+0000","last_peered":"2023-09-23T22:23:28.893214+0000","last_clean":"2023-09-23T22:23:28.893214+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:23:28.893214+0000","last_undegraded":"2023-09-23T22:23:28.893214+0000","last_fullsized":"2023-09-23T22:23:28.893214+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":356,"reported_epoch":249,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:23:29.883272+0000","last_change":"2023-09-23T22:22:18.917671+0000","last_active":"2023-09-23T22:23:29.883272+0000","last_peered":"2023-09-23T22:23:29.883272+0000","last_clean":"2023-09-23T22:15:39.913392+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:23:29.883272+0000","last_undegraded":"2023-09-23T22:22:18.874220+0000","last_fullsized":"2023-09-23T22:23:29.883272+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":209,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.890445+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:23:28.890445+0000","last_peered":"2023-09-23T22:23:28.890445+0000","last_clean":"2023-09-23T22:23:28.890445+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:23:28.890445+0000","last_undegraded":"2023-09-23T22:23:28.890445+0000","last_fullsized":"2023-09-23T22:23:28.890445+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":255,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.892609+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:23:28.892609+0000","last_peered":"2023-09-23T22:23:28.892609+0000","last_clean":"2023-09-23T22:23:28.892609+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:23:28.892609+0000","last_undegraded":"2023-09-23T22:23:28.892609+0000","last_fullsized":"2023-09-23T22:23:28.892609+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"249'8011","reported_seq":8065,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:30.273092+0000","last_change":"2023-09-23T22:20:19.470871+0000","last_active":"2023-09-23T22:23:30.273092+0000","last_peered":"2023-09-23T22:23:30.273092+0000","last_clean":"2023-09-23T22:23:30.273092+0000","last_became_active":"2023-09-23T22:20:19.470597+0000","last_became_peered":"2023-09-23T22:20:19.470597+0000","last_unstale":"2023-09-23T22:23:30.273092+0000","last_undegraded":"2023-09-23T22:23:30.273092+0000","last_fullsized":"2023-09-23T22:23:30.273092+0000","mapping_epoch":203,"log_start":"211'2500","ondisk_log_start":"211'2500","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5511,"log_dups_size":0,"ondisk_log_size":5511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:32.388985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152109056,"num_objects":2321,"num_object_clones":0,"num_object_copies":6963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8011,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":844,"reported_epoch":248,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:28.885294+0000","last_change":"2023-09-23T22:21:58.853256+0000","last_active":"2023-09-23T22:23:28.885294+0000","last_peered":"2023-09-23T22:23:28.885294+0000","last_clean":"2023-09-23T22:21:50.447508+0000","last_became_active":"2023-09-23T22:21:58.852815+0000","last_became_peered":"2023-09-23T22:21:58.852815+0000","last_unstale":"2023-09-23T22:23:28.885294+0000","last_undegraded":"2023-09-23T22:21:58.818640+0000","last_fullsized":"2023-09-23T22:21:58.817793+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:25.619025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":561,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.893351+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:23:28.893351+0000","last_peered":"2023-09-23T22:23:28.893351+0000","last_clean":"2023-09-23T22:23:28.893351+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:23:28.893351+0000","last_undegraded":"2023-09-23T22:23:28.893351+0000","last_fullsized":"2023-09-23T22:23:28.893351+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":174,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.884078+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:23:29.884078+0000","last_peered":"2023-09-23T22:23:29.884078+0000","last_clean":"2023-09-23T22:23:29.884078+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:23:29.884078+0000","last_undegraded":"2023-09-23T22:23:29.884078+0000","last_fullsized":"2023-09-23T22:23:29.884078+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":136,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.953345+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:23:29.953345+0000","last_peered":"2023-09-23T22:23:29.953345+0000","last_clean":"2023-09-23T22:23:29.953345+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:23:29.953345+0000","last_undegraded":"2023-09-23T22:23:29.953345+0000","last_fullsized":"2023-09-23T22:23:29.953345+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"249'7934","reported_seq":8011,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:33.704038+0000","last_change":"2023-09-23T22:22:18.954521+0000","last_active":"2023-09-23T22:23:33.704038+0000","last_peered":"2023-09-23T22:23:33.704038+0000","last_clean":"2023-09-23T22:21:32.869170+0000","last_became_active":"2023-09-23T22:22:18.945835+0000","last_became_peered":"2023-09-23T22:22:18.945835+0000","last_unstale":"2023-09-23T22:23:33.704038+0000","last_undegraded":"2023-09-23T22:22:18.869474+0000","last_fullsized":"2023-09-23T22:22:18.861404+0000","mapping_epoch":231,"log_start":"212'3305","ondisk_log_start":"212'3305","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4629,"log_dups_size":0,"ondisk_log_size":4629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:02.907936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138674176,"num_objects":2116,"num_object_clones":0,"num_object_copies":6348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7934,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,4,5","objects":591},{"shards":"1,5","objects":1525}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":580,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.892136+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:23:28.892136+0000","last_peered":"2023-09-23T22:23:28.892136+0000","last_clean":"2023-09-23T22:23:28.892136+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:23:28.892136+0000","last_undegraded":"2023-09-23T22:23:28.892136+0000","last_fullsized":"2023-09-23T22:23:28.892136+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":599,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.890664+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:23:28.890664+0000","last_peered":"2023-09-23T22:23:28.890664+0000","last_clean":"2023-09-23T22:23:28.890664+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:23:28.890664+0000","last_undegraded":"2023-09-23T22:23:28.890664+0000","last_fullsized":"2023-09-23T22:23:28.890664+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":131,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.882975+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:23:29.882975+0000","last_peered":"2023-09-23T22:23:29.882975+0000","last_clean":"2023-09-23T22:23:29.882975+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:23:29.882975+0000","last_undegraded":"2023-09-23T22:23:29.882975+0000","last_fullsized":"2023-09-23T22:23:29.882975+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":88,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891432+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:23:28.891432+0000","last_peered":"2023-09-23T22:23:28.891432+0000","last_clean":"2023-09-23T22:23:28.891432+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:23:28.891432+0000","last_undegraded":"2023-09-23T22:23:28.891432+0000","last_fullsized":"2023-09-23T22:23:28.891432+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"249'8056","reported_seq":11597,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:33.687815+0000","last_change":"2023-09-23T22:22:45.524797+0000","last_active":"2023-09-23T22:23:33.687815+0000","last_peered":"2023-09-23T22:23:33.687815+0000","last_clean":"2023-09-23T22:21:12.245233+0000","last_became_active":"2023-09-23T22:22:45.454018+0000","last_became_peered":"2023-09-23T22:22:45.454018+0000","last_unstale":"2023-09-23T22:23:33.687815+0000","last_undegraded":"2023-09-23T22:22:45.437539+0000","last_fullsized":"2023-09-23T22:22:45.322462+0000","mapping_epoch":240,"log_start":"212'3430","ondisk_log_start":"212'3430","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4626,"log_dups_size":0,"ondisk_log_size":4626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:10.659044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142999552,"num_objects":2182,"num_object_clones":0,"num_object_copies":6546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8056,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1153,"num_bytes_recovered":75563008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1","objects":1},{"shards":"0,1,5","objects":565},{"shards":"1,2,7","objects":731},{"shards":"1,7","objects":-899}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":570,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891641+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:23:28.891641+0000","last_peered":"2023-09-23T22:23:28.891641+0000","last_clean":"2023-09-23T22:23:28.891641+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:23:28.891641+0000","last_undegraded":"2023-09-23T22:23:28.891641+0000","last_fullsized":"2023-09-23T22:23:28.891641+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":140,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.890973+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:23:28.890973+0000","last_peered":"2023-09-23T22:23:28.890973+0000","last_clean":"2023-09-23T22:23:28.890973+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:23:28.890973+0000","last_undegraded":"2023-09-23T22:23:28.890973+0000","last_fullsized":"2023-09-23T22:23:28.890973+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":200,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.892059+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:23:28.892059+0000","last_peered":"2023-09-23T22:23:28.892059+0000","last_clean":"2023-09-23T22:23:28.892059+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:23:28.892059+0000","last_undegraded":"2023-09-23T22:23:28.892059+0000","last_fullsized":"2023-09-23T22:23:28.892059+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":104,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.884052+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:23:29.884052+0000","last_peered":"2023-09-23T22:23:29.884052+0000","last_clean":"2023-09-23T22:23:29.884052+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:23:29.884052+0000","last_undegraded":"2023-09-23T22:23:29.884052+0000","last_fullsized":"2023-09-23T22:23:29.884052+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"250'8218","reported_seq":14431,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:34.857637+0000","last_change":"2023-09-23T22:22:19.043755+0000","last_active":"2023-09-23T22:23:34.857637+0000","last_peered":"2023-09-23T22:23:34.857637+0000","last_clean":"2023-09-23T22:21:12.214088+0000","last_became_active":"2023-09-23T22:22:18.986181+0000","last_became_peered":"2023-09-23T22:22:18.986181+0000","last_unstale":"2023-09-23T22:23:34.857637+0000","last_undegraded":"2023-09-23T22:22:18.975289+0000","last_fullsized":"2023-09-23T22:22:18.862148+0000","mapping_epoch":231,"log_start":"211'2900","ondisk_log_start":"211'2900","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5318,"log_dups_size":0,"ondisk_log_size":5318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:34.422545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137232384,"num_objects":2094,"num_object_clones":0,"num_object_copies":6282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2543,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8218,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3063,"num_bytes_recovered":200736768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,3,4","objects":661},{"shards":"2,4,7","objects":1349},{"shards":"4,5,7","objects":83},{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":554,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.952411+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:23:29.952411+0000","last_peered":"2023-09-23T22:23:29.952411+0000","last_clean":"2023-09-23T22:23:29.952411+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:23:29.952411+0000","last_undegraded":"2023-09-23T22:23:29.952411+0000","last_fullsized":"2023-09-23T22:23:29.952411+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":263,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.886040+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:23:29.886040+0000","last_peered":"2023-09-23T22:23:29.886040+0000","last_clean":"2023-09-23T22:23:29.886040+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:23:29.886040+0000","last_undegraded":"2023-09-23T22:23:29.886040+0000","last_fullsized":"2023-09-23T22:23:29.886040+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":890,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.892788+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:23:28.892788+0000","last_peered":"2023-09-23T22:23:28.892788+0000","last_clean":"2023-09-23T22:23:28.892788+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:23:28.892788+0000","last_undegraded":"2023-09-23T22:23:28.892788+0000","last_fullsized":"2023-09-23T22:23:28.892788+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":38,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.884346+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:23:29.884346+0000","last_peered":"2023-09-23T22:23:29.884346+0000","last_clean":"2023-09-23T22:23:29.884346+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:23:29.884346+0000","last_undegraded":"2023-09-23T22:23:29.884346+0000","last_fullsized":"2023-09-23T22:23:29.884346+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"249'7917","reported_seq":8114,"reported_epoch":249,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:33.642172+0000","last_change":"2023-09-23T22:22:44.336083+0000","last_active":"2023-09-23T22:23:33.642172+0000","last_peered":"2023-09-23T22:23:33.642172+0000","last_clean":"2023-09-23T22:21:56.808651+0000","last_became_active":"2023-09-23T22:21:59.059105+0000","last_became_peered":"2023-09-23T22:21:59.059105+0000","last_unstale":"2023-09-23T22:23:33.642172+0000","last_undegraded":"2023-09-23T22:21:58.989636+0000","last_fullsized":"2023-09-23T22:21:58.812134+0000","mapping_epoch":226,"log_start":"212'3202","ondisk_log_start":"212'3202","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4715,"log_dups_size":0,"ondisk_log_size":4715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:51.370288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135856128,"num_objects":2073,"num_object_clones":0,"num_object_copies":6219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7917,"num_write_kb":319680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":4259840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":1994},{"shards":"1,7","objects":-2779}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":537,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.883881+0000","last_change":"2023-09-23T22:22:19.883431+0000","last_active":"2023-09-23T22:23:29.883881+0000","last_peered":"2023-09-23T22:23:29.883881+0000","last_clean":"2023-09-23T22:23:29.883881+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:23:29.883881+0000","last_undegraded":"2023-09-23T22:23:29.883881+0000","last_fullsized":"2023-09-23T22:23:29.883881+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:46.903071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:46.903071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":231,"reported_epoch":249,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:23:29.886098+0000","last_change":"2023-09-23T22:22:44.342884+0000","last_active":"2023-09-23T22:23:29.886098+0000","last_peered":"2023-09-23T22:23:29.886098+0000","last_clean":"2023-09-23T22:06:25.449250+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:23:29.886098+0000","last_undegraded":"2023-09-23T22:22:44.329336+0000","last_fullsized":"2023-09-23T22:23:29.886098+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":83,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891568+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:23:28.891568+0000","last_peered":"2023-09-23T22:23:28.891568+0000","last_clean":"2023-09-23T22:23:28.891568+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:23:28.891568+0000","last_undegraded":"2023-09-23T22:23:28.891568+0000","last_fullsized":"2023-09-23T22:23:28.891568+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":213,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.951727+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:23:29.951727+0000","last_peered":"2023-09-23T22:23:29.951727+0000","last_clean":"2023-09-23T22:23:29.951727+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:23:29.951727+0000","last_undegraded":"2023-09-23T22:23:29.951727+0000","last_fullsized":"2023-09-23T22:23:29.951727+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":78,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.890423+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:23:28.890423+0000","last_peered":"2023-09-23T22:23:28.890423+0000","last_clean":"2023-09-23T22:23:28.890423+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:23:28.890423+0000","last_undegraded":"2023-09-23T22:23:28.890423+0000","last_fullsized":"2023-09-23T22:23:28.890423+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":259,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.891969+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:23:28.891969+0000","last_peered":"2023-09-23T22:23:28.891969+0000","last_clean":"2023-09-23T22:23:28.891969+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:23:28.891969+0000","last_undegraded":"2023-09-23T22:23:28.891969+0000","last_fullsized":"2023-09-23T22:23:28.891969+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":211,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.886043+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:23:28.886043+0000","last_peered":"2023-09-23T22:23:28.886043+0000","last_clean":"2023-09-23T22:23:28.886043+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:23:28.886043+0000","last_undegraded":"2023-09-23T22:23:28.886043+0000","last_fullsized":"2023-09-23T22:23:28.886043+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":225,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.952870+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:23:29.952870+0000","last_peered":"2023-09-23T22:23:29.952870+0000","last_clean":"2023-09-23T22:23:29.952870+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:23:29.952870+0000","last_undegraded":"2023-09-23T22:23:29.952870+0000","last_fullsized":"2023-09-23T22:23:29.952870+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":208,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.887289+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:23:28.887289+0000","last_peered":"2023-09-23T22:23:28.887289+0000","last_clean":"2023-09-23T22:23:28.887289+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:23:28.887289+0000","last_undegraded":"2023-09-23T22:23:28.887289+0000","last_fullsized":"2023-09-23T22:23:28.887289+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.955227+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:23:29.955227+0000","last_peered":"2023-09-23T22:23:29.955227+0000","last_clean":"2023-09-23T22:23:29.955227+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:23:29.955227+0000","last_undegraded":"2023-09-23T22:23:29.955227+0000","last_fullsized":"2023-09-23T22:23:29.955227+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":230,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.883207+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:23:29.883207+0000","last_peered":"2023-09-23T22:23:29.883207+0000","last_clean":"2023-09-23T22:23:29.883207+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:23:29.883207+0000","last_undegraded":"2023-09-23T22:23:29.883207+0000","last_fullsized":"2023-09-23T22:23:29.883207+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":271,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.883047+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:23:29.883047+0000","last_peered":"2023-09-23T22:23:29.883047+0000","last_clean":"2023-09-23T22:23:29.883047+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:23:29.883047+0000","last_undegraded":"2023-09-23T22:23:29.883047+0000","last_fullsized":"2023-09-23T22:23:29.883047+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":138,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T22:23:28.892065+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:23:28.892065+0000","last_peered":"2023-09-23T22:23:28.892065+0000","last_clean":"2023-09-23T22:23:28.892065+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:23:28.892065+0000","last_undegraded":"2023-09-23T22:23:28.892065+0000","last_fullsized":"2023-09-23T22:23:28.892065+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":258,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.951307+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:23:29.951307+0000","last_peered":"2023-09-23T22:23:29.951307+0000","last_clean":"2023-09-23T22:23:29.951307+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:23:29.951307+0000","last_undegraded":"2023-09-23T22:23:29.951307+0000","last_fullsized":"2023-09-23T22:23:29.951307+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2232221720,"num_objects":34062,"num_object_clones":0,"num_object_copies":102186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33062,"num_objects_misplaced":3400,"num_objects_unfound":0,"num_objects_dirty":34062,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":128634,"num_write_kb":5206209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19883,"num_bytes_recovered":1252720640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9185275904,"data_stored":9185263688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83009,"ondisk_log_size":83009,"up":48,"acting":36,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":370028,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":88,"num_read_kb":166,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234230,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1482476,"kb_used_data":1304996,"kb_used_omap":0,"kb_used_meta":177472,"kb_avail":92262676,"statfs":{"total":95995035648,"available":94476980224,"internally_reserved":0,"allocated":1336315904,"data_stored":1335311636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":181731328},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:03 2023","interfaces":[{"interface":"back","average":{"1min":9.048,"5min":4.476,"15min":2.2679999999999998},"min":{"1min":0.23000000000000001,"5min":0.22,"15min":0.22},"max":{"1min":158.042,"5min":158.042,"15min":158.042},"last":0.22600000000000001},{"interface":"front","average":{"1min":9.6690000000000005,"5min":4.7149999999999999,"15min":2.4830000000000001},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":157.952,"5min":157.952,"15min":157.952},"last":0.48299999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":4.9329999999999998,"15min":2.9809999999999999},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.246},"max":{"1min":10.127000000000001,"5min":101.145,"15min":123.79300000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.5859999999999999,"5min":4.9909999999999997,"15min":3.1309999999999998},"min":{"1min":0.28199999999999997,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":10.92,"5min":101.13,"15min":123.66500000000001},"last":1.071}]},{"osd":2,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":4.8609999999999998,"15min":3.6600000000000001},"min":{"1min":0.35299999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.5119999999999996,"5min":105.34399999999999,"15min":125.101},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.7470000000000001,"5min":4.6589999999999998,"15min":3.375},"min":{"1min":0.34699999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.0549999999999997,"5min":105.63200000000001,"15min":126.021},"last":6.1479999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:23:12 2023","interfaces":[{"interface":"back","average":{"1min":10.196999999999999,"5min":3.7349999999999999,"15min":2.1890000000000001},"min":{"1min":0.36699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":170.86099999999999,"5min":170.86099999999999,"15min":170.86099999999999},"last":2.5179999999999998},{"interface":"front","average":{"1min":9.5510000000000002,"5min":3.9239999999999999,"15min":2.3170000000000002},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":174.57499999999999,"5min":174.57499999999999,"15min":174.57499999999999},"last":0.61699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":2.3980000000000001,"15min":1.2969999999999999},"min":{"1min":0.29699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":27.922000000000001,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.88600000000000001},{"interface":"front","average":{"1min":2.738,"5min":2.3719999999999999,"15min":1.2669999999999999},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":29.920000000000002,"5min":88.495999999999995,"15min":88.495999999999995},"last":1.107}]},{"osd":5,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":2.4279999999999999,"15min":1.226},"min":{"1min":0.32700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.965,"5min":93.816000000000003,"15min":93.816000000000003},"last":2.8010000000000002},{"interface":"front","average":{"1min":2.4710000000000001,"5min":2.5230000000000001,"15min":1.4419999999999999},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.29799999999999999},"max":{"1min":24.712,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.80200000000000005,"15min":0.71399999999999997},"min":{"1min":0.34200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.343,"5min":7.9459999999999997,"15min":9.0139999999999993},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.153,"15min":0.91500000000000004},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":6.3879999999999999,"5min":10.347,"15min":13.362},"last":0.64300000000000002}]}]},{"osd":6,"up_from":0,"seq":197568495866,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795138,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1372884,"kb_used_data":1067660,"kb_used_omap":7,"kb_used_meta":305208,"kb_avail":92372268,"statfs":{"total":95995035648,"available":94589202432,"internally_reserved":0,"allocated":1093283840,"data_stored":1091942750,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":312533364},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8689999999999998,"5min":2.113,"15min":1.3149999999999999},"min":{"1min":0.255,"5min":0.189,"15min":0.189},"max":{"1min":56.503,"5min":56.503,"15min":61.890000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":5.1550000000000002,"5min":3.0230000000000001,"15min":1.728},"min":{"1min":0.40799999999999997,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":56.560000000000002,"5min":56.560000000000002,"15min":62.689999999999998},"last":2.968}]},{"osd":2,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":2.0179999999999998,"15min":1.1850000000000001},"min":{"1min":0.252,"5min":0.223,"15min":0.216},"max":{"1min":11.179,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.1480000000000001,"5min":3.2370000000000001,"15min":1.5049999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.2670000000000003,"5min":34.781999999999996,"15min":34.781999999999996},"last":6.226}]},{"osd":3,"last update":"Sat Sep 23 22:23:20 2023","interfaces":[{"interface":"back","average":{"1min":8.5220000000000002,"5min":4.3360000000000003,"15min":2.0590000000000002},"min":{"1min":0.47599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":110.417,"5min":110.417,"15min":110.417},"last":6.1230000000000002},{"interface":"front","average":{"1min":8.5239999999999991,"5min":4.1820000000000004,"15min":2.0179999999999998},"min":{"1min":0.33400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":110.721,"5min":110.721,"15min":110.721},"last":4.5019999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:23:20 2023","interfaces":[{"interface":"back","average":{"1min":8.6099999999999994,"5min":4.6239999999999997,"15min":2.056},"min":{"1min":0.314,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":171.09999999999999,"5min":234.79900000000001,"15min":234.79900000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":9.1289999999999996,"5min":5.2699999999999996,"15min":2.3519999999999999},"min":{"1min":0.44700000000000001,"5min":0.33200000000000002,"15min":0.318},"max":{"1min":171.08500000000001,"5min":234.81700000000001,"15min":234.81700000000001},"last":3.0049999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":3.6850000000000001,"15min":1.788},"min":{"1min":0.39100000000000001,"5min":0.31,"15min":0.31},"max":{"1min":9.5709999999999997,"5min":196.79900000000001,"15min":196.79900000000001},"last":3.2509999999999999},{"interface":"front","average":{"1min":2.9340000000000002,"5min":3.6499999999999999,"15min":1.7989999999999999},"min":{"1min":0.41299999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.01,"5min":196.68299999999999,"15min":196.68299999999999},"last":0.61799999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":1.913,"15min":1.2490000000000001},"min":{"1min":0.497,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":10.074999999999999,"5min":10.971,"15min":10.989000000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.448,"5min":1.296,"15min":1.006},"min":{"1min":0.437,"5min":0.35699999999999998,"15min":0.34999999999999998},"max":{"1min":7.7300000000000004,"5min":11.385999999999999,"15min":11.385999999999999},"last":0.53600000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.6970000000000001,"5min":1.9339999999999999,"15min":1.2889999999999999},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.048,"5min":10.933999999999999,"15min":10.948},"last":2.718},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.3200000000000001,"15min":1.0589999999999999},"min":{"1min":0.40699999999999997,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":12.045999999999999,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.502}]}]},{"osd":0,"up_from":93,"seq":399431958672,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":970744,"kb_used_data":703856,"kb_used_omap":8,"kb_used_meta":266807,"kb_avail":92774408,"statfs":{"total":95995035648,"available":95000993792,"internally_reserved":0,"allocated":720748544,"data_stored":719377472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":273211242},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2009999999999996,"5min":4.5810000000000004,"15min":2.274},"min":{"1min":0.21299999999999999,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":51.018999999999998,"5min":169.77199999999999,"15min":169.77199999999999},"last":2.0030000000000001},{"interface":"front","average":{"1min":3.6739999999999999,"5min":5.0860000000000003,"15min":2.5640000000000001},"min":{"1min":0.24099999999999999,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":23.891999999999999,"5min":169.52099999999999,"15min":169.52099999999999},"last":0.69199999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:22:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":6.5620000000000003,"15min":2.5870000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":10.891,"5min":206.495,"15min":206.495},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.8759999999999999,"5min":7.71,"15min":3.1890000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.22800000000000001},"max":{"1min":15.186,"5min":207.34399999999999,"15min":207.34399999999999},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4350000000000001,"5min":3.7160000000000002,"15min":2.0009999999999999},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.3399999999999999,"5min":131.078,"15min":131.078},"last":1.1379999999999999},{"interface":"front","average":{"1min":3.0070000000000001,"5min":3.9540000000000002,"15min":2.1040000000000001},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.438000000000001,"5min":133.083,"15min":133.083},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.23,"15min":1.7849999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":9.673,"5min":82.825000000000003,"15min":82.825000000000003},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.306,"15min":1.2509999999999999},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.308},"max":{"1min":3.129,"5min":75.930999999999997,"15min":75.930999999999997},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":8.1050000000000004,"5min":4.4669999999999996,"15min":2.173},"min":{"1min":0.39200000000000002,"5min":0.34799999999999998,"15min":0.29499999999999998},"max":{"1min":93.685000000000002,"5min":93.685000000000002,"15min":93.685000000000002},"last":1.1559999999999999},{"interface":"front","average":{"1min":7.6550000000000002,"5min":3.5459999999999998,"15min":1.7030000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":93.546000000000006,"5min":93.546000000000006,"15min":93.546000000000006},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":2.3069999999999999,"15min":1.472},"min":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":9.7140000000000004,"5min":10.275,"15min":10.275},"last":1.204},{"interface":"front","average":{"1min":2.3769999999999998,"5min":1.764,"15min":1.115},"min":{"1min":0.43099999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":10.919,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.439}]},{"osd":7,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":4.0570000000000004,"5min":2.5800000000000001,"15min":1.643},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":32.773000000000003,"5min":32.773000000000003,"15min":32.773000000000003},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.069,"5min":2.4249999999999998,"15min":1.6850000000000001},"min":{"1min":0.39900000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5790000000000006,"5min":10.257999999999999,"15min":23.265999999999998},"last":1.1819999999999999}]}]},{"osd":2,"up_from":0,"seq":442381631611,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468085,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1632056,"kb_used_data":1411248,"kb_used_omap":4,"kb_used_meta":220795,"kb_avail":92113096,"statfs":{"total":95995035648,"available":94323810304,"internally_reserved":0,"allocated":1445117952,"data_stored":1443741386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":226094637},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":5.7969999999999997,"15min":2.7069999999999999},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":23.890000000000001,"5min":225.86699999999999,"15min":225.86699999999999},"last":4.8579999999999997},{"interface":"front","average":{"1min":4.4619999999999997,"5min":6.2400000000000002,"15min":2.9340000000000002},"min":{"1min":0.23699999999999999,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":24.053000000000001,"5min":225.84800000000001,"15min":225.84800000000001},"last":0.57599999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":4.0819999999999999,"15min":2.1840000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.209,"5min":82.951999999999998,"15min":82.951999999999998},"last":3.5230000000000001},{"interface":"front","average":{"1min":4.75,"5min":4.4509999999999996,"15min":2.2170000000000001},"min":{"1min":0.34300000000000003,"5min":0.22600000000000001,"15min":0.189},"max":{"1min":13.85,"5min":86.001000000000005,"15min":86.001000000000005},"last":4.8860000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":5.5129999999999999,"15min":3.3849999999999998},"min":{"1min":0.501,"5min":0.252,"15min":0.223},"max":{"1min":8.1180000000000003,"5min":99.962999999999994,"15min":99.962999999999994},"last":3.5640000000000001},{"interface":"front","average":{"1min":2.2890000000000001,"5min":5.6790000000000003,"15min":3.2130000000000001},"min":{"1min":0.41399999999999998,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":9.5909999999999993,"5min":100.429,"15min":100.429},"last":0.30499999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.9850000000000001,"15min":1.1839999999999999},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":6.2960000000000003,"5min":11.911,"15min":11.911},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.764,"15min":1.2030000000000001},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.949999999999999,"5min":10.949999999999999,"15min":12.036},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":2.4220000000000002,"15min":1.391},"min":{"1min":0.39900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":11.028,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.6259999999999999,"5min":2.427,"15min":1.3799999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.883,"5min":49.668999999999997,"15min":49.668999999999997},"last":3.6080000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":2.1259999999999999,"15min":1.3320000000000001},"min":{"1min":0.42699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.992000000000001,"5min":10.992000000000001,"15min":12.068},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.893,"5min":1.8620000000000001,"15min":1.2410000000000001},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35399999999999998},"max":{"1min":5.9219999999999997,"5min":11.68,"15min":11.68},"last":3.6419999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.0110000000000001,"15min":1.3140000000000001},"min":{"1min":0.39700000000000002,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":6.0309999999999997,"5min":16.849,"15min":16.849},"last":3.6709999999999998},{"interface":"front","average":{"1min":2.3290000000000002,"5min":2.0910000000000002,"15min":1.272},"min":{"1min":0.46200000000000002,"5min":0.40400000000000003,"15min":0.36699999999999999},"max":{"1min":6.1749999999999998,"5min":11.945,"15min":12.382},"last":4.8140000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019135,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1020784,"kb_used_data":840872,"kb_used_omap":0,"kb_used_meta":179904,"kb_avail":92724368,"statfs":{"total":95995035648,"available":94949752832,"internally_reserved":0,"allocated":861052928,"data_stored":860045371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":184221696},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":2.4340000000000002,"15min":1.649},"min":{"1min":0.28399999999999997,"5min":0.23000000000000001,"15min":0.218},"max":{"1min":9.2579999999999991,"5min":46.18,"15min":66.355000000000004},"last":0.318},{"interface":"front","average":{"1min":2.379,"5min":2.5019999999999998,"15min":1.7350000000000001},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":9.2390000000000008,"5min":46.207000000000001,"15min":66.332999999999998},"last":0.39400000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":2.0470000000000002,"15min":1.3460000000000001},"min":{"1min":0.26700000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.0750000000000002,"5min":35.409999999999997,"15min":38.878},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.492,"15min":1.694},"min":{"1min":0.28100000000000003,"5min":0.255,"15min":0.255},"max":{"1min":7.8620000000000001,"5min":35.999000000000002,"15min":38.853999999999999},"last":0.55400000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9159999999999999,"5min":4.5730000000000004,"15min":2.1589999999999998},"min":{"1min":0.372,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":6.1020000000000003,"5min":138.06100000000001,"15min":138.06100000000001},"last":1.2010000000000001},{"interface":"front","average":{"1min":1.8120000000000001,"5min":4.6079999999999997,"15min":2.327},"min":{"1min":0.35899999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.9580000000000002,"5min":138.15199999999999,"15min":138.15199999999999},"last":3.9540000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:23:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":1.8220000000000001,"15min":1.196},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":12.129,"5min":12.129,"15min":12.129},"last":0.624},{"interface":"front","average":{"1min":2.5870000000000002,"5min":1.8200000000000001,"15min":1.228},"min":{"1min":0.46300000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4059999999999997,"5min":8.8930000000000007,"15min":9.8109999999999999},"last":0.41499999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:23:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.284,"15min":0.874},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.8570000000000002,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.5680000000000001,"5min":1.3759999999999999,"15min":0.90400000000000003},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":7.298,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.50900000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:23:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.80800000000000005,"15min":0.68999999999999995},"min":{"1min":0.33200000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":2.262,"5min":7.3079999999999998,"15min":7.3079999999999998},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.873,"5min":0.85299999999999998,"15min":0.749},"min":{"1min":0.29099999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.2679999999999998,"5min":6.7279999999999998,"15min":8.5210000000000008},"last":0.44900000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.94799999999999995,"15min":0.83999999999999997},"min":{"1min":0.313,"5min":0.313,"15min":0.307},"max":{"1min":2.754,"5min":6.7480000000000002,"15min":8.577},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.143,"5min":0.94099999999999995,"15min":0.78700000000000003},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":5.1479999999999997,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.57299999999999995}]}]},{"osd":5,"up_from":38,"seq":163208757502,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1199184,"kb_used_data":982728,"kb_used_omap":0,"kb_used_meta":216448,"kb_avail":92545968,"statfs":{"total":95995035648,"available":94767071232,"internally_reserved":0,"allocated":1006313472,"data_stored":1005317436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":221642752},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:55 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":2.1150000000000002,"15min":1.859},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":12.661,"5min":28.741,"15min":133.22499999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":3.198,"5min":1.8959999999999999,"15min":1.8979999999999999},"min":{"1min":0.26100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.324999999999999,"5min":28.873999999999999,"15min":132.78200000000001},"last":0.29699999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:23:16 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":4.3090000000000002,"15min":2.681},"min":{"1min":0.27200000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.385999999999999,"5min":93.981999999999999,"15min":93.981999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.9099999999999999,"5min":4.2329999999999997,"15min":2.6509999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":13.457000000000001,"5min":93.909999999999997,"15min":93.909999999999997},"last":0.435}]},{"osd":2,"last update":"Sat Sep 23 22:22:49 2023","interfaces":[{"interface":"back","average":{"1min":2.972,"5min":6.0650000000000004,"15min":3.4329999999999998},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.534000000000001,"5min":104.254,"15min":104.254},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.661,"5min":6.3380000000000001,"15min":3.7040000000000002},"min":{"1min":0.35199999999999998,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":5.7380000000000004,"5min":104.217,"15min":104.217},"last":0.36699999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:23:16 2023","interfaces":[{"interface":"back","average":{"1min":4.1429999999999998,"5min":3.6789999999999998,"15min":2.23},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.246},"max":{"1min":18.876000000000001,"5min":93.739999999999995,"15min":93.739999999999995},"last":4.3659999999999997},{"interface":"front","average":{"1min":2.6000000000000001,"5min":3.4220000000000002,"15min":2.1699999999999999},"min":{"1min":0.34200000000000003,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":13.773999999999999,"5min":93.197999999999993,"15min":93.197999999999993},"last":0.44700000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.085,"15min":1.03},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":5.1799999999999997,"5min":10.163,"15min":82.530000000000001},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.0629999999999999,"15min":1.0660000000000001},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.2119999999999997,"5min":7.2119999999999997,"15min":85.441000000000003},"last":0.307}]},{"osd":6,"last update":"Sat Sep 23 22:22:35 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.1399999999999999,"15min":1.123},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.26400000000000001},"max":{"1min":4.056,"5min":9.6519999999999992,"15min":114.83499999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1,"15min":1.0469999999999999},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.8690000000000002,"5min":7.4640000000000004,"15min":110.905},"last":0.376}]},{"osd":7,"last update":"Sat Sep 23 22:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.159,"15min":0.84299999999999997},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.9409999999999998,"5min":8.2159999999999993,"15min":8.8030000000000008},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.82499999999999996,"15min":0.745},"min":{"1min":0.39100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5299999999999998,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":0.39800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":705302528,"data_stored":705298456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1083248640,"data_stored":1083244568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2819883008,"data_stored":2819883008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1408958464,"data_stored":1408958464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":842989568,"data_stored":842989568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":993464320,"data_stored":993460248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1331429376,"data_stored":1331429376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:23:37.569 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:37.653 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:37.815 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:37.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:37.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:37.928 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:38.000 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:38.002 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:38.025 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:38 smithi026 ceph-mon[215775]: pgmap v613: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 33089/98382 objects degraded (33.633%); 3131/98382 objects misplaced (3.182%); 7.5 MiB/s, 144 objects/s recovering 2023-09-23T22:23:38.156 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:38.161 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:23:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:23:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:23:38.162 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:23:38.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:38.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:38.280 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:38.361 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 33089/102267 objects degraded (32.356%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:38.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:38 smithi026 ceph-mon[223014]: pgmap v613: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 33089/98382 objects degraded (33.633%); 3131/98382 objects misplaced (3.182%); 7.5 MiB/s, 144 objects/s recovering 2023-09-23T22:23:38.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 33089/102267 objects degraded (32.356%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:38.371 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:38.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:38 smithi070 ceph-mon[192652]: pgmap v613: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.2 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 33089/98382 objects degraded (33.633%); 3131/98382 objects misplaced (3.182%); 7.5 MiB/s, 144 objects/s recovering 2023-09-23T22:23:38.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 33089/102267 objects degraded (32.356%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:39.327 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:39.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:39.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:39.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:39.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:39.436 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:23:39.461 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:39.508 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:23:39.556 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:39.694 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:23:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:23:39] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-23T22:23:40.087 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:23:40.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:40.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:23:40.192 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:40.232 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:40.304 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:40.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:40.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:40.409 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:40.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:40 smithi026 ceph-mon[223014]: pgmap v614: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 33089/96690 objects degraded (34.222%); 3131/96690 objects misplaced (3.238%); 6.4 MiB/s, 123 objects/s recovering 2023-09-23T22:23:40.436 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:40 smithi026 ceph-mon[215775]: pgmap v614: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 33089/96690 objects degraded (34.222%); 3131/96690 objects misplaced (3.238%); 6.4 MiB/s, 123 objects/s recovering 2023-09-23T22:23:40.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:40.493 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:23:40.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:40 smithi070 ceph-mon[192652]: pgmap v614: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 2.0 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 33089/96690 objects degraded (34.222%); 3131/96690 objects misplaced (3.238%); 6.4 MiB/s, 123 objects/s recovering 2023-09-23T22:23:40.827 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:40.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:40.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:40.936 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:40.984 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:41.049 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:41.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:41.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:41.159 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:41.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:41.279 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:41.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:41.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:41.386 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:41.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:41.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:41.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:41.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:41.620 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:41.678 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:41.742 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:41.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:41.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:41.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:41.964 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:42.149 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:42.185 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:42.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:42.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:42.258 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:42.301 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:42.323 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:23:42.342 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:42.373 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:23:42.399 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:42.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:42.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:42.419 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:23:42.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:42 smithi026 ceph-mon[223014]: pgmap v615: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 32890/90306 objects degraded (36.421%); 2936/90306 objects misplaced (3.251%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T22:23:42.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2696437777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:42.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:42 smithi026 ceph-mon[215775]: pgmap v615: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 32890/90306 objects degraded (36.421%); 2936/90306 objects misplaced (3.251%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T22:23:42.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2696437777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:42.484 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:23:42.509 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:42.517 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:23:42.558 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:23:42.605 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:23:42.614 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:42.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:42 smithi070 ceph-mon[192652]: pgmap v615: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 32890/90306 objects degraded (36.421%); 2936/90306 objects misplaced (3.251%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T22:23:42.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2696437777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:42.648 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:23:42.663 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:23:42.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:23:42.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:23:42.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:23:42.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:23:42.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:23:42.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:23:42.664 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:23:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 32890/90306 objects degraded (36.421%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1307997582' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:23:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:43 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:23:43.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 32890/90306 objects degraded (36.421%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1307997582' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:23:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:43 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:23:43.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 32890/90306 objects degraded (36.421%), 16 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:43.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1307997582' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:23:43.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:43 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:23:43.900 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:23:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:44 smithi070 ceph-mon[192652]: pgmap v616: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 32890/89046 objects degraded (36.936%); 2680/89046 objects misplaced (3.010%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:23:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:44 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:23:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:44 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:23:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:44 smithi070 ceph-mon[192652]: osdmap e251: 8 total, 8 up, 6 in 2023-09-23T22:23:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:44 smithi070 ceph-mon[192652]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:44 smithi070 ceph-mon[192652]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:44 smithi070 ceph-mon[192652]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:44 smithi070 ceph-mon[192652]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[223014]: pgmap v616: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 32890/89046 objects degraded (36.936%); 2680/89046 objects misplaced (3.010%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:23:44.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:23:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:23:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[223014]: osdmap e251: 8 total, 8 up, 6 in 2023-09-23T22:23:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[223014]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[223014]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[223014]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[223014]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[215775]: pgmap v616: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.8 GiB data, 6.7 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 631 op/s; 32890/89046 objects degraded (36.936%); 2680/89046 objects misplaced (3.010%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:23:44.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:23:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:23:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[215775]: osdmap e251: 8 total, 8 up, 6 in 2023-09-23T22:23:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[215775]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[215775]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[215775]: failed to encode map e251 with expected crc 2023-09-23T22:23:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:44 smithi026 ceph-mon[215775]: failed to encode map e251 with expected crc 2023-09-23T22:23:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:45 smithi026 ceph-mon[223014]: failed to encode map e251 with expected crc 2023-09-23T22:23:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:45 smithi026 ceph-mon[215775]: failed to encode map e251 with expected crc 2023-09-23T22:23:45.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:45 smithi070 ceph-mon[192652]: failed to encode map e251 with expected crc 2023-09-23T22:23:46.239 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:46.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:46.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:46.344 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:46.375 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:46.430 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:46.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:46.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:46.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:46 smithi026 ceph-mon[223014]: pgmap v618: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 32684/80358 objects degraded (40.673%); 2479/80358 objects misplaced (3.085%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:46.497 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:46 smithi026 ceph-mon[215775]: pgmap v618: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 32684/80358 objects degraded (40.673%); 2479/80358 objects misplaced (3.085%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:46.536 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:46.600 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:46 smithi070 ceph-mon[192652]: pgmap v618: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 730 op/s; 32684/80358 objects degraded (40.673%); 2479/80358 objects misplaced (3.085%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:46.665 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:46.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:46.674 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:46.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:46.785 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:23:46.785 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:23:46.818 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:46.886 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:46.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:46.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:46.995 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:47.035 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:47.098 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:47.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:47.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:47.206 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:47.223 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:23:47.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:47.417 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:47.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:47.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:47.443 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:23:47.449 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:23:47.529 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:47.554 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:47.594 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:47.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:47.601 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:47.701 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:47.750 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:47.831 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:47.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:47.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:47.945 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:47.989 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:48.077 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:48.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:48.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:48.185 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:48.234 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:48.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:48 smithi026 ceph-mon[223014]: pgmap v619: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 32684/79029 objects degraded (41.357%); 2216/79029 objects misplaced (2.804%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:48.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:48 smithi026 ceph-mon[223014]: from='client.35283 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:48.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 32684/79029 objects degraded (41.357%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:48.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:48 smithi026 ceph-mon[215775]: pgmap v619: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 32684/79029 objects degraded (41.357%); 2216/79029 objects misplaced (2.804%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:48.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:48 smithi026 ceph-mon[215775]: from='client.35283 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:48.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 32684/79029 objects degraded (41.357%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:48.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:23:48.562 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:23:48.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:48 smithi070 ceph-mon[192652]: pgmap v619: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 644 op/s; 32684/79029 objects degraded (41.357%); 2216/79029 objects misplaced (2.804%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:48.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:48 smithi070 ceph-mon[192652]: from='client.35283 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:23:48.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 32684/79029 objects degraded (41.357%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:48.719 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:48.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:48.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:48.840 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:48.887 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":619,"stamp":"2023-09-23T22:23:47.268868+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1729149719,"num_objects":26343,"num_object_clones":0,"num_object_copies":79029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32684,"num_objects_misplaced":2216,"num_objects_unfound":0,"num_objects_dirty":26343,"num_whiteouts":0,"num_read":6071,"num_read_kb":12351,"num_write":137331,"num_write_kb":5219374,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21223,"num_bytes_recovered":1339953871,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86057,"ondisk_log_size":86057,"up":297,"acting":283,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6589864,"kb_used_data":5172600,"kb_used_omap":20,"kb_used_meta":1417067,"kb_avail":555881048,"statfs":{"total":575970213888,"available":569222193152,"internally_reserved":0,"allocated":5296742400,"data_stored":5289600445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1451077387},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-422772736,"num_objects":-6451,"num_object_clones":0,"num_object_copies":-19353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-405,"num_objects_misplaced":-915,"num_objects_unfound":0,"num_objects_dirty":-6451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6451,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":924,"num_bytes_recovered":60555264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007524"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":238,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.905852+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:23:33.905852+0000","last_peered":"2023-09-23T22:23:33.905852+0000","last_clean":"2023-09-23T22:23:33.905852+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:23:33.905852+0000","last_undegraded":"2023-09-23T22:23:33.905852+0000","last_fullsized":"2023-09-23T22:23:33.905852+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":229,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914701+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:23:33.914701+0000","last_peered":"2023-09-23T22:23:33.914701+0000","last_clean":"2023-09-23T22:23:33.914701+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:23:33.914701+0000","last_undegraded":"2023-09-23T22:23:33.914701+0000","last_fullsized":"2023-09-23T22:23:33.914701+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":191,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.891368+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:23:29.891368+0000","last_peered":"2023-09-23T22:23:29.891368+0000","last_clean":"2023-09-23T22:23:29.891368+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:23:29.891368+0000","last_undegraded":"2023-09-23T22:23:29.891368+0000","last_fullsized":"2023-09-23T22:23:29.891368+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":108,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.919103+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:23:33.919103+0000","last_peered":"2023-09-23T22:23:33.919103+0000","last_clean":"2023-09-23T22:23:33.919103+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:23:33.919103+0000","last_undegraded":"2023-09-23T22:23:33.919103+0000","last_fullsized":"2023-09-23T22:23:33.919103+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":270,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:39.268220+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:23:39.268220+0000","last_peered":"2023-09-23T22:23:39.268220+0000","last_clean":"2023-09-23T22:23:39.268220+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:23:39.268220+0000","last_undegraded":"2023-09-23T22:23:39.268220+0000","last_fullsized":"2023-09-23T22:23:39.268220+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":206,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.915597+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:23:33.915597+0000","last_peered":"2023-09-23T22:23:33.915597+0000","last_clean":"2023-09-23T22:23:33.915597+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:23:33.915597+0000","last_undegraded":"2023-09-23T22:23:33.915597+0000","last_fullsized":"2023-09-23T22:23:33.915597+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":136,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.919836+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:23:33.919836+0000","last_peered":"2023-09-23T22:23:33.919836+0000","last_clean":"2023-09-23T22:23:33.919836+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:23:33.919836+0000","last_undegraded":"2023-09-23T22:23:33.919836+0000","last_fullsized":"2023-09-23T22:23:33.919836+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":227,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.912852+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:23:33.912852+0000","last_peered":"2023-09-23T22:23:33.912852+0000","last_clean":"2023-09-23T22:23:33.912852+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:23:33.912852+0000","last_undegraded":"2023-09-23T22:23:33.912852+0000","last_fullsized":"2023-09-23T22:23:33.912852+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":211,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914560+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:23:33.914560+0000","last_peered":"2023-09-23T22:23:33.914560+0000","last_clean":"2023-09-23T22:23:33.914560+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:23:33.914560+0000","last_undegraded":"2023-09-23T22:23:33.914560+0000","last_fullsized":"2023-09-23T22:23:33.914560+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":195,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.910779+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:23:33.910779+0000","last_peered":"2023-09-23T22:23:33.910779+0000","last_clean":"2023-09-23T22:23:33.910779+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:23:33.910779+0000","last_undegraded":"2023-09-23T22:23:33.910779+0000","last_fullsized":"2023-09-23T22:23:33.910779+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":194,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914628+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:23:33.914628+0000","last_peered":"2023-09-23T22:23:33.914628+0000","last_clean":"2023-09-23T22:23:33.914628+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:23:33.914628+0000","last_undegraded":"2023-09-23T22:23:33.914628+0000","last_fullsized":"2023-09-23T22:23:33.914628+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":227,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.889151+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:23:29.889151+0000","last_peered":"2023-09-23T22:23:29.889151+0000","last_clean":"2023-09-23T22:23:29.889151+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:23:29.889151+0000","last_undegraded":"2023-09-23T22:23:29.889151+0000","last_fullsized":"2023-09-23T22:23:29.889151+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":255,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.908502+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:23:33.908502+0000","last_peered":"2023-09-23T22:23:33.908502+0000","last_clean":"2023-09-23T22:23:33.908502+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:23:33.908502+0000","last_undegraded":"2023-09-23T22:23:33.908502+0000","last_fullsized":"2023-09-23T22:23:33.908502+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":219,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.905931+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:23:33.905931+0000","last_peered":"2023-09-23T22:23:33.905931+0000","last_clean":"2023-09-23T22:23:33.905931+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:23:33.905931+0000","last_undegraded":"2023-09-23T22:23:33.905931+0000","last_fullsized":"2023-09-23T22:23:33.905931+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":178,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.906045+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:23:33.906045+0000","last_peered":"2023-09-23T22:23:33.906045+0000","last_clean":"2023-09-23T22:23:33.906045+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:23:33.906045+0000","last_undegraded":"2023-09-23T22:23:33.906045+0000","last_fullsized":"2023-09-23T22:23:33.906045+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":218,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.910956+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:23:33.910956+0000","last_peered":"2023-09-23T22:23:33.910956+0000","last_clean":"2023-09-23T22:23:33.910956+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:23:33.910956+0000","last_undegraded":"2023-09-23T22:23:33.910956+0000","last_fullsized":"2023-09-23T22:23:33.910956+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":126,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:41.582625+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:23:41.582625+0000","last_peered":"2023-09-23T22:23:41.582625+0000","last_clean":"2023-09-23T22:23:41.582625+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:23:41.582625+0000","last_undegraded":"2023-09-23T22:23:41.582625+0000","last_fullsized":"2023-09-23T22:23:41.582625+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":230,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.915142+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:23:33.915142+0000","last_peered":"2023-09-23T22:23:33.915142+0000","last_clean":"2023-09-23T22:23:33.915142+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:23:33.915142+0000","last_undegraded":"2023-09-23T22:23:33.915142+0000","last_fullsized":"2023-09-23T22:23:33.915142+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":174,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.906067+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:23:33.906067+0000","last_peered":"2023-09-23T22:23:33.906067+0000","last_clean":"2023-09-23T22:23:33.906067+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:23:33.906067+0000","last_undegraded":"2023-09-23T22:23:33.906067+0000","last_fullsized":"2023-09-23T22:23:33.906067+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":107,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.919333+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:23:33.919333+0000","last_peered":"2023-09-23T22:23:33.919333+0000","last_clean":"2023-09-23T22:23:33.919333+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:23:33.919333+0000","last_undegraded":"2023-09-23T22:23:33.919333+0000","last_fullsized":"2023-09-23T22:23:33.919333+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":268,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.917808+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:23:33.917808+0000","last_peered":"2023-09-23T22:23:33.917808+0000","last_clean":"2023-09-23T22:23:33.917808+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:23:33.917808+0000","last_undegraded":"2023-09-23T22:23:33.917808+0000","last_fullsized":"2023-09-23T22:23:33.917808+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":39,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.920962+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:23:33.920962+0000","last_peered":"2023-09-23T22:23:33.920962+0000","last_clean":"2023-09-23T22:23:33.920962+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:23:33.920962+0000","last_undegraded":"2023-09-23T22:23:33.920962+0000","last_fullsized":"2023-09-23T22:23:33.920962+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"251'8461","reported_seq":15825,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.842901+0000","last_change":"2023-09-23T22:23:29.940235+0000","last_active":"2023-09-23T22:23:44.842901+0000","last_peered":"2023-09-23T22:23:44.842901+0000","last_clean":"2023-09-23T22:23:44.842901+0000","last_became_active":"2023-09-23T22:23:29.892586+0000","last_became_peered":"2023-09-23T22:23:29.892586+0000","last_unstale":"2023-09-23T22:23:44.842901+0000","last_undegraded":"2023-09-23T22:23:44.842901+0000","last_fullsized":"2023-09-23T22:23:44.842901+0000","mapping_epoch":248,"log_start":"211'3000","ondisk_log_start":"211'3000","created":203,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5461,"log_dups_size":0,"ondisk_log_size":5461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:50.397106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104136704,"num_objects":1589,"num_object_clones":0,"num_object_copies":4767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8461,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3642,"num_bytes_recovered":188481536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":246,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.911318+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:23:33.911318+0000","last_peered":"2023-09-23T22:23:33.911318+0000","last_clean":"2023-09-23T22:23:33.911318+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:23:33.911318+0000","last_undegraded":"2023-09-23T22:23:33.911318+0000","last_fullsized":"2023-09-23T22:23:33.911318+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":233,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.917915+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:23:33.917915+0000","last_peered":"2023-09-23T22:23:33.917915+0000","last_clean":"2023-09-23T22:23:33.917915+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:23:33.917915+0000","last_undegraded":"2023-09-23T22:23:33.917915+0000","last_fullsized":"2023-09-23T22:23:33.917915+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"251'8511","reported_seq":16034,"reported_epoch":251,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:23:44.266769+0000","last_change":"2023-09-23T22:22:45.430551+0000","last_active":"2023-09-23T22:23:44.266769+0000","last_peered":"2023-09-23T22:23:44.266769+0000","last_clean":"2023-09-23T22:22:43.315002+0000","last_became_active":"2023-09-23T22:22:45.409613+0000","last_became_peered":"2023-09-23T22:22:45.409613+0000","last_unstale":"2023-09-23T22:23:44.266769+0000","last_undegraded":"2023-09-23T22:23:44.266769+0000","last_fullsized":"2023-09-23T22:23:44.266769+0000","mapping_epoch":240,"log_start":"212'3091","ondisk_log_start":"212'3091","created":203,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5420,"log_dups_size":0,"ondisk_log_size":5420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:03.767060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108199936,"num_objects":1651,"num_object_clones":0,"num_object_copies":4953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1651,"num_objects_unfound":0,"num_objects_dirty":1651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8511,"num_write_kb":325184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3717,"num_bytes_recovered":243597312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1651}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":160,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.911650+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:23:33.911650+0000","last_peered":"2023-09-23T22:23:33.911650+0000","last_clean":"2023-09-23T22:23:33.911650+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:23:33.911650+0000","last_undegraded":"2023-09-23T22:23:33.911650+0000","last_fullsized":"2023-09-23T22:23:33.911650+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":238,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.906044+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:23:33.906044+0000","last_peered":"2023-09-23T22:23:33.906044+0000","last_clean":"2023-09-23T22:23:33.906044+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:23:33.906044+0000","last_undegraded":"2023-09-23T22:23:33.906044+0000","last_fullsized":"2023-09-23T22:23:33.906044+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"251'8699","reported_seq":12421,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:44.218640+0000","last_change":"2023-09-23T22:22:45.550371+0000","last_active":"2023-09-23T22:23:44.218640+0000","last_peered":"2023-09-23T22:23:44.218640+0000","last_clean":"2023-09-23T22:21:32.851371+0000","last_became_active":"2023-09-23T22:22:45.472694+0000","last_became_peered":"2023-09-23T22:22:45.472694+0000","last_unstale":"2023-09-23T22:23:44.218640+0000","last_undegraded":"2023-09-23T22:22:45.456856+0000","last_fullsized":"2023-09-23T22:22:45.317726+0000","mapping_epoch":240,"log_start":"212'3333","ondisk_log_start":"212'3333","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5366,"log_dups_size":0,"ondisk_log_size":5366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.540906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113311744,"num_objects":1729,"num_object_clones":0,"num_object_copies":5187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5210,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8699,"num_write_kb":333696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1215,"num_bytes_recovered":79495168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"0,1,3","objects":88},{"shards":"0,2,3","objects":495},{"shards":"2,3","objects":1},{"shards":"2,3,7","objects":1145},{"shards":"3,7","objects":-2270}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":176,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.915220+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:23:33.915220+0000","last_peered":"2023-09-23T22:23:33.915220+0000","last_clean":"2023-09-23T22:23:33.915220+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:23:33.915220+0000","last_undegraded":"2023-09-23T22:23:33.915220+0000","last_fullsized":"2023-09-23T22:23:33.915220+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":185,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.889233+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:23:29.889233+0000","last_peered":"2023-09-23T22:23:29.889233+0000","last_clean":"2023-09-23T22:23:29.889233+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:23:29.889233+0000","last_undegraded":"2023-09-23T22:23:29.889233+0000","last_fullsized":"2023-09-23T22:23:29.889233+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"251'8690","reported_seq":16213,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:46.568141+0000","last_change":"2023-09-23T22:22:17.938724+0000","last_active":"2023-09-23T22:23:46.568141+0000","last_peered":"2023-09-23T22:23:46.568141+0000","last_clean":"2023-09-23T22:21:12.226652+0000","last_became_active":"2023-09-23T22:21:13.982783+0000","last_became_peered":"2023-09-23T22:21:13.982783+0000","last_unstale":"2023-09-23T22:23:46.568141+0000","last_undegraded":"2023-09-23T22:22:17.930337+0000","last_fullsized":"2023-09-23T22:21:13.795059+0000","mapping_epoch":214,"log_start":"215'3797","ondisk_log_start":"215'3797","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4893,"log_dups_size":0,"ondisk_log_size":4893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.489307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99745792,"num_objects":1522,"num_object_clones":0,"num_object_copies":4566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8690,"num_write_kb":326784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3728,"num_bytes_recovered":244318208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":38},{"shards":"0,7","objects":1484}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":241,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914809+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:23:33.914809+0000","last_peered":"2023-09-23T22:23:33.914809+0000","last_clean":"2023-09-23T22:23:33.914809+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:23:33.914809+0000","last_undegraded":"2023-09-23T22:23:33.914809+0000","last_fullsized":"2023-09-23T22:23:33.914809+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":105,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.915021+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:23:33.915021+0000","last_peered":"2023-09-23T22:23:33.915021+0000","last_clean":"2023-09-23T22:23:33.915021+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:23:33.915021+0000","last_undegraded":"2023-09-23T22:23:33.915021+0000","last_fullsized":"2023-09-23T22:23:33.915021+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"251'8580","reported_seq":10181,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:44.889503+0000","last_change":"2023-09-23T22:21:57.837966+0000","last_active":"2023-09-23T22:23:44.889503+0000","last_peered":"2023-09-23T22:23:44.889503+0000","last_clean":"2023-09-23T22:21:12.225315+0000","last_became_active":"2023-09-23T22:21:13.940088+0000","last_became_peered":"2023-09-23T22:21:13.940088+0000","last_unstale":"2023-09-23T22:23:44.889503+0000","last_undegraded":"2023-09-23T22:21:57.835407+0000","last_fullsized":"2023-09-23T22:21:13.800025+0000","mapping_epoch":214,"log_start":"212'3300","ondisk_log_start":"212'3300","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5280,"log_dups_size":0,"ondisk_log_size":5280,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:26.245766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106823680,"num_objects":1630,"num_object_clones":0,"num_object_copies":4890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8580,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":768,"num_bytes_recovered":50331648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":233},{"shards":"1,4","objects":-1310}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":554,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.910618+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:23:33.910618+0000","last_peered":"2023-09-23T22:23:33.910618+0000","last_clean":"2023-09-23T22:23:33.910618+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:23:33.910618+0000","last_undegraded":"2023-09-23T22:23:33.910618+0000","last_fullsized":"2023-09-23T22:23:33.910618+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":264,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.909315+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:23:33.909315+0000","last_peered":"2023-09-23T22:23:33.909315+0000","last_clean":"2023-09-23T22:23:33.909315+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:23:33.909315+0000","last_undegraded":"2023-09-23T22:23:33.909315+0000","last_fullsized":"2023-09-23T22:23:33.909315+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":219,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.908378+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:23:33.908378+0000","last_peered":"2023-09-23T22:23:33.908378+0000","last_clean":"2023-09-23T22:23:33.908378+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:23:33.908378+0000","last_undegraded":"2023-09-23T22:23:33.908378+0000","last_fullsized":"2023-09-23T22:23:33.908378+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"251'8424","reported_seq":9962,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:44.884176+0000","last_change":"2023-09-23T22:22:20.034548+0000","last_active":"2023-09-23T22:23:44.884176+0000","last_peered":"2023-09-23T22:23:44.884176+0000","last_clean":"2023-09-23T22:21:12.194696+0000","last_became_active":"2023-09-23T22:22:19.958601+0000","last_became_peered":"2023-09-23T22:22:19.958601+0000","last_unstale":"2023-09-23T22:23:44.884176+0000","last_undegraded":"2023-09-23T22:22:19.951596+0000","last_fullsized":"2023-09-23T22:22:19.874428+0000","mapping_epoch":232,"log_start":"212'3100","ondisk_log_start":"212'3100","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5324,"log_dups_size":0,"ondisk_log_size":5324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:10.906601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102105088,"num_objects":1558,"num_object_clones":0,"num_object_copies":4674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4172,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8424,"num_write_kb":319424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":486,"num_bytes_recovered":31850496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[4,3],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"1,4,5","objects":506},{"shards":"2,3,4","objects":234},{"shards":"3,4","objects":-2128}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":549,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:33.917834+0000","last_change":"2023-09-23T22:21:59.026062+0000","last_active":"2023-09-23T22:23:33.917834+0000","last_peered":"2023-09-23T22:23:33.917834+0000","last_clean":"2023-09-23T22:21:50.447236+0000","last_became_active":"2023-09-23T22:21:59.021009+0000","last_became_peered":"2023-09-23T22:21:59.021009+0000","last_unstale":"2023-09-23T22:23:33.917834+0000","last_undegraded":"2023-09-23T22:21:58.818399+0000","last_fullsized":"2023-09-23T22:21:58.817464+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:52.228490+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":167,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.907035+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:23:33.907035+0000","last_peered":"2023-09-23T22:23:33.907035+0000","last_clean":"2023-09-23T22:23:33.907035+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:23:33.907035+0000","last_undegraded":"2023-09-23T22:23:33.907035+0000","last_fullsized":"2023-09-23T22:23:33.907035+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":236,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.906734+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:23:33.906734+0000","last_peered":"2023-09-23T22:23:33.906734+0000","last_clean":"2023-09-23T22:23:33.906734+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:23:33.906734+0000","last_undegraded":"2023-09-23T22:23:33.906734+0000","last_fullsized":"2023-09-23T22:23:33.906734+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"251'8440","reported_seq":8515,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:44.903129+0000","last_change":"2023-09-23T22:22:18.973077+0000","last_active":"2023-09-23T22:23:44.903129+0000","last_peered":"2023-09-23T22:23:44.903129+0000","last_clean":"2023-09-23T22:21:32.779211+0000","last_became_active":"2023-09-23T22:22:18.949524+0000","last_became_peered":"2023-09-23T22:22:18.949524+0000","last_unstale":"2023-09-23T22:23:44.903129+0000","last_undegraded":"2023-09-23T22:22:18.870000+0000","last_fullsized":"2023-09-23T22:22:18.859180+0000","mapping_epoch":231,"log_start":"211'2900","ondisk_log_start":"211'2900","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5540,"log_dups_size":0,"ondisk_log_size":5540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.194549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104595456,"num_objects":1596,"num_object_clones":0,"num_object_copies":4788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":914,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8440,"num_write_kb":321152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"1,5,7","objects":487},{"shards":"5,7","objects":1109}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":212,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.905964+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:23:33.905964+0000","last_peered":"2023-09-23T22:23:33.905964+0000","last_clean":"2023-09-23T22:23:33.905964+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:23:33.905964+0000","last_undegraded":"2023-09-23T22:23:33.905964+0000","last_fullsized":"2023-09-23T22:23:33.905964+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":227,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.911506+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:23:33.911506+0000","last_peered":"2023-09-23T22:23:33.911506+0000","last_clean":"2023-09-23T22:23:33.911506+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:23:33.911506+0000","last_undegraded":"2023-09-23T22:23:33.911506+0000","last_fullsized":"2023-09-23T22:23:33.911506+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"251'8526","reported_seq":8583,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.923057+0000","last_change":"2023-09-23T22:20:19.473637+0000","last_active":"2023-09-23T22:23:44.923057+0000","last_peered":"2023-09-23T22:23:44.923057+0000","last_clean":"2023-09-23T22:23:44.923057+0000","last_became_active":"2023-09-23T22:20:19.473385+0000","last_became_peered":"2023-09-23T22:20:19.473385+0000","last_unstale":"2023-09-23T22:23:44.923057+0000","last_undegraded":"2023-09-23T22:23:44.923057+0000","last_fullsized":"2023-09-23T22:23:44.923057+0000","mapping_epoch":203,"log_start":"211'3000","ondisk_log_start":"211'3000","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5526,"log_dups_size":0,"ondisk_log_size":5526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:01.911861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108331032,"num_objects":1654,"num_object_clones":0,"num_object_copies":4962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1654,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8526,"num_write_kb":325697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":555,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.913608+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:23:33.913608+0000","last_peered":"2023-09-23T22:23:33.913608+0000","last_clean":"2023-09-23T22:23:33.913608+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:23:33.913608+0000","last_undegraded":"2023-09-23T22:23:33.913608+0000","last_fullsized":"2023-09-23T22:23:33.913608+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":30,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.903417+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:23:33.903417+0000","last_peered":"2023-09-23T22:23:33.903417+0000","last_clean":"2023-09-23T22:23:33.903417+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:23:33.903417+0000","last_undegraded":"2023-09-23T22:23:33.903417+0000","last_fullsized":"2023-09-23T22:23:33.903417+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":191,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.890314+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:23:29.890314+0000","last_peered":"2023-09-23T22:23:29.890314+0000","last_clean":"2023-09-23T22:23:29.890314+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:23:29.890314+0000","last_undegraded":"2023-09-23T22:23:29.890314+0000","last_fullsized":"2023-09-23T22:23:29.890314+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"251'8746","reported_seq":14468,"reported_epoch":251,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:46.583258+0000","last_change":"2023-09-23T22:22:43.348452+0000","last_active":"2023-09-23T22:23:46.583258+0000","last_peered":"2023-09-23T22:23:46.583258+0000","last_clean":"2023-09-23T22:21:32.822849+0000","last_became_active":"2023-09-23T22:21:35.128019+0000","last_became_peered":"2023-09-23T22:21:35.128019+0000","last_unstale":"2023-09-23T22:23:46.583258+0000","last_undegraded":"2023-09-23T22:21:34.912513+0000","last_fullsized":"2023-09-23T22:21:34.880959+0000","mapping_epoch":220,"log_start":"217'3847","ondisk_log_start":"217'3847","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4899,"log_dups_size":0,"ondisk_log_size":4899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:58.687801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102367232,"num_objects":1562,"num_object_clones":0,"num_object_copies":4686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1209,"num_objects_misplaced":565,"num_objects_unfound":0,"num_objects_dirty":1562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8746,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2828,"num_bytes_recovered":185335808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":565},{"shards":"0,3","objects":-212}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":546,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.921144+0000","last_change":"2023-09-23T22:22:47.349660+0000","last_active":"2023-09-23T22:23:33.921144+0000","last_peered":"2023-09-23T22:23:33.921144+0000","last_clean":"2023-09-23T22:23:33.921144+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:23:33.921144+0000","last_undegraded":"2023-09-23T22:23:33.921144+0000","last_fullsized":"2023-09-23T22:23:33.921144+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":799,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.889869+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:23:29.889869+0000","last_peered":"2023-09-23T22:23:29.889869+0000","last_clean":"2023-09-23T22:23:29.889869+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:23:29.889869+0000","last_undegraded":"2023-09-23T22:23:29.889869+0000","last_fullsized":"2023-09-23T22:23:29.889869+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":268,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.915000+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:23:33.915000+0000","last_peered":"2023-09-23T22:23:33.915000+0000","last_clean":"2023-09-23T22:23:33.915000+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:23:33.915000+0000","last_undegraded":"2023-09-23T22:23:33.915000+0000","last_fullsized":"2023-09-23T22:23:33.915000+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":35,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.888992+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:23:29.888992+0000","last_peered":"2023-09-23T22:23:29.888992+0000","last_clean":"2023-09-23T22:23:29.888992+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:23:29.888992+0000","last_undegraded":"2023-09-23T22:23:29.888992+0000","last_fullsized":"2023-09-23T22:23:29.888992+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"251'8589","reported_seq":8677,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:44.885082+0000","last_change":"2023-09-23T22:22:18.955534+0000","last_active":"2023-09-23T22:23:44.885082+0000","last_peered":"2023-09-23T22:23:44.885082+0000","last_clean":"2023-09-23T22:21:32.860473+0000","last_became_active":"2023-09-23T22:22:18.941549+0000","last_became_peered":"2023-09-23T22:22:18.941549+0000","last_unstale":"2023-09-23T22:23:44.885082+0000","last_undegraded":"2023-09-23T22:22:18.875275+0000","last_fullsized":"2023-09-23T22:22:18.863381+0000","mapping_epoch":231,"log_start":"212'3300","ondisk_log_start":"212'3300","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5289,"log_dups_size":0,"ondisk_log_size":5289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:27.412277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107675648,"num_objects":1643,"num_object_clones":0,"num_object_copies":4929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":988,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1643,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8589,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,5,7","objects":527},{"shards":"4,7","objects":1116}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":555,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.913561+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:23:33.913561+0000","last_peered":"2023-09-23T22:23:33.913561+0000","last_clean":"2023-09-23T22:23:33.913561+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:23:33.913561+0000","last_undegraded":"2023-09-23T22:23:33.913561+0000","last_fullsized":"2023-09-23T22:23:33.913561+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":983,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.913812+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:23:33.913812+0000","last_peered":"2023-09-23T22:23:33.913812+0000","last_clean":"2023-09-23T22:23:33.913812+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:23:33.913812+0000","last_undegraded":"2023-09-23T22:23:33.913812+0000","last_fullsized":"2023-09-23T22:23:33.913812+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":260,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.905824+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:23:33.905824+0000","last_peered":"2023-09-23T22:23:33.905824+0000","last_clean":"2023-09-23T22:23:33.905824+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:23:33.905824+0000","last_undegraded":"2023-09-23T22:23:33.905824+0000","last_fullsized":"2023-09-23T22:23:33.905824+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":215,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914930+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:23:33.914930+0000","last_peered":"2023-09-23T22:23:33.914930+0000","last_clean":"2023-09-23T22:23:33.914930+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:23:33.914930+0000","last_undegraded":"2023-09-23T22:23:33.914930+0000","last_fullsized":"2023-09-23T22:23:33.914930+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"251'8321","reported_seq":8386,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:44.231610+0000","last_change":"2023-09-23T22:21:59.013998+0000","last_active":"2023-09-23T22:23:44.231610+0000","last_peered":"2023-09-23T22:23:44.231610+0000","last_clean":"2023-09-23T22:21:56.814967+0000","last_became_active":"2023-09-23T22:21:59.002385+0000","last_became_peered":"2023-09-23T22:21:59.002385+0000","last_unstale":"2023-09-23T22:23:44.231610+0000","last_undegraded":"2023-09-23T22:21:58.865154+0000","last_fullsized":"2023-09-23T22:21:58.825566+0000","mapping_epoch":226,"log_start":"212'2920","ondisk_log_start":"212'2920","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:00.273421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108331008,"num_objects":1653,"num_object_clones":0,"num_object_copies":4959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4918,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8321,"num_write_kb":319168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":1584},{"shards":"3,7","objects":-3265}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":580,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.915373+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:23:33.915373+0000","last_peered":"2023-09-23T22:23:33.915373+0000","last_clean":"2023-09-23T22:23:33.915373+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:23:33.915373+0000","last_undegraded":"2023-09-23T22:23:33.915373+0000","last_fullsized":"2023-09-23T22:23:33.915373+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":357,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:23:33.910889+0000","last_change":"2023-09-23T22:22:18.917671+0000","last_active":"2023-09-23T22:23:33.910889+0000","last_peered":"2023-09-23T22:23:33.910889+0000","last_clean":"2023-09-23T22:15:39.913392+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:23:33.910889+0000","last_undegraded":"2023-09-23T22:22:18.874220+0000","last_fullsized":"2023-09-23T22:23:33.910889+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":210,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.889133+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:23:29.889133+0000","last_peered":"2023-09-23T22:23:29.889133+0000","last_clean":"2023-09-23T22:23:29.889133+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:23:29.889133+0000","last_undegraded":"2023-09-23T22:23:29.889133+0000","last_fullsized":"2023-09-23T22:23:29.889133+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":257,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.932045+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:23:33.932045+0000","last_peered":"2023-09-23T22:23:33.932045+0000","last_clean":"2023-09-23T22:23:33.932045+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:23:33.932045+0000","last_undegraded":"2023-09-23T22:23:33.932045+0000","last_fullsized":"2023-09-23T22:23:33.932045+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"251'8651","reported_seq":8707,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.895726+0000","last_change":"2023-09-23T22:20:19.470871+0000","last_active":"2023-09-23T22:23:44.895726+0000","last_peered":"2023-09-23T22:23:44.895726+0000","last_clean":"2023-09-23T22:23:44.895726+0000","last_became_active":"2023-09-23T22:20:19.470597+0000","last_became_peered":"2023-09-23T22:20:19.470597+0000","last_unstale":"2023-09-23T22:23:44.895726+0000","last_undegraded":"2023-09-23T22:23:44.895726+0000","last_fullsized":"2023-09-23T22:23:44.895726+0000","mapping_epoch":203,"log_start":"212'3100","ondisk_log_start":"212'3100","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5551,"log_dups_size":0,"ondisk_log_size":5551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:32.388985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110166016,"num_objects":1681,"num_object_clones":0,"num_object_copies":5043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8651,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":846,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:33.913909+0000","last_change":"2023-09-23T22:21:58.853256+0000","last_active":"2023-09-23T22:23:33.913909+0000","last_peered":"2023-09-23T22:23:33.913909+0000","last_clean":"2023-09-23T22:21:50.447508+0000","last_became_active":"2023-09-23T22:21:58.852815+0000","last_became_peered":"2023-09-23T22:21:58.852815+0000","last_unstale":"2023-09-23T22:23:33.913909+0000","last_undegraded":"2023-09-23T22:21:58.818640+0000","last_fullsized":"2023-09-23T22:21:58.817793+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:25.619025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":563,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.917598+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:23:33.917598+0000","last_peered":"2023-09-23T22:23:33.917598+0000","last_clean":"2023-09-23T22:23:33.917598+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:23:33.917598+0000","last_undegraded":"2023-09-23T22:23:33.917598+0000","last_fullsized":"2023-09-23T22:23:33.917598+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":175,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.910113+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:23:33.910113+0000","last_peered":"2023-09-23T22:23:33.910113+0000","last_clean":"2023-09-23T22:23:33.910113+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:23:33.910113+0000","last_undegraded":"2023-09-23T22:23:33.910113+0000","last_fullsized":"2023-09-23T22:23:33.910113+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":137,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.909976+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:23:33.909976+0000","last_peered":"2023-09-23T22:23:33.909976+0000","last_clean":"2023-09-23T22:23:33.909976+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:23:33.909976+0000","last_undegraded":"2023-09-23T22:23:33.909976+0000","last_fullsized":"2023-09-23T22:23:33.909976+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"250'8343","reported_seq":8421,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:43.712846+0000","last_change":"2023-09-23T22:22:18.954521+0000","last_active":"2023-09-23T22:23:43.712846+0000","last_peered":"2023-09-23T22:23:43.712846+0000","last_clean":"2023-09-23T22:21:32.869170+0000","last_became_active":"2023-09-23T22:22:18.945835+0000","last_became_peered":"2023-09-23T22:22:18.945835+0000","last_unstale":"2023-09-23T22:23:43.712846+0000","last_undegraded":"2023-09-23T22:22:18.869474+0000","last_fullsized":"2023-09-23T22:22:18.861404+0000","mapping_epoch":231,"log_start":"217'3705","ondisk_log_start":"217'3705","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4638,"log_dups_size":0,"ondisk_log_size":4638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:02.907936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111869952,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8343,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,4,5","objects":591},{"shards":"1,5","objects":1116}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":581,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.890458+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:23:29.890458+0000","last_peered":"2023-09-23T22:23:29.890458+0000","last_clean":"2023-09-23T22:23:29.890458+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:23:29.890458+0000","last_undegraded":"2023-09-23T22:23:29.890458+0000","last_fullsized":"2023-09-23T22:23:29.890458+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":600,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.889368+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:23:29.889368+0000","last_peered":"2023-09-23T22:23:29.889368+0000","last_clean":"2023-09-23T22:23:29.889368+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:23:29.889368+0000","last_undegraded":"2023-09-23T22:23:29.889368+0000","last_fullsized":"2023-09-23T22:23:29.889368+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":133,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914981+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:23:33.914981+0000","last_peered":"2023-09-23T22:23:33.914981+0000","last_clean":"2023-09-23T22:23:33.914981+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:23:33.914981+0000","last_undegraded":"2023-09-23T22:23:33.914981+0000","last_fullsized":"2023-09-23T22:23:33.914981+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":90,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.919030+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:23:33.919030+0000","last_peered":"2023-09-23T22:23:33.919030+0000","last_clean":"2023-09-23T22:23:33.919030+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:23:33.919030+0000","last_undegraded":"2023-09-23T22:23:33.919030+0000","last_fullsized":"2023-09-23T22:23:33.919030+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"250'8468","reported_seq":12010,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:43.766781+0000","last_change":"2023-09-23T22:22:45.524797+0000","last_active":"2023-09-23T22:23:43.766781+0000","last_peered":"2023-09-23T22:23:43.766781+0000","last_clean":"2023-09-23T22:21:12.245233+0000","last_became_active":"2023-09-23T22:22:45.454018+0000","last_became_peered":"2023-09-23T22:22:45.454018+0000","last_unstale":"2023-09-23T22:23:43.766781+0000","last_undegraded":"2023-09-23T22:22:45.437539+0000","last_fullsized":"2023-09-23T22:22:45.322462+0000","mapping_epoch":240,"log_start":"217'3830","ondisk_log_start":"217'3830","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4638,"log_dups_size":0,"ondisk_log_size":4638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:10.659044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115998720,"num_objects":1770,"num_object_clones":0,"num_object_copies":5310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8468,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1153,"num_bytes_recovered":75563008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1","objects":1},{"shards":"0,1,5","objects":565},{"shards":"1,2,7","objects":319},{"shards":"1,7","objects":-1311}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":572,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.919262+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:23:33.919262+0000","last_peered":"2023-09-23T22:23:33.919262+0000","last_clean":"2023-09-23T22:23:33.919262+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:23:33.919262+0000","last_undegraded":"2023-09-23T22:23:33.919262+0000","last_fullsized":"2023-09-23T22:23:33.919262+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":142,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.918637+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:23:33.918637+0000","last_peered":"2023-09-23T22:23:33.918637+0000","last_clean":"2023-09-23T22:23:33.918637+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:23:33.918637+0000","last_undegraded":"2023-09-23T22:23:33.918637+0000","last_fullsized":"2023-09-23T22:23:33.918637+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":201,"reported_epoch":249,"state":"active+clean","last_fresh":"2023-09-23T22:23:29.890156+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:23:29.890156+0000","last_peered":"2023-09-23T22:23:29.890156+0000","last_clean":"2023-09-23T22:23:29.890156+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:23:29.890156+0000","last_undegraded":"2023-09-23T22:23:29.890156+0000","last_fullsized":"2023-09-23T22:23:29.890156+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":106,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914951+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:23:33.914951+0000","last_peered":"2023-09-23T22:23:33.914951+0000","last_clean":"2023-09-23T22:23:33.914951+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:23:33.914951+0000","last_undegraded":"2023-09-23T22:23:33.914951+0000","last_fullsized":"2023-09-23T22:23:33.914951+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"251'8624","reported_seq":15648,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:44.859877+0000","last_change":"2023-09-23T22:22:19.043755+0000","last_active":"2023-09-23T22:23:44.859877+0000","last_peered":"2023-09-23T22:23:44.859877+0000","last_clean":"2023-09-23T22:21:12.214088+0000","last_became_active":"2023-09-23T22:22:18.986181+0000","last_became_peered":"2023-09-23T22:22:18.986181+0000","last_unstale":"2023-09-23T22:23:44.859877+0000","last_undegraded":"2023-09-23T22:22:18.975289+0000","last_fullsized":"2023-09-23T22:22:18.862148+0000","mapping_epoch":231,"log_start":"212'3300","ondisk_log_start":"212'3300","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5324,"log_dups_size":0,"ondisk_log_size":5324,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:34.422545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110624768,"num_objects":1688,"num_object_clones":0,"num_object_copies":5064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2138,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8624,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3468,"num_bytes_recovered":227278848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,3,4","objects":256},{"shards":"2,4,7","objects":1349},{"shards":"4,5,7","objects":83}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":555,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.909694+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:23:33.909694+0000","last_peered":"2023-09-23T22:23:33.909694+0000","last_clean":"2023-09-23T22:23:33.909694+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:23:33.909694+0000","last_undegraded":"2023-09-23T22:23:33.909694+0000","last_fullsized":"2023-09-23T22:23:33.909694+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":264,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.921302+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:23:33.921302+0000","last_peered":"2023-09-23T22:23:33.921302+0000","last_clean":"2023-09-23T22:23:33.921302+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:23:33.921302+0000","last_undegraded":"2023-09-23T22:23:33.921302+0000","last_fullsized":"2023-09-23T22:23:33.921302+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":892,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.931333+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:23:33.931333+0000","last_peered":"2023-09-23T22:23:33.931333+0000","last_clean":"2023-09-23T22:23:33.931333+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:23:33.931333+0000","last_undegraded":"2023-09-23T22:23:33.931333+0000","last_fullsized":"2023-09-23T22:23:33.931333+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":39,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.903316+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:23:33.903316+0000","last_peered":"2023-09-23T22:23:33.903316+0000","last_clean":"2023-09-23T22:23:33.903316+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:23:33.903316+0000","last_undegraded":"2023-09-23T22:23:33.903316+0000","last_fullsized":"2023-09-23T22:23:33.903316+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"250'8307","reported_seq":8505,"reported_epoch":250,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:43.722486+0000","last_change":"2023-09-23T22:22:44.336083+0000","last_active":"2023-09-23T22:23:43.722486+0000","last_peered":"2023-09-23T22:23:43.722486+0000","last_clean":"2023-09-23T22:21:56.808651+0000","last_became_active":"2023-09-23T22:21:59.059105+0000","last_became_peered":"2023-09-23T22:21:59.059105+0000","last_unstale":"2023-09-23T22:23:43.722486+0000","last_undegraded":"2023-09-23T22:21:58.989636+0000","last_fullsized":"2023-09-23T22:21:58.812134+0000","mapping_epoch":226,"log_start":"212'3602","ondisk_log_start":"212'3602","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4705,"log_dups_size":0,"ondisk_log_size":4705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:51.370288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110297088,"num_objects":1683,"num_object_clones":0,"num_object_copies":5049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8307,"num_write_kb":319680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":4259840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":1604},{"shards":"1,7","objects":-3169}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":538,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.911134+0000","last_change":"2023-09-23T22:22:19.883431+0000","last_active":"2023-09-23T22:23:33.911134+0000","last_peered":"2023-09-23T22:23:33.911134+0000","last_clean":"2023-09-23T22:23:33.911134+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:23:33.911134+0000","last_undegraded":"2023-09-23T22:23:33.911134+0000","last_fullsized":"2023-09-23T22:23:33.911134+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:46.903071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:46.903071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":232,"reported_epoch":250,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:23:33.920990+0000","last_change":"2023-09-23T22:22:44.342884+0000","last_active":"2023-09-23T22:23:33.920990+0000","last_peered":"2023-09-23T22:23:33.920990+0000","last_clean":"2023-09-23T22:06:25.449250+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:23:33.920990+0000","last_undegraded":"2023-09-23T22:22:44.329336+0000","last_fullsized":"2023-09-23T22:23:33.920990+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":85,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.919182+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:23:33.919182+0000","last_peered":"2023-09-23T22:23:33.919182+0000","last_clean":"2023-09-23T22:23:33.919182+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:23:33.919182+0000","last_undegraded":"2023-09-23T22:23:33.919182+0000","last_fullsized":"2023-09-23T22:23:33.919182+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":214,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.905965+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:23:33.905965+0000","last_peered":"2023-09-23T22:23:33.905965+0000","last_clean":"2023-09-23T22:23:33.905965+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:23:33.905965+0000","last_undegraded":"2023-09-23T22:23:33.905965+0000","last_fullsized":"2023-09-23T22:23:33.905965+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":80,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914928+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:23:33.914928+0000","last_peered":"2023-09-23T22:23:33.914928+0000","last_clean":"2023-09-23T22:23:33.914928+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:23:33.914928+0000","last_undegraded":"2023-09-23T22:23:33.914928+0000","last_fullsized":"2023-09-23T22:23:33.914928+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":261,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.930527+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:23:33.930527+0000","last_peered":"2023-09-23T22:23:33.930527+0000","last_clean":"2023-09-23T22:23:33.930527+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:23:33.930527+0000","last_undegraded":"2023-09-23T22:23:33.930527+0000","last_fullsized":"2023-09-23T22:23:33.930527+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":213,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914727+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:23:33.914727+0000","last_peered":"2023-09-23T22:23:33.914727+0000","last_clean":"2023-09-23T22:23:33.914727+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:23:33.914727+0000","last_undegraded":"2023-09-23T22:23:33.914727+0000","last_fullsized":"2023-09-23T22:23:33.914727+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":226,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.906118+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:23:33.906118+0000","last_peered":"2023-09-23T22:23:33.906118+0000","last_clean":"2023-09-23T22:23:33.906118+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:23:33.906118+0000","last_undegraded":"2023-09-23T22:23:33.906118+0000","last_fullsized":"2023-09-23T22:23:33.906118+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":210,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.915077+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:23:33.915077+0000","last_peered":"2023-09-23T22:23:33.915077+0000","last_clean":"2023-09-23T22:23:33.915077+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:23:33.915077+0000","last_undegraded":"2023-09-23T22:23:33.915077+0000","last_fullsized":"2023-09-23T22:23:33.915077+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.907096+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:23:33.907096+0000","last_peered":"2023-09-23T22:23:33.907096+0000","last_clean":"2023-09-23T22:23:33.907096+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:23:33.907096+0000","last_undegraded":"2023-09-23T22:23:33.907096+0000","last_fullsized":"2023-09-23T22:23:33.907096+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":231,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.914335+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:23:33.914335+0000","last_peered":"2023-09-23T22:23:33.914335+0000","last_clean":"2023-09-23T22:23:33.914335+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:23:33.914335+0000","last_undegraded":"2023-09-23T22:23:33.914335+0000","last_fullsized":"2023-09-23T22:23:33.914335+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":273,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:39.268269+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:23:39.268269+0000","last_peered":"2023-09-23T22:23:39.268269+0000","last_clean":"2023-09-23T22:23:39.268269+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:23:39.268269+0000","last_undegraded":"2023-09-23T22:23:39.268269+0000","last_fullsized":"2023-09-23T22:23:39.268269+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":140,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.919763+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:23:33.919763+0000","last_peered":"2023-09-23T22:23:33.919763+0000","last_clean":"2023-09-23T22:23:33.919763+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:23:33.919763+0000","last_undegraded":"2023-09-23T22:23:33.919763+0000","last_fullsized":"2023-09-23T22:23:33.919763+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":259,"reported_epoch":250,"state":"active+clean","last_fresh":"2023-09-23T22:23:33.905730+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:23:33.905730+0000","last_peered":"2023-09-23T22:23:33.905730+0000","last_clean":"2023-09-23T22:23:33.905730+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:23:33.905730+0000","last_undegraded":"2023-09-23T22:23:33.905730+0000","last_fullsized":"2023-09-23T22:23:33.905730+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1724579864,"num_objects":26316,"num_object_clones":0,"num_object_copies":78948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32657,"num_objects_misplaced":2216,"num_objects_unfound":0,"num_objects_dirty":26316,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136380,"num_write_kb":5206209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21076,"num_bytes_recovered":1330905088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7978299392,"data_stored":7978287176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83255,"ondisk_log_size":83255,"up":48,"acting":36,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":370028,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":88,"num_read_kb":166,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234232,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1316024,"kb_used_data":1137776,"kb_used_omap":0,"kb_used_meta":178240,"kb_avail":92429128,"statfs":{"total":95995035648,"available":94647427072,"internally_reserved":0,"allocated":1165082624,"data_stored":1164074580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":182517760},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:03 2023","interfaces":[{"interface":"back","average":{"1min":9.048,"5min":4.476,"15min":2.2679999999999998},"min":{"1min":0.23000000000000001,"5min":0.22,"15min":0.22},"max":{"1min":158.042,"5min":158.042,"15min":158.042},"last":0.46200000000000002},{"interface":"front","average":{"1min":9.6690000000000005,"5min":4.7149999999999999,"15min":2.4830000000000001},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":157.952,"5min":157.952,"15min":157.952},"last":0.436}]},{"osd":1,"last update":"Sat Sep 23 22:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":4.9329999999999998,"15min":2.9809999999999999},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.246},"max":{"1min":10.127000000000001,"5min":101.145,"15min":123.79300000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.5859999999999999,"5min":4.9909999999999997,"15min":3.1309999999999998},"min":{"1min":0.28199999999999997,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":10.92,"5min":101.13,"15min":123.66500000000001},"last":1.0069999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":4.8609999999999998,"15min":3.6600000000000001},"min":{"1min":0.35299999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.5119999999999996,"5min":105.34399999999999,"15min":125.101},"last":2.1989999999999998},{"interface":"front","average":{"1min":1.7470000000000001,"5min":4.6589999999999998,"15min":3.375},"min":{"1min":0.34699999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.0549999999999997,"5min":105.63200000000001,"15min":126.021},"last":2.1269999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:23:12 2023","interfaces":[{"interface":"back","average":{"1min":10.196999999999999,"5min":3.7349999999999999,"15min":2.1890000000000001},"min":{"1min":0.36699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":170.86099999999999,"5min":170.86099999999999,"15min":170.86099999999999},"last":1.1359999999999999},{"interface":"front","average":{"1min":9.5510000000000002,"5min":3.9239999999999999,"15min":2.3170000000000002},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":174.57499999999999,"5min":174.57499999999999,"15min":174.57499999999999},"last":1.0409999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4180000000000001,"5min":2.3980000000000001,"15min":1.2969999999999999},"min":{"1min":0.29699999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":27.922000000000001,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.738,"5min":2.3719999999999999,"15min":1.2669999999999999},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":29.920000000000002,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.311}]},{"osd":5,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":2.4279999999999999,"15min":1.226},"min":{"1min":0.32700000000000001,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":24.965,"5min":93.816000000000003,"15min":93.816000000000003},"last":0.34699999999999998},{"interface":"front","average":{"1min":2.4710000000000001,"5min":2.5230000000000001,"15min":1.4419999999999999},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.29799999999999999},"max":{"1min":24.712,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.80200000000000005,"15min":0.71399999999999997},"min":{"1min":0.34200000000000003,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.343,"5min":7.9459999999999997,"15min":9.0139999999999993},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.153,"15min":0.91500000000000004},"min":{"1min":0.33700000000000002,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":6.3879999999999999,"5min":10.347,"15min":13.362},"last":0.38500000000000001}]}]},{"osd":6,"up_from":0,"seq":197568495868,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795140,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1191968,"kb_used_data":890776,"kb_used_omap":7,"kb_used_meta":301112,"kb_avail":92553184,"statfs":{"total":95995035648,"available":94774460416,"internally_reserved":0,"allocated":912154624,"data_stored":910809758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":308339060},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8689999999999998,"5min":2.113,"15min":1.3149999999999999},"min":{"1min":0.255,"5min":0.189,"15min":0.189},"max":{"1min":56.503,"5min":56.503,"15min":61.890000000000001},"last":1.369},{"interface":"front","average":{"1min":5.1550000000000002,"5min":3.0230000000000001,"15min":1.728},"min":{"1min":0.40799999999999997,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":56.560000000000002,"5min":56.560000000000002,"15min":62.689999999999998},"last":2.4620000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":2.0179999999999998,"15min":1.1850000000000001},"min":{"1min":0.252,"5min":0.223,"15min":0.216},"max":{"1min":11.179,"5min":33.902000000000001,"15min":33.902000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":2.1480000000000001,"5min":3.2370000000000001,"15min":1.5049999999999999},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.2670000000000003,"5min":34.781999999999996,"15min":34.781999999999996},"last":2.5070000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:23:20 2023","interfaces":[{"interface":"back","average":{"1min":8.5220000000000002,"5min":4.3360000000000003,"15min":2.0590000000000002},"min":{"1min":0.47599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":110.417,"5min":110.417,"15min":110.417},"last":2.5880000000000001},{"interface":"front","average":{"1min":8.5239999999999991,"5min":4.1820000000000004,"15min":2.0179999999999998},"min":{"1min":0.33400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":110.721,"5min":110.721,"15min":110.721},"last":3.0960000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:23:20 2023","interfaces":[{"interface":"back","average":{"1min":8.6099999999999994,"5min":4.6239999999999997,"15min":2.056},"min":{"1min":0.314,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":171.09999999999999,"5min":234.79900000000001,"15min":234.79900000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":9.1289999999999996,"5min":5.2699999999999996,"15min":2.3519999999999999},"min":{"1min":0.44700000000000001,"5min":0.33200000000000002,"15min":0.318},"max":{"1min":171.08500000000001,"5min":234.81700000000001,"15min":234.81700000000001},"last":2.54}]},{"osd":5,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":3.6850000000000001,"15min":1.788},"min":{"1min":0.39100000000000001,"5min":0.31,"15min":0.31},"max":{"1min":9.5709999999999997,"5min":196.79900000000001,"15min":196.79900000000001},"last":2.5659999999999998},{"interface":"front","average":{"1min":2.9340000000000002,"5min":3.6499999999999999,"15min":1.7989999999999999},"min":{"1min":0.41299999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.01,"5min":196.68299999999999,"15min":196.68299999999999},"last":3.198}]},{"osd":6,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":1.913,"15min":1.2490000000000001},"min":{"1min":0.497,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":10.074999999999999,"5min":10.971,"15min":10.989000000000001},"last":3.238},{"interface":"front","average":{"1min":1.448,"5min":1.296,"15min":1.006},"min":{"1min":0.437,"5min":0.35699999999999998,"15min":0.34999999999999998},"max":{"1min":7.7300000000000004,"5min":11.385999999999999,"15min":11.385999999999999},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.6970000000000001,"5min":1.9339999999999999,"15min":1.2889999999999999},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.048,"5min":10.933999999999999,"15min":10.948},"last":4.2610000000000001},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.3200000000000001,"15min":1.0589999999999999},"min":{"1min":0.40699999999999997,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":12.045999999999999,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.42599999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958675,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":846168,"kb_used_data":544912,"kb_used_omap":8,"kb_used_meta":301239,"kb_avail":92898984,"statfs":{"total":95995035648,"available":95128559616,"internally_reserved":0,"allocated":557989888,"data_stored":556603676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":308469610},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2009999999999996,"5min":4.5810000000000004,"15min":2.274},"min":{"1min":0.21299999999999999,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":51.018999999999998,"5min":169.77199999999999,"15min":169.77199999999999},"last":0.437},{"interface":"front","average":{"1min":3.6739999999999999,"5min":5.0860000000000003,"15min":2.5640000000000001},"min":{"1min":0.24099999999999999,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":23.891999999999999,"5min":169.52099999999999,"15min":169.52099999999999},"last":0.60099999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:22:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":6.5620000000000003,"15min":2.5870000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":10.891,"5min":206.495,"15min":206.495},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.8759999999999999,"5min":7.71,"15min":3.1890000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.22800000000000001},"max":{"1min":15.186,"5min":207.34399999999999,"15min":207.34399999999999},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4350000000000001,"5min":3.7160000000000002,"15min":2.0009999999999999},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.3399999999999999,"5min":131.078,"15min":131.078},"last":0.65500000000000003},{"interface":"front","average":{"1min":3.0070000000000001,"5min":3.9540000000000002,"15min":2.1040000000000001},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.438000000000001,"5min":133.083,"15min":133.083},"last":0.54800000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.23,"15min":1.7849999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":9.673,"5min":82.825000000000003,"15min":82.825000000000003},"last":0.79300000000000004},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.306,"15min":1.2509999999999999},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.308},"max":{"1min":3.129,"5min":75.930999999999997,"15min":75.930999999999997},"last":7.4370000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":8.1050000000000004,"5min":4.4669999999999996,"15min":2.173},"min":{"1min":0.39200000000000002,"5min":0.34799999999999998,"15min":0.29499999999999998},"max":{"1min":93.685000000000002,"5min":93.685000000000002,"15min":93.685000000000002},"last":0.84499999999999997},{"interface":"front","average":{"1min":7.6550000000000002,"5min":3.5459999999999998,"15min":1.7030000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":93.546000000000006,"5min":93.546000000000006,"15min":93.546000000000006},"last":3.5059999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":2.3069999999999999,"15min":1.472},"min":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":9.7140000000000004,"5min":10.275,"15min":10.275},"last":0.82099999999999995},{"interface":"front","average":{"1min":2.3769999999999998,"5min":1.764,"15min":1.115},"min":{"1min":0.43099999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":10.919,"5min":22.263000000000002,"15min":22.263000000000002},"last":5.3390000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":4.0570000000000004,"5min":2.5800000000000001,"15min":1.643},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":32.773000000000003,"5min":32.773000000000003,"15min":32.773000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.069,"5min":2.4249999999999998,"15min":1.6850000000000001},"min":{"1min":0.39900000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5790000000000006,"5min":10.257999999999999,"15min":23.265999999999998},"last":2.5030000000000001}]}]},{"osd":2,"up_from":0,"seq":442381631614,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468087,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1351496,"kb_used_data":1116672,"kb_used_omap":4,"kb_used_meta":234747,"kb_avail":92393656,"statfs":{"total":95995035648,"available":94611103744,"internally_reserved":0,"allocated":1143472128,"data_stored":1142087992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":240381485},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":5.7969999999999997,"15min":2.7069999999999999},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":23.890000000000001,"5min":225.86699999999999,"15min":225.86699999999999},"last":8.7240000000000002},{"interface":"front","average":{"1min":4.4619999999999997,"5min":6.2400000000000002,"15min":2.9340000000000002},"min":{"1min":0.23699999999999999,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":24.053000000000001,"5min":225.84800000000001,"15min":225.84800000000001},"last":0.91500000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":4.0819999999999999,"15min":2.1840000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.209,"5min":82.951999999999998,"15min":82.951999999999998},"last":5.2610000000000001},{"interface":"front","average":{"1min":4.75,"5min":4.4509999999999996,"15min":2.2170000000000001},"min":{"1min":0.34300000000000003,"5min":0.22600000000000001,"15min":0.189},"max":{"1min":13.85,"5min":86.001000000000005,"15min":86.001000000000005},"last":6.4690000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":5.5129999999999999,"15min":3.3849999999999998},"min":{"1min":0.501,"5min":0.252,"15min":0.223},"max":{"1min":8.1180000000000003,"5min":99.962999999999994,"15min":99.962999999999994},"last":0.63100000000000001},{"interface":"front","average":{"1min":2.2890000000000001,"5min":5.6790000000000003,"15min":3.2130000000000001},"min":{"1min":0.41399999999999998,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":9.5909999999999993,"5min":100.429,"15min":100.429},"last":6.5220000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.9850000000000001,"15min":1.1839999999999999},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":6.2960000000000003,"5min":11.911,"15min":11.911},"last":6.556},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.764,"15min":1.2030000000000001},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.949999999999999,"5min":10.949999999999999,"15min":12.036},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":2.4220000000000002,"15min":1.391},"min":{"1min":0.39900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":11.028,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.6259999999999999,"5min":2.427,"15min":1.3799999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.883,"5min":49.668999999999997,"15min":49.668999999999997},"last":0.55000000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":2.1259999999999999,"15min":1.3320000000000001},"min":{"1min":0.42699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.992000000000001,"5min":10.992000000000001,"15min":12.068},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.893,"5min":1.8620000000000001,"15min":1.2410000000000001},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35399999999999998},"max":{"1min":5.9219999999999997,"5min":11.68,"15min":11.68},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.0110000000000001,"15min":1.3140000000000001},"min":{"1min":0.39700000000000002,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":6.0309999999999997,"5min":16.849,"15min":16.849},"last":0.58499999999999996},{"interface":"front","average":{"1min":2.3290000000000002,"5min":2.0910000000000002,"15min":1.272},"min":{"1min":0.46200000000000002,"5min":0.40400000000000003,"15min":0.36699999999999999},"max":{"1min":6.1749999999999998,"5min":11.945,"15min":12.382},"last":6.5899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019137,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":858812,"kb_used_data":684212,"kb_used_omap":0,"kb_used_meta":174592,"kb_avail":92886340,"statfs":{"total":95995035648,"available":95115612160,"internally_reserved":0,"allocated":700633088,"data_stored":699621755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":178782208},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":2.4340000000000002,"15min":1.649},"min":{"1min":0.28399999999999997,"5min":0.23000000000000001,"15min":0.218},"max":{"1min":9.2579999999999991,"5min":46.18,"15min":66.355000000000004},"last":23.902000000000001},{"interface":"front","average":{"1min":2.379,"5min":2.5019999999999998,"15min":1.7350000000000001},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":9.2390000000000008,"5min":46.207000000000001,"15min":66.332999999999998},"last":23.867000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":2.0470000000000002,"15min":1.3460000000000001},"min":{"1min":0.26700000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.0750000000000002,"5min":35.409999999999997,"15min":38.878},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.492,"15min":1.694},"min":{"1min":0.28100000000000003,"5min":0.255,"15min":0.255},"max":{"1min":7.8620000000000001,"5min":35.999000000000002,"15min":38.853999999999999},"last":0.30099999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:23:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":3.472,"15min":2.2970000000000002},"min":{"1min":0.36099999999999999,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":10.051,"5min":138.06100000000001,"15min":138.06100000000001},"last":0.442},{"interface":"front","average":{"1min":2.1030000000000002,"5min":3.3380000000000001,"15min":2.4340000000000002},"min":{"1min":0.35499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.2510000000000003,"5min":138.15199999999999,"15min":138.15199999999999},"last":0.5}]},{"osd":3,"last update":"Sat Sep 23 22:23:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":1.8220000000000001,"15min":1.196},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":12.129,"5min":12.129,"15min":12.129},"last":0.63100000000000001},{"interface":"front","average":{"1min":2.5870000000000002,"5min":1.8200000000000001,"15min":1.228},"min":{"1min":0.46300000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4059999999999997,"5min":8.8930000000000007,"15min":9.8109999999999999},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:23:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.284,"15min":0.874},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.8570000000000002,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.5680000000000001,"5min":1.3759999999999999,"15min":0.90400000000000003},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":7.298,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.34300000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:23:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.80800000000000005,"15min":0.68999999999999995},"min":{"1min":0.33200000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":2.262,"5min":7.3079999999999998,"15min":7.3079999999999998},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.873,"5min":0.85299999999999998,"15min":0.749},"min":{"1min":0.29099999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.2679999999999998,"5min":6.7279999999999998,"15min":8.5210000000000008},"last":0.53200000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.94799999999999995,"15min":0.83999999999999997},"min":{"1min":0.313,"5min":0.313,"15min":0.307},"max":{"1min":2.754,"5min":6.7480000000000002,"15min":8.577},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.143,"5min":0.94099999999999995,"15min":0.78700000000000003},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":5.1479999999999997,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.70199999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757505,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1025396,"kb_used_data":798252,"kb_used_omap":0,"kb_used_meta":227136,"kb_avail":92719756,"statfs":{"total":95995035648,"available":94945030144,"internally_reserved":0,"allocated":817410048,"data_stored":816402684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":232587264},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:22:55 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":2.1150000000000002,"15min":1.859},"min":{"1min":0.251,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":12.661,"5min":28.741,"15min":133.22499999999999},"last":2.5030000000000001},{"interface":"front","average":{"1min":3.198,"5min":1.8959999999999999,"15min":1.8979999999999999},"min":{"1min":0.26100000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.324999999999999,"5min":28.873999999999999,"15min":132.78200000000001},"last":0.61399999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:23:16 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":4.3090000000000002,"15min":2.681},"min":{"1min":0.27200000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.385999999999999,"5min":93.981999999999999,"15min":93.981999999999999},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.9099999999999999,"5min":4.2329999999999997,"15min":2.6509999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":13.457000000000001,"5min":93.909999999999997,"15min":93.909999999999997},"last":0.58899999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:22:49 2023","interfaces":[{"interface":"back","average":{"1min":2.972,"5min":6.0650000000000004,"15min":3.4329999999999998},"min":{"1min":0.32300000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.534000000000001,"5min":104.254,"15min":104.254},"last":2.5379999999999998},{"interface":"front","average":{"1min":1.661,"5min":6.3380000000000001,"15min":3.7040000000000002},"min":{"1min":0.35199999999999998,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":5.7380000000000004,"5min":104.217,"15min":104.217},"last":0.33600000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:23:16 2023","interfaces":[{"interface":"back","average":{"1min":4.1429999999999998,"5min":3.6789999999999998,"15min":2.23},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.246},"max":{"1min":18.876000000000001,"5min":93.739999999999995,"15min":93.739999999999995},"last":2.7090000000000001},{"interface":"front","average":{"1min":2.6000000000000001,"5min":3.4220000000000002,"15min":2.1699999999999999},"min":{"1min":0.34200000000000003,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":13.773999999999999,"5min":93.197999999999993,"15min":93.197999999999993},"last":2.347}]},{"osd":4,"last update":"Sat Sep 23 22:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.085,"15min":1.03},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":5.1799999999999997,"5min":10.163,"15min":82.530000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.0629999999999999,"15min":1.0660000000000001},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.2119999999999997,"5min":7.2119999999999997,"15min":85.441000000000003},"last":2.6789999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:23:39 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.82199999999999995,"15min":1.1559999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":7.0289999999999999,"5min":7.0289999999999999,"15min":114.83499999999999},"last":2.569},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.73499999999999999,"15min":1.0669999999999999},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.2250000000000001,"5min":4.5220000000000002,"15min":110.905},"last":0.70899999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:22:55 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.159,"15min":0.84299999999999997},"min":{"1min":0.33800000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.9409999999999998,"5min":8.2159999999999993,"15min":8.8030000000000008},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.79700000000000004,"5min":0.82499999999999996,"15min":0.745},"min":{"1min":0.39100000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5299999999999998,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":0.36899999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":543952896,"data_stored":543948824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":899223552,"data_stored":899219480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2790391808,"data_stored":2790391808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1115291648,"data_stored":1115291648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":682295296,"data_stored":682295296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":806817792,"data_stored":806813720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1140326400,"data_stored":1140326400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:23:48.894 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:23:48.921 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:49.010 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:49.031 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:49.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:49.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:49.142 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:49.180 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:49.237 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:49.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:49.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:49.344 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:49.385 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:49.458 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:49.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:49.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:49.564 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:23:49.600 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:49.684 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:23:49.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:49.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:23:49.693 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:49.694 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:49.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:23:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:23:49] "GET /metrics HTTP/1.1" 200 36414 "" "Prometheus/2.43.0" 2023-09-23T22:23:49.790 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:49.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:50.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:50 smithi070 ceph-mon[192652]: pgmap v620: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 652 op/s; 32684/77112 objects degraded (42.385%); 2216/77112 objects misplaced (2.874%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:50.735 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:50.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:50 smithi026 ceph-mon[215775]: pgmap v620: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 652 op/s; 32684/77112 objects degraded (42.385%); 2216/77112 objects misplaced (2.874%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:50 smithi026 ceph-mon[223014]: pgmap v620: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.6 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 652 op/s; 32684/77112 objects degraded (42.385%); 2216/77112 objects misplaced (2.874%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:23:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:50.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:50.839 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:23:50.889 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:50.938 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:50.959 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:23:50.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:50.970 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:23:51.070 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:51.198 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:51.686 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:51.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:51.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:51.794 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:51.836 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:51.909 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:51.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:51.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:52.019 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:52.106 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:52.210 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:23:52.232 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:23:52.278 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:23:52.344 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:23:52.377 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:23:52.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:52 smithi026 ceph-mon[215775]: pgmap v621: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 32466/70377 objects degraded (46.132%); 1999/70377 objects misplaced (2.840%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:23:52.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3250198636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:52.433 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:23:52.472 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:23:52.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:23:52.540 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:23:52.540 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:23:52.540 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:23:52.540 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:23:52.540 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:23:52.540 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:23:52.541 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:23:52.541 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:23:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:52 smithi026 ceph-mon[223014]: pgmap v621: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 32466/70377 objects degraded (46.132%); 1999/70377 objects misplaced (2.840%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:23:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3250198636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:52 smithi070 ceph-mon[192652]: pgmap v621: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 663 op/s; 32466/70377 objects degraded (46.132%); 1999/70377 objects misplaced (2.840%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:23:52.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3250198636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:23:53.672 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:53.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:53.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:53.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 32466/70377 objects degraded (46.132%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:53.688 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 32466/70377 objects degraded (46.132%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:53.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:53.838 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:53.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 32466/70377 objects degraded (46.132%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:54.585 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:54.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:54.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:54.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:54.721 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:54.782 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:54.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:54.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:54.886 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:23:54.919 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:54.975 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:23:54.985 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:54 smithi070 ceph-mon[192652]: pgmap v622: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 662 op/s; 32466/69159 objects degraded (46.944%); 1706/69159 objects misplaced (2.467%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:23:54.985 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:54.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:54.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:23:54.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:54 smithi026 ceph-mon[223014]: pgmap v622: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 662 op/s; 32466/69159 objects degraded (46.944%); 1706/69159 objects misplaced (2.467%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:23:54.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:54.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:54 smithi026 ceph-mon[215775]: pgmap v622: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.4 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 662 op/s; 32466/69159 objects degraded (46.944%); 1706/69159 objects misplaced (2.467%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:23:54.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:23:55.087 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:23:55.123 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:55.201 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:23:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:55.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:23:55.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:55.401 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:55.774 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:55.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:55.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:55.883 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:55.977 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:56.119 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:56.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:56.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:56.145 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:23:56.145 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19248 5.b 5:d5f2e858:::benchmark_data_smithi026_286012_object19247:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.612054+0000", 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 507.52767685999999, 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0715806959999998, 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.146 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19248 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612054+0000", 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612054+0000", 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612052+0000", 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612092+0000", 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9764000000000001e-05 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612110+0000", 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7623999999999999e-05 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612118+0000", 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5050000000000007e-06 2023-09-23T22:23:56.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612164+0000", 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5317e-05 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612290+0000", 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012669800000000001 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612417+0000", 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012688099999999999 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.616214+0000", 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003796935 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.617099+0000", 2023-09-23T22:23:56.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00088451700000000003 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619412+0000", 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023133870000000001 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619428+0000", 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5701000000000001e-05 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.707610+0000", 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0881826130000001 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.707626+0000", 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5586999999999998e-05 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.683528+0000", 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97590195599999996 2023-09-23T22:23:56.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.683634+0000", 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106642 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19250 5.b 5:dabccd57:::benchmark_data_smithi026_286012_object19249:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.612132+0000", 2023-09-23T22:23:56.154 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 507.527598974, 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0806422269999998, 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19250 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612132+0000", 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612132+0000", 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612131+0000", 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612170+0000", 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9413999999999999e-05 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612181+0000", 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0156000000000001e-05 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612187+0000", 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5660000000000003e-06 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.613342+0000", 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001154831 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.613465+0000", 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012269399999999999 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.613597+0000", 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013208999999999999 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.617105+0000", 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035081029999999998 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619462+0000", 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023571519999999999 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619472+0000", 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0101e-05 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.707670+0000", 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.08819804 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.707685+0000", 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.522e-05 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.684305+0000", 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97661962499999999 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:23:56.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.685980+0000", 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016749449999999999 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.692666+0000", 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066856199999999998 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.692774+0000", 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010827000000000001 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.162 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19253 5.b 5:d53d0f6a:::benchmark_data_smithi026_286012_object19252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.612252+0000", 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 507.527478761, 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.081510379, 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.163 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19253 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612252+0000", 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612252+0000", 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612251+0000", 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612271+0000", 2023-09-23T22:23:56.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9383000000000001e-05 2023-09-23T22:23:56.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612278+0000", 2023-09-23T22:23:56.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2590000000000003e-06 2023-09-23T22:23:56.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612284+0000", 2023-09-23T22:23:56.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8130000000000001e-06 2023-09-23T22:23:56.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.614283+0000", 2023-09-23T22:23:56.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019988549999999999 2023-09-23T22:23:56.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.614394+0000", 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011096500000000001 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.614529+0000", 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135604 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.617110+0000", 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025808239999999998 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619513+0000", 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024025990000000001 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619523+0000", 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9490000000000008e-06 2023-09-23T22:23:56.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.707739+0000", 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.088216224 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.707750+0000", 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0709e-05 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.686037+0000", 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97828704899999996 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.686467+0000", 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043086699999999998 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.693665+0000", 2023-09-23T22:23:56.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071971819999999999 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.693762+0000", 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7639999999999994e-05 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:19254 5.b 5:d5071eaa:::benchmark_data_smithi026_286012_object19253:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:23:56.172 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:15:28.612288+0000", 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 507.52744225200001, 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.082522532, 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 19254 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612288+0000", 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612288+0000", 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612288+0000", 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:56.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612310+0000", 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2419e-05 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612317+0000", 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8600000000000004e-06 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.612322+0000", 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2039999999999996e-06 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.615257+0000", 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002934407 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.615377+0000", 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012002699999999999 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.615507+0000", 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012959700000000001 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619556+0000", 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040492829999999999 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:28.619565+0000", 2023-09-23T22:23:56.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6870000000000006e-06 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.707812+0000", 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.088246279 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:29.707823+0000", 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1426e-05 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.686532+0000", 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97870919499999998 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.687068+0000", 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00053536800000000004 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.694727+0000", 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076595070000000003 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:15:30.694811+0000", 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3694000000000002e-05 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150387 5.b 5:d9a1ac1d:::benchmark_data_smithi026_286012_object56579:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.180 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.248274+0000", 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.89145658299998, 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.210601423, 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150387 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248274+0000", 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248274+0000", 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248274+0000", 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:56.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248277+0000", 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6859999999999999e-06 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248285+0000", 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6499999999999996e-06 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248291+0000", 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5720000000000002e-06 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.257620+0000", 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093298359999999993 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.257632+0000", 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1358999999999999e-05 2023-09-23T22:23:56.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202876+0000", 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94524393299999998 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202888+0000", 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.224e-05 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.443162+0000", 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.240274028 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.443328+0000", 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000166458 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.443469+0000", 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140727 2023-09-23T22:23:56.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.452748+0000", 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092784719999999994 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.452762+0000", 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4769e-05 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.457005+0000", 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042431370000000001 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.457082+0000", 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6798000000000006e-05 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.458875+0000", 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017932340000000001 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.188 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150389 5.b 5:d9ddf50c:::benchmark_data_smithi026_286012_object56581:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.248301+0000", 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.89143000500002, 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2246119269999998, 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.189 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150389 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248301+0000", 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248301+0000", 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248300+0000", 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248303+0000", 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4959999999999999e-06 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248308+0000", 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3900000000000001e-06 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248313+0000", 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.763e-06 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.257664+0000", 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093506289999999992 2023-09-23T22:23:56.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.257672+0000", 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3000000000000002e-06 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202927+0000", 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94525478299999999 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202936+0000", 2023-09-23T22:23:56.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5699999999999999e-06 2023-09-23T22:23:56.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.449098+0000", 2023-09-23T22:23:56.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2461615049999999 2023-09-23T22:23:56.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.449282+0000", 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018425700000000001 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.449482+0000", 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019988100000000001 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.458961+0000", 2023-09-23T22:23:56.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094792399999999999 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.458975+0000", 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4269e-05 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.470682+0000", 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011706553999999999 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.470753+0000", 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1092000000000003e-05 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.472913+0000", 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021595579999999998 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150438 5.b 5:d8e4acd1:::benchmark_data_smithi026_286012_object56630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.315867+0000", 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.82386389800001, 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1589761699999999, 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150438 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.198 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315867+0000", 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315867+0000", 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315865+0000", 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315872+0000", 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8040000000000004e-06 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315888+0000", 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5611999999999999e-05 2023-09-23T22:23:56.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315894+0000", 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5860000000000001e-06 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316024+0000", 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012938899999999999 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316035+0000", 2023-09-23T22:23:56.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1059e-05 2023-09-23T22:23:56.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202965+0000", 2023-09-23T22:23:56.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.88693064899999996 2023-09-23T22:23:56.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202973+0000", 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9109999999999993e-06 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.452813+0000", 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.249840042 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.453004+0000", 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019102799999999999 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.453165+0000", 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000160285 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.459043+0000", 2023-09-23T22:23:56.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058780220000000001 2023-09-23T22:23:56.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.463292+0000", 2023-09-23T22:23:56.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042492659999999998 2023-09-23T22:23:56.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.473010+0000", 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097182450000000004 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.473084+0000", 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3503000000000004e-05 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.474843+0000", 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001759172 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150443 5.b 5:d57a26b4:::benchmark_data_smithi026_286012_object56635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.316285+0000", 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.823445924, 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1896902159999998, 2023-09-23T22:23:56.207 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150443 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316285+0000", 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316285+0000", 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316284+0000", 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316289+0000", 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.147e-06 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316301+0000", 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1287999999999999e-05 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316310+0000", 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5119999999999997e-06 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316368+0000", 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8335999999999998e-05 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316381+0000", 2023-09-23T22:23:56.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.253e-05 2023-09-23T22:23:56.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203015+0000", 2023-09-23T22:23:56.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.88663416500000003 2023-09-23T22:23:56.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203024+0000", 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6200000000000005e-06 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.459094+0000", 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.256070204 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.459264+0000", 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000170175 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.459450+0000", 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000186274 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.470566+0000", 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011115583 2023-09-23T22:23:56.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.470633+0000", 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7287999999999997e-05 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503835+0000", 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033201738000000001 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503894+0000", 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9120999999999999e-05 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.505975+0000", 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020808419999999998 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150448 5.b 5:da46b32b:::benchmark_data_smithi026_286012_object56640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.323598+0000", 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.81613265599998, 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1845984239999998, 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150448 2023-09-23T22:23:56.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323598+0000", 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323598+0000", 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323598+0000", 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323600+0000", 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1299999999999999e-06 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323605+0000", 2023-09-23T22:23:56.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.259e-06 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323610+0000", 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4850000000000003e-06 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.330889+0000", 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072798610000000003 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.330902+0000", 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2853e-05 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203057+0000", 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87215510799999996 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203066+0000", 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6379999999999993e-06 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.463330+0000", 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2602637059999999 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.463498+0000", 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016803400000000001 2023-09-23T22:23:56.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.463595+0000", 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7663000000000003e-05 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.474872+0000", 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011276243 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479113+0000", 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042416440000000001 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.506089+0000", 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026975932000000001 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.506124+0000", 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5131e-05 2023-09-23T22:23:56.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508196+0000", 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020720909999999999 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.226 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150486 5.b 5:d9f35565:::benchmark_data_smithi026_286012_object56678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.381625+0000", 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.75810525000003, 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1288887700000001, 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150486 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381625+0000", 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381625+0000", 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381624+0000", 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381631+0000", 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3270000000000003e-06 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381646+0000", 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4357e-05 2023-09-23T22:23:56.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381651+0000", 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.665e-06 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381699+0000", 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7746e-05 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381728+0000", 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9057999999999999e-05 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203099+0000", 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82137101700000004 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203110+0000", 2023-09-23T22:23:56.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0749e-05 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.466717+0000", 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2636070150000001 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.466867+0000", 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015026800000000001 2023-09-23T22:23:56.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.466991+0000", 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000123554 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479184+0000", 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012193633000000001 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479265+0000", 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0585000000000002e-05 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508322+0000", 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029057554999999999 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508388+0000", 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5982999999999995e-05 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510514+0000", 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021257820000000001 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.235 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150488 5.b 5:df0377d0:::benchmark_data_smithi026_286012_object56680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.381878+0000", 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.75785250899997, 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.142269556, 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150488 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381878+0000", 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381878+0000", 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381877+0000", 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381883+0000", 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2889999999999997e-06 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381894+0000", 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1110999999999999e-05 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381901+0000", 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3119999999999996e-06 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381958+0000", 2023-09-23T22:23:56.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6611999999999999e-05 2023-09-23T22:23:56.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381967+0000", 2023-09-23T22:23:56.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1880000000000008e-06 2023-09-23T22:23:56.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203147+0000", 2023-09-23T22:23:56.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82117972299999997 2023-09-23T22:23:56.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203155+0000", 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9129999999999992e-06 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.474913+0000", 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.271757247 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.475101+0000", 2023-09-23T22:23:56.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018805800000000001 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.475273+0000", 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000171979 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479252+0000", 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039795200000000003 2023-09-23T22:23:56.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.487178+0000", 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079256190000000001 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.522088+0000", 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034909664999999999 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.522141+0000", 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3514999999999998e-05 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524148+0000", 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020066900000000002 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150489 5.b 5:d03d5463:::benchmark_data_smithi026_286012_object56681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.382063+0000", 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.757667637, 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.144385561, 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150489 2023-09-23T22:23:56.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382063+0000", 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382063+0000", 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382062+0000", 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382068+0000", 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.339e-06 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382079+0000", 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1643e-05 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382086+0000", 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0740000000000004e-06 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382133+0000", 2023-09-23T22:23:56.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6912000000000001e-05 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382142+0000", 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.217e-06 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203183+0000", 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82104097300000001 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203188+0000", 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8319999999999996e-06 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479295+0000", 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2761071349999999 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479482+0000", 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000186773 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479658+0000", 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017554100000000001 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491200+0000", 2023-09-23T22:23:56.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011542458 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491216+0000", 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6084999999999999e-05 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524314+0000", 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033097416999999997 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524363+0000", 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9353e-05 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.526449+0000", 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020855959999999999 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.254 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150492 5.b 5:d18b0458:::benchmark_data_smithi026_286012_object56684:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.385473+0000", 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.75425789299999, 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1432154150000002, 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.255 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150492 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385473+0000", 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385473+0000", 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385471+0000", 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385478+0000", 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5880000000000003e-06 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385495+0000", 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.694e-05 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385503+0000", 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7430000000000002e-06 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385566+0000", 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2924999999999998e-05 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385576+0000", 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0856000000000001e-05 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203218+0000", 2023-09-23T22:23:56.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81764150400000002 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203226+0000", 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8599999999999993e-06 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.483349+0000", 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2801233700000001 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.483508+0000", 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015833399999999999 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.483686+0000", 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017833700000000001 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.495359+0000", 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011672854 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503787+0000", 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084286329999999996 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.526527+0000", 2023-09-23T22:23:56.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022739796 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.526586+0000", 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8517999999999998e-05 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.528688+0000", 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021025269999999999 2023-09-23T22:23:56.263 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150497 5.b 5:d9455892:::benchmark_data_smithi026_286012_object56689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.391903+0000", 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.74782727100001, 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.139149631, 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.264 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150497 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391903+0000", 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391903+0000", 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391903+0000", 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391906+0000", 2023-09-23T22:23:56.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0060000000000001e-06 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391912+0000", 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1639999999999998e-06 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391918+0000", 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4940000000000003e-06 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391971+0000", 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3808999999999999e-05 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391982+0000", 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0264e-05 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203251+0000", 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81126955899999997 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203256+0000", 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3359999999999997e-06 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.487231+0000", 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2839750679999999 2023-09-23T22:23:56.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.487402+0000", 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000171072 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.487509+0000", 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106952 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.499494+0000", 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011985350000000001 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.506047+0000", 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065530850000000002 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.528764+0000", 2023-09-23T22:23:56.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022716584000000001 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.528832+0000", 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8011999999999999e-05 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.531053+0000", 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002221326 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.272 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150513 5.b 5:dc4d2ce0:::benchmark_data_smithi026_286012_object56705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.412359+0000", 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.727372049, 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1316144010000002, 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.273 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150513 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412359+0000", 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412359+0000", 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412358+0000", 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412362+0000", 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1350000000000002e-06 2023-09-23T22:23:56.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412373+0000", 2023-09-23T22:23:56.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0803e-05 2023-09-23T22:23:56.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.276 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:23:56.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412380+0000", 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5100000000000001e-06 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412439+0000", 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8786999999999998e-05 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412449+0000", 2023-09-23T22:23:56.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7869999999999996e-06 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203287+0000", 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.79083858200000001 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203296+0000", 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9949999999999994e-06 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491247+0000", 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2879502519999999 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491442+0000", 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000195491 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491636+0000", 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019384900000000001 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503737+0000", 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012101031999999999 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508281+0000", 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004544289 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.541884+0000", 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033602434 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.541940+0000", 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6039000000000003e-05 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.543973+0000", 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020332409999999999 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150541 5.b 5:d3c31e9d:::benchmark_data_smithi026_286012_object56733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.465404+0000", 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.67432660399999, 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0846849810000001, 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.284 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150541 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465404+0000", 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465404+0000", 2023-09-23T22:23:56.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465403+0000", 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465409+0000", 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8209999999999998e-06 2023-09-23T22:23:56.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465420+0000", 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1038e-05 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465427+0000", 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3130000000000001e-06 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465483+0000", 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5687999999999999e-05 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465496+0000", 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2700999999999999e-05 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203318+0000", 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.73782281000000005 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203323+0000", 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2910000000000001e-06 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.495388+0000", 2023-09-23T22:23:56.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.292065206 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.495557+0000", 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016910800000000001 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.495740+0000", 2023-09-23T22:23:56.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018281699999999999 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503797+0000", 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080567409999999992 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510565+0000", 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067689430000000004 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.547914+0000", 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037348627000000002 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.547980+0000", 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6103000000000001e-05 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.550089+0000", 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021088069999999999 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150552 5.b 5:d9f2880b:::benchmark_data_smithi026_286012_object56744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.490846+0000", 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.64888435199998, 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0703078010000002, 2023-09-23T22:23:56.293 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150552 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490846+0000", 2023-09-23T22:23:56.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490846+0000", 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490845+0000", 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490850+0000", 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8600000000000001e-06 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490858+0000", 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3850000000000002e-06 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490864+0000", 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0629999999999996e-06 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490904+0000", 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0195999999999999e-05 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490911+0000", 2023-09-23T22:23:56.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3239999999999997e-06 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203358+0000", 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.71244706400000002 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203363+0000", 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9620000000000001e-06 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.499527+0000", 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.296164345 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.499713+0000", 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018603700000000001 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.499895+0000", 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018175900000000001 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508222+0000", 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083265400000000003 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510605+0000", 2023-09-23T22:23:56.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002382941 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.558890+0000", 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048285010000000003 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.558948+0000", 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8162000000000003e-05 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.561154+0000", 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022062309999999999 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150566 5.b 5:daf8d5e7:::benchmark_data_smithi026_286012_object56758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.516941+0000", 2023-09-23T22:23:56.302 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.62278932499999, 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0461972589999999, 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150566 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516941+0000", 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516941+0000", 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516940+0000", 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:23:56.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516948+0000", 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6199999999999999e-06 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516964+0000", 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6416999999999999e-05 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516973+0000", 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8179999999999993e-06 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.517041+0000", 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8503999999999993e-05 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.517051+0000", 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6600000000000007e-06 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203395+0000", 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68634447600000004 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203402+0000", 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6699999999999997e-06 2023-09-23T22:23:56.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510628+0000", 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3072261539999999 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510814+0000", 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018611499999999999 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510998+0000", 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018392300000000001 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524167+0000", 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013168875999999999 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524243+0000", 2023-09-23T22:23:56.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6271000000000002e-05 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.561248+0000", 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037004598999999999 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.561312+0000", 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3862999999999995e-05 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.563139+0000", 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018266560000000001 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150579 5.b 5:d1157291:::benchmark_data_smithi026_286012_object56771:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.555041+0000", 2023-09-23T22:23:56.311 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.58469000299999, 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0120606849999998, 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150579 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555041+0000", 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555041+0000", 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555040+0000", 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:23:56.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555044+0000", 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.357e-06 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555055+0000", 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0241000000000001e-05 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555059+0000", 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.763e-06 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555119+0000", 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0109000000000003e-05 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555133+0000", 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3774999999999999e-05 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203428+0000", 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.64829514799999999 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203433+0000", 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2390000000000004e-06 2023-09-23T22:23:56.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.514827+0000", 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3113947669999999 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.515060+0000", 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000232835 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.515295+0000", 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000235157 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524177+0000", 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088820050000000001 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.531120+0000", 2023-09-23T22:23:56.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069427389999999999 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.564932+0000", 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033812138999999998 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.564997+0000", 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4709000000000007e-05 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.567101+0000", 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021043540000000001 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150581 5.b 5:d6413485:::benchmark_data_smithi026_286012_object56773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.564269+0000", 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 316.57546172100001, 2023-09-23T22:23:56.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0047219690000002, 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150581 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564269+0000", 2023-09-23T22:23:56.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564269+0000", 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564267+0000", 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:23:56.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564274+0000", 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4239999999999998e-06 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564288+0000", 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3917000000000001e-05 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564297+0000", 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9420000000000001e-06 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564349+0000", 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2439999999999999e-05 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564359+0000", 2023-09-23T22:23:56.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9909999999999997e-06 2023-09-23T22:23:56.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203460+0000", 2023-09-23T22:23:56.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.639100432 2023-09-23T22:23:56.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203471+0000", 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.165e-05 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.518685+0000", 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.315213328 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.518873+0000", 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000188099 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.519036+0000", 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016286600000000001 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.526472+0000", 2023-09-23T22:23:56.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074364390000000004 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.531180+0000", 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047083710000000003 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.567156+0000", 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.035975205000000003 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.567214+0000", 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7972e-05 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.568991+0000", 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001777362 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.329 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:23:56.330 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:23:56.330 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:23:56.397 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:23:56.397 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 32428/59814 objects degraded (54.215%), 16 pgs degraded, 14 pgs undersized","count":30},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":820,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":251,"num_osds":8,"num_up_osds":8,"osd_up_since":1695507239,"num_in_osds":6,"osd_in_since":1695507716,"num_remapped_pgs":15},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":82},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":13},{"state_name":"active+recovery_wait+degraded","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":99,"num_pools":5,"num_objects":19938,"data_bytes":1309391639,"bytes_used":5659566080,"bytes_avail":570310647808,"bytes_total":575970213888,"degraded_objects":32428,"degraded_total":59814,"degraded_ratio":0.54214732336911087,"misplaced_objects":1485,"misplaced_total":59814,"misplaced_ratio":0.024826963587120068,"recovering_objects_per_sec":89,"recovering_bytes_per_sec":5860475,"recovering_keys_per_sec":0,"num_objects_recovered":1018,"num_bytes_recovered":66715648,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":799},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T22:14:59.012277+0000","services":{}},"progress_events":{"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"f9b296f7-36c4-4993-8ca1-c3652b75187b":{"message":"Global Recovery Event (2m)\n [=======================.....] (remaining: 33s)","progress":0.82828283309936523,"add_to_ceph_s":true}}} 2023-09-23T22:23:56.423 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:56.524 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:23:56.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:56.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:23:56.627 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:56.696 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:56.958 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:23:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:56 smithi026 ceph-mon[223014]: pgmap v623: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 799 op/s; 32428/59814 objects degraded (54.215%); 1485/59814 objects misplaced (2.483%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T22:23:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1632478433' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:56.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1474704231' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:23:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:56 smithi026 ceph-mon[215775]: pgmap v623: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 799 op/s; 32428/59814 objects degraded (54.215%); 1485/59814 objects misplaced (2.483%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T22:23:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1632478433' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:56.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1474704231' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:23:57.124 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:23:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:56 smithi070 ceph-mon[192652]: pgmap v623: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 799 op/s; 32428/59814 objects degraded (54.215%); 1485/59814 objects misplaced (2.483%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T22:23:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1632478433' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:23:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1474704231' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:23:58.180 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:58.182 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:23:58.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[223014]: failed to encode map e252 with expected crc 2023-09-23T22:23:58.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1632478433' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:23:58.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[223014]: osdmap e252: 8 total, 8 up, 6 in 2023-09-23T22:23:58.183 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 32428/59814 objects degraded (54.215%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:58.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:23:58.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[215775]: failed to encode map e252 with expected crc 2023-09-23T22:23:58.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1632478433' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:23:58.183 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[215775]: osdmap e252: 8 total, 8 up, 6 in 2023-09-23T22:23:58.184 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:57 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 32428/59814 objects degraded (54.215%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:58.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:58.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:58.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:58.319 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:58.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:57 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:23:58.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:57 smithi070 ceph-mon[192652]: failed to encode map e252 with expected crc 2023-09-23T22:23:58.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1632478433' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:23:58.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:57 smithi070 ceph-mon[192652]: osdmap e252: 8 total, 8 up, 6 in 2023-09-23T22:23:58.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:57 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 32428/59814 objects degraded (54.215%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:23:58.472 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:23:59.107 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:59.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:59.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:59.190 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:23:59.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:23:59.222 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:23:59.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:23:58 smithi026 ceph-mon[215775]: pgmap v625: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 684 op/s; 32337/58482 objects degraded (55.294%); 1213/58482 objects misplaced (2.074%); 5.1 MiB/s, 91 objects/s recovering 2023-09-23T22:23:59.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:23:58 smithi026 ceph-mon[223014]: pgmap v625: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 684 op/s; 32337/58482 objects degraded (55.294%); 1213/58482 objects misplaced (2.074%); 5.1 MiB/s, 91 objects/s recovering 2023-09-23T22:23:59.325 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:59.351 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:59.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:23:58 smithi070 ceph-mon[192652]: pgmap v625: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 684 op/s; 32337/58482 objects degraded (55.294%); 1213/58482 objects misplaced (2.074%); 5.1 MiB/s, 91 objects/s recovering 2023-09-23T22:23:59.499 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:23:59.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:59.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:23:59.558 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:23:59.618 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:23:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:23:59] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.43.0" 2023-09-23T22:23:59.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:23:59.730 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:23:59.892 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:23:59.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:23:59.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:23:59.997 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:00.033 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:00.094 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:00.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:00.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:00.202 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:00.248 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:00.334 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:00.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:00.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:00.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:00.482 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:01.145 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:24:01.228 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:00 smithi026 ceph-mon[223014]: pgmap v626: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 679 op/s; 32337/56706 objects degraded (57.026%); 1213/56706 objects misplaced (2.139%); 5.1 MiB/s, 91 objects/s recovering 2023-09-23T22:24:01.228 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:00 smithi026 ceph-mon[215775]: pgmap v626: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 679 op/s; 32337/56706 objects degraded (57.026%); 1213/56706 objects misplaced (2.139%); 5.1 MiB/s, 91 objects/s recovering 2023-09-23T22:24:01.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:00 smithi070 ceph-mon[192652]: pgmap v626: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.2 GiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 679 op/s; 32337/56706 objects degraded (57.026%); 1213/56706 objects misplaced (2.139%); 5.1 MiB/s, 91 objects/s recovering 2023-09-23T22:24:01.753 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:01.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:01.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:01.910 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:01.988 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:01.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:01.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:02.009 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:02.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:02.196 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:02.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:02 smithi070 ceph-mon[192652]: pgmap v627: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 32337/50115 objects degraded (64.526%); 1012/50115 objects misplaced (2.019%); 3.8 MiB/s, 69 objects/s recovering 2023-09-23T22:24:02.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:02 smithi026 ceph-mon[223014]: pgmap v627: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 32337/50115 objects degraded (64.526%); 1012/50115 objects misplaced (2.019%); 3.8 MiB/s, 69 objects/s recovering 2023-09-23T22:24:02.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:02 smithi026 ceph-mon[215775]: pgmap v627: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1.0 GiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 32337/50115 objects degraded (64.526%); 1012/50115 objects misplaced (2.019%); 3.8 MiB/s, 69 objects/s recovering 2023-09-23T22:24:02.556 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:02.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:02.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:02.663 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:02.722 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:03.230 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:24:03.255 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:03.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 32337/50115 objects degraded (64.526%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:03.263 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 32337/50115 objects degraded (64.526%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:03.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:03.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:03.266 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:24:03.318 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:24:03.359 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:24:03.364 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:03.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 32337/50115 objects degraded (64.526%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:03.402 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:24:03.405 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:03.444 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:24:03.472 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:03.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:03.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:03.485 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:24:03.518 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:24:03.528 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:24:03.529 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:24:03.529 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:24:03.529 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:24:03.529 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:24:03.529 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:24:03.529 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:24:03.529 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:24:03.580 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:03.634 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:03.991 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:03.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:03.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:04.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:04.095 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:04.113 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:24:04.113 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:24:04.206 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:04.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/999340641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:04.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:04 smithi070 ceph-mon[192652]: pgmap v628: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1021 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 31844/48867 objects degraded (65.165%); 1012/48867 objects misplaced (2.071%); 1.9 MiB/s, 89 objects/s recovering 2023-09-23T22:24:04.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/999340641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:04.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:04 smithi026 ceph-mon[215775]: pgmap v628: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1021 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 31844/48867 objects degraded (65.165%); 1012/48867 objects misplaced (2.071%); 1.9 MiB/s, 89 objects/s recovering 2023-09-23T22:24:04.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/999340641' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:04.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:04 smithi026 ceph-mon[223014]: pgmap v628: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 1021 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 675 op/s; 31844/48867 objects degraded (65.165%); 1012/48867 objects misplaced (2.071%); 1.9 MiB/s, 89 objects/s recovering 2023-09-23T22:24:04.754 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:04.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:04.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:04.864 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:04.931 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:05.251 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:05.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:05.363 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:05.402 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:05.448 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:24:05.462 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:05.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:05.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:05.568 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:05.677 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:05.835 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:05.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:05.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:05.958 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:06.057 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:06.258 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:06.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:06.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:06.319 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:24:06.321 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:24:06.367 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:06.447 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:06.549 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:06.555 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:06 smithi026 ceph-mon[223014]: pgmap v629: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 822 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 31844/39318 objects degraded (80.991%); 804/39318 objects misplaced (2.045%); 1.7 MiB/s, 85 objects/s recovering 2023-09-23T22:24:06.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:06 smithi026 ceph-mon[215775]: pgmap v629: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 822 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 31844/39318 objects degraded (80.991%); 804/39318 objects misplaced (2.045%); 1.7 MiB/s, 85 objects/s recovering 2023-09-23T22:24:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:06.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:06.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:06 smithi070 ceph-mon[192652]: pgmap v629: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 822 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 682 op/s; 31844/39318 objects degraded (80.991%); 804/39318 objects misplaced (2.045%); 1.7 MiB/s, 85 objects/s recovering 2023-09-23T22:24:06.657 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:06.734 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:07.291 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:07.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:07.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:07.399 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:07.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:07 smithi026 ceph-mon[215775]: from='client.55078 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:07.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:07 smithi026 ceph-mon[223014]: from='client.55078 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:07.426 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:07.452 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:24:07.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:07 smithi070 ceph-mon[192652]: from='client.55078 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:07.749 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:07.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:07.856 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:07.864 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":629,"stamp":"2023-09-23T22:24:05.285510+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":861649687,"num_objects":13106,"num_object_clones":0,"num_object_copies":39318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31844,"num_objects_misplaced":804,"num_objects_unfound":0,"num_objects_dirty":13106,"num_whiteouts":0,"num_read":6071,"num_read_kb":12351,"num_write":150568,"num_write_kb":5219374,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22628,"num_bytes_recovered":1393758927,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86094,"ondisk_log_size":86094,"up":297,"acting":283,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":4373720,"kb_used_data":2931432,"kb_used_omap":20,"kb_used_meta":1442219,"kb_avail":558097192,"statfs":{"total":575970213888,"available":571491524608,"internally_reserved":0,"allocated":3001786368,"data_stored":2994599029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1476833035},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,2,5,8,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-447741952,"num_objects":-6832,"num_object_clones":0,"num_object_copies":-20496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-584,"num_objects_misplaced":-681,"num_objects_unfound":0,"num_objects_dirty":-6832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6832,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":856,"num_bytes_recovered":17825792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.011020"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":239,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934874+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:23:43.934874+0000","last_peered":"2023-09-23T22:23:43.934874+0000","last_clean":"2023-09-23T22:23:43.934874+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:23:43.934874+0000","last_undegraded":"2023-09-23T22:23:43.934874+0000","last_fullsized":"2023-09-23T22:23:43.934874+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":231,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:23:56.940205+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:23:56.940205+0000","last_peered":"2023-09-23T22:23:56.940205+0000","last_clean":"2023-09-23T22:23:56.940205+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:23:56.940205+0000","last_undegraded":"2023-09-23T22:23:56.940205+0000","last_fullsized":"2023-09-23T22:23:56.940205+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":193,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.897891+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:23:43.897891+0000","last_peered":"2023-09-23T22:23:43.897891+0000","last_clean":"2023-09-23T22:23:43.897891+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:23:43.897891+0000","last_undegraded":"2023-09-23T22:23:43.897891+0000","last_fullsized":"2023-09-23T22:23:43.897891+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":109,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.021428+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:23:44.021428+0000","last_peered":"2023-09-23T22:23:44.021428+0000","last_clean":"2023-09-23T22:23:44.021428+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:23:44.021428+0000","last_undegraded":"2023-09-23T22:23:44.021428+0000","last_fullsized":"2023-09-23T22:23:44.021428+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":271,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892260+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:23:43.892260+0000","last_peered":"2023-09-23T22:23:43.892260+0000","last_clean":"2023-09-23T22:23:43.892260+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:23:43.892260+0000","last_undegraded":"2023-09-23T22:23:43.892260+0000","last_fullsized":"2023-09-23T22:23:43.892260+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":207,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.938201+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:23:43.938201+0000","last_peered":"2023-09-23T22:23:43.938201+0000","last_clean":"2023-09-23T22:23:43.938201+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:23:43.938201+0000","last_undegraded":"2023-09-23T22:23:43.938201+0000","last_fullsized":"2023-09-23T22:23:43.938201+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":138,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:51.613168+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:23:51.613168+0000","last_peered":"2023-09-23T22:23:51.613168+0000","last_clean":"2023-09-23T22:23:51.613168+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:23:51.613168+0000","last_undegraded":"2023-09-23T22:23:51.613168+0000","last_fullsized":"2023-09-23T22:23:51.613168+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":228,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892328+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:23:43.892328+0000","last_peered":"2023-09-23T22:23:43.892328+0000","last_clean":"2023-09-23T22:23:43.892328+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:23:43.892328+0000","last_undegraded":"2023-09-23T22:23:43.892328+0000","last_fullsized":"2023-09-23T22:23:43.892328+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":212,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.931857+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:23:43.931857+0000","last_peered":"2023-09-23T22:23:43.931857+0000","last_clean":"2023-09-23T22:23:43.931857+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:23:43.931857+0000","last_undegraded":"2023-09-23T22:23:43.931857+0000","last_fullsized":"2023-09-23T22:23:43.931857+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":196,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892012+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:23:43.892012+0000","last_peered":"2023-09-23T22:23:43.892012+0000","last_clean":"2023-09-23T22:23:43.892012+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:23:43.892012+0000","last_undegraded":"2023-09-23T22:23:43.892012+0000","last_fullsized":"2023-09-23T22:23:43.892012+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":195,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.893027+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:23:43.893027+0000","last_peered":"2023-09-23T22:23:43.893027+0000","last_clean":"2023-09-23T22:23:43.893027+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:23:43.893027+0000","last_undegraded":"2023-09-23T22:23:43.893027+0000","last_fullsized":"2023-09-23T22:23:43.893027+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":229,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.895355+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:23:43.895355+0000","last_peered":"2023-09-23T22:23:43.895355+0000","last_clean":"2023-09-23T22:23:43.895355+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:23:43.895355+0000","last_undegraded":"2023-09-23T22:23:43.895355+0000","last_fullsized":"2023-09-23T22:23:43.895355+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":256,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.935311+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:23:43.935311+0000","last_peered":"2023-09-23T22:23:43.935311+0000","last_clean":"2023-09-23T22:23:43.935311+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:23:43.935311+0000","last_undegraded":"2023-09-23T22:23:43.935311+0000","last_fullsized":"2023-09-23T22:23:43.935311+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":220,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934671+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:23:43.934671+0000","last_peered":"2023-09-23T22:23:43.934671+0000","last_clean":"2023-09-23T22:23:43.934671+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:23:43.934671+0000","last_undegraded":"2023-09-23T22:23:43.934671+0000","last_fullsized":"2023-09-23T22:23:43.934671+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":179,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934747+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:23:43.934747+0000","last_peered":"2023-09-23T22:23:43.934747+0000","last_clean":"2023-09-23T22:23:43.934747+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:23:43.934747+0000","last_undegraded":"2023-09-23T22:23:43.934747+0000","last_fullsized":"2023-09-23T22:23:43.934747+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":219,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892549+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:23:43.892549+0000","last_peered":"2023-09-23T22:23:43.892549+0000","last_clean":"2023-09-23T22:23:43.892549+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:23:43.892549+0000","last_undegraded":"2023-09-23T22:23:43.892549+0000","last_fullsized":"2023-09-23T22:23:43.892549+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":127,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.021553+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:23:44.021553+0000","last_peered":"2023-09-23T22:23:44.021553+0000","last_clean":"2023-09-23T22:23:44.021553+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:23:44.021553+0000","last_undegraded":"2023-09-23T22:23:44.021553+0000","last_fullsized":"2023-09-23T22:23:44.021553+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":231,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934737+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:23:43.934737+0000","last_peered":"2023-09-23T22:23:43.934737+0000","last_clean":"2023-09-23T22:23:43.934737+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:23:43.934737+0000","last_undegraded":"2023-09-23T22:23:43.934737+0000","last_fullsized":"2023-09-23T22:23:43.934737+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":175,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.936654+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:23:43.936654+0000","last_peered":"2023-09-23T22:23:43.936654+0000","last_clean":"2023-09-23T22:23:43.936654+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:23:43.936654+0000","last_undegraded":"2023-09-23T22:23:43.936654+0000","last_fullsized":"2023-09-23T22:23:43.936654+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":109,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:51.613142+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:23:51.613142+0000","last_peered":"2023-09-23T22:23:51.613142+0000","last_clean":"2023-09-23T22:23:51.613142+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:23:51.613142+0000","last_undegraded":"2023-09-23T22:23:51.613142+0000","last_fullsized":"2023-09-23T22:23:51.613142+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":270,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:51.613010+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:23:51.613010+0000","last_peered":"2023-09-23T22:23:51.613010+0000","last_clean":"2023-09-23T22:23:51.613010+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:23:51.613010+0000","last_undegraded":"2023-09-23T22:23:51.613010+0000","last_fullsized":"2023-09-23T22:23:51.613010+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":40,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892592+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:23:43.892592+0000","last_peered":"2023-09-23T22:23:43.892592+0000","last_clean":"2023-09-23T22:23:43.892592+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:23:43.892592+0000","last_undegraded":"2023-09-23T22:23:43.892592+0000","last_fullsized":"2023-09-23T22:23:43.892592+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"252'9250","reported_seq":16615,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:24:04.896125+0000","last_change":"2023-09-23T22:23:29.940235+0000","last_active":"2023-09-23T22:24:04.896125+0000","last_peered":"2023-09-23T22:24:04.896125+0000","last_clean":"2023-09-23T22:24:04.896125+0000","last_became_active":"2023-09-23T22:23:29.892586+0000","last_became_peered":"2023-09-23T22:23:29.892586+0000","last_unstale":"2023-09-23T22:24:04.896125+0000","last_undegraded":"2023-09-23T22:24:04.896125+0000","last_fullsized":"2023-09-23T22:24:04.896125+0000","mapping_epoch":248,"log_start":"216'3700","ondisk_log_start":"216'3700","created":203,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5550,"log_dups_size":0,"ondisk_log_size":5550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:50.397106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52428800,"num_objects":800,"num_object_clones":0,"num_object_copies":2400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9250,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3642,"num_bytes_recovered":188481536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":247,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.937484+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:23:43.937484+0000","last_peered":"2023-09-23T22:23:43.937484+0000","last_clean":"2023-09-23T22:23:43.937484+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:23:43.937484+0000","last_undegraded":"2023-09-23T22:23:43.937484+0000","last_fullsized":"2023-09-23T22:23:43.937484+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":234,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.931469+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:23:43.931469+0000","last_peered":"2023-09-23T22:23:43.931469+0000","last_clean":"2023-09-23T22:23:43.931469+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:23:43.931469+0000","last_undegraded":"2023-09-23T22:23:43.931469+0000","last_fullsized":"2023-09-23T22:23:43.931469+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"252'9358","reported_seq":16882,"reported_epoch":252,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:24:04.300131+0000","last_change":"2023-09-23T22:22:45.430551+0000","last_active":"2023-09-23T22:24:04.300131+0000","last_peered":"2023-09-23T22:24:04.300131+0000","last_clean":"2023-09-23T22:22:43.315002+0000","last_became_active":"2023-09-23T22:22:45.409613+0000","last_became_peered":"2023-09-23T22:22:45.409613+0000","last_unstale":"2023-09-23T22:24:04.300131+0000","last_undegraded":"2023-09-23T22:24:04.300131+0000","last_fullsized":"2023-09-23T22:24:04.300131+0000","mapping_epoch":240,"log_start":"217'3991","ondisk_log_start":"217'3991","created":203,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5367,"log_dups_size":0,"ondisk_log_size":5367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:03.767060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52690944,"num_objects":804,"num_object_clones":0,"num_object_copies":2412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":804,"num_objects_unfound":0,"num_objects_dirty":804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9358,"num_write_kb":325184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3717,"num_bytes_recovered":243597312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":804}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":161,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.936124+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:23:43.936124+0000","last_peered":"2023-09-23T22:23:43.936124+0000","last_clean":"2023-09-23T22:23:43.936124+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:23:43.936124+0000","last_undegraded":"2023-09-23T22:23:43.936124+0000","last_fullsized":"2023-09-23T22:23:43.936124+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":239,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.936580+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:23:43.936580+0000","last_peered":"2023-09-23T22:23:43.936580+0000","last_clean":"2023-09-23T22:23:43.936580+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:23:43.936580+0000","last_undegraded":"2023-09-23T22:23:43.936580+0000","last_fullsized":"2023-09-23T22:23:43.936580+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"252'9565","reported_seq":13288,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:04.297601+0000","last_change":"2023-09-23T22:22:45.550371+0000","last_active":"2023-09-23T22:24:04.297601+0000","last_peered":"2023-09-23T22:24:04.297601+0000","last_clean":"2023-09-23T22:21:32.851371+0000","last_became_active":"2023-09-23T22:22:45.472694+0000","last_became_peered":"2023-09-23T22:22:45.472694+0000","last_unstale":"2023-09-23T22:24:04.297601+0000","last_undegraded":"2023-09-23T22:22:45.456856+0000","last_fullsized":"2023-09-23T22:22:45.317726+0000","mapping_epoch":240,"log_start":"217'4133","ondisk_log_start":"217'4133","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5432,"log_dups_size":0,"ondisk_log_size":5432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.540906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56557568,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5210,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9565,"num_write_kb":333696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1215,"num_bytes_recovered":79495168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"0,1,3","objects":88},{"shards":"0,2,3","objects":495},{"shards":"2,3","objects":1},{"shards":"2,3,7","objects":279},{"shards":"3,7","objects":-3136}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":177,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934817+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:23:43.934817+0000","last_peered":"2023-09-23T22:23:43.934817+0000","last_clean":"2023-09-23T22:23:43.934817+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:23:43.934817+0000","last_undegraded":"2023-09-23T22:23:43.934817+0000","last_fullsized":"2023-09-23T22:23:43.934817+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":187,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.895439+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:23:43.895439+0000","last_peered":"2023-09-23T22:23:43.895439+0000","last_clean":"2023-09-23T22:23:43.895439+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:23:43.895439+0000","last_undegraded":"2023-09-23T22:23:43.895439+0000","last_fullsized":"2023-09-23T22:23:43.895439+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"252'9327","reported_seq":16851,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:01.575809+0000","last_change":"2023-09-23T22:22:17.938724+0000","last_active":"2023-09-23T22:24:01.575809+0000","last_peered":"2023-09-23T22:24:01.575809+0000","last_clean":"2023-09-23T22:21:12.226652+0000","last_became_active":"2023-09-23T22:21:13.982783+0000","last_became_peered":"2023-09-23T22:21:13.982783+0000","last_unstale":"2023-09-23T22:24:01.575809+0000","last_undegraded":"2023-09-23T22:22:17.930337+0000","last_fullsized":"2023-09-23T22:21:13.795059+0000","mapping_epoch":214,"log_start":"218'4397","ondisk_log_start":"218'4397","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4930,"log_dups_size":0,"ondisk_log_size":4930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.489307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57999360,"num_objects":885,"num_object_clones":0,"num_object_copies":2655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9327,"num_write_kb":326784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3728,"num_bytes_recovered":244318208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,7","objects":847}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":242,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.933236+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:23:43.933236+0000","last_peered":"2023-09-23T22:23:43.933236+0000","last_clean":"2023-09-23T22:23:43.933236+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:23:43.933236+0000","last_undegraded":"2023-09-23T22:23:43.933236+0000","last_fullsized":"2023-09-23T22:23:43.933236+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":107,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:23:56.948438+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:23:56.948438+0000","last_peered":"2023-09-23T22:23:56.948438+0000","last_clean":"2023-09-23T22:23:56.948438+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:23:56.948438+0000","last_undegraded":"2023-09-23T22:23:56.948438+0000","last_fullsized":"2023-09-23T22:23:56.948438+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"252'9412","reported_seq":11015,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:04.897426+0000","last_change":"2023-09-23T22:21:57.837966+0000","last_active":"2023-09-23T22:24:04.897426+0000","last_peered":"2023-09-23T22:24:04.897426+0000","last_clean":"2023-09-23T22:21:12.225315+0000","last_became_active":"2023-09-23T22:21:13.940088+0000","last_became_peered":"2023-09-23T22:21:13.940088+0000","last_unstale":"2023-09-23T22:24:04.897426+0000","last_undegraded":"2023-09-23T22:21:57.835407+0000","last_fullsized":"2023-09-23T22:21:13.800025+0000","mapping_epoch":214,"log_start":"217'4100","ondisk_log_start":"217'4100","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5312,"log_dups_size":0,"ondisk_log_size":5312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:26.245766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52297728,"num_objects":798,"num_object_clones":0,"num_object_copies":2394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9412,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":768,"num_bytes_recovered":50331648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,4","objects":-2142}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":555,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.891925+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:23:43.891925+0000","last_peered":"2023-09-23T22:23:43.891925+0000","last_clean":"2023-09-23T22:23:43.891925+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:23:43.891925+0000","last_undegraded":"2023-09-23T22:23:43.891925+0000","last_fullsized":"2023-09-23T22:23:43.891925+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":265,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892460+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:23:43.892460+0000","last_peered":"2023-09-23T22:23:43.892460+0000","last_clean":"2023-09-23T22:23:43.892460+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:23:43.892460+0000","last_undegraded":"2023-09-23T22:23:43.892460+0000","last_fullsized":"2023-09-23T22:23:43.892460+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":220,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.935168+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:23:43.935168+0000","last_peered":"2023-09-23T22:23:43.935168+0000","last_clean":"2023-09-23T22:23:43.935168+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:23:43.935168+0000","last_undegraded":"2023-09-23T22:23:43.935168+0000","last_fullsized":"2023-09-23T22:23:43.935168+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"252'9252","reported_seq":10791,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:04.896891+0000","last_change":"2023-09-23T22:22:20.034548+0000","last_active":"2023-09-23T22:24:04.896891+0000","last_peered":"2023-09-23T22:24:04.896891+0000","last_clean":"2023-09-23T22:21:12.194696+0000","last_became_active":"2023-09-23T22:22:19.958601+0000","last_became_peered":"2023-09-23T22:22:19.958601+0000","last_unstale":"2023-09-23T22:24:04.896891+0000","last_undegraded":"2023-09-23T22:22:19.951596+0000","last_fullsized":"2023-09-23T22:22:19.874428+0000","mapping_epoch":232,"log_start":"217'3900","ondisk_log_start":"217'3900","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5352,"log_dups_size":0,"ondisk_log_size":5352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:10.906601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47841280,"num_objects":730,"num_object_clones":0,"num_object_copies":2190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4172,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9252,"num_write_kb":319424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":486,"num_bytes_recovered":31850496,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[4,3],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"1,4,5","objects":506},{"shards":"3,4","objects":-2956}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":550,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:43.931441+0000","last_change":"2023-09-23T22:21:59.026062+0000","last_active":"2023-09-23T22:23:43.931441+0000","last_peered":"2023-09-23T22:23:43.931441+0000","last_clean":"2023-09-23T22:21:50.447236+0000","last_became_active":"2023-09-23T22:21:59.021009+0000","last_became_peered":"2023-09-23T22:21:59.021009+0000","last_unstale":"2023-09-23T22:23:43.931441+0000","last_undegraded":"2023-09-23T22:21:58.818399+0000","last_fullsized":"2023-09-23T22:21:58.817464+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:52.228490+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":168,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934706+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:23:43.934706+0000","last_peered":"2023-09-23T22:23:43.934706+0000","last_clean":"2023-09-23T22:23:43.934706+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:23:43.934706+0000","last_undegraded":"2023-09-23T22:23:43.934706+0000","last_fullsized":"2023-09-23T22:23:43.934706+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":237,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934700+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:23:43.934700+0000","last_peered":"2023-09-23T22:23:43.934700+0000","last_clean":"2023-09-23T22:23:43.934700+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:23:43.934700+0000","last_undegraded":"2023-09-23T22:23:43.934700+0000","last_fullsized":"2023-09-23T22:23:43.934700+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"252'9289","reported_seq":9365,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:04.884135+0000","last_change":"2023-09-23T22:22:18.973077+0000","last_active":"2023-09-23T22:24:04.884135+0000","last_peered":"2023-09-23T22:24:04.884135+0000","last_clean":"2023-09-23T22:21:32.779211+0000","last_became_active":"2023-09-23T22:22:18.949524+0000","last_became_peered":"2023-09-23T22:22:18.949524+0000","last_unstale":"2023-09-23T22:24:04.884135+0000","last_undegraded":"2023-09-23T22:22:18.870000+0000","last_fullsized":"2023-09-23T22:22:18.859180+0000","mapping_epoch":231,"log_start":"217'3800","ondisk_log_start":"217'3800","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5489,"log_dups_size":0,"ondisk_log_size":5489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.194549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48955392,"num_objects":747,"num_object_clones":0,"num_object_copies":2241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":914,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9289,"num_write_kb":321152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"1,5,7","objects":487},{"shards":"5,7","objects":260}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":213,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.936394+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:23:43.936394+0000","last_peered":"2023-09-23T22:23:43.936394+0000","last_clean":"2023-09-23T22:23:43.936394+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:23:43.936394+0000","last_undegraded":"2023-09-23T22:23:43.936394+0000","last_fullsized":"2023-09-23T22:23:43.936394+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":228,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.935932+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:23:43.935932+0000","last_peered":"2023-09-23T22:23:43.935932+0000","last_clean":"2023-09-23T22:23:43.935932+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:23:43.935932+0000","last_undegraded":"2023-09-23T22:23:43.935932+0000","last_fullsized":"2023-09-23T22:23:43.935932+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"252'9397","reported_seq":9455,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:24:04.886835+0000","last_change":"2023-09-23T22:20:19.473637+0000","last_active":"2023-09-23T22:24:04.886835+0000","last_peered":"2023-09-23T22:24:04.886835+0000","last_clean":"2023-09-23T22:24:04.886835+0000","last_became_active":"2023-09-23T22:20:19.473385+0000","last_became_peered":"2023-09-23T22:20:19.473385+0000","last_unstale":"2023-09-23T22:24:04.886835+0000","last_undegraded":"2023-09-23T22:24:04.886835+0000","last_fullsized":"2023-09-23T22:24:04.886835+0000","mapping_epoch":203,"log_start":"217'3900","ondisk_log_start":"217'3900","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5497,"log_dups_size":0,"ondisk_log_size":5497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:01.911861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51249176,"num_objects":783,"num_object_clones":0,"num_object_copies":2349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":783,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9397,"num_write_kb":325697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":556,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.931847+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:23:43.931847+0000","last_peered":"2023-09-23T22:23:43.931847+0000","last_clean":"2023-09-23T22:23:43.931847+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:23:43.931847+0000","last_undegraded":"2023-09-23T22:23:43.931847+0000","last_fullsized":"2023-09-23T22:23:43.931847+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":31,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.891999+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:23:43.891999+0000","last_peered":"2023-09-23T22:23:43.891999+0000","last_clean":"2023-09-23T22:23:43.891999+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:23:43.891999+0000","last_undegraded":"2023-09-23T22:23:43.891999+0000","last_fullsized":"2023-09-23T22:23:43.891999+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":193,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.896081+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:23:43.896081+0000","last_peered":"2023-09-23T22:23:43.896081+0000","last_clean":"2023-09-23T22:23:43.896081+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:23:43.896081+0000","last_undegraded":"2023-09-23T22:23:43.896081+0000","last_fullsized":"2023-09-23T22:23:43.896081+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"252'9375","reported_seq":17395,"reported_epoch":252,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:01.616600+0000","last_change":"2023-09-23T22:22:43.348452+0000","last_active":"2023-09-23T22:24:01.616600+0000","last_peered":"2023-09-23T22:24:01.616600+0000","last_clean":"2023-09-23T22:21:32.822849+0000","last_became_active":"2023-09-23T22:21:35.128019+0000","last_became_peered":"2023-09-23T22:21:35.128019+0000","last_unstale":"2023-09-23T22:24:01.616600+0000","last_undegraded":"2023-09-23T22:21:34.912513+0000","last_fullsized":"2023-09-23T22:21:34.880959+0000","mapping_epoch":220,"log_start":"218'4447","ondisk_log_start":"218'4447","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4928,"log_dups_size":0,"ondisk_log_size":4928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:09:58.687801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61145088,"num_objects":933,"num_object_clones":0,"num_object_copies":2799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":625,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9375,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3977,"num_bytes_recovered":222363648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,3","objects":308}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":547,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892685+0000","last_change":"2023-09-23T22:22:47.349660+0000","last_active":"2023-09-23T22:23:43.892685+0000","last_peered":"2023-09-23T22:23:43.892685+0000","last_clean":"2023-09-23T22:23:43.892685+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:23:43.892685+0000","last_undegraded":"2023-09-23T22:23:43.892685+0000","last_fullsized":"2023-09-23T22:23:43.892685+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":801,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.895409+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:23:43.895409+0000","last_peered":"2023-09-23T22:23:43.895409+0000","last_clean":"2023-09-23T22:23:43.895409+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:23:43.895409+0000","last_undegraded":"2023-09-23T22:23:43.895409+0000","last_fullsized":"2023-09-23T22:23:43.895409+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":269,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934573+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:23:43.934573+0000","last_peered":"2023-09-23T22:23:43.934573+0000","last_clean":"2023-09-23T22:23:43.934573+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:23:43.934573+0000","last_undegraded":"2023-09-23T22:23:43.934573+0000","last_fullsized":"2023-09-23T22:23:43.934573+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":38,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:23:56.933758+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:23:56.933758+0000","last_peered":"2023-09-23T22:23:56.933758+0000","last_clean":"2023-09-23T22:23:56.933758+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:23:56.933758+0000","last_undegraded":"2023-09-23T22:23:56.933758+0000","last_fullsized":"2023-09-23T22:23:56.933758+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"252'9468","reported_seq":9557,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:04.908931+0000","last_change":"2023-09-23T22:22:18.955534+0000","last_active":"2023-09-23T22:24:04.908931+0000","last_peered":"2023-09-23T22:24:04.908931+0000","last_clean":"2023-09-23T22:21:32.860473+0000","last_became_active":"2023-09-23T22:22:18.941549+0000","last_became_peered":"2023-09-23T22:22:18.941549+0000","last_unstale":"2023-09-23T22:24:04.908931+0000","last_undegraded":"2023-09-23T22:22:18.875275+0000","last_fullsized":"2023-09-23T22:22:18.863381+0000","mapping_epoch":231,"log_start":"217'4200","ondisk_log_start":"217'4200","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5268,"log_dups_size":0,"ondisk_log_size":5268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:27.412277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50069504,"num_objects":764,"num_object_clones":0,"num_object_copies":2292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":988,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9468,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,5,7","objects":527},{"shards":"4,7","objects":237}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":556,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.931791+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:23:43.931791+0000","last_peered":"2023-09-23T22:23:43.931791+0000","last_clean":"2023-09-23T22:23:43.931791+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:23:43.931791+0000","last_undegraded":"2023-09-23T22:23:43.931791+0000","last_fullsized":"2023-09-23T22:23:43.931791+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":984,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.932042+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:23:43.932042+0000","last_peered":"2023-09-23T22:23:43.932042+0000","last_clean":"2023-09-23T22:23:43.932042+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:23:43.932042+0000","last_undegraded":"2023-09-23T22:23:43.932042+0000","last_fullsized":"2023-09-23T22:23:43.932042+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":261,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934603+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:23:43.934603+0000","last_peered":"2023-09-23T22:23:43.934603+0000","last_clean":"2023-09-23T22:23:43.934603+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:23:43.934603+0000","last_undegraded":"2023-09-23T22:23:43.934603+0000","last_fullsized":"2023-09-23T22:23:43.934603+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":216,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.933346+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:23:43.933346+0000","last_peered":"2023-09-23T22:23:43.933346+0000","last_clean":"2023-09-23T22:23:43.933346+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:23:43.933346+0000","last_undegraded":"2023-09-23T22:23:43.933346+0000","last_fullsized":"2023-09-23T22:23:43.933346+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"252'9195","reported_seq":9261,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:04.298762+0000","last_change":"2023-09-23T22:21:59.013998+0000","last_active":"2023-09-23T22:24:04.298762+0000","last_peered":"2023-09-23T22:24:04.298762+0000","last_clean":"2023-09-23T22:21:56.814967+0000","last_became_active":"2023-09-23T22:21:59.002385+0000","last_became_peered":"2023-09-23T22:21:59.002385+0000","last_unstale":"2023-09-23T22:24:04.298762+0000","last_undegraded":"2023-09-23T22:21:58.865154+0000","last_fullsized":"2023-09-23T22:21:58.825566+0000","mapping_epoch":226,"log_start":"217'3820","ondisk_log_start":"217'3820","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5375,"log_dups_size":0,"ondisk_log_size":5375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:00.273421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51052544,"num_objects":779,"num_object_clones":0,"num_object_copies":2337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4918,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9195,"num_write_kb":319168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":710},{"shards":"3,7","objects":-4139}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":581,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.021398+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:23:44.021398+0000","last_peered":"2023-09-23T22:23:44.021398+0000","last_clean":"2023-09-23T22:23:44.021398+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:23:44.021398+0000","last_undegraded":"2023-09-23T22:23:44.021398+0000","last_fullsized":"2023-09-23T22:23:44.021398+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":358,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:23:43.892113+0000","last_change":"2023-09-23T22:22:18.917671+0000","last_active":"2023-09-23T22:23:43.892113+0000","last_peered":"2023-09-23T22:23:43.892113+0000","last_clean":"2023-09-23T22:15:39.913392+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:23:43.892113+0000","last_undegraded":"2023-09-23T22:22:18.874220+0000","last_fullsized":"2023-09-23T22:23:43.892113+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":212,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.895939+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:23:43.895939+0000","last_peered":"2023-09-23T22:23:43.895939+0000","last_clean":"2023-09-23T22:23:43.895939+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:23:43.895939+0000","last_undegraded":"2023-09-23T22:23:43.895939+0000","last_fullsized":"2023-09-23T22:23:43.895939+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":259,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.896207+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:23:43.896207+0000","last_peered":"2023-09-23T22:23:43.896207+0000","last_clean":"2023-09-23T22:23:43.896207+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:23:43.896207+0000","last_undegraded":"2023-09-23T22:23:43.896207+0000","last_fullsized":"2023-09-23T22:23:43.896207+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"252'9519","reported_seq":9576,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:24:04.899335+0000","last_change":"2023-09-23T22:20:19.470871+0000","last_active":"2023-09-23T22:24:04.899335+0000","last_peered":"2023-09-23T22:24:04.899335+0000","last_clean":"2023-09-23T22:24:04.899335+0000","last_became_active":"2023-09-23T22:20:19.470597+0000","last_became_peered":"2023-09-23T22:20:19.470597+0000","last_unstale":"2023-09-23T22:24:04.899335+0000","last_undegraded":"2023-09-23T22:24:04.899335+0000","last_fullsized":"2023-09-23T22:24:04.899335+0000","mapping_epoch":203,"log_start":"217'4000","ondisk_log_start":"217'4000","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:32.388985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53280768,"num_objects":813,"num_object_clones":0,"num_object_copies":2439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9519,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":847,"reported_epoch":251,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:23:43.932088+0000","last_change":"2023-09-23T22:21:58.853256+0000","last_active":"2023-09-23T22:23:43.932088+0000","last_peered":"2023-09-23T22:23:43.932088+0000","last_clean":"2023-09-23T22:21:50.447508+0000","last_became_active":"2023-09-23T22:21:58.852815+0000","last_became_peered":"2023-09-23T22:21:58.852815+0000","last_unstale":"2023-09-23T22:23:43.932088+0000","last_undegraded":"2023-09-23T22:21:58.818640+0000","last_fullsized":"2023-09-23T22:21:58.817793+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:25.619025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":564,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.035078+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:23:44.035078+0000","last_peered":"2023-09-23T22:23:44.035078+0000","last_clean":"2023-09-23T22:23:44.035078+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:23:44.035078+0000","last_undegraded":"2023-09-23T22:23:44.035078+0000","last_fullsized":"2023-09-23T22:23:44.035078+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":176,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892023+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:23:43.892023+0000","last_peered":"2023-09-23T22:23:43.892023+0000","last_clean":"2023-09-23T22:23:43.892023+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:23:43.892023+0000","last_undegraded":"2023-09-23T22:23:43.892023+0000","last_fullsized":"2023-09-23T22:23:43.892023+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":138,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934898+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:23:43.934898+0000","last_peered":"2023-09-23T22:23:43.934898+0000","last_clean":"2023-09-23T22:23:43.934898+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:23:43.934898+0000","last_undegraded":"2023-09-23T22:23:43.934898+0000","last_fullsized":"2023-09-23T22:23:43.934898+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"252'9179","reported_seq":9259,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:03.782854+0000","last_change":"2023-09-23T22:22:18.954521+0000","last_active":"2023-09-23T22:24:03.782854+0000","last_peered":"2023-09-23T22:24:03.782854+0000","last_clean":"2023-09-23T22:21:32.869170+0000","last_became_active":"2023-09-23T22:22:18.945835+0000","last_became_peered":"2023-09-23T22:22:18.945835+0000","last_unstale":"2023-09-23T22:24:03.782854+0000","last_undegraded":"2023-09-23T22:22:18.869474+0000","last_fullsized":"2023-09-23T22:22:18.861404+0000","mapping_epoch":231,"log_start":"223'4505","ondisk_log_start":"223'4505","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4674,"log_dups_size":0,"ondisk_log_size":4674,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:02.907936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57081856,"num_objects":871,"num_object_clones":0,"num_object_copies":2613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9179,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,4,5","objects":591},{"shards":"1,5","objects":280}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":583,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.897044+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:23:43.897044+0000","last_peered":"2023-09-23T22:23:43.897044+0000","last_clean":"2023-09-23T22:23:43.897044+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:23:43.897044+0000","last_undegraded":"2023-09-23T22:23:43.897044+0000","last_fullsized":"2023-09-23T22:23:43.897044+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":602,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.896149+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:23:43.896149+0000","last_peered":"2023-09-23T22:23:43.896149+0000","last_clean":"2023-09-23T22:23:43.896149+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:23:43.896149+0000","last_undegraded":"2023-09-23T22:23:43.896149+0000","last_fullsized":"2023-09-23T22:23:43.896149+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":134,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.024887+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:23:44.024887+0000","last_peered":"2023-09-23T22:23:44.024887+0000","last_clean":"2023-09-23T22:23:44.024887+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:23:44.024887+0000","last_undegraded":"2023-09-23T22:23:44.024887+0000","last_fullsized":"2023-09-23T22:23:44.024887+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":91,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.021364+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:23:44.021364+0000","last_peered":"2023-09-23T22:23:44.021364+0000","last_clean":"2023-09-23T22:23:44.021364+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:23:44.021364+0000","last_undegraded":"2023-09-23T22:23:44.021364+0000","last_fullsized":"2023-09-23T22:23:44.021364+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"252'9354","reported_seq":12898,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:03.756199+0000","last_change":"2023-09-23T22:22:45.524797+0000","last_active":"2023-09-23T22:24:03.756199+0000","last_peered":"2023-09-23T22:24:03.756199+0000","last_clean":"2023-09-23T22:21:12.245233+0000","last_became_active":"2023-09-23T22:22:45.454018+0000","last_became_peered":"2023-09-23T22:22:45.454018+0000","last_unstale":"2023-09-23T22:24:03.756199+0000","last_undegraded":"2023-09-23T22:22:45.437539+0000","last_fullsized":"2023-09-23T22:22:45.322462+0000","mapping_epoch":240,"log_start":"223'4730","ondisk_log_start":"223'4730","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4624,"log_dups_size":0,"ondisk_log_size":4624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:10.659044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57933824,"num_objects":884,"num_object_clones":0,"num_object_copies":2652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9354,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1153,"num_bytes_recovered":75563008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1","objects":1},{"shards":"0,1,5","objects":565},{"shards":"1,7","objects":-2197}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":573,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.021520+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:23:44.021520+0000","last_peered":"2023-09-23T22:23:44.021520+0000","last_clean":"2023-09-23T22:23:44.021520+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:23:44.021520+0000","last_undegraded":"2023-09-23T22:23:44.021520+0000","last_fullsized":"2023-09-23T22:23:44.021520+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":143,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.021758+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:23:44.021758+0000","last_peered":"2023-09-23T22:23:44.021758+0000","last_clean":"2023-09-23T22:23:44.021758+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:23:44.021758+0000","last_undegraded":"2023-09-23T22:23:44.021758+0000","last_fullsized":"2023-09-23T22:23:44.021758+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":203,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.895673+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:23:43.895673+0000","last_peered":"2023-09-23T22:23:43.895673+0000","last_clean":"2023-09-23T22:23:43.895673+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:23:43.895673+0000","last_undegraded":"2023-09-23T22:23:43.895673+0000","last_fullsized":"2023-09-23T22:23:43.895673+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":107,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.025957+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:23:44.025957+0000","last_peered":"2023-09-23T22:23:44.025957+0000","last_clean":"2023-09-23T22:23:44.025957+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:23:44.025957+0000","last_undegraded":"2023-09-23T22:23:44.025957+0000","last_fullsized":"2023-09-23T22:23:44.025957+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"252'9529","reported_seq":17066,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:04.874935+0000","last_change":"2023-09-23T22:22:19.043755+0000","last_active":"2023-09-23T22:24:04.874935+0000","last_peered":"2023-09-23T22:24:04.874935+0000","last_clean":"2023-09-23T22:21:12.214088+0000","last_became_active":"2023-09-23T22:22:18.986181+0000","last_became_peered":"2023-09-23T22:22:18.986181+0000","last_unstale":"2023-09-23T22:24:04.874935+0000","last_undegraded":"2023-09-23T22:22:18.975289+0000","last_fullsized":"2023-09-23T22:22:18.862148+0000","mapping_epoch":231,"log_start":"217'4200","ondisk_log_start":"217'4200","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5329,"log_dups_size":0,"ondisk_log_size":5329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:34.422545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51314688,"num_objects":783,"num_object_clones":0,"num_object_copies":2349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1882,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9529,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3724,"num_bytes_recovered":244056064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":700},{"shards":"4,5,7","objects":83},{"shards":"4,7","objects":-649}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":556,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934564+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:23:43.934564+0000","last_peered":"2023-09-23T22:23:43.934564+0000","last_clean":"2023-09-23T22:23:43.934564+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:23:43.934564+0000","last_undegraded":"2023-09-23T22:23:43.934564+0000","last_fullsized":"2023-09-23T22:23:43.934564+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":265,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.894868+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:23:43.894868+0000","last_peered":"2023-09-23T22:23:43.894868+0000","last_clean":"2023-09-23T22:23:43.894868+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:23:43.894868+0000","last_undegraded":"2023-09-23T22:23:43.894868+0000","last_fullsized":"2023-09-23T22:23:43.894868+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":894,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.895590+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:23:43.895590+0000","last_peered":"2023-09-23T22:23:43.895590+0000","last_clean":"2023-09-23T22:23:43.895590+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:23:43.895590+0000","last_undegraded":"2023-09-23T22:23:43.895590+0000","last_fullsized":"2023-09-23T22:23:43.895590+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":40,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.891762+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:23:43.891762+0000","last_peered":"2023-09-23T22:23:43.891762+0000","last_clean":"2023-09-23T22:23:43.891762+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:23:43.891762+0000","last_undegraded":"2023-09-23T22:23:43.891762+0000","last_fullsized":"2023-09-23T22:23:43.891762+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"252'9148","reported_seq":9348,"reported_epoch":252,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:03.805572+0000","last_change":"2023-09-23T22:22:44.336083+0000","last_active":"2023-09-23T22:24:03.805572+0000","last_peered":"2023-09-23T22:24:03.805572+0000","last_clean":"2023-09-23T22:21:56.808651+0000","last_became_active":"2023-09-23T22:21:59.059105+0000","last_became_peered":"2023-09-23T22:21:59.059105+0000","last_unstale":"2023-09-23T22:24:03.805572+0000","last_undegraded":"2023-09-23T22:21:58.989636+0000","last_fullsized":"2023-09-23T22:21:58.812134+0000","mapping_epoch":226,"log_start":"223'4502","ondisk_log_start":"223'4502","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4646,"log_dups_size":0,"ondisk_log_size":4646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:51.370288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55181312,"num_objects":842,"num_object_clones":0,"num_object_copies":2526,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":842,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9148,"num_write_kb":319680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":4259840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":763},{"shards":"1,7","objects":-4010}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":539,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892783+0000","last_change":"2023-09-23T22:22:19.883431+0000","last_active":"2023-09-23T22:23:43.892783+0000","last_peered":"2023-09-23T22:23:43.892783+0000","last_clean":"2023-09-23T22:23:43.892783+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:23:43.892783+0000","last_undegraded":"2023-09-23T22:23:43.892783+0000","last_fullsized":"2023-09-23T22:23:43.892783+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:46.903071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:46.903071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":233,"reported_epoch":251,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:23:43.894876+0000","last_change":"2023-09-23T22:22:44.342884+0000","last_active":"2023-09-23T22:23:43.894876+0000","last_peered":"2023-09-23T22:23:43.894876+0000","last_clean":"2023-09-23T22:06:25.449250+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:23:43.894876+0000","last_undegraded":"2023-09-23T22:22:44.329336+0000","last_fullsized":"2023-09-23T22:23:43.894876+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":86,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.021459+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:23:44.021459+0000","last_peered":"2023-09-23T22:23:44.021459+0000","last_clean":"2023-09-23T22:23:44.021459+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:23:44.021459+0000","last_undegraded":"2023-09-23T22:23:44.021459+0000","last_fullsized":"2023-09-23T22:23:44.021459+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":215,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.936463+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:23:43.936463+0000","last_peered":"2023-09-23T22:23:43.936463+0000","last_clean":"2023-09-23T22:23:43.936463+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:23:43.936463+0000","last_undegraded":"2023-09-23T22:23:43.936463+0000","last_fullsized":"2023-09-23T22:23:43.936463+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":82,"reported_epoch":252,"state":"active+clean","last_fresh":"2023-09-23T22:23:56.948290+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:23:56.948290+0000","last_peered":"2023-09-23T22:23:56.948290+0000","last_clean":"2023-09-23T22:23:56.948290+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:23:56.948290+0000","last_undegraded":"2023-09-23T22:23:56.948290+0000","last_fullsized":"2023-09-23T22:23:56.948290+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":263,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.895595+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:23:43.895595+0000","last_peered":"2023-09-23T22:23:43.895595+0000","last_clean":"2023-09-23T22:23:43.895595+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:23:43.895595+0000","last_undegraded":"2023-09-23T22:23:43.895595+0000","last_fullsized":"2023-09-23T22:23:43.895595+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":214,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.933164+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:23:43.933164+0000","last_peered":"2023-09-23T22:23:43.933164+0000","last_clean":"2023-09-23T22:23:43.933164+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:23:43.933164+0000","last_undegraded":"2023-09-23T22:23:43.933164+0000","last_fullsized":"2023-09-23T22:23:43.933164+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":227,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934814+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:23:43.934814+0000","last_peered":"2023-09-23T22:23:43.934814+0000","last_clean":"2023-09-23T22:23:43.934814+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:23:43.934814+0000","last_undegraded":"2023-09-23T22:23:43.934814+0000","last_fullsized":"2023-09-23T22:23:43.934814+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":211,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934661+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:23:43.934661+0000","last_peered":"2023-09-23T22:23:43.934661+0000","last_clean":"2023-09-23T22:23:43.934661+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:23:43.934661+0000","last_undegraded":"2023-09-23T22:23:43.934661+0000","last_fullsized":"2023-09-23T22:23:43.934661+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934777+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:23:43.934777+0000","last_peered":"2023-09-23T22:23:43.934777+0000","last_clean":"2023-09-23T22:23:43.934777+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:23:43.934777+0000","last_undegraded":"2023-09-23T22:23:43.934777+0000","last_fullsized":"2023-09-23T22:23:43.934777+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":232,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.892788+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:23:43.892788+0000","last_peered":"2023-09-23T22:23:43.892788+0000","last_clean":"2023-09-23T22:23:43.892788+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:23:43.892788+0000","last_undegraded":"2023-09-23T22:23:43.892788+0000","last_fullsized":"2023-09-23T22:23:43.892788+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":274,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.891988+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:23:43.891988+0000","last_peered":"2023-09-23T22:23:43.891988+0000","last_clean":"2023-09-23T22:23:43.891988+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:23:43.891988+0000","last_undegraded":"2023-09-23T22:23:43.891988+0000","last_fullsized":"2023-09-23T22:23:43.891988+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":141,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:44.021564+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:23:44.021564+0000","last_peered":"2023-09-23T22:23:44.021564+0000","last_clean":"2023-09-23T22:23:44.021564+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:23:44.021564+0000","last_undegraded":"2023-09-23T22:23:44.021564+0000","last_fullsized":"2023-09-23T22:23:44.021564+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":260,"reported_epoch":251,"state":"active+clean","last_fresh":"2023-09-23T22:23:43.934451+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:23:43.934451+0000","last_peered":"2023-09-23T22:23:43.934451+0000","last_clean":"2023-09-23T22:23:43.934451+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:23:43.934451+0000","last_undegraded":"2023-09-23T22:23:43.934451+0000","last_fullsized":"2023-09-23T22:23:43.934451+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":857079832,"num_objects":13079,"num_object_clones":0,"num_object_copies":39237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31817,"num_objects_misplaced":804,"num_objects_unfound":0,"num_objects_dirty":13079,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149617,"num_write_kb":5206209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22481,"num_bytes_recovered":1384710144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5648822272,"data_stored":5648810056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83292,"ondisk_log_size":83292,"up":48,"acting":36,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":370028,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":88,"num_read_kb":166,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234236,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":893700,"kb_used_data":709628,"kb_used_omap":0,"kb_used_meta":184064,"kb_avail":92851452,"statfs":{"total":95995035648,"available":95079886848,"internally_reserved":0,"allocated":726659072,"data_stored":725647252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":188481536},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:03 2023","interfaces":[{"interface":"back","average":{"1min":9.048,"5min":4.476,"15min":2.2679999999999998},"min":{"1min":0.23000000000000001,"5min":0.22,"15min":0.22},"max":{"1min":158.042,"5min":158.042,"15min":158.042},"last":1.7849999999999999},{"interface":"front","average":{"1min":9.6690000000000005,"5min":4.7149999999999999,"15min":2.4830000000000001},"min":{"1min":0.26300000000000001,"5min":0.217,"15min":0.217},"max":{"1min":157.952,"5min":157.952,"15min":157.952},"last":5.5359999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":4.9329999999999998,"15min":2.9809999999999999},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.246},"max":{"1min":10.127000000000001,"5min":101.145,"15min":123.79300000000001},"last":3.8180000000000001},{"interface":"front","average":{"1min":2.5859999999999999,"5min":4.9909999999999997,"15min":3.1309999999999998},"min":{"1min":0.28199999999999997,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":10.92,"5min":101.13,"15min":123.66500000000001},"last":5.5819999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":4.1260000000000003,"15min":3.7650000000000001},"min":{"1min":0.307,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.7939999999999996,"5min":105.34399999999999,"15min":125.101},"last":5.6429999999999998},{"interface":"front","average":{"1min":2.6440000000000001,"5min":4.0960000000000001,"15min":3.5110000000000001},"min":{"1min":0.30499999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.3769999999999998,"5min":105.63200000000001,"15min":126.021},"last":2.0099999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:23:12 2023","interfaces":[{"interface":"back","average":{"1min":10.196999999999999,"5min":3.7349999999999999,"15min":2.1890000000000001},"min":{"1min":0.36699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":170.86099999999999,"5min":170.86099999999999,"15min":170.86099999999999},"last":8.8170000000000002},{"interface":"front","average":{"1min":9.5510000000000002,"5min":3.9239999999999999,"15min":2.3170000000000002},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":174.57499999999999,"5min":174.57499999999999,"15min":174.57499999999999},"last":5.7919999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":2.2120000000000002,"15min":1.3280000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.5169999999999999,"5min":88.543000000000006,"15min":88.543000000000006},"last":5.8200000000000003},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.29,"15min":1.3009999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.6719999999999997,"5min":88.495999999999995,"15min":88.495999999999995},"last":2.0600000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":2.2530000000000001,"15min":1.2509999999999999},"min":{"1min":0.30299999999999999,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":3.21,"5min":93.816000000000003,"15min":93.816000000000003},"last":1.857},{"interface":"front","average":{"1min":0.81499999999999995,"5min":2.2970000000000002,"15min":1.466},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.544,"5min":95.579999999999998,"15min":95.579999999999998},"last":5.8529999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.68100000000000005,"15min":0.74099999999999999},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.7120000000000002,"5min":5.343,"15min":9.0139999999999993},"last":1.8879999999999999},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.86299999999999999,"15min":0.93899999999999995},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":3.6339999999999999,"5min":6.3879999999999999,"15min":13.362},"last":5.6130000000000004}]}]},{"osd":6,"up_from":0,"seq":197568495872,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795144,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":783676,"kb_used_data":478452,"kb_used_omap":7,"kb_used_meta":305208,"kb_avail":92961476,"statfs":{"total":95995035648,"available":95192551424,"internally_reserved":0,"allocated":489934848,"data_stored":488578636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":312533364},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,0,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8689999999999998,"5min":2.113,"15min":1.3149999999999999},"min":{"1min":0.255,"5min":0.189,"15min":0.189},"max":{"1min":56.503,"5min":56.503,"15min":61.890000000000001},"last":7.0279999999999996},{"interface":"front","average":{"1min":5.1550000000000002,"5min":3.0230000000000001,"15min":1.728},"min":{"1min":0.40799999999999997,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":56.560000000000002,"5min":56.560000000000002,"15min":62.689999999999998},"last":5.407}]},{"osd":2,"last update":"Sat Sep 23 22:23:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7749999999999999,"5min":1.294,"15min":1.2789999999999999},"min":{"1min":0.27600000000000002,"5min":0.223,"15min":0.216},"max":{"1min":7.694,"5min":11.179,"15min":33.902000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":3.734,"5min":2.4660000000000002,"15min":1.726},"min":{"1min":0.312,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.8369999999999997,"5min":15.015000000000001,"15min":34.781999999999996},"last":5.3630000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:23:20 2023","interfaces":[{"interface":"back","average":{"1min":8.5220000000000002,"5min":4.3360000000000003,"15min":2.0590000000000002},"min":{"1min":0.47599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":110.417,"5min":110.417,"15min":110.417},"last":7.7759999999999998},{"interface":"front","average":{"1min":8.5239999999999991,"5min":4.1820000000000004,"15min":2.0179999999999998},"min":{"1min":0.33400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":110.721,"5min":110.721,"15min":110.721},"last":9.6270000000000007}]},{"osd":4,"last update":"Sat Sep 23 22:23:20 2023","interfaces":[{"interface":"back","average":{"1min":8.6099999999999994,"5min":4.6239999999999997,"15min":2.056},"min":{"1min":0.314,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":171.09999999999999,"5min":234.79900000000001,"15min":234.79900000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":9.1289999999999996,"5min":5.2699999999999996,"15min":2.3519999999999999},"min":{"1min":0.44700000000000001,"5min":0.33200000000000002,"15min":0.318},"max":{"1min":171.08500000000001,"5min":234.81700000000001,"15min":234.81700000000001},"last":2.3889999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":3.6850000000000001,"15min":1.788},"min":{"1min":0.39100000000000001,"5min":0.31,"15min":0.31},"max":{"1min":9.5709999999999997,"5min":196.79900000000001,"15min":196.79900000000001},"last":2.4279999999999999},{"interface":"front","average":{"1min":2.9340000000000002,"5min":3.6499999999999999,"15min":1.7989999999999999},"min":{"1min":0.41299999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.01,"5min":196.68299999999999,"15min":196.68299999999999},"last":2.4630000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":1.913,"15min":1.2490000000000001},"min":{"1min":0.497,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":10.074999999999999,"5min":10.971,"15min":10.989000000000001},"last":2.512},{"interface":"front","average":{"1min":1.448,"5min":1.296,"15min":1.006},"min":{"1min":0.437,"5min":0.35699999999999998,"15min":0.34999999999999998},"max":{"1min":7.7300000000000004,"5min":11.385999999999999,"15min":11.385999999999999},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.6970000000000001,"5min":1.9339999999999999,"15min":1.2889999999999999},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.048,"5min":10.933999999999999,"15min":10.948},"last":2.4889999999999999},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.3200000000000001,"15min":1.0589999999999999},"min":{"1min":0.40699999999999997,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":12.045999999999999,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.47999999999999998}]}]},{"osd":0,"up_from":93,"seq":399431958678,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":657768,"kb_used_data":360608,"kb_used_omap":8,"kb_used_meta":297143,"kb_avail":93087384,"statfs":{"total":95995035648,"available":95321481216,"internally_reserved":0,"allocated":369262592,"data_stored":367868810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":304275306},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2009999999999996,"5min":4.5810000000000004,"15min":2.274},"min":{"1min":0.21299999999999999,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":51.018999999999998,"5min":169.77199999999999,"15min":169.77199999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":3.6739999999999999,"5min":5.0860000000000003,"15min":2.5640000000000001},"min":{"1min":0.24099999999999999,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":23.891999999999999,"5min":169.52099999999999,"15min":169.52099999999999},"last":0.82499999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":6.1360000000000001,"15min":2.6800000000000002},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":16.097999999999999,"5min":206.495,"15min":206.495},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.258,"5min":6.6130000000000004,"15min":3.3050000000000002},"min":{"1min":0.29299999999999998,"5min":0.28499999999999998,"15min":0.22800000000000001},"max":{"1min":11.138,"5min":207.34399999999999,"15min":207.34399999999999},"last":0.66000000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4350000000000001,"5min":3.7160000000000002,"15min":2.0009999999999999},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.3399999999999999,"5min":131.078,"15min":131.078},"last":0.374},{"interface":"front","average":{"1min":3.0070000000000001,"5min":3.9540000000000002,"15min":2.1040000000000001},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.438000000000001,"5min":133.083,"15min":133.083},"last":0.85999999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.23,"15min":1.7849999999999999},"min":{"1min":0.33900000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":9.673,"5min":82.825000000000003,"15min":82.825000000000003},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.0129999999999999,"5min":2.306,"15min":1.2509999999999999},"min":{"1min":0.32500000000000001,"5min":0.313,"15min":0.308},"max":{"1min":3.129,"5min":75.930999999999997,"15min":75.930999999999997},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":8.1050000000000004,"5min":4.4669999999999996,"15min":2.173},"min":{"1min":0.39200000000000002,"5min":0.34799999999999998,"15min":0.29499999999999998},"max":{"1min":93.685000000000002,"5min":93.685000000000002,"15min":93.685000000000002},"last":0.46300000000000002},{"interface":"front","average":{"1min":7.6550000000000002,"5min":3.5459999999999998,"15min":1.7030000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":93.546000000000006,"5min":93.546000000000006,"15min":93.546000000000006},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":2.3069999999999999,"15min":1.472},"min":{"1min":0.40100000000000002,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":9.7140000000000004,"5min":10.275,"15min":10.275},"last":0.5},{"interface":"front","average":{"1min":2.3769999999999998,"5min":1.764,"15min":1.115},"min":{"1min":0.43099999999999999,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":10.919,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.51900000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:23:04 2023","interfaces":[{"interface":"back","average":{"1min":4.0570000000000004,"5min":2.5800000000000001,"15min":1.643},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":32.773000000000003,"5min":32.773000000000003,"15min":32.773000000000003},"last":0.629},{"interface":"front","average":{"1min":2.069,"5min":2.4249999999999998,"15min":1.6850000000000001},"min":{"1min":0.39900000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":9.5790000000000006,"5min":10.257999999999999,"15min":23.265999999999998},"last":1.022}]}]},{"osd":2,"up_from":0,"seq":442381631617,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468091,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":782696,"kb_used_data":552032,"kb_used_omap":4,"kb_used_meta":230651,"kb_avail":92962456,"statfs":{"total":95995035648,"available":95193554944,"internally_reserved":0,"allocated":565280768,"data_stored":563881492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":236187181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":5.7969999999999997,"15min":2.7069999999999999},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":23.890000000000001,"5min":225.86699999999999,"15min":225.86699999999999},"last":5.3220000000000001},{"interface":"front","average":{"1min":4.4619999999999997,"5min":6.2400000000000002,"15min":2.9340000000000002},"min":{"1min":0.23699999999999999,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":24.053000000000001,"5min":225.84800000000001,"15min":225.84800000000001},"last":5.7889999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":4.0819999999999999,"15min":2.1840000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.209,"5min":82.951999999999998,"15min":82.951999999999998},"last":4.3239999999999998},{"interface":"front","average":{"1min":4.75,"5min":4.4509999999999996,"15min":2.2170000000000001},"min":{"1min":0.34300000000000003,"5min":0.22600000000000001,"15min":0.189},"max":{"1min":13.85,"5min":86.001000000000005,"15min":86.001000000000005},"last":5.5899999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":5.5129999999999999,"15min":3.3849999999999998},"min":{"1min":0.501,"5min":0.252,"15min":0.223},"max":{"1min":8.1180000000000003,"5min":99.962999999999994,"15min":99.962999999999994},"last":3.2410000000000001},{"interface":"front","average":{"1min":2.2890000000000001,"5min":5.6790000000000003,"15min":3.2130000000000001},"min":{"1min":0.41399999999999998,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":9.5909999999999993,"5min":100.429,"15min":100.429},"last":5.4279999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.9850000000000001,"15min":1.1839999999999999},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":6.2960000000000003,"5min":11.911,"15min":11.911},"last":5.1980000000000004},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.764,"15min":1.2030000000000001},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.949999999999999,"5min":10.949999999999999,"15min":12.036},"last":3.8090000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":2.4220000000000002,"15min":1.391},"min":{"1min":0.39900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":11.028,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.6259999999999999,"5min":2.427,"15min":1.3799999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.883,"5min":49.668999999999997,"15min":49.668999999999997},"last":3.2730000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":2.1259999999999999,"15min":1.3320000000000001},"min":{"1min":0.42699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.992000000000001,"5min":10.992000000000001,"15min":12.068},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.893,"5min":1.8620000000000001,"15min":1.2410000000000001},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35399999999999998},"max":{"1min":5.9219999999999997,"5min":11.68,"15min":11.68},"last":3.7360000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.0110000000000001,"15min":1.3140000000000001},"min":{"1min":0.39700000000000002,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":6.0309999999999997,"5min":16.849,"15min":16.849},"last":3.3140000000000001},{"interface":"front","average":{"1min":2.3290000000000002,"5min":2.0910000000000002,"15min":1.272},"min":{"1min":0.46200000000000002,"5min":0.40400000000000003,"15min":0.36699999999999999},"max":{"1min":6.1749999999999998,"5min":11.945,"15min":12.382},"last":5.0910000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019141,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":536200,"kb_used_data":351552,"kb_used_omap":0,"kb_used_meta":184640,"kb_avail":93208952,"statfs":{"total":95995035648,"available":95445966848,"internally_reserved":0,"allocated":359989248,"data_stored":358974139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":189071360},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":2.4079999999999999,"5min":2.4340000000000002,"15min":1.649},"min":{"1min":0.28399999999999997,"5min":0.23000000000000001,"15min":0.218},"max":{"1min":9.2579999999999991,"5min":46.18,"15min":66.355000000000004},"last":4.5060000000000002},{"interface":"front","average":{"1min":2.379,"5min":2.5019999999999998,"15min":1.7350000000000001},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":9.2390000000000008,"5min":46.207000000000001,"15min":66.332999999999998},"last":4.516}]},{"osd":1,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":2.0470000000000002,"15min":1.3460000000000001},"min":{"1min":0.26700000000000002,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":5.0750000000000002,"5min":35.409999999999997,"15min":38.878},"last":3.984},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.492,"15min":1.694},"min":{"1min":0.28100000000000003,"5min":0.255,"15min":0.255},"max":{"1min":7.8620000000000001,"5min":35.999000000000002,"15min":38.853999999999999},"last":1.6160000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:23:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":3.472,"15min":2.2970000000000002},"min":{"1min":0.36099999999999999,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":10.051,"5min":138.06100000000001,"15min":138.06100000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.1030000000000002,"5min":3.3380000000000001,"15min":2.4340000000000002},"min":{"1min":0.35499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.2510000000000003,"5min":138.15199999999999,"15min":138.15199999999999},"last":4.1219999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:23:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":1.8220000000000001,"15min":1.196},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":12.129,"5min":12.129,"15min":12.129},"last":3.1640000000000001},{"interface":"front","average":{"1min":2.5870000000000002,"5min":1.8200000000000001,"15min":1.228},"min":{"1min":0.46300000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4059999999999997,"5min":8.8930000000000007,"15min":9.8109999999999999},"last":1.6930000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:23:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.284,"15min":0.874},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.8570000000000002,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.5680000000000001,"5min":1.3759999999999999,"15min":0.90400000000000003},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":7.298,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.35699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:23:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.80800000000000005,"15min":0.68999999999999995},"min":{"1min":0.33200000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":2.262,"5min":7.3079999999999998,"15min":7.3079999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.873,"5min":0.85299999999999998,"15min":0.749},"min":{"1min":0.29099999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.2679999999999998,"5min":6.7279999999999998,"15min":8.5210000000000008},"last":0.318}]},{"osd":7,"last update":"Sat Sep 23 22:23:08 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":0.94799999999999995,"15min":0.83999999999999997},"min":{"1min":0.313,"5min":0.313,"15min":0.307},"max":{"1min":2.754,"5min":6.7480000000000002,"15min":8.577},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.143,"5min":0.94099999999999995,"15min":0.78700000000000003},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":5.1479999999999997,"5min":7.2679999999999998,"15min":7.2679999999999998},"last":0.56499999999999995}]}]},{"osd":5,"up_from":38,"seq":163208757509,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":719680,"kb_used_data":479160,"kb_used_omap":0,"kb_used_meta":240512,"kb_avail":93025472,"statfs":{"total":95995035648,"available":95258083328,"internally_reserved":0,"allocated":490659840,"data_stored":489648700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":246284288},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:57 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":2.7989999999999999,"15min":2.1579999999999999},"min":{"1min":0.23999999999999999,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":64.344999999999999,"5min":64.344999999999999,"15min":133.22499999999999},"last":0.93899999999999995},{"interface":"front","average":{"1min":4.891,"5min":2.528,"15min":2.181},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":64.022000000000006,"5min":64.022000000000006,"15min":132.78200000000001},"last":6.141}]},{"osd":1,"last update":"Sat Sep 23 22:23:16 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":4.3090000000000002,"15min":2.681},"min":{"1min":0.27200000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.385999999999999,"5min":93.981999999999999,"15min":93.981999999999999},"last":0.311},{"interface":"front","average":{"1min":1.9099999999999999,"5min":4.2329999999999997,"15min":2.6509999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":13.457000000000001,"5min":93.909999999999997,"15min":93.909999999999997},"last":2.3100000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":3.9239999999999999,"15min":3.504},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.3079999999999998,"5min":95.790999999999997,"15min":104.254},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.355,"5min":4.0119999999999996,"15min":3.7549999999999999},"min":{"1min":0.32500000000000001,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":5.3479999999999999,"5min":95.366,"15min":104.217},"last":2.327}]},{"osd":3,"last update":"Sat Sep 23 22:23:16 2023","interfaces":[{"interface":"back","average":{"1min":4.1429999999999998,"5min":3.6789999999999998,"15min":2.23},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.246},"max":{"1min":18.876000000000001,"5min":93.739999999999995,"15min":93.739999999999995},"last":4.1879999999999997},{"interface":"front","average":{"1min":2.6000000000000001,"5min":3.4220000000000002,"15min":2.1699999999999999},"min":{"1min":0.34200000000000003,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":13.773999999999999,"5min":93.197999999999993,"15min":93.197999999999993},"last":4.2569999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.085,"15min":1.03},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":5.1799999999999997,"5min":10.163,"15min":82.530000000000001},"last":2.2669999999999999},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.0629999999999999,"15min":1.0660000000000001},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.2119999999999997,"5min":7.2119999999999997,"15min":85.441000000000003},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:23:39 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.82199999999999995,"15min":1.1559999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":7.0289999999999999,"5min":7.0289999999999999,"15min":114.83499999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.73499999999999999,"15min":1.0669999999999999},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.2250000000000001,"5min":4.5220000000000002,"15min":110.905},"last":0.42999999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.86799999999999999,"15min":0.878},"min":{"1min":0.33300000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.2759999999999998,"5min":4.3760000000000003,"15min":8.8030000000000008},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.74099999999999999,"15min":0.76200000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3479999999999999,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":2.2909999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":350818304,"data_stored":350814232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":478285824,"data_stored":478281752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2760900608,"data_stored":2760900608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":537919488,"data_stored":537919488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":344195072,"data_stored":344195072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":477106176,"data_stored":477102104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":699596800,"data_stored":699596800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:24:07.888 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:07.958 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:07.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:07.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:08.065 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:08.097 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:08.156 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:08.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:08.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:08.219 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:08.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:08.319 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:08.327 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:08.421 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:08.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:08.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:08.526 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:08.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:08 smithi070 ceph-mon[192652]: pgmap v630: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 794 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 659 op/s; 31350/37971 objects degraded (82.563%); 804/37971 objects misplaced (2.117%); 0 B/s, 95 objects/s recovering 2023-09-23T22:24:08.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 31350/37971 objects degraded (82.563%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:08.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:08 smithi070 ceph-mon[192652]: osdmap e253: 8 total, 8 up, 6 in 2023-09-23T22:24:08.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:08 smithi070 ceph-mon[192652]: failed to encode map e253 with expected crc 2023-09-23T22:24:08.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:08 smithi070 ceph-mon[192652]: failed to encode map e253 with expected crc 2023-09-23T22:24:08.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:08 smithi070 ceph-mon[192652]: failed to encode map e253 with expected crc 2023-09-23T22:24:08.552 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:08.594 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:08.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:08.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[223014]: pgmap v630: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 794 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 659 op/s; 31350/37971 objects degraded (82.563%); 804/37971 objects misplaced (2.117%); 0 B/s, 95 objects/s recovering 2023-09-23T22:24:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 31350/37971 objects degraded (82.563%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[223014]: osdmap e253: 8 total, 8 up, 6 in 2023-09-23T22:24:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[223014]: failed to encode map e253 with expected crc 2023-09-23T22:24:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[223014]: failed to encode map e253 with expected crc 2023-09-23T22:24:08.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[223014]: failed to encode map e253 with expected crc 2023-09-23T22:24:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[215775]: pgmap v630: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 794 MiB data, 4.1 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 659 op/s; 31350/37971 objects degraded (82.563%); 804/37971 objects misplaced (2.117%); 0 B/s, 95 objects/s recovering 2023-09-23T22:24:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 31350/37971 objects degraded (82.563%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[215775]: osdmap e253: 8 total, 8 up, 6 in 2023-09-23T22:24:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[215775]: failed to encode map e253 with expected crc 2023-09-23T22:24:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[215775]: failed to encode map e253 with expected crc 2023-09-23T22:24:08.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:08 smithi026 ceph-mon[215775]: failed to encode map e253 with expected crc 2023-09-23T22:24:08.700 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:08.725 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:08.766 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:08.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:08.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:08.872 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:08.913 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:24:09.143 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:09 smithi070 ceph-mon[192652]: failed to encode map e253 with expected crc 2023-09-23T22:24:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:09 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1443279586' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:24:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:09 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:24:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:09 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:09 smithi070 ceph-mon[192652]: failed to encode map e254 with expected crc 2023-09-23T22:24:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:09 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:24:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:09 smithi070 ceph-mon[192652]: osdmap e254: 8 total, 8 up, 6 in 2023-09-23T22:24:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[215775]: failed to encode map e253 with expected crc 2023-09-23T22:24:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1443279586' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:24:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:24:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[215775]: failed to encode map e254 with expected crc 2023-09-23T22:24:09.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:24:09.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[215775]: osdmap e254: 8 total, 8 up, 6 in 2023-09-23T22:24:09.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:09.737 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:24:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:24:09] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T22:24:09.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[223014]: failed to encode map e253 with expected crc 2023-09-23T22:24:09.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1443279586' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:24:09.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:24:09.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:09.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[223014]: failed to encode map e254 with expected crc 2023-09-23T22:24:09.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:24:09.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[223014]: osdmap e254: 8 total, 8 up, 6 in 2023-09-23T22:24:09.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:10.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:10 smithi070 ceph-mon[192652]: failed to encode map e254 with expected crc 2023-09-23T22:24:10.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:10 smithi070 ceph-mon[192652]: pgmap v633: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 756 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 579 op/s; 31350/36186 objects degraded (86.636%); 804/36186 objects misplaced (2.222%); 0 B/s, 123 objects/s recovering 2023-09-23T22:24:10.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:10 smithi026 ceph-mon[223014]: failed to encode map e254 with expected crc 2023-09-23T22:24:10.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:10 smithi026 ceph-mon[223014]: pgmap v633: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 756 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 579 op/s; 31350/36186 objects degraded (86.636%); 804/36186 objects misplaced (2.222%); 0 B/s, 123 objects/s recovering 2023-09-23T22:24:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:10 smithi026 ceph-mon[215775]: failed to encode map e254 with expected crc 2023-09-23T22:24:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:10 smithi026 ceph-mon[215775]: pgmap v633: 99 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 756 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 579 op/s; 31350/36186 objects degraded (86.636%); 804/36186 objects misplaced (2.222%); 0 B/s, 123 objects/s recovering 2023-09-23T22:24:10.872 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:24:11.802 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:11.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:11.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:11.915 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:11.965 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:12.036 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:12.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:12.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:12.142 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:12.451 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:12.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:12 smithi070 ceph-mon[192652]: pgmap v634: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 625 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 30799/29889 objects degraded (103.045%); 1077/29889 objects misplaced (3.603%); 712 KiB/s, 72 objects/s recovering 2023-09-23T22:24:12.690 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:12.713 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:12 smithi026 ceph-mon[215775]: pgmap v634: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 625 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 30799/29889 objects degraded (103.045%); 1077/29889 objects misplaced (3.603%); 712 KiB/s, 72 objects/s recovering 2023-09-23T22:24:12.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:12.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:12.715 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:12 smithi026 ceph-mon[223014]: pgmap v634: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 82 active+clean; 625 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 30799/29889 objects degraded (103.045%); 1077/29889 objects misplaced (3.603%); 712 KiB/s, 72 objects/s recovering 2023-09-23T22:24:12.816 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:12.909 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:13.314 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:13.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:13.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:13.420 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:13.455 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:24:13.455 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:24:13.508 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:13.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 30799/29889 objects degraded (103.045%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:13.685 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 30799/29889 objects degraded (103.045%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:13.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 30799/29889 objects degraded (103.045%), 16 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:13.973 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:13.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:13.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:14.010 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:24:14.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:24:14.078 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:14.144 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:24:14.144 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:24:14.198 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:24:14.245 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:24:14.355 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:24:14.355 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:24:14.355 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:14.507 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:24:14.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:24:14.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:24:14.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:24:14.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:24:14.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:24:14.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:24:14.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:24:14.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:24:15.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:14 smithi026 ceph-mon[215775]: pgmap v635: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 604 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 30668/28845 objects degraded (106.320%); 1077/28845 objects misplaced (3.734%); 712 KiB/s, 89 objects/s recovering 2023-09-23T22:24:15.128 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:14 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/415645922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:15.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:14 smithi026 ceph-mon[223014]: pgmap v635: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 604 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 30668/28845 objects degraded (106.320%); 1077/28845 objects misplaced (3.734%); 712 KiB/s, 89 objects/s recovering 2023-09-23T22:24:15.129 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:14 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/415645922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:15.129 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:15.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:14 smithi070 ceph-mon[192652]: pgmap v635: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 604 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 30668/28845 objects degraded (106.320%); 1077/28845 objects misplaced (3.734%); 712 KiB/s, 89 objects/s recovering 2023-09-23T22:24:15.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:14 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/415645922' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:15.528 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:15.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:15.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:15.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:15.807 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:16.578 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:24:16.800 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:16.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:16.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:16.912 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:16.934 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:16 smithi026 ceph-mon[215775]: pgmap v636: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 427 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 30267/20388 objects degraded (148.455%); 672/20388 objects misplaced (3.296%); 5.3 MiB/s, 101 objects/s recovering 2023-09-23T22:24:16.934 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:16 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:16.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:16 smithi026 ceph-mon[223014]: pgmap v636: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 427 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 30267/20388 objects degraded (148.455%); 672/20388 objects misplaced (3.296%); 5.3 MiB/s, 101 objects/s recovering 2023-09-23T22:24:16.935 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:16 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:17.023 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:17.160 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:17.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:17.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:17.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:16 smithi070 ceph-mon[192652]: pgmap v636: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 427 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 732 op/s; 30267/20388 objects degraded (148.455%); 672/20388 objects misplaced (3.296%); 5.3 MiB/s, 101 objects/s recovering 2023-09-23T22:24:17.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:16 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:24:17.375 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:18.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:17 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 30267/20388 objects degraded (148.455%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:18.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:17 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 30267/20388 objects degraded (148.455%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:18.306 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:18.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:18.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:18.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:17 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 30267/20388 objects degraded (148.455%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:18.413 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:18.469 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:18.553 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:18.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:18.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:18.662 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:18.708 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:18.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:18 smithi026 ceph-mon[215775]: pgmap v637: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 402 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 30267/19179 objects degraded (157.813%); 672/19179 objects misplaced (3.504%); 4.6 MiB/s, 86 objects/s recovering 2023-09-23T22:24:18.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:18 smithi026 ceph-mon[223014]: pgmap v637: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 402 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 30267/19179 objects degraded (157.813%); 672/19179 objects misplaced (3.504%); 4.6 MiB/s, 86 objects/s recovering 2023-09-23T22:24:19.060 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:19.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:19.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:19.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:24:19.167 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:19.169 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:19.177 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:19.202 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:19.313 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:19.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:18 smithi070 ceph-mon[192652]: pgmap v637: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 402 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 30267/19179 objects degraded (157.813%); 672/19179 objects misplaced (3.504%); 4.6 MiB/s, 86 objects/s recovering 2023-09-23T22:24:19.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:19.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:19.419 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:19.509 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:19.775 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:19.858 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:24:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:24:19] "GET /metrics HTTP/1.1" 200 36399 "" "Prometheus/2.43.0" 2023-09-23T22:24:19.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:19.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:19.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:20.077 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:20.535 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:20.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:20.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:20.644 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:20.695 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:20.755 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:20.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:20.861 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:20.918 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:20.959 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:24:20.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:20 smithi026 ceph-mon[215775]: pgmap v638: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 364 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 604 op/s; 30064/17343 objects degraded (173.349%); 672/17343 objects misplaced (3.875%); 5.3 MiB/s, 98 objects/s recovering 2023-09-23T22:24:20.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:20.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:20.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:20 smithi070 ceph-mon[192652]: pgmap v638: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 364 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 604 op/s; 30064/17343 objects degraded (173.349%); 672/17343 objects misplaced (3.875%); 5.3 MiB/s, 98 objects/s recovering 2023-09-23T22:24:20.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:20.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:20.985 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:20.989 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:24:20.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:20.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:21.097 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:21.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:20 smithi026 ceph-mon[223014]: pgmap v638: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 364 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 604 op/s; 30064/17343 objects degraded (173.349%); 672/17343 objects misplaced (3.875%); 5.3 MiB/s, 98 objects/s recovering 2023-09-23T22:24:21.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:21.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:21.297 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:22.103 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:22.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:21 smithi026 ceph-mon[215775]: from='client.55093 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:22.106 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:21 smithi026 ceph-mon[223014]: from='client.55093 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:22.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:22.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:22.208 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:22.292 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:21 smithi070 ceph-mon[192652]: from='client.55093 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:23.224 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:22 smithi026 ceph-mon[215775]: pgmap v639: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 236 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 693 op/s; 29676/11193 objects degraded (265.130%); 270/11193 objects misplaced (2.412%); 7.8 MiB/s, 135 objects/s recovering 2023-09-23T22:24:23.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:22 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 29676/11193 objects degraded (265.130%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:23.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:22 smithi026 ceph-mon[223014]: pgmap v639: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 236 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 693 op/s; 29676/11193 objects degraded (265.130%); 270/11193 objects misplaced (2.412%); 7.8 MiB/s, 135 objects/s recovering 2023-09-23T22:24:23.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:22 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 29676/11193 objects degraded (265.130%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:23.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:22 smithi070 ceph-mon[192652]: pgmap v639: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 236 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 693 op/s; 29676/11193 objects degraded (265.130%); 270/11193 objects misplaced (2.412%); 7.8 MiB/s, 135 objects/s recovering 2023-09-23T22:24:23.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:22 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 29676/11193 objects degraded (265.130%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:23.708 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":638,"stamp":"2023-09-23T22:24:19.291787+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":381598487,"num_objects":5781,"num_object_clones":0,"num_object_copies":17343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30064,"num_objects_misplaced":672,"num_objects_unfound":0,"num_objects_dirty":5781,"num_whiteouts":0,"num_read":6071,"num_read_kb":12351,"num_write":157893,"num_write_kb":5219374,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24141,"num_bytes_recovered":1451954895,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85819,"ondisk_log_size":85819,"up":297,"acting":284,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":343,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":3146748,"kb_used_data":1670732,"kb_used_omap":20,"kb_used_meta":1475819,"kb_avail":559324164,"statfs":{"total":575970213888,"available":572747943936,"internally_reserved":0,"allocated":1710829568,"data_stored":1703577681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1511239435},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,5,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-411631616,"num_objects":-6281,"num_object_clones":0,"num_object_copies":-18843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1286,"num_objects_misplaced":-132,"num_objects_unfound":0,"num_objects_dirty":-6281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6281,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1019,"num_bytes_recovered":58195968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.387299"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":241,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923758+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:24:07.923758+0000","last_peered":"2023-09-23T22:24:07.923758+0000","last_clean":"2023-09-23T22:24:07.923758+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:24:07.923758+0000","last_undegraded":"2023-09-23T22:24:07.923758+0000","last_fullsized":"2023-09-23T22:24:07.923758+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":233,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.912728+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:24:07.912728+0000","last_peered":"2023-09-23T22:24:07.912728+0000","last_clean":"2023-09-23T22:24:07.912728+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:24:07.912728+0000","last_undegraded":"2023-09-23T22:24:07.912728+0000","last_fullsized":"2023-09-23T22:24:07.912728+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":195,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.924692+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:24:07.924692+0000","last_peered":"2023-09-23T22:24:07.924692+0000","last_clean":"2023-09-23T22:24:07.924692+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:24:07.924692+0000","last_undegraded":"2023-09-23T22:24:07.924692+0000","last_fullsized":"2023-09-23T22:24:07.924692+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":111,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.896839+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:24:07.896839+0000","last_peered":"2023-09-23T22:24:07.896839+0000","last_clean":"2023-09-23T22:24:07.896839+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:24:07.896839+0000","last_undegraded":"2023-09-23T22:24:07.896839+0000","last_fullsized":"2023-09-23T22:24:07.896839+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":273,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.911823+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:24:07.911823+0000","last_peered":"2023-09-23T22:24:07.911823+0000","last_clean":"2023-09-23T22:24:07.911823+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:24:07.911823+0000","last_undegraded":"2023-09-23T22:24:07.911823+0000","last_fullsized":"2023-09-23T22:24:07.911823+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":209,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.916030+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:24:07.916030+0000","last_peered":"2023-09-23T22:24:07.916030+0000","last_clean":"2023-09-23T22:24:07.916030+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:24:07.916030+0000","last_undegraded":"2023-09-23T22:24:07.916030+0000","last_fullsized":"2023-09-23T22:24:07.916030+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":140,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.897599+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:24:07.897599+0000","last_peered":"2023-09-23T22:24:07.897599+0000","last_clean":"2023-09-23T22:24:07.897599+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:24:07.897599+0000","last_undegraded":"2023-09-23T22:24:07.897599+0000","last_fullsized":"2023-09-23T22:24:07.897599+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":230,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.911905+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:24:07.911905+0000","last_peered":"2023-09-23T22:24:07.911905+0000","last_clean":"2023-09-23T22:24:07.911905+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:24:07.911905+0000","last_undegraded":"2023-09-23T22:24:07.911905+0000","last_fullsized":"2023-09-23T22:24:07.911905+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":214,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.915641+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:24:07.915641+0000","last_peered":"2023-09-23T22:24:07.915641+0000","last_clean":"2023-09-23T22:24:07.915641+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:24:07.915641+0000","last_undegraded":"2023-09-23T22:24:07.915641+0000","last_fullsized":"2023-09-23T22:24:07.915641+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":198,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.913536+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:24:07.913536+0000","last_peered":"2023-09-23T22:24:07.913536+0000","last_clean":"2023-09-23T22:24:07.913536+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:24:07.913536+0000","last_undegraded":"2023-09-23T22:24:07.913536+0000","last_fullsized":"2023-09-23T22:24:07.913536+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":197,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.912651+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:24:07.912651+0000","last_peered":"2023-09-23T22:24:07.912651+0000","last_clean":"2023-09-23T22:24:07.912651+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:24:07.912651+0000","last_undegraded":"2023-09-23T22:24:07.912651+0000","last_fullsized":"2023-09-23T22:24:07.912651+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":231,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.924275+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:24:07.924275+0000","last_peered":"2023-09-23T22:24:07.924275+0000","last_clean":"2023-09-23T22:24:07.924275+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:24:07.924275+0000","last_undegraded":"2023-09-23T22:24:07.924275+0000","last_fullsized":"2023-09-23T22:24:07.924275+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":258,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923029+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:24:07.923029+0000","last_peered":"2023-09-23T22:24:07.923029+0000","last_clean":"2023-09-23T22:24:07.923029+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:24:07.923029+0000","last_undegraded":"2023-09-23T22:24:07.923029+0000","last_fullsized":"2023-09-23T22:24:07.923029+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":222,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.922935+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:24:07.922935+0000","last_peered":"2023-09-23T22:24:07.922935+0000","last_clean":"2023-09-23T22:24:07.922935+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:24:07.922935+0000","last_undegraded":"2023-09-23T22:24:07.922935+0000","last_fullsized":"2023-09-23T22:24:07.922935+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":181,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923025+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:24:07.923025+0000","last_peered":"2023-09-23T22:24:07.923025+0000","last_clean":"2023-09-23T22:24:07.923025+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:24:07.923025+0000","last_undegraded":"2023-09-23T22:24:07.923025+0000","last_fullsized":"2023-09-23T22:24:07.923025+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":221,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.912894+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:24:07.912894+0000","last_peered":"2023-09-23T22:24:07.912894+0000","last_clean":"2023-09-23T22:24:07.912894+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:24:07.912894+0000","last_undegraded":"2023-09-23T22:24:07.912894+0000","last_fullsized":"2023-09-23T22:24:07.912894+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":129,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.897552+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:24:07.897552+0000","last_peered":"2023-09-23T22:24:07.897552+0000","last_clean":"2023-09-23T22:24:07.897552+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:24:07.897552+0000","last_undegraded":"2023-09-23T22:24:07.897552+0000","last_fullsized":"2023-09-23T22:24:07.897552+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":233,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.915912+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:24:07.915912+0000","last_peered":"2023-09-23T22:24:07.915912+0000","last_clean":"2023-09-23T22:24:07.915912+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:24:07.915912+0000","last_undegraded":"2023-09-23T22:24:07.915912+0000","last_fullsized":"2023-09-23T22:24:07.915912+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":177,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.922126+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:24:07.922126+0000","last_peered":"2023-09-23T22:24:07.922126+0000","last_clean":"2023-09-23T22:24:07.922126+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:24:07.922126+0000","last_undegraded":"2023-09-23T22:24:07.922126+0000","last_fullsized":"2023-09-23T22:24:07.922126+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":111,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.897106+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:24:07.897106+0000","last_peered":"2023-09-23T22:24:07.897106+0000","last_clean":"2023-09-23T22:24:07.897106+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:24:07.897106+0000","last_undegraded":"2023-09-23T22:24:07.897106+0000","last_fullsized":"2023-09-23T22:24:07.897106+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":272,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.904998+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:24:07.904998+0000","last_peered":"2023-09-23T22:24:07.904998+0000","last_clean":"2023-09-23T22:24:07.904998+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:24:07.904998+0000","last_undegraded":"2023-09-23T22:24:07.904998+0000","last_fullsized":"2023-09-23T22:24:07.904998+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":42,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.914756+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:24:07.914756+0000","last_peered":"2023-09-23T22:24:07.914756+0000","last_clean":"2023-09-23T22:24:07.914756+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:24:07.914756+0000","last_undegraded":"2023-09-23T22:24:07.914756+0000","last_fullsized":"2023-09-23T22:24:07.914756+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"254'9655","reported_seq":17022,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:24:14.873622+0000","last_change":"2023-09-23T22:23:29.940235+0000","last_active":"2023-09-23T22:24:14.873622+0000","last_peered":"2023-09-23T22:24:14.873622+0000","last_clean":"2023-09-23T22:24:14.873622+0000","last_became_active":"2023-09-23T22:23:29.892586+0000","last_became_peered":"2023-09-23T22:23:29.892586+0000","last_unstale":"2023-09-23T22:24:14.873622+0000","last_undegraded":"2023-09-23T22:24:14.873622+0000","last_fullsized":"2023-09-23T22:24:14.873622+0000","mapping_epoch":248,"log_start":"218'4200","ondisk_log_start":"218'4200","created":203,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5455,"log_dups_size":0,"ondisk_log_size":5455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:50.397106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25886720,"num_objects":395,"num_object_clones":0,"num_object_copies":1185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9655,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3642,"num_bytes_recovered":188481536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":249,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923651+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:24:07.923651+0000","last_peered":"2023-09-23T22:24:07.923651+0000","last_clean":"2023-09-23T22:24:07.923651+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:24:07.923651+0000","last_undegraded":"2023-09-23T22:24:07.923651+0000","last_fullsized":"2023-09-23T22:24:07.923651+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":236,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.916028+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:24:07.916028+0000","last_peered":"2023-09-23T22:24:07.916028+0000","last_clean":"2023-09-23T22:24:07.916028+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:24:07.916028+0000","last_undegraded":"2023-09-23T22:24:07.916028+0000","last_fullsized":"2023-09-23T22:24:07.916028+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"254'9756","reported_seq":17282,"reported_epoch":254,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:24:14.340260+0000","last_change":"2023-09-23T22:22:45.430551+0000","last_active":"2023-09-23T22:24:14.340260+0000","last_peered":"2023-09-23T22:24:14.340260+0000","last_clean":"2023-09-23T22:22:43.315002+0000","last_became_active":"2023-09-23T22:22:45.409613+0000","last_became_peered":"2023-09-23T22:22:45.409613+0000","last_unstale":"2023-09-23T22:24:14.340260+0000","last_undegraded":"2023-09-23T22:24:14.340260+0000","last_fullsized":"2023-09-23T22:24:14.340260+0000","mapping_epoch":240,"log_start":"218'4391","ondisk_log_start":"218'4391","created":203,"last_epoch_clean":237,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5365,"log_dups_size":0,"ondisk_log_size":5365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:03.767060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26607616,"num_objects":406,"num_object_clones":0,"num_object_copies":1218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":406,"num_objects_unfound":0,"num_objects_dirty":406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9756,"num_write_kb":325184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3717,"num_bytes_recovered":243597312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":406}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":163,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.924865+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:24:07.924865+0000","last_peered":"2023-09-23T22:24:07.924865+0000","last_clean":"2023-09-23T22:24:07.924865+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:24:07.924865+0000","last_undegraded":"2023-09-23T22:24:07.924865+0000","last_fullsized":"2023-09-23T22:24:07.924865+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":241,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.922051+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:24:07.922051+0000","last_peered":"2023-09-23T22:24:07.922051+0000","last_clean":"2023-09-23T22:24:07.922051+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:24:07.922051+0000","last_undegraded":"2023-09-23T22:24:07.922051+0000","last_fullsized":"2023-09-23T22:24:07.922051+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"254'9994","reported_seq":14021,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:14.341126+0000","last_change":"2023-09-23T22:22:45.550371+0000","last_active":"2023-09-23T22:24:14.341126+0000","last_peered":"2023-09-23T22:24:14.341126+0000","last_clean":"2023-09-23T22:21:32.851371+0000","last_became_active":"2023-09-23T22:22:45.472694+0000","last_became_peered":"2023-09-23T22:22:45.472694+0000","last_unstale":"2023-09-23T22:24:14.341126+0000","last_undegraded":"2023-09-23T22:22:45.456856+0000","last_fullsized":"2023-09-23T22:22:45.317726+0000","mapping_epoch":240,"log_start":"221'4633","ondisk_log_start":"221'4633","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5361,"log_dups_size":0,"ondisk_log_size":5361,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.540906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28442624,"num_objects":434,"num_object_clones":0,"num_object_copies":1302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5059,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9994,"num_write_kb":333696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1365,"num_bytes_recovered":89325568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7],"avail_no_missing":["3","2"],"object_location_counts":[{"shards":"0,1,3","objects":88},{"shards":"0,2,3","objects":346},{"shards":"3,7","objects":-3415}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":179,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.916002+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:24:07.916002+0000","last_peered":"2023-09-23T22:24:07.916002+0000","last_clean":"2023-09-23T22:24:07.916002+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:24:07.916002+0000","last_undegraded":"2023-09-23T22:24:07.916002+0000","last_fullsized":"2023-09-23T22:24:07.916002+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":189,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.924367+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:24:07.924367+0000","last_peered":"2023-09-23T22:24:07.924367+0000","last_clean":"2023-09-23T22:24:07.924367+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:24:07.924367+0000","last_undegraded":"2023-09-23T22:24:07.924367+0000","last_fullsized":"2023-09-23T22:24:07.924367+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.9","version":"254'9907","reported_seq":17433,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:16.585225+0000","last_change":"2023-09-23T22:22:17.938724+0000","last_active":"2023-09-23T22:24:16.585225+0000","last_peered":"2023-09-23T22:24:16.585225+0000","last_clean":"2023-09-23T22:21:12.226652+0000","last_became_active":"2023-09-23T22:21:13.982783+0000","last_became_peered":"2023-09-23T22:21:13.982783+0000","last_unstale":"2023-09-23T22:24:16.585225+0000","last_undegraded":"2023-09-23T22:22:17.930337+0000","last_fullsized":"2023-09-23T22:21:13.795059+0000","mapping_epoch":214,"log_start":"224'4997","ondisk_log_start":"224'4997","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4910,"log_dups_size":0,"ondisk_log_size":4910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:40.489307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19988480,"num_objects":305,"num_object_clones":0,"num_object_copies":915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9907,"num_write_kb":326784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3728,"num_bytes_recovered":244318208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,7","objects":267}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":244,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.916309+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:24:07.916309+0000","last_peered":"2023-09-23T22:24:07.916309+0000","last_clean":"2023-09-23T22:24:07.916309+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:24:07.916309+0000","last_undegraded":"2023-09-23T22:24:07.916309+0000","last_fullsized":"2023-09-23T22:24:07.916309+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":109,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.898116+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:24:07.898116+0000","last_peered":"2023-09-23T22:24:07.898116+0000","last_clean":"2023-09-23T22:24:07.898116+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:24:07.898116+0000","last_undegraded":"2023-09-23T22:24:07.898116+0000","last_fullsized":"2023-09-23T22:24:07.898116+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"254'9792","reported_seq":11397,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:14.887572+0000","last_change":"2023-09-23T22:21:57.837966+0000","last_active":"2023-09-23T22:24:14.887572+0000","last_peered":"2023-09-23T22:24:14.887572+0000","last_clean":"2023-09-23T22:21:12.225315+0000","last_became_active":"2023-09-23T22:21:13.940088+0000","last_became_peered":"2023-09-23T22:21:13.940088+0000","last_unstale":"2023-09-23T22:24:14.887572+0000","last_undegraded":"2023-09-23T22:21:57.835407+0000","last_fullsized":"2023-09-23T22:21:13.800025+0000","mapping_epoch":214,"log_start":"218'4500","ondisk_log_start":"218'4500","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5292,"log_dups_size":0,"ondisk_log_size":5292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:26.245766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27394048,"num_objects":418,"num_object_clones":0,"num_object_copies":1254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2940,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9792,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":768,"num_bytes_recovered":50331648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,4","objects":-2522}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":557,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.913415+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:24:07.913415+0000","last_peered":"2023-09-23T22:24:07.913415+0000","last_clean":"2023-09-23T22:24:07.913415+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:24:07.913415+0000","last_undegraded":"2023-09-23T22:24:07.913415+0000","last_fullsized":"2023-09-23T22:24:07.913415+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":267,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.915897+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:24:07.915897+0000","last_peered":"2023-09-23T22:24:07.915897+0000","last_clean":"2023-09-23T22:24:07.915897+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:24:07.915897+0000","last_undegraded":"2023-09-23T22:24:07.915897+0000","last_fullsized":"2023-09-23T22:24:07.915897+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":222,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.922866+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:24:07.922866+0000","last_peered":"2023-09-23T22:24:07.922866+0000","last_clean":"2023-09-23T22:24:07.922866+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:24:07.922866+0000","last_undegraded":"2023-09-23T22:24:07.922866+0000","last_fullsized":"2023-09-23T22:24:07.922866+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"254'9610","reported_seq":11419,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:14.847098+0000","last_change":"2023-09-23T22:22:20.034548+0000","last_active":"2023-09-23T22:24:14.847098+0000","last_peered":"2023-09-23T22:24:14.847098+0000","last_clean":"2023-09-23T22:21:12.194696+0000","last_became_active":"2023-09-23T22:22:19.958601+0000","last_became_peered":"2023-09-23T22:22:19.958601+0000","last_unstale":"2023-09-23T22:24:14.847098+0000","last_undegraded":"2023-09-23T22:22:19.951596+0000","last_fullsized":"2023-09-23T22:22:19.874428+0000","mapping_epoch":232,"log_start":"218'4300","ondisk_log_start":"218'4300","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5310,"log_dups_size":0,"ondisk_log_size":5310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:10.906601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24379392,"num_objects":372,"num_object_clones":0,"num_object_copies":1116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4038,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9610,"num_write_kb":319424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":620,"num_bytes_recovered":40632320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[4,3],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"1,4,5","objects":372},{"shards":"3,4","objects":-3180}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":552,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:07.915800+0000","last_change":"2023-09-23T22:21:59.026062+0000","last_active":"2023-09-23T22:24:07.915800+0000","last_peered":"2023-09-23T22:24:07.915800+0000","last_clean":"2023-09-23T22:21:50.447236+0000","last_became_active":"2023-09-23T22:21:59.021009+0000","last_became_peered":"2023-09-23T22:21:59.021009+0000","last_unstale":"2023-09-23T22:24:07.915800+0000","last_undegraded":"2023-09-23T22:21:58.818399+0000","last_fullsized":"2023-09-23T22:21:58.817464+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:52.228490+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3],"avail_no_missing":["5","2","3"],"object_location_counts":[{"shards":"2,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":170,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923597+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:24:07.923597+0000","last_peered":"2023-09-23T22:24:07.923597+0000","last_clean":"2023-09-23T22:24:07.923597+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:24:07.923597+0000","last_undegraded":"2023-09-23T22:24:07.923597+0000","last_fullsized":"2023-09-23T22:24:07.923597+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.922452+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:24:07.922452+0000","last_peered":"2023-09-23T22:24:07.922452+0000","last_clean":"2023-09-23T22:24:07.922452+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:24:07.922452+0000","last_undegraded":"2023-09-23T22:24:07.922452+0000","last_fullsized":"2023-09-23T22:24:07.922452+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"254'9694","reported_seq":9771,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:14.924218+0000","last_change":"2023-09-23T22:22:18.973077+0000","last_active":"2023-09-23T22:24:14.924218+0000","last_peered":"2023-09-23T22:24:14.924218+0000","last_clean":"2023-09-23T22:21:32.779211+0000","last_became_active":"2023-09-23T22:22:18.949524+0000","last_became_peered":"2023-09-23T22:22:18.949524+0000","last_unstale":"2023-09-23T22:24:14.924218+0000","last_undegraded":"2023-09-23T22:22:18.870000+0000","last_fullsized":"2023-09-23T22:22:18.859180+0000","mapping_epoch":231,"log_start":"218'4200","ondisk_log_start":"218'4200","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5494,"log_dups_size":0,"ondisk_log_size":5494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:11.194549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22413312,"num_objects":342,"num_object_clones":0,"num_object_copies":1026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":914,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9694,"num_write_kb":321152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"1,5,7","objects":342},{"shards":"5,7","objects":-145}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":215,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.922193+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:24:07.922193+0000","last_peered":"2023-09-23T22:24:07.922193+0000","last_clean":"2023-09-23T22:24:07.922193+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:24:07.922193+0000","last_undegraded":"2023-09-23T22:24:07.922193+0000","last_fullsized":"2023-09-23T22:24:07.922193+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":230,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.924746+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:24:07.924746+0000","last_peered":"2023-09-23T22:24:07.924746+0000","last_clean":"2023-09-23T22:24:07.924746+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:24:07.924746+0000","last_undegraded":"2023-09-23T22:24:07.924746+0000","last_fullsized":"2023-09-23T22:24:07.924746+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"254'9773","reported_seq":9833,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:24:14.894249+0000","last_change":"2023-09-23T22:20:19.473637+0000","last_active":"2023-09-23T22:24:14.894249+0000","last_peered":"2023-09-23T22:24:14.894249+0000","last_clean":"2023-09-23T22:24:14.894249+0000","last_became_active":"2023-09-23T22:20:19.473385+0000","last_became_peered":"2023-09-23T22:20:19.473385+0000","last_unstale":"2023-09-23T22:24:14.894249+0000","last_undegraded":"2023-09-23T22:24:14.894249+0000","last_fullsized":"2023-09-23T22:24:14.894249+0000","mapping_epoch":203,"log_start":"218'4300","ondisk_log_start":"218'4300","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:01.911861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26607640,"num_objects":407,"num_object_clones":0,"num_object_copies":1221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":407,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9773,"num_write_kb":325697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":558,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.919898+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:24:07.919898+0000","last_peered":"2023-09-23T22:24:07.919898+0000","last_clean":"2023-09-23T22:24:07.919898+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:24:07.919898+0000","last_undegraded":"2023-09-23T22:24:07.919898+0000","last_fullsized":"2023-09-23T22:24:07.919898+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":33,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.910438+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:24:07.910438+0000","last_peered":"2023-09-23T22:24:07.910438+0000","last_clean":"2023-09-23T22:24:07.910438+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:24:07.910438+0000","last_undegraded":"2023-09-23T22:24:07.910438+0000","last_fullsized":"2023-09-23T22:24:07.910438+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":196,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:08.809624+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:24:08.809624+0000","last_peered":"2023-09-23T22:24:08.809624+0000","last_clean":"2023-09-23T22:24:08.809624+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:24:08.809624+0000","last_undegraded":"2023-09-23T22:24:08.809624+0000","last_fullsized":"2023-09-23T22:24:08.809624+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"254'9995","reported_seq":19278,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:24:16.576429+0000","last_change":"2023-09-23T22:24:08.940015+0000","last_active":"2023-09-23T22:24:16.576429+0000","last_peered":"2023-09-23T22:24:16.576429+0000","last_clean":"2023-09-23T22:24:16.576429+0000","last_became_active":"2023-09-23T22:24:08.914451+0000","last_became_peered":"2023-09-23T22:24:08.914451+0000","last_unstale":"2023-09-23T22:24:16.576429+0000","last_undegraded":"2023-09-23T22:24:16.576429+0000","last_fullsized":"2023-09-23T22:24:16.576429+0000","mapping_epoch":253,"log_start":"229'5147","ondisk_log_start":"229'5147","created":203,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4848,"log_dups_size":0,"ondisk_log_size":4848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:13.769518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20512768,"num_objects":313,"num_object_clones":0,"num_object_copies":939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9995,"num_write_kb":329856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4602,"num_bytes_recovered":222363648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":549,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.915017+0000","last_change":"2023-09-23T22:22:47.349660+0000","last_active":"2023-09-23T22:24:07.915017+0000","last_peered":"2023-09-23T22:24:07.915017+0000","last_clean":"2023-09-23T22:24:07.915017+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:24:07.915017+0000","last_undegraded":"2023-09-23T22:24:07.915017+0000","last_fullsized":"2023-09-23T22:24:07.915017+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":803,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.924797+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:24:07.924797+0000","last_peered":"2023-09-23T22:24:07.924797+0000","last_clean":"2023-09-23T22:24:07.924797+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:24:07.924797+0000","last_undegraded":"2023-09-23T22:24:07.924797+0000","last_fullsized":"2023-09-23T22:24:07.924797+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":271,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.915745+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:24:07.915745+0000","last_peered":"2023-09-23T22:24:07.915745+0000","last_clean":"2023-09-23T22:24:07.915745+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:24:07.915745+0000","last_undegraded":"2023-09-23T22:24:07.915745+0000","last_fullsized":"2023-09-23T22:24:07.915745+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":40,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923978+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:24:07.923978+0000","last_peered":"2023-09-23T22:24:07.923978+0000","last_clean":"2023-09-23T22:24:07.923978+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:24:07.923978+0000","last_undegraded":"2023-09-23T22:24:07.923978+0000","last_fullsized":"2023-09-23T22:24:07.923978+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"254'9857","reported_seq":9948,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:14.890440+0000","last_change":"2023-09-23T22:22:18.955534+0000","last_active":"2023-09-23T22:24:14.890440+0000","last_peered":"2023-09-23T22:24:14.890440+0000","last_clean":"2023-09-23T22:21:32.860473+0000","last_became_active":"2023-09-23T22:22:18.941549+0000","last_became_peered":"2023-09-23T22:22:18.941549+0000","last_unstale":"2023-09-23T22:24:14.890440+0000","last_undegraded":"2023-09-23T22:22:18.875275+0000","last_fullsized":"2023-09-23T22:22:18.863381+0000","mapping_epoch":231,"log_start":"218'4500","ondisk_log_start":"218'4500","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5357,"log_dups_size":0,"ondisk_log_size":5357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:14:27.412277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24576000,"num_objects":375,"num_object_clones":0,"num_object_copies":1125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":988,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9857,"num_write_kb":327424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,5,7","objects":375},{"shards":"4,7","objects":-152}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":558,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.915317+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:24:07.915317+0000","last_peered":"2023-09-23T22:24:07.915317+0000","last_clean":"2023-09-23T22:24:07.915317+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:24:07.915317+0000","last_undegraded":"2023-09-23T22:24:07.915317+0000","last_fullsized":"2023-09-23T22:24:07.915317+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":986,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.920104+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:24:07.920104+0000","last_peered":"2023-09-23T22:24:07.920104+0000","last_clean":"2023-09-23T22:24:07.920104+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:24:07.920104+0000","last_undegraded":"2023-09-23T22:24:07.920104+0000","last_fullsized":"2023-09-23T22:24:07.920104+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":263,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.922864+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:24:07.922864+0000","last_peered":"2023-09-23T22:24:07.922864+0000","last_clean":"2023-09-23T22:24:07.922864+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:24:07.922864+0000","last_undegraded":"2023-09-23T22:24:07.922864+0000","last_fullsized":"2023-09-23T22:24:07.922864+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":218,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.916367+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:24:07.916367+0000","last_peered":"2023-09-23T22:24:07.916367+0000","last_clean":"2023-09-23T22:24:07.916367+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:24:07.916367+0000","last_undegraded":"2023-09-23T22:24:07.916367+0000","last_fullsized":"2023-09-23T22:24:07.916367+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.5","version":"254'9590","reported_seq":10229,"reported_epoch":254,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:14.327862+0000","last_change":"2023-09-23T22:24:07.703920+0000","last_active":"2023-09-23T22:24:14.327862+0000","last_peered":"2023-09-23T22:24:14.327862+0000","last_clean":"2023-09-23T22:21:56.814967+0000","last_became_active":"2023-09-23T22:21:59.002385+0000","last_became_peered":"2023-09-23T22:21:59.002385+0000","last_unstale":"2023-09-23T22:24:14.327862+0000","last_undegraded":"2023-09-23T22:21:58.865154+0000","last_fullsized":"2023-09-23T22:21:58.825566+0000","mapping_epoch":226,"log_start":"218'4220","ondisk_log_start":"218'4220","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5370,"log_dups_size":0,"ondisk_log_size":5370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:00.273421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25165824,"num_objects":384,"num_object_clones":0,"num_object_copies":1152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4367,"num_objects_misplaced":266,"num_objects_unfound":0,"num_objects_dirty":384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9590,"num_write_kb":319168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":285,"num_bytes_recovered":18677760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7],"avail_no_missing":["3","2","7"],"object_location_counts":[{"shards":"2,3,7","objects":266},{"shards":"3,7","objects":-4249}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":583,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.897412+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:24:07.897412+0000","last_peered":"2023-09-23T22:24:07.897412+0000","last_clean":"2023-09-23T22:24:07.897412+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:24:07.897412+0000","last_undegraded":"2023-09-23T22:24:07.897412+0000","last_fullsized":"2023-09-23T22:24:07.897412+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":360,"reported_epoch":253,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:24:07.913680+0000","last_change":"2023-09-23T22:22:18.917671+0000","last_active":"2023-09-23T22:24:07.913680+0000","last_peered":"2023-09-23T22:24:07.913680+0000","last_clean":"2023-09-23T22:15:39.913392+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:24:07.913680+0000","last_undegraded":"2023-09-23T22:22:18.874220+0000","last_fullsized":"2023-09-23T22:24:07.913680+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":214,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923668+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:24:07.923668+0000","last_peered":"2023-09-23T22:24:07.923668+0000","last_clean":"2023-09-23T22:24:07.923668+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:24:07.923668+0000","last_undegraded":"2023-09-23T22:24:07.923668+0000","last_fullsized":"2023-09-23T22:24:07.923668+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":261,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923827+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:24:07.923827+0000","last_peered":"2023-09-23T22:24:07.923827+0000","last_clean":"2023-09-23T22:24:07.923827+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:24:07.923827+0000","last_undegraded":"2023-09-23T22:24:07.923827+0000","last_fullsized":"2023-09-23T22:24:07.923827+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"254'9927","reported_seq":9986,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T22:24:14.924913+0000","last_change":"2023-09-23T22:20:19.470871+0000","last_active":"2023-09-23T22:24:14.924913+0000","last_peered":"2023-09-23T22:24:14.924913+0000","last_clean":"2023-09-23T22:24:14.924913+0000","last_became_active":"2023-09-23T22:20:19.470597+0000","last_became_peered":"2023-09-23T22:20:19.470597+0000","last_unstale":"2023-09-23T22:24:14.924913+0000","last_undegraded":"2023-09-23T22:24:14.924913+0000","last_fullsized":"2023-09-23T22:24:14.924913+0000","mapping_epoch":203,"log_start":"218'4400","ondisk_log_start":"218'4400","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5527,"log_dups_size":0,"ondisk_log_size":5527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:32.388985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26542080,"num_objects":405,"num_object_clones":0,"num_object_copies":1215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9927,"num_write_kb":330624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":849,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:07.915619+0000","last_change":"2023-09-23T22:21:58.853256+0000","last_active":"2023-09-23T22:24:07.915619+0000","last_peered":"2023-09-23T22:24:07.915619+0000","last_clean":"2023-09-23T22:21:50.447508+0000","last_became_active":"2023-09-23T22:21:58.852815+0000","last_became_peered":"2023-09-23T22:21:58.852815+0000","last_unstale":"2023-09-23T22:24:07.915619+0000","last_undegraded":"2023-09-23T22:21:58.818640+0000","last_fullsized":"2023-09-23T22:21:58.817793+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:25.619025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":566,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.904851+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:24:07.904851+0000","last_peered":"2023-09-23T22:24:07.904851+0000","last_clean":"2023-09-23T22:24:07.904851+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:24:07.904851+0000","last_undegraded":"2023-09-23T22:24:07.904851+0000","last_fullsized":"2023-09-23T22:24:07.904851+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":178,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.910674+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:24:07.910674+0000","last_peered":"2023-09-23T22:24:07.910674+0000","last_clean":"2023-09-23T22:24:07.910674+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:24:07.910674+0000","last_undegraded":"2023-09-23T22:24:07.910674+0000","last_fullsized":"2023-09-23T22:24:07.910674+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":140,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.922659+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:24:07.922659+0000","last_peered":"2023-09-23T22:24:07.922659+0000","last_clean":"2023-09-23T22:24:07.922659+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:24:07.922659+0000","last_undegraded":"2023-09-23T22:24:07.922659+0000","last_fullsized":"2023-09-23T22:24:07.922659+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"254'9756","reported_seq":9838,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:18.826835+0000","last_change":"2023-09-23T22:22:18.954521+0000","last_active":"2023-09-23T22:24:18.826835+0000","last_peered":"2023-09-23T22:24:18.826835+0000","last_clean":"2023-09-23T22:21:32.869170+0000","last_became_active":"2023-09-23T22:22:18.945835+0000","last_became_peered":"2023-09-23T22:22:18.945835+0000","last_unstale":"2023-09-23T22:24:18.826835+0000","last_undegraded":"2023-09-23T22:22:18.869474+0000","last_fullsized":"2023-09-23T22:22:18.861404+0000","mapping_epoch":231,"log_start":"229'5105","ondisk_log_start":"229'5105","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4651,"log_dups_size":0,"ondisk_log_size":4651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:30:02.907936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19267584,"num_objects":294,"num_object_clones":0,"num_object_copies":882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1045,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9756,"num_write_kb":321600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,4,5","objects":294},{"shards":"1,5","objects":-297}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":585,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.924805+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:24:07.924805+0000","last_peered":"2023-09-23T22:24:07.924805+0000","last_clean":"2023-09-23T22:24:07.924805+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:24:07.924805+0000","last_undegraded":"2023-09-23T22:24:07.924805+0000","last_fullsized":"2023-09-23T22:24:07.924805+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":604,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923873+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:24:07.923873+0000","last_peered":"2023-09-23T22:24:07.923873+0000","last_clean":"2023-09-23T22:24:07.923873+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:24:07.923873+0000","last_undegraded":"2023-09-23T22:24:07.923873+0000","last_fullsized":"2023-09-23T22:24:07.923873+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":136,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.898395+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:24:07.898395+0000","last_peered":"2023-09-23T22:24:07.898395+0000","last_clean":"2023-09-23T22:24:07.898395+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:24:07.898395+0000","last_undegraded":"2023-09-23T22:24:07.898395+0000","last_fullsized":"2023-09-23T22:24:07.898395+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":93,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.896759+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:24:07.896759+0000","last_peered":"2023-09-23T22:24:07.896759+0000","last_clean":"2023-09-23T22:24:07.896759+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:24:07.896759+0000","last_undegraded":"2023-09-23T22:24:07.896759+0000","last_fullsized":"2023-09-23T22:24:07.896759+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"254'9990","reported_seq":14172,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:18.832644+0000","last_change":"2023-09-23T22:22:45.524797+0000","last_active":"2023-09-23T22:24:18.832644+0000","last_peered":"2023-09-23T22:24:18.832644+0000","last_clean":"2023-09-23T22:21:12.245233+0000","last_became_active":"2023-09-23T22:22:45.454018+0000","last_became_peered":"2023-09-23T22:22:45.454018+0000","last_unstale":"2023-09-23T22:24:18.832644+0000","last_undegraded":"2023-09-23T22:22:45.437539+0000","last_fullsized":"2023-09-23T22:22:45.322462+0000","mapping_epoch":240,"log_start":"229'5330","ondisk_log_start":"229'5330","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4660,"log_dups_size":0,"ondisk_log_size":4660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:10.659044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16252928,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3914,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9990,"num_write_kb":327616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1472,"num_bytes_recovered":96468992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,5","objects":247},{"shards":"1,7","objects":-2514}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":575,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.897024+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:24:07.897024+0000","last_peered":"2023-09-23T22:24:07.897024+0000","last_clean":"2023-09-23T22:24:07.897024+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:24:07.897024+0000","last_undegraded":"2023-09-23T22:24:07.897024+0000","last_fullsized":"2023-09-23T22:24:07.897024+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":145,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.897046+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:24:07.897046+0000","last_peered":"2023-09-23T22:24:07.897046+0000","last_clean":"2023-09-23T22:24:07.897046+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:24:07.897046+0000","last_undegraded":"2023-09-23T22:24:07.897046+0000","last_fullsized":"2023-09-23T22:24:07.897046+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":205,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.925112+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:24:07.925112+0000","last_peered":"2023-09-23T22:24:07.925112+0000","last_clean":"2023-09-23T22:24:07.925112+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:24:07.925112+0000","last_undegraded":"2023-09-23T22:24:07.925112+0000","last_fullsized":"2023-09-23T22:24:07.925112+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":109,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.898036+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:24:07.898036+0000","last_peered":"2023-09-23T22:24:07.898036+0000","last_clean":"2023-09-23T22:24:07.898036+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:24:07.898036+0000","last_undegraded":"2023-09-23T22:24:07.898036+0000","last_fullsized":"2023-09-23T22:24:07.898036+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"254'9902","reported_seq":17441,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:14.906240+0000","last_change":"2023-09-23T22:22:19.043755+0000","last_active":"2023-09-23T22:24:14.906240+0000","last_peered":"2023-09-23T22:24:14.906240+0000","last_clean":"2023-09-23T22:21:12.214088+0000","last_became_active":"2023-09-23T22:22:18.986181+0000","last_became_peered":"2023-09-23T22:22:18.986181+0000","last_unstale":"2023-09-23T22:24:14.906240+0000","last_undegraded":"2023-09-23T22:22:18.975289+0000","last_fullsized":"2023-09-23T22:22:18.862148+0000","mapping_epoch":231,"log_start":"218'4600","ondisk_log_start":"218'4600","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":5302,"log_dups_size":0,"ondisk_log_size":5302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:34.422545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26869760,"num_objects":410,"num_object_clones":0,"num_object_copies":1230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1882,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9902,"num_write_kb":329984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3724,"num_bytes_recovered":244056064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":327},{"shards":"4,5,7","objects":83},{"shards":"4,7","objects":-1022}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":558,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.922309+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:24:07.922309+0000","last_peered":"2023-09-23T22:24:07.922309+0000","last_clean":"2023-09-23T22:24:07.922309+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:24:07.922309+0000","last_undegraded":"2023-09-23T22:24:07.922309+0000","last_fullsized":"2023-09-23T22:24:07.922309+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":267,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.915230+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:24:07.915230+0000","last_peered":"2023-09-23T22:24:07.915230+0000","last_clean":"2023-09-23T22:24:07.915230+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:24:07.915230+0000","last_undegraded":"2023-09-23T22:24:07.915230+0000","last_fullsized":"2023-09-23T22:24:07.915230+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":896,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.924353+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:24:07.924353+0000","last_peered":"2023-09-23T22:24:07.924353+0000","last_clean":"2023-09-23T22:24:07.924353+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:24:07.924353+0000","last_undegraded":"2023-09-23T22:24:07.924353+0000","last_fullsized":"2023-09-23T22:24:07.924353+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":42,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.910214+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:24:07.910214+0000","last_peered":"2023-09-23T22:24:07.910214+0000","last_clean":"2023-09-23T22:24:07.910214+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:24:07.910214+0000","last_undegraded":"2023-09-23T22:24:07.910214+0000","last_fullsized":"2023-09-23T22:24:07.910214+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"254'9744","reported_seq":9947,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:24:18.742311+0000","last_change":"2023-09-23T22:22:44.336083+0000","last_active":"2023-09-23T22:24:18.742311+0000","last_peered":"2023-09-23T22:24:18.742311+0000","last_clean":"2023-09-23T22:21:56.808651+0000","last_became_active":"2023-09-23T22:21:59.059105+0000","last_became_peered":"2023-09-23T22:21:59.059105+0000","last_unstale":"2023-09-23T22:24:18.742311+0000","last_undegraded":"2023-09-23T22:21:58.989636+0000","last_fullsized":"2023-09-23T22:21:58.812134+0000","mapping_epoch":226,"log_start":"229'5102","ondisk_log_start":"229'5102","created":203,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:20:18.446560+0000","last_clean_scrub_stamp":"2023-09-23T22:20:18.446560+0000","objects_scrubbed":0,"log_size":4642,"log_dups_size":0,"ondisk_log_size":4642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:51.370288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16121856,"num_objects":246,"num_object_clones":0,"num_object_copies":738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4852,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9744,"num_write_kb":319680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":4259840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":167},{"shards":"1,7","objects":-4606}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":541,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.913058+0000","last_change":"2023-09-23T22:22:19.883431+0000","last_active":"2023-09-23T22:24:07.913058+0000","last_peered":"2023-09-23T22:24:07.913058+0000","last_clean":"2023-09-23T22:24:07.913058+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:24:07.913058+0000","last_undegraded":"2023-09-23T22:24:07.913058+0000","last_fullsized":"2023-09-23T22:24:07.913058+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:46.903071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:46.903071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":236,"reported_epoch":254,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:24:08.912022+0000","last_change":"2023-09-23T22:22:44.342884+0000","last_active":"2023-09-23T22:24:08.912022+0000","last_peered":"2023-09-23T22:24:08.912022+0000","last_clean":"2023-09-23T22:06:25.449250+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:24:08.912022+0000","last_undegraded":"2023-09-23T22:22:44.329336+0000","last_fullsized":"2023-09-23T22:24:08.912022+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":["3","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":88,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.896933+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:24:07.896933+0000","last_peered":"2023-09-23T22:24:07.896933+0000","last_clean":"2023-09-23T22:24:07.896933+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:24:07.896933+0000","last_undegraded":"2023-09-23T22:24:07.896933+0000","last_fullsized":"2023-09-23T22:24:07.896933+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":217,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.921915+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:24:07.921915+0000","last_peered":"2023-09-23T22:24:07.921915+0000","last_clean":"2023-09-23T22:24:07.921915+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:24:07.921915+0000","last_undegraded":"2023-09-23T22:24:07.921915+0000","last_fullsized":"2023-09-23T22:24:07.921915+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":84,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.897960+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:24:07.897960+0000","last_peered":"2023-09-23T22:24:07.897960+0000","last_clean":"2023-09-23T22:24:07.897960+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:24:07.897960+0000","last_undegraded":"2023-09-23T22:24:07.897960+0000","last_fullsized":"2023-09-23T22:24:07.897960+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":265,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.925026+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:24:07.925026+0000","last_peered":"2023-09-23T22:24:07.925026+0000","last_clean":"2023-09-23T22:24:07.925026+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:24:07.925026+0000","last_undegraded":"2023-09-23T22:24:07.925026+0000","last_fullsized":"2023-09-23T22:24:07.925026+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":216,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.916229+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:24:07.916229+0000","last_peered":"2023-09-23T22:24:07.916229+0000","last_clean":"2023-09-23T22:24:07.916229+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:24:07.916229+0000","last_undegraded":"2023-09-23T22:24:07.916229+0000","last_fullsized":"2023-09-23T22:24:07.916229+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":229,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923095+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:24:07.923095+0000","last_peered":"2023-09-23T22:24:07.923095+0000","last_clean":"2023-09-23T22:24:07.923095+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:24:07.923095+0000","last_undegraded":"2023-09-23T22:24:07.923095+0000","last_fullsized":"2023-09-23T22:24:07.923095+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":213,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.915833+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:24:07.915833+0000","last_peered":"2023-09-23T22:24:07.915833+0000","last_clean":"2023-09-23T22:24:07.915833+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:24:07.915833+0000","last_undegraded":"2023-09-23T22:24:07.915833+0000","last_fullsized":"2023-09-23T22:24:07.915833+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":88,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.923674+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:24:07.923674+0000","last_peered":"2023-09-23T22:24:07.923674+0000","last_clean":"2023-09-23T22:24:07.923674+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:24:07.923674+0000","last_undegraded":"2023-09-23T22:24:07.923674+0000","last_fullsized":"2023-09-23T22:24:07.923674+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":234,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.915036+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:24:07.915036+0000","last_peered":"2023-09-23T22:24:07.915036+0000","last_clean":"2023-09-23T22:24:07.915036+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:24:07.915036+0000","last_undegraded":"2023-09-23T22:24:07.915036+0000","last_fullsized":"2023-09-23T22:24:07.915036+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":276,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.913479+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:24:07.913479+0000","last_peered":"2023-09-23T22:24:07.913479+0000","last_clean":"2023-09-23T22:24:07.913479+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:24:07.913479+0000","last_undegraded":"2023-09-23T22:24:07.913479+0000","last_fullsized":"2023-09-23T22:24:07.913479+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":143,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.897520+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:24:07.897520+0000","last_peered":"2023-09-23T22:24:07.897520+0000","last_clean":"2023-09-23T22:24:07.897520+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:24:07.897520+0000","last_undegraded":"2023-09-23T22:24:07.897520+0000","last_fullsized":"2023-09-23T22:24:07.897520+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":262,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T22:24:07.921700+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:24:07.921700+0000","last_peered":"2023-09-23T22:24:07.921700+0000","last_clean":"2023-09-23T22:24:07.921700+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:24:07.921700+0000","last_undegraded":"2023-09-23T22:24:07.921700+0000","last_fullsized":"2023-09-23T22:24:07.921700+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":377028632,"num_objects":5754,"num_object_clones":0,"num_object_copies":17262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":30037,"num_objects_misplaced":672,"num_objects_unfound":0,"num_objects_dirty":5754,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":156942,"num_write_kb":5206209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23994,"num_bytes_recovered":1442906112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4344590336,"data_stored":4344578120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83017,"ondisk_log_size":83017,"up":48,"acting":37,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":370028,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":88,"num_read_kb":166,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":649699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234238,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":711004,"kb_used_data":522068,"kb_used_omap":0,"kb_used_meta":188928,"kb_avail":93034148,"statfs":{"total":95995035648,"available":95266967552,"internally_reserved":0,"allocated":534597632,"data_stored":533578202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":193462272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":4.1870000000000003,"15min":2.3159999999999998},"min":{"1min":0.22600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.5499999999999998,"5min":158.042,"15min":158.042},"last":3.6989999999999998},{"interface":"front","average":{"1min":2.4369999999999998,"5min":4.6920000000000002,"15min":2.6070000000000002},"min":{"1min":0.28199999999999997,"5min":0.217,"15min":0.217},"max":{"1min":6.6719999999999997,"5min":157.952,"15min":157.952},"last":0.38900000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:23:12 2023","interfaces":[{"interface":"back","average":{"1min":2.8479999999999999,"5min":4.9329999999999998,"15min":2.9809999999999999},"min":{"1min":0.28299999999999997,"5min":0.254,"15min":0.246},"max":{"1min":10.127000000000001,"5min":101.145,"15min":123.79300000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.5859999999999999,"5min":4.9909999999999997,"15min":3.1309999999999998},"min":{"1min":0.28199999999999997,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":10.92,"5min":101.13,"15min":123.66500000000001},"last":0.71499999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":4.1260000000000003,"15min":3.7650000000000001},"min":{"1min":0.307,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.7939999999999996,"5min":105.34399999999999,"15min":125.101},"last":0.309},{"interface":"front","average":{"1min":2.6440000000000001,"5min":4.0960000000000001,"15min":3.5110000000000001},"min":{"1min":0.30499999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.3769999999999998,"5min":105.63200000000001,"15min":126.021},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:23:12 2023","interfaces":[{"interface":"back","average":{"1min":10.196999999999999,"5min":3.7349999999999999,"15min":2.1890000000000001},"min":{"1min":0.36699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":170.86099999999999,"5min":170.86099999999999,"15min":170.86099999999999},"last":4.4530000000000003},{"interface":"front","average":{"1min":9.5510000000000002,"5min":3.9239999999999999,"15min":2.3170000000000002},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":174.57499999999999,"5min":174.57499999999999,"15min":174.57499999999999},"last":4.6660000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":2.2120000000000002,"15min":1.3280000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.5169999999999999,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.29,"15min":1.3009999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.6719999999999997,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.28299999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":2.2530000000000001,"15min":1.2509999999999999},"min":{"1min":0.30299999999999999,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":3.21,"5min":93.816000000000003,"15min":93.816000000000003},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.81499999999999995,"5min":2.2970000000000002,"15min":1.466},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.544,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.32900000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.68100000000000005,"15min":0.74099999999999999},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.7120000000000002,"5min":5.343,"15min":9.0139999999999993},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.86299999999999999,"15min":0.93899999999999995},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":3.6339999999999999,"5min":6.3879999999999999,"15min":13.362},"last":0.35899999999999999}]}]},{"osd":6,"up_from":0,"seq":197568495874,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795147,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":501532,"kb_used_data":178196,"kb_used_omap":7,"kb_used_meta":323256,"kb_avail":93243620,"statfs":{"total":95995035648,"available":95481466880,"internally_reserved":0,"allocated":182472704,"data_stored":181101331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":331014516},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,3,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":4.8689999999999998,"5min":2.113,"15min":1.3149999999999999},"min":{"1min":0.255,"5min":0.189,"15min":0.189},"max":{"1min":56.503,"5min":56.503,"15min":61.890000000000001},"last":10.125999999999999},{"interface":"front","average":{"1min":5.1550000000000002,"5min":3.0230000000000001,"15min":1.728},"min":{"1min":0.40799999999999997,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":56.560000000000002,"5min":56.560000000000002,"15min":62.689999999999998},"last":15.965999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:23:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7749999999999999,"5min":1.294,"15min":1.2789999999999999},"min":{"1min":0.27600000000000002,"5min":0.223,"15min":0.216},"max":{"1min":7.694,"5min":11.179,"15min":33.902000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":3.734,"5min":2.4660000000000002,"15min":1.726},"min":{"1min":0.312,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.8369999999999997,"5min":15.015000000000001,"15min":34.781999999999996},"last":16.053000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:23:20 2023","interfaces":[{"interface":"back","average":{"1min":8.5220000000000002,"5min":4.3360000000000003,"15min":2.0590000000000002},"min":{"1min":0.47599999999999998,"5min":0.252,"15min":0.252},"max":{"1min":110.417,"5min":110.417,"15min":110.417},"last":13.871},{"interface":"front","average":{"1min":8.5239999999999991,"5min":4.1820000000000004,"15min":2.0179999999999998},"min":{"1min":0.33400000000000002,"5min":0.223,"15min":0.223},"max":{"1min":110.721,"5min":110.721,"15min":110.721},"last":16.346}]},{"osd":4,"last update":"Sat Sep 23 22:23:20 2023","interfaces":[{"interface":"back","average":{"1min":8.6099999999999994,"5min":4.6239999999999997,"15min":2.056},"min":{"1min":0.314,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":171.09999999999999,"5min":234.79900000000001,"15min":234.79900000000001},"last":0.434},{"interface":"front","average":{"1min":9.1289999999999996,"5min":5.2699999999999996,"15min":2.3519999999999999},"min":{"1min":0.44700000000000001,"5min":0.33200000000000002,"15min":0.318},"max":{"1min":171.08500000000001,"5min":234.81700000000001,"15min":234.81700000000001},"last":2.5619999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":3.6850000000000001,"15min":1.788},"min":{"1min":0.39100000000000001,"5min":0.31,"15min":0.31},"max":{"1min":9.5709999999999997,"5min":196.79900000000001,"15min":196.79900000000001},"last":2.6059999999999999},{"interface":"front","average":{"1min":2.9340000000000002,"5min":3.6499999999999999,"15min":1.7989999999999999},"min":{"1min":0.41299999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.01,"5min":196.68299999999999,"15min":196.68299999999999},"last":16.207000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.8940000000000001,"5min":1.913,"15min":1.2490000000000001},"min":{"1min":0.497,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":10.074999999999999,"5min":10.971,"15min":10.989000000000001},"last":16.314},{"interface":"front","average":{"1min":1.448,"5min":1.296,"15min":1.006},"min":{"1min":0.437,"5min":0.35699999999999998,"15min":0.34999999999999998},"max":{"1min":7.7300000000000004,"5min":11.385999999999999,"15min":11.385999999999999},"last":0.48699999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.6970000000000001,"5min":1.9339999999999999,"15min":1.2889999999999999},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.048,"5min":10.933999999999999,"15min":10.948},"last":16.265000000000001},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.3200000000000001,"15min":1.0589999999999999},"min":{"1min":0.40699999999999997,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":12.045999999999999,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.46400000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958681,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":434888,"kb_used_data":137664,"kb_used_omap":8,"kb_used_meta":297143,"kb_avail":93310264,"statfs":{"total":95995035648,"available":95549710336,"internally_reserved":0,"allocated":140967936,"data_stored":139558993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":304275306},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":5.2009999999999996,"5min":4.5810000000000004,"15min":2.274},"min":{"1min":0.21299999999999999,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":51.018999999999998,"5min":169.77199999999999,"15min":169.77199999999999},"last":0.30299999999999999},{"interface":"front","average":{"1min":3.6739999999999999,"5min":5.0860000000000003,"15min":2.5640000000000001},"min":{"1min":0.24099999999999999,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":23.891999999999999,"5min":169.52099999999999,"15min":169.52099999999999},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":6.1360000000000001,"15min":2.6800000000000002},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":16.097999999999999,"5min":206.495,"15min":206.495},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.258,"5min":6.6130000000000004,"15min":3.3050000000000002},"min":{"1min":0.29299999999999998,"5min":0.28499999999999998,"15min":0.22800000000000001},"max":{"1min":11.138,"5min":207.34399999999999,"15min":207.34399999999999},"last":1.1579999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":2.4350000000000001,"5min":3.7160000000000002,"15min":2.0009999999999999},"min":{"1min":0.29299999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.3399999999999999,"5min":131.078,"15min":131.078},"last":4.7469999999999999},{"interface":"front","average":{"1min":3.0070000000000001,"5min":3.9540000000000002,"15min":2.1040000000000001},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":10.438000000000001,"5min":133.083,"15min":133.083},"last":4.7830000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":3.7839999999999998,"5min":3.1299999999999999,"15min":1.994},"min":{"1min":0.33500000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":16.347999999999999,"5min":82.825000000000003,"15min":82.825000000000003},"last":0.55400000000000005},{"interface":"front","average":{"1min":2.5910000000000002,"5min":2.2269999999999999,"15min":1.3799999999999999},"min":{"1min":0.377,"5min":0.313,"15min":0.308},"max":{"1min":16.184000000000001,"5min":75.930999999999997,"15min":75.930999999999997},"last":0.52700000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":3.8180000000000001,"5min":4.2919999999999998,"15min":2.3809999999999998},"min":{"1min":0.44,"5min":0.34799999999999998,"15min":0.29499999999999998},"max":{"1min":16.323,"5min":93.685000000000002,"15min":93.685000000000002},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.431,"5min":3.4980000000000002,"15min":1.819},"min":{"1min":0.38600000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":16.152000000000001,"5min":93.546000000000006,"15min":93.546000000000006},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":4.093,"5min":2.2130000000000001,"15min":1.6990000000000001},"min":{"1min":0.47799999999999998,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":16.366,"5min":16.366,"15min":16.366},"last":1.488},{"interface":"front","average":{"1min":2.3519999999999999,"5min":1.661,"15min":1.224},"min":{"1min":0.439,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":16.225999999999999,"5min":16.225999999999999,"15min":22.263000000000002},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":2.21,"15min":1.746},"min":{"1min":0.44600000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.718,"5min":32.773000000000003,"15min":32.773000000000003},"last":0.432},{"interface":"front","average":{"1min":4.1059999999999999,"5min":2.343,"15min":1.9079999999999999},"min":{"1min":0.50600000000000001,"5min":0.39900000000000002,"15min":0.38900000000000001},"max":{"1min":16.381,"5min":16.381,"15min":23.265999999999998},"last":0.63}]}]},{"osd":2,"up_from":0,"seq":442381631620,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468093,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":551044,"kb_used_data":320380,"kb_used_omap":4,"kb_used_meta":230651,"kb_avail":93194108,"statfs":{"total":95995035648,"available":95430766592,"internally_reserved":0,"allocated":328069120,"data_stored":326658448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":236187181},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":5.7969999999999997,"15min":2.7069999999999999},"min":{"1min":0.23699999999999999,"5min":0.23000000000000001,"15min":0.19900000000000001},"max":{"1min":23.890000000000001,"5min":225.86699999999999,"15min":225.86699999999999},"last":3.782},{"interface":"front","average":{"1min":4.4619999999999997,"5min":6.2400000000000002,"15min":2.9340000000000002},"min":{"1min":0.23699999999999999,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":24.053000000000001,"5min":225.84800000000001,"15min":225.84800000000001},"last":3.7440000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.427,"5min":4.0819999999999999,"15min":2.1840000000000002},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.209,"5min":82.951999999999998,"15min":82.951999999999998},"last":2.2549999999999999},{"interface":"front","average":{"1min":4.75,"5min":4.4509999999999996,"15min":2.2170000000000001},"min":{"1min":0.34300000000000003,"5min":0.22600000000000001,"15min":0.189},"max":{"1min":13.85,"5min":86.001000000000005,"15min":86.001000000000005},"last":1.331}]},{"osd":2,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":5.5129999999999999,"15min":3.3849999999999998},"min":{"1min":0.501,"5min":0.252,"15min":0.223},"max":{"1min":8.1180000000000003,"5min":99.962999999999994,"15min":99.962999999999994},"last":2.2829999999999999},{"interface":"front","average":{"1min":2.2890000000000001,"5min":5.6790000000000003,"15min":3.2130000000000001},"min":{"1min":0.41399999999999998,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":9.5909999999999993,"5min":100.429,"15min":100.429},"last":1.296}]},{"osd":4,"last update":"Sat Sep 23 22:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.9850000000000001,"15min":1.1839999999999999},"min":{"1min":0.38400000000000001,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":6.2960000000000003,"5min":11.911,"15min":11.911},"last":1.2330000000000001},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.764,"15min":1.2030000000000001},"min":{"1min":0.36599999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":10.949999999999999,"5min":10.949999999999999,"15min":12.036},"last":3.1320000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.7650000000000001,"5min":2.4220000000000002,"15min":1.391},"min":{"1min":0.39900000000000002,"5min":0.309,"15min":0.309},"max":{"1min":11.028,"5min":49.695999999999998,"15min":49.695999999999998},"last":3.1539999999999999},{"interface":"front","average":{"1min":1.6259999999999999,"5min":2.427,"15min":1.3799999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.883,"5min":49.668999999999997,"15min":49.668999999999997},"last":2.3010000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":2.496,"5min":2.1259999999999999,"15min":1.3320000000000001},"min":{"1min":0.42699999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":10.992000000000001,"5min":10.992000000000001,"15min":12.068},"last":3.2010000000000001},{"interface":"front","average":{"1min":1.893,"5min":1.8620000000000001,"15min":1.2410000000000001},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35399999999999998},"max":{"1min":5.9219999999999997,"5min":11.68,"15min":11.68},"last":2.3370000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:23:13 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.0110000000000001,"15min":1.3140000000000001},"min":{"1min":0.39700000000000002,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":6.0309999999999997,"5min":16.849,"15min":16.849},"last":2.3199999999999998},{"interface":"front","average":{"1min":2.3290000000000002,"5min":2.0910000000000002,"15min":1.272},"min":{"1min":0.46200000000000002,"5min":0.40400000000000003,"15min":0.36699999999999999},"max":{"1min":6.1749999999999998,"5min":11.945,"15min":12.382},"last":1.27}]}]},{"osd":4,"up_from":30,"seq":128849019143,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":396704,"kb_used_data":201176,"kb_used_omap":0,"kb_used_meta":195520,"kb_avail":93348448,"statfs":{"total":95995035648,"available":95588810752,"internally_reserved":0,"allocated":206004224,"data_stored":204981505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":200212480},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7069999999999999,"5min":2.9369999999999998,"15min":1.8540000000000001},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.218},"max":{"1min":25.425000000000001,"5min":46.18,"15min":66.355000000000004},"last":0.39900000000000002},{"interface":"front","average":{"1min":3.444,"5min":2.9039999999999999,"15min":1.921},"min":{"1min":0.28799999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":25.451000000000001,"5min":46.207000000000001,"15min":66.332999999999998},"last":0.32000000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":5.4039999999999999,"5min":2.7160000000000002,"15min":1.6699999999999999},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":85.662000000000006,"5min":85.662000000000006,"15min":85.662000000000006},"last":3.8460000000000001},{"interface":"front","average":{"1min":5.5999999999999996,"5min":3.0899999999999999,"15min":2.0289999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":81.195999999999998,"5min":81.195999999999998,"15min":81.195999999999998},"last":2}]},{"osd":2,"last update":"Sat Sep 23 22:23:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":3.472,"15min":2.2970000000000002},"min":{"1min":0.36099999999999999,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":10.051,"5min":138.06100000000001,"15min":138.06100000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.1030000000000002,"5min":3.3380000000000001,"15min":2.4340000000000002},"min":{"1min":0.35499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.2510000000000003,"5min":138.15199999999999,"15min":138.15199999999999},"last":3.6509999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:23:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":1.8220000000000001,"15min":1.196},"min":{"1min":0.33000000000000002,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":12.129,"5min":12.129,"15min":12.129},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.5870000000000002,"5min":1.8200000000000001,"15min":1.228},"min":{"1min":0.46300000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4059999999999997,"5min":8.8930000000000007,"15min":9.8109999999999999},"last":1.8899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:23:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.284,"15min":0.874},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.8570000000000002,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.5680000000000001,"5min":1.3759999999999999,"15min":0.90400000000000003},"min":{"1min":0.30599999999999999,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":7.298,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.373}]},{"osd":6,"last update":"Sat Sep 23 22:23:34 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":0.80800000000000005,"15min":0.68999999999999995},"min":{"1min":0.33200000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":2.262,"5min":7.3079999999999998,"15min":7.3079999999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.873,"5min":0.85299999999999998,"15min":0.749},"min":{"1min":0.29099999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.2679999999999998,"5min":6.7279999999999998,"15min":8.5210000000000008},"last":1.7150000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":0.92900000000000005,"15min":0.878},"min":{"1min":0.33300000000000002,"5min":0.313,"15min":0.307},"max":{"1min":3.347,"5min":6.7480000000000002,"15min":8.577},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.377,"5min":1.0149999999999999,"15min":0.83999999999999997},"min":{"1min":0.34200000000000003,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":7.8109999999999999,"5min":7.8109999999999999,"15min":7.8109999999999999},"last":0.55000000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757511,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":551576,"kb_used_data":311248,"kb_used_omap":0,"kb_used_meta":240320,"kb_avail":93193576,"statfs":{"total":95995035648,"available":95430221824,"internally_reserved":0,"allocated":318717952,"data_stored":317699202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":246087680},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:57 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":2.7989999999999999,"15min":2.1579999999999999},"min":{"1min":0.23999999999999999,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":64.344999999999999,"5min":64.344999999999999,"15min":133.22499999999999},"last":1.931},{"interface":"front","average":{"1min":4.891,"5min":2.528,"15min":2.181},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":64.022000000000006,"5min":64.022000000000006,"15min":132.78200000000001},"last":0.248}]},{"osd":1,"last update":"Sat Sep 23 22:23:16 2023","interfaces":[{"interface":"back","average":{"1min":2.2850000000000001,"5min":4.3090000000000002,"15min":2.681},"min":{"1min":0.27200000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":13.385999999999999,"5min":93.981999999999999,"15min":93.981999999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.9099999999999999,"5min":4.2329999999999997,"15min":2.6509999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":13.457000000000001,"5min":93.909999999999997,"15min":93.909999999999997},"last":10.949}]},{"osd":2,"last update":"Sat Sep 23 22:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":3.9239999999999999,"15min":3.504},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.3079999999999998,"5min":95.790999999999997,"15min":104.254},"last":0.373},{"interface":"front","average":{"1min":1.355,"5min":4.0119999999999996,"15min":3.7549999999999999},"min":{"1min":0.32500000000000001,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":5.3479999999999999,"5min":95.366,"15min":104.217},"last":8.1010000000000009}]},{"osd":3,"last update":"Sat Sep 23 22:23:16 2023","interfaces":[{"interface":"back","average":{"1min":4.1429999999999998,"5min":3.6789999999999998,"15min":2.23},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.246},"max":{"1min":18.876000000000001,"5min":93.739999999999995,"15min":93.739999999999995},"last":2.948},{"interface":"front","average":{"1min":2.6000000000000001,"5min":3.4220000000000002,"15min":2.1699999999999999},"min":{"1min":0.34200000000000003,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":13.773999999999999,"5min":93.197999999999993,"15min":93.197999999999993},"last":3.1709999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:23:11 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.085,"15min":1.03},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":5.1799999999999997,"5min":10.163,"15min":82.530000000000001},"last":2.569},{"interface":"front","average":{"1min":1.6000000000000001,"5min":1.0629999999999999,"15min":1.0660000000000001},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.2119999999999997,"5min":7.2119999999999997,"15min":85.441000000000003},"last":0.32000000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:23:39 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.82199999999999995,"15min":1.1559999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":7.0289999999999999,"5min":7.0289999999999999,"15min":114.83499999999999},"last":1.0640000000000001},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.73499999999999999,"15min":1.0669999999999999},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.2250000000000001,"5min":4.5220000000000002,"15min":110.905},"last":0.437}]},{"osd":7,"last update":"Sat Sep 23 22:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.86799999999999999,"15min":0.878},"min":{"1min":0.33300000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.2759999999999998,"5min":4.3760000000000003,"15min":8.8030000000000008},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.74099999999999999,"15min":0.76200000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3479999999999999,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":3.6299999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":133369856,"data_stored":133365784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":169480192,"data_stored":169476120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2715746304,"data_stored":2715746304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":310444032,"data_stored":310444032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":193789952,"data_stored":193789952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":303894528,"data_stored":303890456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":517865472,"data_stored":517865472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:24:23.715 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:24:23.730 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:23.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:23.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:23.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:23.962 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:24.227 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:24.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:24.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:24.272 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:24.353 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:24.378 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:24:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:24:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:24:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:24:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:85381 6.8 6:1a746b4c:::benchmark_data_smithi026_355573_object4031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T22:24:24.379 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:10.165170+0000", 2023-09-23T22:24:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 134.20574517, 2023-09-23T22:24:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.37987881699999998, 2023-09-23T22:24:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.380 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 85381 2023-09-23T22:24:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.165170+0000", 2023-09-23T22:24:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.165170+0000", 2023-09-23T22:24:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.165169+0000", 2023-09-23T22:24:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.165175+0000", 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5110000000000003e-06 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.165187+0000", 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2021000000000001e-05 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.165193+0000", 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7510000000000002e-06 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.165238+0000", 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4159999999999997e-05 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:24:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.165781+0000", 2023-09-23T22:24:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054354900000000005 2023-09-23T22:24:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.524926+0000", 2023-09-23T22:24:24.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.359145198 2023-09-23T22:24:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.525035+0000", 2023-09-23T22:24:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108879 2023-09-23T22:24:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.525179+0000", 2023-09-23T22:24:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014342700000000001 2023-09-23T22:24:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.531279+0000", 2023-09-23T22:24:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061004099999999997 2023-09-23T22:24:24.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.539919+0000", 2023-09-23T22:24:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086397169999999999 2023-09-23T22:24:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.541503+0000", 2023-09-23T22:24:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015844839999999999 2023-09-23T22:24:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.541561+0000", 2023-09-23T22:24:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7661999999999998e-05 2023-09-23T22:24:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:10.545049+0000", 2023-09-23T22:24:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034883420000000002 2023-09-23T22:24:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93554 6.b 6:d2b835f0:::benchmark_data_smithi026_355573_object12204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.390 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.178872+0000", 2023-09-23T22:24:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 115.192043559, 2023-09-23T22:24:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0056578679999999, 2023-09-23T22:24:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.391 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.392 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93554 2023-09-23T22:24:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178872+0000", 2023-09-23T22:24:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178872+0000", 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178870+0000", 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:24:24.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178878+0000", 2023-09-23T22:24:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3919999999999997e-06 2023-09-23T22:24:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178894+0000", 2023-09-23T22:24:24.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5838999999999999e-05 2023-09-23T22:24:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178903+0000", 2023-09-23T22:24:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2730000000000008e-06 2023-09-23T22:24:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.184954+0000", 2023-09-23T22:24:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060513269999999996 2023-09-23T22:24:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.184967+0000", 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2462e-05 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.173392+0000", 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.988425043 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.173530+0000", 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138851 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.173688+0000", 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000157379 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178167+0000", 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044787869999999997 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178182+0000", 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5404999999999999e-05 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184445+0000", 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062626799999999996 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184491+0000", 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6737999999999999e-05 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184530+0000", 2023-09-23T22:24:24.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8407e-05 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93584 6.b 6:d49f848e:::benchmark_data_smithi026_355573_object12234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.233323+0000", 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 115.137592313, 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95130220499999996, 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.403 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93584 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233323+0000", 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233323+0000", 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233322+0000", 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233328+0000", 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5429999999999999e-06 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233341+0000", 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3261000000000001e-05 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233351+0000", 2023-09-23T22:24:24.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2459999999999992e-06 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233412+0000", 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0812e-05 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233423+0000", 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1721e-05 2023-09-23T22:24:24.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.176192+0000", 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94276894499999997 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.176311+0000", 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119238 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.176383+0000", 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1946999999999994e-05 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178220+0000", 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001836458 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184379+0000", 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061588290000000002 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184566+0000", 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000187314 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184597+0000", 2023-09-23T22:24:24.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1398e-05 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184625+0000", 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8113e-05 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93607 6.b 6:d3fcbdef:::benchmark_data_smithi026_355573_object12257:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.273341+0000", 2023-09-23T22:24:24.411 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 115.09757408999999, 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91138181100000004, 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93607 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273341+0000", 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273341+0000", 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273340+0000", 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:24:24.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273347+0000", 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1049999999999997e-06 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273360+0000", 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2604e-05 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273367+0000", 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9489999999999996e-06 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273441+0000", 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3835e-05 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273452+0000", 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0873000000000001e-05 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178246+0000", 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90479371200000003 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178360+0000", 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114561 2023-09-23T22:24:24.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178486+0000", 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012549100000000001 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184364+0000", 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058778379999999998 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184384+0000", 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9867000000000001e-05 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184658+0000", 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027395400000000003 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184687+0000", 2023-09-23T22:24:24.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9799e-05 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184723+0000", 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5917999999999997e-05 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.419 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93621 6.b 6:d0e06b28:::benchmark_data_smithi026_355573_object12271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.329808+0000", 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 115.04110790999999, 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.863240699, 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93621 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.420 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329808+0000", 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329808+0000", 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329806+0000", 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329814+0000", 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9070000000000007e-06 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329834+0000", 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9731000000000001e-05 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329842+0000", 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8400000000000001e-06 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329932+0000", 2023-09-23T22:24:24.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9689000000000001e-05 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329945+0000", 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2563e-05 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.181153+0000", 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.85120826599999999 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.181276+0000", 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012343 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.181402+0000", 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125334 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184414+0000", 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030121119999999999 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.192892+0000", 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084780399999999992 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.192974+0000", 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1744999999999997e-05 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193006+0000", 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2427e-05 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193048+0000", 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2385000000000002e-05 2023-09-23T22:24:24.427 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93643 6.b 6:db9397f6:::benchmark_data_smithi026_355573_object12293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.382892+0000", 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.98802363900001, 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81032079400000001, 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.428 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93643 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382892+0000", 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382892+0000", 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382891+0000", 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382895+0000", 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8350000000000003e-06 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382907+0000", 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1114999999999999e-05 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382914+0000", 2023-09-23T22:24:24.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1760000000000004e-06 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.383035+0000", 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121657 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.383045+0000", 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0022e-05 2023-09-23T22:24:24.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184771+0000", 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80172589999999999 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184906+0000", 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013509899999999999 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.185041+0000", 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134182 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193064+0000", 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080234079999999992 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193119+0000", 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5152999999999999e-05 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193146+0000", 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.686e-05 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193175+0000", 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8615e-05 2023-09-23T22:24:24.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193213+0000", 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8120000000000001e-05 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93650 6.b 6:de3511b3:::benchmark_data_smithi026_355573_object12300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.385904+0000", 2023-09-23T22:24:24.436 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.98501197100001, 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81944647999999998, 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93650 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385904+0000", 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385904+0000", 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385903+0000", 2023-09-23T22:24:24.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385906+0000", 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8569999999999999e-06 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385912+0000", 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6999999999999996e-06 2023-09-23T22:24:24.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385917+0000", 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7849999999999999e-06 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.387207+0000", 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001290081 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.387217+0000", 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0086e-05 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.187923+0000", 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80070577899999995 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.188054+0000", 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013196200000000001 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.188190+0000", 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013539300000000001 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193068+0000", 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048783979999999999 2023-09-23T22:24:24.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.199041+0000", 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059726700000000002 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205266+0000", 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062251429999999998 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205303+0000", 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6878999999999997e-05 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205350+0000", 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7138999999999997e-05 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93680 6.b 6:dc4f0c3e:::benchmark_data_smithi026_355573_object12330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.473197+0000", 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.897718497, 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.73235443200000006, 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93680 2023-09-23T22:24:24.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.473197+0000", 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.473197+0000", 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.473195+0000", 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.473206+0000", 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1015e-05 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.473220+0000", 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3105000000000001e-05 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.473227+0000", 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8720000000000002e-06 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.473281+0000", 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3176e-05 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.473289+0000", 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9030000000000004e-06 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.190731+0000", 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.71744223799999995 2023-09-23T22:24:24.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.190832+0000", 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101301 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.190909+0000", 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6508000000000007e-05 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193228+0000", 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002319499 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205223+0000", 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011995257 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205460+0000", 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023634099999999999 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205507+0000", 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7647e-05 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205552+0000", 2023-09-23T22:24:24.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4200999999999999e-05 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93684 6.b 6:dec3cf4e:::benchmark_data_smithi026_355573_object12334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.492778+0000", 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.87813744899999, 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.71292562299999995, 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.453 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93684 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.492778+0000", 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.492778+0000", 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.492777+0000", 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.492785+0000", 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9219999999999997e-06 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.492805+0000", 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0568999999999999e-05 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.492816+0000", 2023-09-23T22:24:24.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0978000000000001e-05 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.492887+0000", 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0696000000000002e-05 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.492895+0000", 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2470000000000008e-06 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193270+0000", 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.70037532999999996 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193367+0000", 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7147999999999999e-05 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193439+0000", 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1204999999999993e-05 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205171+0000", 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011732134999999999 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205411+0000", 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00024050499999999999 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205628+0000", 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021618700000000001 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205669+0000", 2023-09-23T22:24:24.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1938999999999999e-05 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205704+0000", 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4319999999999997e-05 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93690 6.b 6:d874485d:::benchmark_data_smithi026_355573_object12340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.461 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.498290+0000", 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.872625498, 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.70754565800000002, 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93690 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.498290+0000", 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.498290+0000", 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.498288+0000", 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.498295+0000", 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2269999999999998e-06 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.498311+0000", 2023-09-23T22:24:24.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6116999999999998e-05 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.498319+0000", 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1790000000000008e-06 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.498385+0000", 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5531000000000007e-05 2023-09-23T22:24:24.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.498399+0000", 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4073999999999999e-05 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.195959+0000", 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.69756015100000002 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.196063+0000", 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104638 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.196170+0000", 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010616900000000001 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205234+0000", 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0090649489999999992 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205596+0000", 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036104699999999997 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205756+0000", 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000160433 2023-09-23T22:24:24.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205801+0000", 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5364000000000001e-05 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205836+0000", 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4406999999999998e-05 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.469 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93722 6.b 6:d2f2dfee:::benchmark_data_smithi026_355573_object12372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.585855+0000", 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.78506041999999, 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.63935741400000001, 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.470 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93722 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.585855+0000", 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.585855+0000", 2023-09-23T22:24:24.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.585853+0000", 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.585862+0000", 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0849999999999997e-06 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.585876+0000", 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4365000000000001e-05 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.585885+0000", 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1090000000000004e-06 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.585961+0000", 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5548000000000002e-05 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.585974+0000", 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3251000000000001e-05 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.199081+0000", 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.61310757599999999 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.199197+0000", 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011575500000000001 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.199309+0000", 2023-09-23T22:24:24.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011169 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205240+0000", 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059308379999999999 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.212111+0000", 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006870859 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.225125+0000", 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013014005 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.225171+0000", 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6613000000000002e-05 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.225213+0000", 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1409999999999998e-05 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93727 6.b 6:dcedadc8:::benchmark_data_smithi026_355573_object12377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.600699+0000", 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.770216519, 2023-09-23T22:24:24.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.62462116700000003, 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93727 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.600699+0000", 2023-09-23T22:24:24.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.600699+0000", 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.600698+0000", 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:24.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.600706+0000", 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1920000000000005e-06 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.600742+0000", 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5967000000000003e-05 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.600751+0000", 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1300000000000007e-06 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.600823+0000", 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1859000000000004e-05 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.600835+0000", 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2500999999999999e-05 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.202018+0000", 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60118258199999997 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.202170+0000", 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015242599999999999 2023-09-23T22:24:24.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.202328+0000", 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015815299999999999 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.212147+0000", 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0098191179999999999 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.225061+0000", 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012913997999999999 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.225257+0000", 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019534899999999999 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.225290+0000", 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2947999999999997e-05 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.225320+0000", 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0476000000000001e-05 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93770 6.b 6:d04370ef:::benchmark_data_smithi026_355573_object12420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.718118+0000", 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.65279792600001, 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.521039849, 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93770 2023-09-23T22:24:24.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.718118+0000", 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.718118+0000", 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.718116+0000", 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.718124+0000", 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0830000000000004e-06 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.718141+0000", 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6722e-05 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.718150+0000", 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5869999999999993e-06 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.718220+0000", 2023-09-23T22:24:24.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0757e-05 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.718234+0000", 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3786e-05 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205865+0000", 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.48763070600000002 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205990+0000", 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012496999999999999 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.206112+0000", 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121846 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.212185+0000", 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060731159999999999 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.226991+0000", 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014806367000000001 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.239066+0000", 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012074461999999999 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.239117+0000", 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1131999999999998e-05 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.239158+0000", 2023-09-23T22:24:24.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0806999999999997e-05 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93775 6.b 6:d973c2fc:::benchmark_data_smithi026_355573_object12425:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.734801+0000", 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.63611480500001, 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.51222282500000005, 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.495 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93775 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.734801+0000", 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.734801+0000", 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.734799+0000", 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.734806+0000", 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.452e-06 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.734831+0000", 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4505e-05 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.734840+0000", 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8099999999999992e-06 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.734900+0000", 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9763999999999999e-05 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.734912+0000", 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.221e-05 2023-09-23T22:24:24.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.208972+0000", 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.474059797 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.209098+0000", 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012582899999999999 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.209237+0000", 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013928700000000001 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.221564+0000", 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012326861999999999 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.239165+0000", 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017600843000000001 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.246927+0000", 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077620850000000002 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.246979+0000", 2023-09-23T22:24:24.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1678999999999998e-05 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.247024+0000", 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4762000000000003e-05 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93790 6.b 6:d1a1d818:::benchmark_data_smithi026_355573_object12440:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:24:24.503 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.798519+0000", 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 114.572396113, 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.44860919999999999, 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93790 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.798519+0000", 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.798519+0000", 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.798518+0000", 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.798527+0000", 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1039999999999996e-06 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.798548+0000", 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1665e-05 2023-09-23T22:24:24.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.798558+0000", 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4639999999999995e-06 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.798628+0000", 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0283000000000005e-05 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.798643+0000", 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4919999999999999e-05 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.212211+0000", 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.413567511 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.212313+0000", 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000102673 2023-09-23T22:24:24.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.212389+0000", 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5356000000000002e-05 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.227024+0000", 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014635326000000001 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.239170+0000", 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012145850999999999 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.247061+0000", 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078910130000000005 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.247097+0000", 2023-09-23T22:24:24.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5729000000000003e-05 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.247129+0000", 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2162999999999997e-05 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:120339 6.8 6:179a27b4:::benchmark_data_smithi026_355573_object38989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T22:24:24.511 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:18.854800+0000", 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 65.516115928000005, 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.3278684, 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 120339 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.854800+0000", 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.854800+0000", 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.854798+0000", 2023-09-23T22:24:24.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.854806+0000", 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.977e-06 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.854820+0000", 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3475000000000001e-05 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.854829+0000", 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5959999999999993e-06 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.854915+0000", 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6020000000000007e-05 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.855092+0000", 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017650700000000001 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.855253+0000", 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016148099999999999 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.862563+0000", 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073099819999999996 2023-09-23T22:24:24.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.866595+0000", 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040316140000000002 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.179531+0000", 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31293627699999998 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.179588+0000", 2023-09-23T22:24:24.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6749000000000002e-05 2023-09-23T22:24:24.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.182668+0000", 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030801800000000001 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:120346 6.0 e246/220 6:0ef776c7:::benchmark_data_smithi026_355573_object38996:head v 246'7632, mlcod=0'0)", 2023-09-23T22:24:24.519 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:18.867422+0000", 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 65.503493988000002, 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31202095299999999, 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.867422+0000", 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.867422+0000", 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.867421+0000", 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.867427+0000", 2023-09-23T22:24:24.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3709999999999999e-06 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.867433+0000", 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.874e-06 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.867437+0000", 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5390000000000001e-06 2023-09-23T22:24:24.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.867753+0000", 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00031623000000000003 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.867777+0000", 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3172000000000001e-05 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.872907+0000", 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051303410000000001 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.179443+0000", 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.30653570699999999 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:120342 6.2 e246/231 6:43feeae8:::benchmark_data_smithi026_355573_object38992:head v 246'7613, mlcod=211'2456)", 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:18.868019+0000", 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 65.502896113000006, 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31148664300000001, 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.868019+0000", 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.868019+0000", 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.868019+0000", 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:24:24.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.868024+0000", 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0130000000000001e-06 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.868031+0000", 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6390000000000003e-06 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.868036+0000", 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9320000000000004e-06 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.869882+0000", 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018464 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.869911+0000", 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9009999999999998e-05 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.179477+0000", 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.30956555499999999 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.179506+0000", 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9431e-05 2023-09-23T22:24:24.529 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:120349 6.0 e246/220 6:09c9ecb7:::benchmark_data_smithi026_355573_object38999:head v 246'7633, mlcod=0'0)", 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:18.869386+0000", 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 65.501529286999997, 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31008909299999998, 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.530 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.869386+0000", 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.869386+0000", 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.869386+0000", 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.869393+0000", 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8800000000000008e-06 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.869403+0000", 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9480000000000003e-06 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.869410+0000", 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1219999999999997e-06 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.870213+0000", 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00080231300000000003 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.870235+0000", 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2557000000000001e-05 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.179448+0000", 2023-09-23T22:24:24.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.309212875 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.179475+0000", 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7166e-05 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45002.0:120355 6.7 e246/203 6:ecf4cdb2:::benchmark_data_smithi026_355573_object39005:head v 246'7634, mlcod=246'7632)", 2023-09-23T22:24:24.535 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:23:18.871298+0000", 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 65.499617481000001, 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31210281299999998, 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.871298+0000", 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.871298+0000", 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.871297+0000", 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.871304+0000", 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1570000000000003e-06 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.871315+0000", 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0933e-05 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.871322+0000", 2023-09-23T22:24:24.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7660000000000001e-06 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.871378+0000", 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5413e-05 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:18.871397+0000", 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9635000000000001e-05 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.183354+0000", 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.31195636500000001 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:23:19.183401+0000", 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7358999999999998e-05 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.540 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:24:24.541 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:24:24.541 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:24:24.541 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:25.299 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 81347 objects 2023-09-23T22:24:25.299 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :143.869 2023-09-23T22:24:25.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:24 smithi070 ceph-mon[192652]: pgmap v640: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 215 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 29676/10182 objects degraded (291.456%); 270/10182 objects misplaced (2.652%); 7.3 MiB/s, 127 objects/s recovering 2023-09-23T22:24:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:24 smithi026 ceph-mon[215775]: pgmap v640: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 215 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 29676/10182 objects degraded (291.456%); 270/10182 objects misplaced (2.652%); 7.3 MiB/s, 127 objects/s recovering 2023-09-23T22:24:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:25.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:24 smithi026 ceph-mon[223014]: pgmap v640: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 215 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 547 op/s; 29676/10182 objects degraded (291.456%); 270/10182 objects misplaced (2.652%); 7.3 MiB/s, 127 objects/s recovering 2023-09-23T22:24:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:26.001 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:24:26.305 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4290400875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:24:26.305 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4290400875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:24:26.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4290400875' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:24:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[215775]: pgmap v641: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 48 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 740 op/s; 28952/2172 objects degraded (1332.965%); 30/2172 objects misplaced (1.381%); 10 MiB/s, 181 objects/s recovering 2023-09-23T22:24:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:27.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[215775]: failed to encode map e255 with expected crc 2023-09-23T22:24:27.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[215775]: failed to encode map e255 with expected crc 2023-09-23T22:24:27.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4290400875' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:24:27.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[215775]: osdmap e255: 8 total, 8 up, 6 in 2023-09-23T22:24:27.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[223014]: pgmap v641: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 48 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 740 op/s; 28952/2172 objects degraded (1332.965%); 30/2172 objects misplaced (1.381%); 10 MiB/s, 181 objects/s recovering 2023-09-23T22:24:27.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:27.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[223014]: failed to encode map e255 with expected crc 2023-09-23T22:24:27.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[223014]: failed to encode map e255 with expected crc 2023-09-23T22:24:27.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4290400875' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:24:27.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:26 smithi026 ceph-mon[223014]: osdmap e255: 8 total, 8 up, 6 in 2023-09-23T22:24:27.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:26 smithi070 ceph-mon[192652]: pgmap v641: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 48 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 740 op/s; 28952/2172 objects degraded (1332.965%); 30/2172 objects misplaced (1.381%); 10 MiB/s, 181 objects/s recovering 2023-09-23T22:24:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:26 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:26 smithi070 ceph-mon[192652]: failed to encode map e255 with expected crc 2023-09-23T22:24:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:26 smithi070 ceph-mon[192652]: failed to encode map e255 with expected crc 2023-09-23T22:24:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4290400875' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:24:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:26 smithi070 ceph-mon[192652]: osdmap e255: 8 total, 8 up, 6 in 2023-09-23T22:24:27.410 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:24:27.752 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:27.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:27.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:27.758 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:27.856 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:27.883 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:28.027 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:28.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:28.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:28.131 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:28.165 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:27 smithi070 ceph-mon[192652]: pgmap v643: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 30 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 594 op/s; 28952/1329 objects degraded (2178.480%); 30/1329 objects misplaced (2.257%); 8.8 MiB/s, 158 objects/s recovering 2023-09-23T22:24:28.166 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:27 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 28952/2172 objects degraded (1332.965%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:28.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:27 smithi026 ceph-mon[215775]: pgmap v643: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 30 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 594 op/s; 28952/1329 objects degraded (2178.480%); 30/1329 objects misplaced (2.257%); 8.8 MiB/s, 158 objects/s recovering 2023-09-23T22:24:28.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:27 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 28952/2172 objects degraded (1332.965%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:28.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:27 smithi026 ceph-mon[223014]: pgmap v643: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 30 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 594 op/s; 28952/1329 objects degraded (2178.480%); 30/1329 objects misplaced (2.257%); 8.8 MiB/s, 158 objects/s recovering 2023-09-23T22:24:28.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:27 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 28952/2172 objects degraded (1332.965%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:28.317 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:28.926 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:28.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:28.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:29.031 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:29.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:29.103 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:29.104 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:29.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:29.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:29.207 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:29.233 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:29.275 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:29.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:29.287 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T22:24:29.287 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T22:24:29.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:29.410 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:29.569 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:29.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:29.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:29.673 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:29.707 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:29.813 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:29.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:29.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:29.818 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:24:29.842 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:24:29.864 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:24:29.892 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:24:29.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:29.923 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:24:29.949 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:24:29.949 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:24:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:24:29] "GET /metrics HTTP/1.1" 200 36428 "" "Prometheus/2.43.0" 2023-09-23T22:24:29.953 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:29.979 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:24:30.000 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:24:30.001 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:24:30.001 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:24:30.001 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:24:30.002 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:24:30.002 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:24:30.002 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:24:30.002 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:24:30.002 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:24:30.460 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:30.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:30.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:30.616 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:30.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:30 smithi070 ceph-mon[192652]: pgmap v644: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 18 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 28899/762 objects degraded (3792.520%); 30/762 objects misplaced (3.937%); 7.9 MiB/s, 143 objects/s recovering 2023-09-23T22:24:30.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2147844439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:30.643 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:30.685 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:30.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:30.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:30.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:30 smithi026 ceph-mon[215775]: pgmap v644: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 18 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 28899/762 objects degraded (3792.520%); 30/762 objects misplaced (3.937%); 7.9 MiB/s, 143 objects/s recovering 2023-09-23T22:24:30.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2147844439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:30.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:30 smithi026 ceph-mon[223014]: pgmap v644: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 18 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 28899/762 objects degraded (3792.520%); 30/762 objects misplaced (3.937%); 7.9 MiB/s, 143 objects/s recovering 2023-09-23T22:24:30.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2147844439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:30.741 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:24:30.788 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:30.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:30.855 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:30.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:30.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:30.958 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:30.983 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:31.025 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:31.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:31.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:31.129 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:31.155 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:31.409 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:31.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:31.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:31.513 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:31.541 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:31.634 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:31.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:31.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:31.737 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:31.771 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:31.918 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:31.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:31.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:31.973 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:24:31.973 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:24:32.022 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:32.053 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:32.252 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:32.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:32.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:32.355 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:32.381 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:32.422 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:32.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:32.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:32.526 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:32.555 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:32.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:32 smithi070 ceph-mon[192652]: pgmap v645: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 370 op/s; 28236/81 objects degraded (34859.259%); 4.5 MiB/s, 148 objects/s recovering 2023-09-23T22:24:32.726 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:24:32.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:32 smithi026 ceph-mon[215775]: pgmap v645: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 370 op/s; 28236/81 objects degraded (34859.259%); 4.5 MiB/s, 148 objects/s recovering 2023-09-23T22:24:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:32 smithi026 ceph-mon[223014]: pgmap v645: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 370 op/s; 28236/81 objects degraded (34859.259%); 4.5 MiB/s, 148 objects/s recovering 2023-09-23T22:24:32.953 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:32.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:32.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:33.056 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:33.072 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:33.086 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:33.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:33.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:33.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:33.179 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:33.237 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:33.273 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:33.425 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:33.534 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:33.537 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 28236/81 objects degraded (34859.259%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:33.537 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 28236/81 objects degraded (34859.259%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:33.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:33.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 28236/81 objects degraded (34859.259%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:33.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:33.668 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:33.713 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:33.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:33.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:33.816 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:33.851 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:33.915 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:24:33.916 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 28236/81 objects degraded (34859.259%), 15 pgs degraded, 13 pgs undersized","count":28},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":857,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":255,"num_osds":8,"num_up_osds":8,"osd_up_since":1695507239,"num_in_osds":6,"osd_in_since":1695507716,"num_remapped_pgs":14},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":83},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":12},{"state_name":"active+recovery_wait+degraded","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":99,"num_pools":5,"num_objects":27,"data_bytes":4569855,"bytes_used":2246311936,"bytes_avail":573723901952,"bytes_total":575970213888,"degraded_objects":28236,"degraded_total":81,"degraded_ratio":348.59259259259261,"recovering_objects_per_sec":148,"recovering_bytes_per_sec":4670444,"recovering_keys_per_sec":0,"num_objects_recovered":1488,"num_bytes_recovered":46727168,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":336},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T22:14:59.012277+0000","services":{}},"progress_events":{"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"f9b296f7-36c4-4993-8ca1-c3652b75187b":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 37s)","progress":0.83838385343551636,"add_to_ceph_s":true}}} 2023-09-23T22:24:34.700 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:24:34.700 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:24:35.019 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:24:35.060 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:35.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:35.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:35.164 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:35.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:34 smithi026 ceph-mon[223014]: pgmap v646: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 336 op/s; 28236/81 objects degraded (34859.259%); 4.5 MiB/s, 148 objects/s recovering 2023-09-23T22:24:35.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:34 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3865825804' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:24:35.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:34 smithi026 ceph-mon[215775]: pgmap v646: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 336 op/s; 28236/81 objects degraded (34859.259%); 4.5 MiB/s, 148 objects/s recovering 2023-09-23T22:24:35.191 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:34 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3865825804' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:24:35.193 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:34 smithi070 ceph-mon[192652]: pgmap v646: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 336 op/s; 28236/81 objects degraded (34859.259%); 4.5 MiB/s, 148 objects/s recovering 2023-09-23T22:24:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:34 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3865825804' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:24:35.475 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:24:35.832 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:36.949 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:36.952 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:24:36.952 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:24:36.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:36.954 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:37.056 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:37.085 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:37.153 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:36 smithi070 ceph-mon[192652]: pgmap v647: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 27482/81 objects degraded (33928.395%); 678 KiB/s, 146 objects/s recovering 2023-09-23T22:24:37.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:36 smithi026 ceph-mon[223014]: pgmap v647: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 27482/81 objects degraded (33928.395%); 678 KiB/s, 146 objects/s recovering 2023-09-23T22:24:37.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:36 smithi026 ceph-mon[215775]: pgmap v647: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 69 op/s; 27482/81 objects degraded (33928.395%); 678 KiB/s, 146 objects/s recovering 2023-09-23T22:24:37.183 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:37.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:37.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:37.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:37.328 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:37.366 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:37.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:37.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:37.382 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:24:37.383 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:24:37.473 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:37.514 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:37.625 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:37.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:37.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:37.728 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:37.760 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:37.911 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_1' removed 2023-09-23T22:24:37.956 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:37.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:37.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:38.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:38.119 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:38.149 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:24:38.531 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:38.922 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: pgmap v648: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 61 op/s; 27482/81 objects degraded (33928.395%); 600 KiB/s, 130 objects/s recovering 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/287782101' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 27482/81 objects degraded (33928.395%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: osdmap e256: 8 total, 8 up, 6 in 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.924 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:38 smithi070 ceph-mon[192652]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: pgmap v648: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 61 op/s; 27482/81 objects degraded (33928.395%); 600 KiB/s, 130 objects/s recovering 2023-09-23T22:24:38.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:38.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:38.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/287782101' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:24:38.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:24:38.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 27482/81 objects degraded (33928.395%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:38.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:24:38.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: osdmap e256: 8 total, 8 up, 6 in 2023-09-23T22:24:38.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[223014]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: pgmap v648: 99 pgs: 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 61 op/s; 27482/81 objects degraded (33928.395%); 600 KiB/s, 130 objects/s recovering 2023-09-23T22:24:38.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:38.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:38.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/287782101' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:24:38.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:24:38.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 27482/81 objects degraded (33928.395%), 15 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T22:24:38.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:24:38.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: osdmap e256: 8 total, 8 up, 6 in 2023-09-23T22:24:38.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: failed to encode map e256 with expected crc 2023-09-23T22:24:38.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:38 smithi026 ceph-mon[215775]: failed to encode map e256 with expected crc 2023-09-23T22:24:39.385 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:39.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:39.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:39.409 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:39.489 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:39.519 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:39.599 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:39.668 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:39.672 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:24:39.673 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:24:39.673 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T22:24:39.673 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T22:24:39.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:39.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:39.776 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:39.805 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:39.899 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:39.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:39.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:39.906 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:24:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:24:39] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.43.0" 2023-09-23T22:24:40.003 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:40.031 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:40.075 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:40.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:40.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:40.113 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:24:40.113 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:24:40.180 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:40.204 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:39 smithi070 ceph-mon[192652]: failed to encode map e257 with expected crc 2023-09-23T22:24:40.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:39 smithi070 ceph-mon[192652]: failed to encode map e257 with expected crc 2023-09-23T22:24:40.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:39 smithi070 ceph-mon[192652]: osdmap e257: 8 total, 8 up, 6 in 2023-09-23T22:24:40.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:39 smithi070 ceph-mon[192652]: failed to encode map e257 with expected crc 2023-09-23T22:24:40.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:40.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:40.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:40.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:40.207 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[215775]: failed to encode map e257 with expected crc 2023-09-23T22:24:40.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[215775]: failed to encode map e257 with expected crc 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[215775]: osdmap e257: 8 total, 8 up, 6 in 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[215775]: failed to encode map e257 with expected crc 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[223014]: failed to encode map e257 with expected crc 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[223014]: failed to encode map e257 with expected crc 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[223014]: osdmap e257: 8 total, 8 up, 6 in 2023-09-23T22:24:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[223014]: failed to encode map e257 with expected crc 2023-09-23T22:24:40.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:40.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:24:40.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:24:40.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:40.351 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:40.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:40.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:40.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:40.483 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:40.774 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:24:40.780 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:40.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:40.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:40.885 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:40.911 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:40.955 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:40.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:41.060 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:41.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:40 smithi070 ceph-mon[192652]: pgmap v651: 83 pgs: 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 27/81 objects degraded (33.333%) 2023-09-23T22:24:41.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:40 smithi070 ceph-mon[192652]: failed to encode map e258 with expected crc 2023-09-23T22:24:41.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:40 smithi070 ceph-mon[192652]: failed to encode map e258 with expected crc 2023-09-23T22:24:41.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:40 smithi070 ceph-mon[192652]: osdmap e258: 8 total, 8 up, 6 in 2023-09-23T22:24:41.085 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:40 smithi070 ceph-mon[192652]: failed to encode map e258 with expected crc 2023-09-23T22:24:41.088 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:41.130 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:41.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:41.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:41.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[215775]: pgmap v651: 83 pgs: 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 27/81 objects degraded (33.333%) 2023-09-23T22:24:41.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[215775]: failed to encode map e258 with expected crc 2023-09-23T22:24:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[215775]: failed to encode map e258 with expected crc 2023-09-23T22:24:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[215775]: osdmap e258: 8 total, 8 up, 6 in 2023-09-23T22:24:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[215775]: failed to encode map e258 with expected crc 2023-09-23T22:24:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[223014]: pgmap v651: 83 pgs: 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 79 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 27/81 objects degraded (33.333%) 2023-09-23T22:24:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[223014]: failed to encode map e258 with expected crc 2023-09-23T22:24:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[223014]: failed to encode map e258 with expected crc 2023-09-23T22:24:41.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[223014]: osdmap e258: 8 total, 8 up, 6 in 2023-09-23T22:24:41.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:40 smithi026 ceph-mon[223014]: failed to encode map e258 with expected crc 2023-09-23T22:24:41.242 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:41.268 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:41.361 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:41.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:41.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:41.464 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:41.495 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:41.549 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:41.643 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:41.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:41.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:41.749 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:41.780 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:41.965 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:24:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/129430157' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:24:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:41 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:24:42.175 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:24:42.177 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:24:42.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/129430157' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:24:42.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:41 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:24:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/129430157' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:24:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:41 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:24:42.693 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:24:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:24:42.693 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:24:42.694 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:24:42.694 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:24:42.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[215775]: pgmap v653: 83 pgs: 2 peering, 81 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:24:43.102 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:24:43.102 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:24:43.102 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:24:43.102 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:24:43.102 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:24:43.102 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:24:43.103 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:24:43.168 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":654,"stamp":"2023-09-23T22:24:41.957403+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4569855,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":6070,"num_read_kb":12350,"num_write":951,"num_write_kb":13165,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":9311471,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2802,"ondisk_log_size":2802,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":329,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2190328,"kb_used_data":679816,"kb_used_omap":20,"kb_used_meta":1510443,"kb_avail":560280584,"statfs":{"total":575970213888,"available":573727318016,"internally_reserved":0,"allocated":696131584,"data_stored":688828059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1546694411},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":32,"apply_latency_ms":32,"commit_latency_ns":32000000,"apply_latency_ns":32000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-27482,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-162696,"num_write_kb":-5206209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-26815,"num_bytes_recovered":-1529085408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.657687"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":244,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.934723+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:24:37.934723+0000","last_peered":"2023-09-23T22:24:37.934723+0000","last_clean":"2023-09-23T22:24:37.934723+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:24:37.934723+0000","last_undegraded":"2023-09-23T22:24:37.934723+0000","last_fullsized":"2023-09-23T22:24:37.934723+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":236,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.919546+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:24:37.919546+0000","last_peered":"2023-09-23T22:24:37.919546+0000","last_clean":"2023-09-23T22:24:37.919546+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:24:37.919546+0000","last_undegraded":"2023-09-23T22:24:37.919546+0000","last_fullsized":"2023-09-23T22:24:37.919546+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":197,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.621955+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:24:26.621955+0000","last_peered":"2023-09-23T22:24:26.621955+0000","last_clean":"2023-09-23T22:24:26.621955+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:24:26.621955+0000","last_undegraded":"2023-09-23T22:24:26.621955+0000","last_fullsized":"2023-09-23T22:24:26.621955+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":115,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016316+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:24:39.016316+0000","last_peered":"2023-09-23T22:24:39.016316+0000","last_clean":"2023-09-23T22:24:39.016316+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:24:39.016316+0000","last_undegraded":"2023-09-23T22:24:39.016316+0000","last_fullsized":"2023-09-23T22:24:39.016316+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":276,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.919949+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:24:37.919949+0000","last_peered":"2023-09-23T22:24:37.919949+0000","last_clean":"2023-09-23T22:24:37.919949+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:24:37.919949+0000","last_undegraded":"2023-09-23T22:24:37.919949+0000","last_fullsized":"2023-09-23T22:24:37.919949+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":212,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.928823+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:24:37.928823+0000","last_peered":"2023-09-23T22:24:37.928823+0000","last_clean":"2023-09-23T22:24:37.928823+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:24:37.928823+0000","last_undegraded":"2023-09-23T22:24:37.928823+0000","last_fullsized":"2023-09-23T22:24:37.928823+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":144,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016943+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:24:39.016943+0000","last_peered":"2023-09-23T22:24:39.016943+0000","last_clean":"2023-09-23T22:24:39.016943+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:24:39.016943+0000","last_undegraded":"2023-09-23T22:24:39.016943+0000","last_fullsized":"2023-09-23T22:24:39.016943+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":233,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.920026+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:24:37.920026+0000","last_peered":"2023-09-23T22:24:37.920026+0000","last_clean":"2023-09-23T22:24:37.920026+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:24:37.920026+0000","last_undegraded":"2023-09-23T22:24:37.920026+0000","last_fullsized":"2023-09-23T22:24:37.920026+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":217,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.928539+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:24:37.928539+0000","last_peered":"2023-09-23T22:24:37.928539+0000","last_clean":"2023-09-23T22:24:37.928539+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:24:37.928539+0000","last_undegraded":"2023-09-23T22:24:37.928539+0000","last_fullsized":"2023-09-23T22:24:37.928539+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":201,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.918568+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:24:37.918568+0000","last_peered":"2023-09-23T22:24:37.918568+0000","last_clean":"2023-09-23T22:24:37.918568+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:24:37.918568+0000","last_undegraded":"2023-09-23T22:24:37.918568+0000","last_fullsized":"2023-09-23T22:24:37.918568+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":200,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.919473+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:24:37.919473+0000","last_peered":"2023-09-23T22:24:37.919473+0000","last_clean":"2023-09-23T22:24:37.919473+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:24:37.919473+0000","last_undegraded":"2023-09-23T22:24:37.919473+0000","last_fullsized":"2023-09-23T22:24:37.919473+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":234,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:33.853453+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:24:33.853453+0000","last_peered":"2023-09-23T22:24:33.853453+0000","last_clean":"2023-09-23T22:24:33.853453+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:24:33.853453+0000","last_undegraded":"2023-09-23T22:24:33.853453+0000","last_fullsized":"2023-09-23T22:24:33.853453+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":261,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935803+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:24:37.935803+0000","last_peered":"2023-09-23T22:24:37.935803+0000","last_clean":"2023-09-23T22:24:37.935803+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:24:37.935803+0000","last_undegraded":"2023-09-23T22:24:37.935803+0000","last_fullsized":"2023-09-23T22:24:37.935803+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":225,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935814+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:24:37.935814+0000","last_peered":"2023-09-23T22:24:37.935814+0000","last_clean":"2023-09-23T22:24:37.935814+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:24:37.935814+0000","last_undegraded":"2023-09-23T22:24:37.935814+0000","last_fullsized":"2023-09-23T22:24:37.935814+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":184,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935906+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:24:37.935906+0000","last_peered":"2023-09-23T22:24:37.935906+0000","last_clean":"2023-09-23T22:24:37.935906+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:24:37.935906+0000","last_undegraded":"2023-09-23T22:24:37.935906+0000","last_fullsized":"2023-09-23T22:24:37.935906+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":224,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.925507+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:24:37.925507+0000","last_peered":"2023-09-23T22:24:37.925507+0000","last_clean":"2023-09-23T22:24:37.925507+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:24:37.925507+0000","last_undegraded":"2023-09-23T22:24:37.925507+0000","last_fullsized":"2023-09-23T22:24:37.925507+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":133,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.017287+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:24:39.017287+0000","last_peered":"2023-09-23T22:24:39.017287+0000","last_clean":"2023-09-23T22:24:39.017287+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:24:39.017287+0000","last_undegraded":"2023-09-23T22:24:39.017287+0000","last_fullsized":"2023-09-23T22:24:39.017287+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":236,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.928695+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:24:37.928695+0000","last_peered":"2023-09-23T22:24:37.928695+0000","last_clean":"2023-09-23T22:24:37.928695+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:24:37.928695+0000","last_undegraded":"2023-09-23T22:24:37.928695+0000","last_fullsized":"2023-09-23T22:24:37.928695+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":180,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935146+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:24:37.935146+0000","last_peered":"2023-09-23T22:24:37.935146+0000","last_clean":"2023-09-23T22:24:37.935146+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:24:37.935146+0000","last_undegraded":"2023-09-23T22:24:37.935146+0000","last_fullsized":"2023-09-23T22:24:37.935146+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":115,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016620+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:24:39.016620+0000","last_peered":"2023-09-23T22:24:39.016620+0000","last_clean":"2023-09-23T22:24:39.016620+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:24:39.016620+0000","last_undegraded":"2023-09-23T22:24:39.016620+0000","last_fullsized":"2023-09-23T22:24:39.016620+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":276,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016371+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:24:39.016371+0000","last_peered":"2023-09-23T22:24:39.016371+0000","last_clean":"2023-09-23T22:24:39.016371+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:24:39.016371+0000","last_undegraded":"2023-09-23T22:24:39.016371+0000","last_fullsized":"2023-09-23T22:24:39.016371+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":45,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.920168+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:24:37.920168+0000","last_peered":"2023-09-23T22:24:37.920168+0000","last_clean":"2023-09-23T22:24:37.920168+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:24:37.920168+0000","last_undegraded":"2023-09-23T22:24:37.920168+0000","last_fullsized":"2023-09-23T22:24:37.920168+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":252,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.942475+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:24:37.942475+0000","last_peered":"2023-09-23T22:24:37.942475+0000","last_clean":"2023-09-23T22:24:37.942475+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:24:37.942475+0000","last_undegraded":"2023-09-23T22:24:37.942475+0000","last_fullsized":"2023-09-23T22:24:37.942475+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":239,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.928234+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:24:37.928234+0000","last_peered":"2023-09-23T22:24:37.928234+0000","last_clean":"2023-09-23T22:24:37.928234+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:24:37.928234+0000","last_undegraded":"2023-09-23T22:24:37.928234+0000","last_fullsized":"2023-09-23T22:24:37.928234+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":166,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.947483+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:24:37.947483+0000","last_peered":"2023-09-23T22:24:37.947483+0000","last_clean":"2023-09-23T22:24:37.947483+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:24:37.947483+0000","last_undegraded":"2023-09-23T22:24:37.947483+0000","last_fullsized":"2023-09-23T22:24:37.947483+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":244,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935118+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:24:37.935118+0000","last_peered":"2023-09-23T22:24:37.935118+0000","last_clean":"2023-09-23T22:24:37.935118+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:24:37.935118+0000","last_undegraded":"2023-09-23T22:24:37.935118+0000","last_fullsized":"2023-09-23T22:24:37.935118+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":182,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.928770+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:24:37.928770+0000","last_peered":"2023-09-23T22:24:37.928770+0000","last_clean":"2023-09-23T22:24:37.928770+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:24:37.928770+0000","last_undegraded":"2023-09-23T22:24:37.928770+0000","last_fullsized":"2023-09-23T22:24:37.928770+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":191,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.622306+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:24:26.622306+0000","last_peered":"2023-09-23T22:24:26.622306+0000","last_clean":"2023-09-23T22:24:26.622306+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:24:26.622306+0000","last_undegraded":"2023-09-23T22:24:26.622306+0000","last_fullsized":"2023-09-23T22:24:26.622306+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":247,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.931894+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:24:37.931894+0000","last_peered":"2023-09-23T22:24:37.931894+0000","last_clean":"2023-09-23T22:24:37.931894+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:24:37.931894+0000","last_undegraded":"2023-09-23T22:24:37.931894+0000","last_fullsized":"2023-09-23T22:24:37.931894+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":113,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016455+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:24:39.016455+0000","last_peered":"2023-09-23T22:24:39.016455+0000","last_clean":"2023-09-23T22:24:39.016455+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:24:39.016455+0000","last_undegraded":"2023-09-23T22:24:39.016455+0000","last_fullsized":"2023-09-23T22:24:39.016455+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":561,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:38.951653+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:24:38.951653+0000","last_peered":"2023-09-23T22:24:38.951653+0000","last_clean":"2023-09-23T22:24:38.951653+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:24:38.951653+0000","last_undegraded":"2023-09-23T22:24:38.951653+0000","last_fullsized":"2023-09-23T22:24:38.951653+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":270,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.926058+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:24:37.926058+0000","last_peered":"2023-09-23T22:24:37.926058+0000","last_clean":"2023-09-23T22:24:37.926058+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:24:37.926058+0000","last_undegraded":"2023-09-23T22:24:37.926058+0000","last_fullsized":"2023-09-23T22:24:37.926058+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":225,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935289+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:24:37.935289+0000","last_peered":"2023-09-23T22:24:37.935289+0000","last_clean":"2023-09-23T22:24:37.935289+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:24:37.935289+0000","last_undegraded":"2023-09-23T22:24:37.935289+0000","last_fullsized":"2023-09-23T22:24:37.935289+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":561,"reported_epoch":257,"state":"peering","last_fresh":"2023-09-23T22:24:38.949771+0000","last_change":"2023-09-23T22:24:38.950007+0000","last_active":"2023-09-23T22:24:38.949739+0000","last_peered":"2023-09-23T22:24:38.009736+0000","last_clean":"2023-09-23T22:21:50.447236+0000","last_became_active":"2023-09-23T22:21:59.021009+0000","last_became_peered":"2023-09-23T22:21:59.021009+0000","last_unstale":"2023-09-23T22:24:38.949771+0000","last_undegraded":"2023-09-23T22:24:38.949771+0000","last_fullsized":"2023-09-23T22:24:38.949771+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:01.044536+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3,4],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":173,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.938296+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:24:37.938296+0000","last_peered":"2023-09-23T22:24:37.938296+0000","last_clean":"2023-09-23T22:24:37.938296+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:24:37.938296+0000","last_undegraded":"2023-09-23T22:24:37.938296+0000","last_fullsized":"2023-09-23T22:24:37.938296+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":242,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.934734+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:24:37.934734+0000","last_peered":"2023-09-23T22:24:37.934734+0000","last_clean":"2023-09-23T22:24:37.934734+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:24:37.934734+0000","last_undegraded":"2023-09-23T22:24:37.934734+0000","last_fullsized":"2023-09-23T22:24:37.934734+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":218,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935008+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:24:37.935008+0000","last_peered":"2023-09-23T22:24:37.935008+0000","last_clean":"2023-09-23T22:24:37.935008+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:24:37.935008+0000","last_undegraded":"2023-09-23T22:24:37.935008+0000","last_fullsized":"2023-09-23T22:24:37.935008+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":233,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.947395+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:24:37.947395+0000","last_peered":"2023-09-23T22:24:37.947395+0000","last_clean":"2023-09-23T22:24:37.947395+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:24:37.947395+0000","last_undegraded":"2023-09-23T22:24:37.947395+0000","last_fullsized":"2023-09-23T22:24:37.947395+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":561,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.930955+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:24:37.930955+0000","last_peered":"2023-09-23T22:24:37.930955+0000","last_clean":"2023-09-23T22:24:37.930955+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:24:37.930955+0000","last_undegraded":"2023-09-23T22:24:37.930955+0000","last_fullsized":"2023-09-23T22:24:37.930955+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":37,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.917450+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:24:37.917450+0000","last_peered":"2023-09-23T22:24:37.917450+0000","last_clean":"2023-09-23T22:24:37.917450+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:24:37.917450+0000","last_undegraded":"2023-09-23T22:24:37.917450+0000","last_fullsized":"2023-09-23T22:24:37.917450+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":198,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.623430+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:24:26.623430+0000","last_peered":"2023-09-23T22:24:26.623430+0000","last_clean":"2023-09-23T22:24:26.623430+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:24:26.623430+0000","last_undegraded":"2023-09-23T22:24:26.623430+0000","last_fullsized":"2023-09-23T22:24:26.623430+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":553,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:38.953011+0000","last_change":"2023-09-23T22:22:47.349660+0000","last_active":"2023-09-23T22:24:38.953011+0000","last_peered":"2023-09-23T22:24:38.953011+0000","last_clean":"2023-09-23T22:24:38.953011+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:24:38.953011+0000","last_undegraded":"2023-09-23T22:24:38.953011+0000","last_fullsized":"2023-09-23T22:24:38.953011+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":805,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.622184+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:24:26.622184+0000","last_peered":"2023-09-23T22:24:26.622184+0000","last_clean":"2023-09-23T22:24:26.622184+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:24:26.622184+0000","last_undegraded":"2023-09-23T22:24:26.622184+0000","last_fullsized":"2023-09-23T22:24:26.622184+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":274,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.928539+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:24:37.928539+0000","last_peered":"2023-09-23T22:24:37.928539+0000","last_clean":"2023-09-23T22:24:37.928539+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:24:37.928539+0000","last_undegraded":"2023-09-23T22:24:37.928539+0000","last_fullsized":"2023-09-23T22:24:37.928539+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":42,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.621785+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:24:26.621785+0000","last_peered":"2023-09-23T22:24:26.621785+0000","last_clean":"2023-09-23T22:24:26.621785+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:24:26.621785+0000","last_undegraded":"2023-09-23T22:24:26.621785+0000","last_fullsized":"2023-09-23T22:24:26.621785+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":561,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.927964+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:24:37.927964+0000","last_peered":"2023-09-23T22:24:37.927964+0000","last_clean":"2023-09-23T22:24:37.927964+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:24:37.927964+0000","last_undegraded":"2023-09-23T22:24:37.927964+0000","last_fullsized":"2023-09-23T22:24:37.927964+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":989,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.931078+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:24:37.931078+0000","last_peered":"2023-09-23T22:24:37.931078+0000","last_clean":"2023-09-23T22:24:37.931078+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:24:37.931078+0000","last_undegraded":"2023-09-23T22:24:37.931078+0000","last_fullsized":"2023-09-23T22:24:37.931078+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":266,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935733+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:24:37.935733+0000","last_peered":"2023-09-23T22:24:37.935733+0000","last_clean":"2023-09-23T22:24:37.935733+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:24:37.935733+0000","last_undegraded":"2023-09-23T22:24:37.935733+0000","last_fullsized":"2023-09-23T22:24:37.935733+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":221,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.931989+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:24:37.931989+0000","last_peered":"2023-09-23T22:24:37.931989+0000","last_clean":"2023-09-23T22:24:37.931989+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:24:37.931989+0000","last_undegraded":"2023-09-23T22:24:37.931989+0000","last_fullsized":"2023-09-23T22:24:37.931989+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":587,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.017198+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:24:39.017198+0000","last_peered":"2023-09-23T22:24:39.017198+0000","last_clean":"2023-09-23T22:24:39.017198+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:24:39.017198+0000","last_undegraded":"2023-09-23T22:24:39.017198+0000","last_fullsized":"2023-09-23T22:24:39.017198+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":392,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:38.072456+0000","last_change":"2023-09-23T22:24:38.072053+0000","last_active":"2023-09-23T22:24:38.072456+0000","last_peered":"2023-09-23T22:24:38.072456+0000","last_clean":"2023-09-23T22:24:38.072456+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:24:38.072456+0000","last_undegraded":"2023-09-23T22:24:38.072456+0000","last_fullsized":"2023-09-23T22:24:38.072456+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":216,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.622143+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:24:26.622143+0000","last_peered":"2023-09-23T22:24:26.622143+0000","last_clean":"2023-09-23T22:24:26.622143+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:24:26.622143+0000","last_undegraded":"2023-09-23T22:24:26.622143+0000","last_fullsized":"2023-09-23T22:24:26.622143+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":263,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.623513+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:24:26.623513+0000","last_peered":"2023-09-23T22:24:26.623513+0000","last_clean":"2023-09-23T22:24:26.623513+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:24:26.623513+0000","last_undegraded":"2023-09-23T22:24:26.623513+0000","last_fullsized":"2023-09-23T22:24:26.623513+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":876,"reported_epoch":257,"state":"peering","last_fresh":"2023-09-23T22:24:38.949667+0000","last_change":"2023-09-23T22:24:38.949933+0000","last_active":"2023-09-23T22:24:38.949631+0000","last_peered":"2023-09-23T22:24:37.993500+0000","last_clean":"2023-09-23T22:21:50.447508+0000","last_became_active":"2023-09-23T22:21:58.852815+0000","last_became_peered":"2023-09-23T22:21:58.852815+0000","last_unstale":"2023-09-23T22:24:38.949667+0000","last_undegraded":"2023-09-23T22:24:38.949667+0000","last_fullsized":"2023-09-23T22:24:38.949667+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:04.504118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":570,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016192+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:24:39.016192+0000","last_peered":"2023-09-23T22:24:39.016192+0000","last_clean":"2023-09-23T22:24:39.016192+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:24:39.016192+0000","last_undegraded":"2023-09-23T22:24:39.016192+0000","last_fullsized":"2023-09-23T22:24:39.016192+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.926735+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:24:37.926735+0000","last_peered":"2023-09-23T22:24:37.926735+0000","last_clean":"2023-09-23T22:24:37.926735+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:24:37.926735+0000","last_undegraded":"2023-09-23T22:24:37.926735+0000","last_fullsized":"2023-09-23T22:24:37.926735+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":143,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.936089+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:24:37.936089+0000","last_peered":"2023-09-23T22:24:37.936089+0000","last_clean":"2023-09-23T22:24:37.936089+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:24:37.936089+0000","last_undegraded":"2023-09-23T22:24:37.936089+0000","last_fullsized":"2023-09-23T22:24:37.936089+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":588,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:28.851641+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:24:28.851641+0000","last_peered":"2023-09-23T22:24:28.851641+0000","last_clean":"2023-09-23T22:24:28.851641+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:24:28.851641+0000","last_undegraded":"2023-09-23T22:24:28.851641+0000","last_fullsized":"2023-09-23T22:24:28.851641+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":606,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.622090+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:24:26.622090+0000","last_peered":"2023-09-23T22:24:26.622090+0000","last_clean":"2023-09-23T22:24:26.622090+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:24:26.622090+0000","last_undegraded":"2023-09-23T22:24:26.622090+0000","last_fullsized":"2023-09-23T22:24:26.622090+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":140,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016679+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:24:39.016679+0000","last_peered":"2023-09-23T22:24:39.016679+0000","last_clean":"2023-09-23T22:24:39.016679+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:24:39.016679+0000","last_undegraded":"2023-09-23T22:24:39.016679+0000","last_fullsized":"2023-09-23T22:24:39.016679+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":97,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016843+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:24:39.016843+0000","last_peered":"2023-09-23T22:24:39.016843+0000","last_clean":"2023-09-23T22:24:39.016843+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:24:39.016843+0000","last_undegraded":"2023-09-23T22:24:39.016843+0000","last_fullsized":"2023-09-23T22:24:39.016843+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":579,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016530+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:24:39.016530+0000","last_peered":"2023-09-23T22:24:39.016530+0000","last_clean":"2023-09-23T22:24:39.016530+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:24:39.016530+0000","last_undegraded":"2023-09-23T22:24:39.016530+0000","last_fullsized":"2023-09-23T22:24:39.016530+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":149,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016836+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:24:39.016836+0000","last_peered":"2023-09-23T22:24:39.016836+0000","last_clean":"2023-09-23T22:24:39.016836+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:24:39.016836+0000","last_undegraded":"2023-09-23T22:24:39.016836+0000","last_fullsized":"2023-09-23T22:24:39.016836+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":207,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.622412+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:24:26.622412+0000","last_peered":"2023-09-23T22:24:26.622412+0000","last_clean":"2023-09-23T22:24:26.622412+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:24:26.622412+0000","last_undegraded":"2023-09-23T22:24:26.622412+0000","last_fullsized":"2023-09-23T22:24:26.622412+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":113,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016369+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:24:39.016369+0000","last_peered":"2023-09-23T22:24:39.016369+0000","last_clean":"2023-09-23T22:24:39.016369+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:24:39.016369+0000","last_undegraded":"2023-09-23T22:24:39.016369+0000","last_fullsized":"2023-09-23T22:24:39.016369+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":561,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935933+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:24:37.935933+0000","last_peered":"2023-09-23T22:24:37.935933+0000","last_clean":"2023-09-23T22:24:37.935933+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:24:37.935933+0000","last_undegraded":"2023-09-23T22:24:37.935933+0000","last_fullsized":"2023-09-23T22:24:37.935933+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":270,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.924093+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:24:37.924093+0000","last_peered":"2023-09-23T22:24:37.924093+0000","last_clean":"2023-09-23T22:24:37.924093+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:24:37.924093+0000","last_undegraded":"2023-09-23T22:24:37.924093+0000","last_fullsized":"2023-09-23T22:24:37.924093+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":898,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.623121+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:24:26.623121+0000","last_peered":"2023-09-23T22:24:26.623121+0000","last_clean":"2023-09-23T22:24:26.623121+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:24:26.623121+0000","last_undegraded":"2023-09-23T22:24:26.623121+0000","last_fullsized":"2023-09-23T22:24:26.623121+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":46,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.916744+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:24:37.916744+0000","last_peered":"2023-09-23T22:24:37.916744+0000","last_clean":"2023-09-23T22:24:37.916744+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:24:37.916744+0000","last_undegraded":"2023-09-23T22:24:37.916744+0000","last_fullsized":"2023-09-23T22:24:37.916744+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":545,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.925786+0000","last_change":"2023-09-23T22:22:19.883431+0000","last_active":"2023-09-23T22:24:37.925786+0000","last_peered":"2023-09-23T22:24:37.925786+0000","last_clean":"2023-09-23T22:24:37.925786+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:24:37.925786+0000","last_undegraded":"2023-09-23T22:24:37.925786+0000","last_fullsized":"2023-09-23T22:24:37.925786+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:46.903071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:46.903071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":251,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:38.076530+0000","last_change":"2023-09-23T22:24:38.076201+0000","last_active":"2023-09-23T22:24:38.076530+0000","last_peered":"2023-09-23T22:24:38.076530+0000","last_clean":"2023-09-23T22:24:38.076530+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:24:38.076530+0000","last_undegraded":"2023-09-23T22:24:38.076530+0000","last_fullsized":"2023-09-23T22:24:38.076530+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":92,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016424+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:24:39.016424+0000","last_peered":"2023-09-23T22:24:39.016424+0000","last_clean":"2023-09-23T22:24:39.016424+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:24:39.016424+0000","last_undegraded":"2023-09-23T22:24:39.016424+0000","last_fullsized":"2023-09-23T22:24:39.016424+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":220,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935039+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:24:37.935039+0000","last_peered":"2023-09-23T22:24:37.935039+0000","last_clean":"2023-09-23T22:24:37.935039+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:24:37.935039+0000","last_undegraded":"2023-09-23T22:24:37.935039+0000","last_fullsized":"2023-09-23T22:24:37.935039+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":88,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016275+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:24:39.016275+0000","last_peered":"2023-09-23T22:24:39.016275+0000","last_clean":"2023-09-23T22:24:39.016275+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:24:39.016275+0000","last_undegraded":"2023-09-23T22:24:39.016275+0000","last_fullsized":"2023-09-23T22:24:39.016275+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":267,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T22:24:26.622360+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:24:26.622360+0000","last_peered":"2023-09-23T22:24:26.622360+0000","last_clean":"2023-09-23T22:24:26.622360+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:24:26.622360+0000","last_undegraded":"2023-09-23T22:24:26.622360+0000","last_fullsized":"2023-09-23T22:24:26.622360+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":219,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.931844+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:24:37.931844+0000","last_peered":"2023-09-23T22:24:37.931844+0000","last_clean":"2023-09-23T22:24:37.931844+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:24:37.931844+0000","last_undegraded":"2023-09-23T22:24:37.931844+0000","last_fullsized":"2023-09-23T22:24:37.931844+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":232,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935896+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:24:37.935896+0000","last_peered":"2023-09-23T22:24:37.935896+0000","last_clean":"2023-09-23T22:24:37.935896+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:24:37.935896+0000","last_undegraded":"2023-09-23T22:24:37.935896+0000","last_fullsized":"2023-09-23T22:24:37.935896+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":216,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.928624+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:24:37.928624+0000","last_peered":"2023-09-23T22:24:37.928624+0000","last_clean":"2023-09-23T22:24:37.928624+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:24:37.928624+0000","last_undegraded":"2023-09-23T22:24:37.928624+0000","last_fullsized":"2023-09-23T22:24:37.928624+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":91,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.938366+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:24:37.938366+0000","last_peered":"2023-09-23T22:24:37.938366+0000","last_clean":"2023-09-23T22:24:37.938366+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:24:37.938366+0000","last_undegraded":"2023-09-23T22:24:37.938366+0000","last_fullsized":"2023-09-23T22:24:37.938366+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":238,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.926539+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:24:37.926539+0000","last_peered":"2023-09-23T22:24:37.926539+0000","last_clean":"2023-09-23T22:24:37.926539+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:24:37.926539+0000","last_undegraded":"2023-09-23T22:24:37.926539+0000","last_fullsized":"2023-09-23T22:24:37.926539+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":279,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.918492+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:24:37.918492+0000","last_peered":"2023-09-23T22:24:37.918492+0000","last_clean":"2023-09-23T22:24:37.918492+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:24:37.918492+0000","last_undegraded":"2023-09-23T22:24:37.918492+0000","last_fullsized":"2023-09-23T22:24:37.918492+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":147,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.016889+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:24:39.016889+0000","last_peered":"2023-09-23T22:24:39.016889+0000","last_clean":"2023-09-23T22:24:39.016889+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:24:39.016889+0000","last_undegraded":"2023-09-23T22:24:39.016889+0000","last_fullsized":"2023-09-23T22:24:39.016889+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":265,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T22:24:37.935004+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:24:37.935004+0000","last_peered":"2023-09-23T22:24:37.935004+0000","last_clean":"2023-09-23T22:24:37.935004+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:24:37.935004+0000","last_undegraded":"2023-09-23T22:24:37.935004+0000","last_fullsized":"2023-09-23T22:24:37.935004+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":370028,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":88,"num_read_kb":166,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":912387,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":126,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234243,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":550972,"kb_used_data":354036,"kb_used_omap":0,"kb_used_meta":196928,"kb_avail":93194180,"statfs":{"total":95995035648,"available":95430840320,"internally_reserved":0,"allocated":362532864,"data_stored":361504988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":201654272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":4.1870000000000003,"15min":2.3159999999999998},"min":{"1min":0.22600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.5499999999999998,"5min":158.042,"15min":158.042},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.4369999999999998,"5min":4.6920000000000002,"15min":2.6070000000000002},"min":{"1min":0.28199999999999997,"5min":0.217,"15min":0.217},"max":{"1min":6.6719999999999997,"5min":157.952,"15min":157.952},"last":0.54900000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":4.4050000000000002,"15min":3.0630000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":6.2839999999999998,"5min":101.145,"15min":123.79300000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.8759999999999999,"5min":4.4969999999999999,"15min":3.2170000000000001},"min":{"1min":0.33200000000000002,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":8.4149999999999991,"5min":101.13,"15min":123.66500000000001},"last":0.40799999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":4.1260000000000003,"15min":3.7650000000000001},"min":{"1min":0.307,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.7939999999999996,"5min":105.34399999999999,"15min":125.101},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.6440000000000001,"5min":4.0960000000000001,"15min":3.5110000000000001},"min":{"1min":0.30499999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.3769999999999998,"5min":105.63200000000001,"15min":126.021},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:24:16 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":3.9329999999999998,"15min":2.335},"min":{"1min":0.46800000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.6479999999999997,"5min":170.86099999999999,"15min":170.86099999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.0619999999999998,"5min":3.7410000000000001,"15min":2.4100000000000001},"min":{"1min":0.44800000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.7919999999999998,"5min":174.57499999999999,"15min":174.57499999999999},"last":0.53000000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":2.2120000000000002,"15min":1.3280000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.5169999999999999,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.38},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.29,"15min":1.3009999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.6719999999999997,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":2.2530000000000001,"15min":1.2509999999999999},"min":{"1min":0.30299999999999999,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":3.21,"5min":93.816000000000003,"15min":93.816000000000003},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.81499999999999995,"5min":2.2970000000000002,"15min":1.466},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.544,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.68100000000000005,"15min":0.74099999999999999},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.7120000000000002,"5min":5.343,"15min":9.0139999999999993},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.86299999999999999,"15min":0.93899999999999995},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":3.6339999999999999,"5min":6.3879999999999999,"15min":13.362},"last":0.497}]}]},{"osd":6,"up_from":0,"seq":197568495879,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795151,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":355764,"kb_used_data":40684,"kb_used_omap":7,"kb_used_meta":315064,"kb_avail":93389388,"statfs":{"total":95995035648,"available":95630733312,"internally_reserved":0,"allocated":41660416,"data_stored":40281899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":322625908},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":2.2770000000000001,"15min":1.4750000000000001},"min":{"1min":0.22700000000000001,"5min":0.189,"15min":0.189},"max":{"1min":10.125999999999999,"5min":56.503,"15min":61.890000000000001},"last":0.218},{"interface":"front","average":{"1min":4.2060000000000004,"5min":3.1179999999999999,"15min":1.982},"min":{"1min":0.28199999999999997,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":15.965999999999999,"5min":56.560000000000002,"15min":62.689999999999998},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:23:45 2023","interfaces":[{"interface":"back","average":{"1min":1.7749999999999999,"5min":1.294,"15min":1.2789999999999999},"min":{"1min":0.27600000000000002,"5min":0.223,"15min":0.216},"max":{"1min":7.694,"5min":11.179,"15min":33.902000000000001},"last":0.312},{"interface":"front","average":{"1min":3.734,"5min":2.4660000000000002,"15min":1.726},"min":{"1min":0.312,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.8369999999999997,"5min":15.015000000000001,"15min":34.781999999999996},"last":0.28899999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":5.5910000000000002,"5min":4.6269999999999998,"15min":2.4049999999999998},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":13.871,"5min":110.417,"15min":110.417},"last":0.46600000000000003},{"interface":"front","average":{"1min":5.29,"5min":4.6589999999999998,"15min":2.343},"min":{"1min":0.28299999999999997,"5min":0.223,"15min":0.223},"max":{"1min":16.346,"5min":110.721,"15min":110.721},"last":0.496}]},{"osd":4,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":4.5359999999999996,"15min":2.0779999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27000000000000002},"max":{"1min":5.0869999999999997,"5min":234.79900000000001,"15min":234.79900000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.153,"5min":5.1689999999999996,"15min":2.4510000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":6.7160000000000002,"5min":234.81700000000001,"15min":234.81700000000001},"last":0.50700000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":3.5310000000000001,"15min":1.889},"min":{"1min":0.34000000000000002,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":6.7480000000000002,"5min":196.79900000000001,"15min":196.79900000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.5190000000000001,"5min":3.8450000000000002,"15min":1.9239999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":16.207000000000001,"5min":196.68299999999999,"15min":196.68299999999999},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":2.1240000000000001,"15min":1.3759999999999999},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":16.314,"5min":16.314,"15min":16.314},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.218,"5min":1.206,"15min":1.042},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.1310000000000002,"5min":11.385999999999999,"15min":11.385999999999999},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":2.2909999999999999,"15min":1.484},"min":{"1min":0.57699999999999996,"5min":0.33800000000000002,"15min":0.32900000000000001},"max":{"1min":16.265000000000001,"5min":16.265000000000001,"15min":16.265000000000001},"last":0.44},{"interface":"front","average":{"1min":1.087,"5min":1.21,"15min":1.0860000000000001},"min":{"1min":0.42199999999999999,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":5.194,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.48199999999999998}]}]},{"osd":0,"up_from":93,"seq":399431958686,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":314352,"kb_used_data":7336,"kb_used_omap":8,"kb_used_meta":306999,"kb_avail":93430800,"statfs":{"total":95995035648,"available":95673139200,"internally_reserved":0,"allocated":7512064,"data_stored":6094941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":314367850},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0619999999999998,"5min":4.5759999999999996,"15min":2.4550000000000001},"min":{"1min":0.214,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":16.253,"5min":169.77199999999999,"15min":169.77199999999999},"last":4.7199999999999998},{"interface":"front","average":{"1min":3.089,"5min":4.7729999999999997,"15min":2.746},"min":{"1min":0.34999999999999998,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":13.108000000000001,"5min":169.52099999999999,"15min":169.52099999999999},"last":0.83699999999999997}]},{"osd":2,"last update":"Sat Sep 23 22:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":6.1360000000000001,"15min":2.6800000000000002},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":16.097999999999999,"5min":206.495,"15min":206.495},"last":4.8579999999999997},{"interface":"front","average":{"1min":2.258,"5min":6.6130000000000004,"15min":3.3050000000000002},"min":{"1min":0.29299999999999998,"5min":0.28499999999999998,"15min":0.22800000000000001},"max":{"1min":11.138,"5min":207.34399999999999,"15min":207.34399999999999},"last":3.153}]},{"osd":3,"last update":"Sat Sep 23 22:24:23 2023","interfaces":[{"interface":"back","average":{"1min":11.73,"5min":5.4100000000000001,"15min":2.754},"min":{"1min":0.35899999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":146.87299999999999,"5min":146.87299999999999,"15min":146.87299999999999},"last":4.9139999999999997},{"interface":"front","average":{"1min":10.827,"5min":5.2699999999999996,"15min":2.7959999999999998},"min":{"1min":0.33600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":145.48099999999999,"5min":145.48099999999999,"15min":145.48099999999999},"last":23.085999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":3.7839999999999998,"5min":3.1299999999999999,"15min":1.994},"min":{"1min":0.33500000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":16.347999999999999,"5min":82.825000000000003,"15min":82.825000000000003},"last":0.627},{"interface":"front","average":{"1min":2.5910000000000002,"5min":2.2269999999999999,"15min":1.3799999999999999},"min":{"1min":0.377,"5min":0.313,"15min":0.308},"max":{"1min":16.184000000000001,"5min":75.930999999999997,"15min":75.930999999999997},"last":4.798}]},{"osd":5,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":3.8180000000000001,"5min":4.2919999999999998,"15min":2.3809999999999998},"min":{"1min":0.44,"5min":0.34799999999999998,"15min":0.29499999999999998},"max":{"1min":16.323,"5min":93.685000000000002,"15min":93.685000000000002},"last":0.67400000000000004},{"interface":"front","average":{"1min":2.431,"5min":3.4980000000000002,"15min":1.819},"min":{"1min":0.38600000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":16.152000000000001,"5min":93.546000000000006,"15min":93.546000000000006},"last":4.8310000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":4.093,"5min":2.2130000000000001,"15min":1.6990000000000001},"min":{"1min":0.47799999999999998,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":16.366,"5min":16.366,"15min":16.366},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.3519999999999999,"5min":1.661,"15min":1.224},"min":{"1min":0.439,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":16.225999999999999,"5min":16.225999999999999,"15min":22.263000000000002},"last":4.883}]},{"osd":7,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":2.21,"15min":1.746},"min":{"1min":0.44600000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.718,"5min":32.773000000000003,"15min":32.773000000000003},"last":0.89000000000000001},{"interface":"front","average":{"1min":4.1059999999999999,"5min":2.343,"15min":1.9079999999999999},"min":{"1min":0.50600000000000001,"5min":0.39900000000000002,"15min":0.38900000000000001},"max":{"1min":16.381,"5min":16.381,"15min":23.265999999999998},"last":0.75900000000000001}]}]},{"osd":2,"up_from":0,"seq":442381631625,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468098,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":389024,"kb_used_data":140312,"kb_used_omap":4,"kb_used_meta":248699,"kb_avail":93356128,"statfs":{"total":95995035648,"available":95596675072,"internally_reserved":0,"allocated":143679488,"data_stored":142257320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":254668333},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1020000000000003,"5min":5.7489999999999997,"15min":2.9550000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.19900000000000001},"max":{"1min":10.547000000000001,"5min":225.86699999999999,"15min":225.86699999999999},"last":0.27500000000000002},{"interface":"front","average":{"1min":4.3819999999999997,"5min":6.1420000000000003,"15min":3.202},"min":{"1min":0.39100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":10.976000000000001,"5min":225.84800000000001,"15min":225.84800000000001},"last":0.29499999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2080000000000002,"5min":3.2309999999999999,"15min":2.3740000000000001},"min":{"1min":0.371,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":13.374000000000001,"5min":37.002000000000002,"15min":82.951999999999998},"last":5.9249999999999998},{"interface":"front","average":{"1min":3.3439999999999999,"5min":3.2290000000000001,"15min":2.4129999999999998},"min":{"1min":0.25700000000000001,"5min":0.22800000000000001,"15min":0.189},"max":{"1min":13.723000000000001,"5min":33.119999999999997,"15min":86.001000000000005},"last":0.313}]},{"osd":2,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.806,"5min":2.5270000000000001,"15min":3.5459999999999998},"min":{"1min":0.40300000000000002,"5min":0.252,"15min":0.223},"max":{"1min":13.518000000000001,"5min":29.076000000000001,"15min":99.962999999999994},"last":5.9740000000000002},{"interface":"front","average":{"1min":3.6030000000000002,"5min":2.6840000000000002,"15min":3.4249999999999998},"min":{"1min":0.29199999999999998,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":13.692,"5min":13.692,"15min":100.429},"last":0.372}]},{"osd":4,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.903,"15min":1.282},"min":{"1min":0.314,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":7.2039999999999997,"5min":11.911,"15min":11.911},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.798,"5min":1.9179999999999999,"15min":1.3460000000000001},"min":{"1min":0.34399999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.2940000000000005,"5min":10.949999999999999,"15min":12.036},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8300000000000001,"5min":2.5920000000000001,"15min":1.538},"min":{"1min":0.41499999999999998,"5min":0.309,"15min":0.309},"max":{"1min":9.4179999999999993,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.5070000000000001,"5min":2.383,"15min":1.506},"min":{"1min":0.376,"5min":0.31,"15min":0.31},"max":{"1min":13.494,"5min":49.668999999999997,"15min":49.668999999999997},"last":6.0179999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.9020000000000001,"5min":2.1829999999999998,"15min":1.482},"min":{"1min":0.436,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.3439999999999994,"5min":10.992000000000001,"15min":12.068},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.9650000000000001,"15min":1.373},"min":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.35399999999999998},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":6.0919999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.1030000000000002,"15min":1.4470000000000001},"min":{"1min":0.434,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":13.467000000000001,"5min":16.849,"15min":16.849},"last":6.0549999999999997},{"interface":"front","average":{"1min":2.375,"5min":2.0649999999999999,"15min":1.385},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.36699999999999999},"max":{"1min":7.2480000000000002,"5min":11.945,"15min":12.382},"last":0.42199999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019148,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":229252,"kb_used_data":37116,"kb_used_omap":0,"kb_used_meta":192128,"kb_avail":93515900,"statfs":{"total":95995035648,"available":95760281600,"internally_reserved":0,"allocated":38006784,"data_stored":36972067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":196739072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7069999999999999,"5min":2.9369999999999998,"15min":1.8540000000000001},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.218},"max":{"1min":25.425000000000001,"5min":46.18,"15min":66.355000000000004},"last":0.32200000000000001},{"interface":"front","average":{"1min":3.444,"5min":2.9039999999999999,"15min":1.921},"min":{"1min":0.28799999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":25.451000000000001,"5min":46.207000000000001,"15min":66.332999999999998},"last":0.40100000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":5.4039999999999999,"5min":2.7160000000000002,"15min":1.6699999999999999},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":85.662000000000006,"5min":85.662000000000006,"15min":85.662000000000006},"last":0.38300000000000001},{"interface":"front","average":{"1min":5.5999999999999996,"5min":3.0899999999999999,"15min":2.0289999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":81.195999999999998,"5min":81.195999999999998,"15min":81.195999999999998},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:23:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":3.472,"15min":2.2970000000000002},"min":{"1min":0.36099999999999999,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":10.051,"5min":138.06100000000001,"15min":138.06100000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.1030000000000002,"5min":3.3380000000000001,"15min":2.4340000000000002},"min":{"1min":0.35499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.2510000000000003,"5min":138.15199999999999,"15min":138.15199999999999},"last":0.45800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.669,"15min":1.2709999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":5.0140000000000002,"5min":12.129,"15min":12.129},"last":0.499},{"interface":"front","average":{"1min":2.3599999999999999,"5min":1.8580000000000001,"15min":1.347},"min":{"1min":0.32300000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.5600000000000005,"5min":8.5600000000000005,"15min":9.8109999999999999},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.3069999999999999,"15min":0.91000000000000003},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.165,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.26,"5min":1.4550000000000001,"15min":0.95499999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27900000000000003},"max":{"1min":5.601,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.44}]},{"osd":6,"last update":"Sat Sep 23 22:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":0.98299999999999998,"15min":0.77600000000000002},"min":{"1min":0.33800000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":7.8550000000000004,"5min":7.8550000000000004,"15min":7.8550000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.81599999999999995,"15min":0.77500000000000002},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.802,"5min":3.9239999999999999,"15min":8.5210000000000008},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":0.92900000000000005,"15min":0.878},"min":{"1min":0.33300000000000002,"5min":0.313,"15min":0.307},"max":{"1min":3.347,"5min":6.7480000000000002,"15min":8.577},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.377,"5min":1.0149999999999999,"15min":0.83999999999999997},"min":{"1min":0.34200000000000003,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":7.8109999999999999,"5min":7.8109999999999999,"15min":7.8109999999999999},"last":0.52400000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757516,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":350964,"kb_used_data":100332,"kb_used_omap":0,"kb_used_meta":250624,"kb_avail":93394188,"statfs":{"total":95995035648,"available":95635648512,"internally_reserved":0,"allocated":102739968,"data_stored":101716844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":256638976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:57 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":2.7989999999999999,"15min":2.1579999999999999},"min":{"1min":0.23999999999999999,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":64.344999999999999,"5min":64.344999999999999,"15min":133.22499999999999},"last":0.376},{"interface":"front","average":{"1min":4.891,"5min":2.528,"15min":2.181},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":64.022000000000006,"5min":64.022000000000006,"15min":132.78200000000001},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":3.1040000000000001,"15min":2.7589999999999999},"min":{"1min":0.27500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.7870000000000008,"5min":93.981999999999999,"15min":93.981999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.885,"5min":3.0310000000000001,"15min":2.7389999999999999},"min":{"1min":0.35499999999999998,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":10.949,"5min":93.909999999999997,"15min":93.909999999999997},"last":0.44500000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":3.9239999999999999,"15min":3.504},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.3079999999999998,"5min":95.790999999999997,"15min":104.254},"last":0.434},{"interface":"front","average":{"1min":1.355,"5min":4.0119999999999996,"15min":3.7549999999999999},"min":{"1min":0.32500000000000001,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":5.3479999999999999,"5min":95.366,"15min":104.217},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1499999999999999,"5min":3.786,"15min":2.335},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.246},"max":{"1min":5.0839999999999996,"5min":93.739999999999995,"15min":93.739999999999995},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.9199999999999999,"5min":3.3279999999999998,"15min":2.2589999999999999},"min":{"1min":0.38200000000000001,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":4.2569999999999997,"5min":93.197999999999993,"15min":93.197999999999993},"last":0.38400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:24:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":1.0089999999999999,"15min":1.046},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.7160000000000002,"5min":10.163,"15min":82.530000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.02,"15min":1.0880000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.8900000000000001,"5min":7.2119999999999997,"15min":85.441000000000003},"last":0.29999999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:23:39 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.82199999999999995,"15min":1.1559999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":7.0289999999999999,"5min":7.0289999999999999,"15min":114.83499999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.73499999999999999,"15min":1.0669999999999999},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.2250000000000001,"5min":4.5220000000000002,"15min":110.905},"last":0.32900000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.86799999999999999,"15min":0.878},"min":{"1min":0.33300000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.2759999999999998,"5min":4.3760000000000003,"15min":8.8030000000000008},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.74099999999999999,"15min":0.76200000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3479999999999999,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":0.47599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:24:43.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[223014]: pgmap v653: 83 pgs: 2 peering, 81 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:24:43.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/81 objects degraded (33.333%), 4 pgs degraded, 2 pgs undersized) 2023-09-23T22:24:43.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:43.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[223014]: failed to encode map e259 with expected crc 2023-09-23T22:24:43.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:24:43.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[223014]: osdmap e259: 8 total, 8 up, 6 in 2023-09-23T22:24:43.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1514505266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:43.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/81 objects degraded (33.333%), 4 pgs degraded, 2 pgs undersized) 2023-09-23T22:24:43.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:43.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[215775]: failed to encode map e259 with expected crc 2023-09-23T22:24:43.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:24:43.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[215775]: osdmap e259: 8 total, 8 up, 6 in 2023-09-23T22:24:43.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1514505266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:43.332 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:43.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:43.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:42 smithi070 ceph-mon[192652]: pgmap v653: 83 pgs: 2 peering, 81 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:24:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:42 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 27/81 objects degraded (33.333%), 4 pgs degraded, 2 pgs undersized) 2023-09-23T22:24:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:42 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:42 smithi070 ceph-mon[192652]: failed to encode map e259 with expected crc 2023-09-23T22:24:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:42 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:24:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:42 smithi070 ceph-mon[192652]: osdmap e259: 8 total, 8 up, 6 in 2023-09-23T22:24:43.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1514505266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:43.383 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:24:43.436 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:43.463 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:43.513 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:43.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:43.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:43.623 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:43.652 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:43.696 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:43.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:43.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:43.742 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:43.803 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:43.837 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:44.214 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:44.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:43 smithi026 ceph-mon[223014]: from='client.45290 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:44.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:43 smithi026 ceph-mon[223014]: failed to encode map e259 with expected crc 2023-09-23T22:24:44.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:43 smithi026 ceph-mon[223014]: failed to encode map e259 with expected crc 2023-09-23T22:24:44.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:43 smithi026 ceph-mon[215775]: from='client.45290 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:44.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:43 smithi026 ceph-mon[215775]: failed to encode map e259 with expected crc 2023-09-23T22:24:44.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:43 smithi026 ceph-mon[215775]: failed to encode map e259 with expected crc 2023-09-23T22:24:44.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:44.217 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:44.318 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:44.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:43 smithi070 ceph-mon[192652]: from='client.45290 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:44.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:43 smithi070 ceph-mon[192652]: failed to encode map e259 with expected crc 2023-09-23T22:24:44.342 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:43 smithi070 ceph-mon[192652]: failed to encode map e259 with expected crc 2023-09-23T22:24:44.346 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:44.390 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:44.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:44.494 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:44.524 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:44.572 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:44.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:44.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:44.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:44.705 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:44.752 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:44.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:44.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:44.856 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:44.884 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:44.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:44.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:44.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:45.039 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:45.068 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:45.143 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:44 smithi070 ceph-mon[192652]: failed to encode map e259 with expected crc 2023-09-23T22:24:45.143 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:44 smithi070 ceph-mon[192652]: pgmap v655: 83 pgs: 2 peering, 81 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 48 KiB/s, 5 objects/s recovering 2023-09-23T22:24:45.161 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:45.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:45.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:45.206 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:24:45.231 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:44 smithi026 ceph-mon[215775]: failed to encode map e259 with expected crc 2023-09-23T22:24:45.232 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:44 smithi026 ceph-mon[215775]: pgmap v655: 83 pgs: 2 peering, 81 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 48 KiB/s, 5 objects/s recovering 2023-09-23T22:24:45.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:44 smithi026 ceph-mon[223014]: failed to encode map e259 with expected crc 2023-09-23T22:24:45.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:44 smithi026 ceph-mon[223014]: pgmap v655: 83 pgs: 2 peering, 81 active+clean; 4.4 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 48 KiB/s, 5 objects/s recovering 2023-09-23T22:24:45.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:45.291 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:45.343 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:45.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:45.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:45.396 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:24:45.397 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:24:45.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:24:45.397 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:24:45.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.397 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19259 5.7 5:e3ec1517:::benchmark_data_smithi026_286012_object19258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T22:24:45.397 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.612249+0000", 2023-09-23T22:24:45.397 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 556.78141508900001, 2023-09-23T22:24:45.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0598800580000001, 2023-09-23T22:24:45.397 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.398 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:24:45.398 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.398 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.398 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.398 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19259 2023-09-23T22:24:45.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.398 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612249+0000", 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612249+0000", 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612247+0000", 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612300+0000", 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2468e-05 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612319+0000", 2023-09-23T22:24:45.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9431000000000001e-05 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612325+0000", 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4820000000000002e-06 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612377+0000", 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1932999999999999e-05 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612500+0000", 2023-09-23T22:24:45.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122898 2023-09-23T22:24:45.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.612640+0000", 2023-09-23T22:24:45.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000140168 2023-09-23T22:24:45.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.613682+0000", 2023-09-23T22:24:45.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010420690000000001 2023-09-23T22:24:45.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.617494+0000", 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038125429999999998 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.617503+0000", 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3459999999999994e-06 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627652+0000", 2023-09-23T22:24:45.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.010149754 2023-09-23T22:24:45.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627667+0000", 2023-09-23T22:24:45.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4209000000000001e-05 2023-09-23T22:24:45.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672028+0000", 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0443611349999999 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672129+0000", 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010098 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.407 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:19268 5.7 5:e4a7ba5f:::benchmark_data_smithi026_286012_object19267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T22:24:45.407 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:28.635055+0000", 2023-09-23T22:24:45.407 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 556.75860916500005, 2023-09-23T22:24:45.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.042290714, 2023-09-23T22:24:45.407 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.407 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.407 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.407 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 19268 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635055+0000", 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635055+0000", 2023-09-23T22:24:45.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635053+0000", 2023-09-23T22:24:45.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635096+0000", 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2938999999999997e-05 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635110+0000", 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3417e-05 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635116+0000", 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.139e-06 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635172+0000", 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6567999999999998e-05 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:28.635186+0000", 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3509999999999999e-05 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627702+0000", 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99251634700000002 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:29.627708+0000", 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3789999999999997e-06 2023-09-23T22:24:45.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672153+0000", 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.044445216 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672284+0000", 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013079899999999999 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.672410+0000", 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012686899999999999 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.673809+0000", 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013983579999999999 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.674260+0000", 2023-09-23T22:24:45.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045079799999999998 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.676705+0000", 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024456840000000001 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.676749+0000", 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4087e-05 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:30.677345+0000", 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059599999999999996 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:36982 5.7 5:ee40b8fd:::benchmark_data_smithi026_286012_object36981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.905294+0000", 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 542.48837025299997, 2023-09-23T22:24:45.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0459705229999998, 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36982 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905294+0000", 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905294+0000", 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905292+0000", 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905348+0000", 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5515e-05 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905359+0000", 2023-09-23T22:24:45.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1467e-05 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905366+0000", 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.229e-06 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905414+0000", 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8909000000000002e-05 2023-09-23T22:24:45.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905562+0000", 2023-09-23T22:24:45.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014794 2023-09-23T22:24:45.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:24:45.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.905720+0000", 2023-09-23T22:24:45.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015746799999999999 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.907741+0000", 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020212910000000001 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.912362+0000", 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046209500000000004 2023-09-23T22:24:45.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.918820+0000", 2023-09-23T22:24:45.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064581580000000003 2023-09-23T22:24:45.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.918832+0000", 2023-09-23T22:24:45.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1997000000000001e-05 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915724+0000", 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99689173499999995 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915741+0000", 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7453999999999999e-05 2023-09-23T22:24:45.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951176+0000", 2023-09-23T22:24:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0354348980000001 2023-09-23T22:24:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951264+0000", 2023-09-23T22:24:45.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7725000000000005e-05 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:36996 5.7 5:e8cdce36:::benchmark_data_smithi026_286012_object36995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e123)", 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.923128+0000", 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 542.47053570100002, 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0387094659999998, 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.426 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36996 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923128+0000", 2023-09-23T22:24:45.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923128+0000", 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923127+0000", 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923158+0000", 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1696999999999998e-05 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923168+0000", 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8609999999999993e-06 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923175+0000", 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3940000000000001e-06 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923656+0000", 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048170500000000002 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.923667+0000", 2023-09-23T22:24:45.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0723999999999999e-05 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915782+0000", 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99211468700000005 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915789+0000", 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7309999999999993e-06 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951301+0000", 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.035511493 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951393+0000", 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2347000000000002e-05 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.951500+0000", 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010628600000000001 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956154+0000", 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046545290000000001 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961149+0000", 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049946900000000004 2023-09-23T22:24:45.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961196+0000", 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6995e-05 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961241+0000", 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5599000000000002e-05 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961838+0000", 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059618699999999998 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37004 5.7 5:eeedc2c2:::benchmark_data_smithi026_286012_object37003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.940630+0000", 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 542.45303420899995, 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0219609119999999, 2023-09-23T22:24:45.435 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37004 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940630+0000", 2023-09-23T22:24:45.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940630+0000", 2023-09-23T22:24:45.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940622+0000", 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999924 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940657+0000", 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5079000000000001e-05 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940672+0000", 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5036e-05 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940678+0000", 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2940000000000002e-06 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940726+0000", 2023-09-23T22:24:45.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8319000000000003e-05 2023-09-23T22:24:45.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.940734+0000", 2023-09-23T22:24:45.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1199999999999996e-06 2023-09-23T22:24:45.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915819+0000", 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97508563500000001 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915826+0000", 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4749999999999998e-06 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.952797+0000", 2023-09-23T22:24:45.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0369709229999999 2023-09-23T22:24:45.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.952925+0000", 2023-09-23T22:24:45.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128796 2023-09-23T22:24:45.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.953102+0000", 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000177011 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.958840+0000", 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057373930000000004 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961876+0000", 2023-09-23T22:24:45.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003036029 2023-09-23T22:24:45.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962106+0000", 2023-09-23T22:24:45.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022989700000000001 2023-09-23T22:24:45.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962141+0000", 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5138999999999997e-05 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962590+0000", 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044962099999999998 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.445 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37005 5.7 5:ebf3df01:::benchmark_data_smithi026_286012_object37004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:42.942342+0000", 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 542.45132217399998, 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0208824060000001, 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.446 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37005 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942342+0000", 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942342+0000", 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942340+0000", 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942383+0000", 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2715e-05 2023-09-23T22:24:45.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942409+0000", 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6746000000000001e-05 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942421+0000", 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1542999999999999e-05 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942572+0000", 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015078199999999999 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:42.942607+0000", 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5651999999999997e-05 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915850+0000", 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97324232600000005 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:43.915855+0000", 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8389999999999997e-06 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.954081+0000", 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.038225908 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.954178+0000", 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7398000000000005e-05 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.954284+0000", 2023-09-23T22:24:45.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106477 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.958847+0000", 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045624619999999998 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961899+0000", 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030523680000000002 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962660+0000", 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00076089199999999999 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962727+0000", 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7230000000000005e-05 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963224+0000", 2023-09-23T22:24:45.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049668100000000001 2023-09-23T22:24:45.455 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.455 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.455 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.455 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37018 5.7 5:e9bfc706:::benchmark_data_smithi026_286012_object37017:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:24:45.455 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.022925+0000", 2023-09-23T22:24:45.455 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 541.37073920700004, 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94092104499999996, 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37018 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022925+0000", 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022925+0000", 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022923+0000", 2023-09-23T22:24:45.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022979+0000", 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5467000000000003e-05 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022992+0000", 2023-09-23T22:24:45.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3006999999999999e-05 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.022999+0000", 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0439999999999998e-06 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.023072+0000", 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3577000000000004e-05 2023-09-23T22:24:45.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.023087+0000", 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4313e-05 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.955164+0000", 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93207778100000005 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.955263+0000", 2023-09-23T22:24:45.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8320999999999996e-05 2023-09-23T22:24:45.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.955401+0000", 2023-09-23T22:24:45.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013850399999999999 2023-09-23T22:24:45.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.959959+0000", 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045578490000000001 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961930+0000", 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001971048 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963292+0000", 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013615999999999999 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963355+0000", 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3847e-05 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963846+0000", 2023-09-23T22:24:45.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049014800000000004 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37021 5.7 5:e7df05ef:::benchmark_data_smithi026_286012_object37020:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.024001+0000", 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 541.36966287600001, 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94048630600000005, 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.464 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37021 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024001+0000", 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024001+0000", 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024000+0000", 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024054+0000", 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4146e-05 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024071+0000", 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6731999999999998e-05 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024078+0000", 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2099999999999996e-06 2023-09-23T22:24:45.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024135+0000", 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7099999999999999e-05 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.024146+0000", 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.084e-05 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956202+0000", 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93205636999999997 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956349+0000", 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147093 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.956522+0000", 2023-09-23T22:24:45.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017295900000000001 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.959972+0000", 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034492390000000002 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961978+0000", 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020065759999999999 2023-09-23T22:24:45.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963918+0000", 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001940034 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.963991+0000", 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2616999999999999e-05 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964487+0000", 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049637900000000003 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:37045 5.7 5:e066772a:::benchmark_data_smithi026_286012_object37044:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:44.042677+0000", 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 541.35098630699997, 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92228575899999998, 2023-09-23T22:24:45.472 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 37045 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042677+0000", 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042677+0000", 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042676+0000", 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042737+0000", 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0795999999999999e-05 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042751+0000", 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4453e-05 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042759+0000", 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1729999999999998e-06 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042834+0000", 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5841999999999997e-05 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.042847+0000", 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.255e-05 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957481+0000", 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91463394099999995 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957625+0000", 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000144027 2023-09-23T22:24:45.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.957796+0000", 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017132399999999999 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.961086+0000", 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003289882 2023-09-23T22:24:45.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.962006+0000", 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00091965299999999999 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964538+0000", 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025323949999999998 2023-09-23T22:24:45.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964578+0000", 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9359e-05 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:44.964963+0000", 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038571299999999998 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.480 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.481 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.481 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59924 5.7 5:e199b184:::benchmark_data_smithi026_286012_object59923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T22:24:45.481 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.616936+0000", 2023-09-23T22:24:45.481 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.77672764600004, 2023-09-23T22:24:45.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0339116370000001, 2023-09-23T22:24:45.481 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.481 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59924 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616936+0000", 2023-09-23T22:24:45.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616936+0000", 2023-09-23T22:24:45.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616935+0000", 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616983+0000", 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8554999999999998e-05 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.616996+0000", 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3098000000000001e-05 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.617003+0000", 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8950000000000001e-06 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.618902+0000", 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001899022 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.618913+0000", 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0487e-05 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.629818+0000", 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.01090558 2023-09-23T22:24:45.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.629940+0000", 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121614 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.630050+0000", 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110517 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636053+0000", 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060029760000000001 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650747+0000", 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014693632999999999 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650777+0000", 2023-09-23T22:24:45.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9669999999999999e-05 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650811+0000", 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4023999999999998e-05 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650848+0000", 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7058999999999999e-05 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.489 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59932 5.7 5:eebeefdc:::benchmark_data_smithi026_286012_object59931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.623830+0000", 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.76983406299996, 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.027128673, 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.490 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59932 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623830+0000", 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623830+0000", 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623829+0000", 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623852+0000", 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3107000000000001e-05 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623863+0000", 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1671999999999999e-05 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623868+0000", 2023-09-23T22:24:45.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1880000000000001e-06 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623915+0000", 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6591999999999997e-05 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.623926+0000", 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0712e-05 2023-09-23T22:24:45.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.632001+0000", 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0080748070000001 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.632141+0000", 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014057300000000001 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.632325+0000", 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000183525 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638341+0000", 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060166400000000002 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650882+0000", 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012540689000000001 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650908+0000", 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5919e-05 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650934+0000", 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6418999999999999e-05 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650958+0000", 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4048000000000001e-05 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59934 5.7 5:e79a8a57:::benchmark_data_smithi026_286012_object59933:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.625976+0000", 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.76768734999996, 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.025169207, 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.499 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59934 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.625976+0000", 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.625976+0000", 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.625975+0000", 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626005+0000", 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0679999999999998e-05 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626023+0000", 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7546000000000001e-05 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626031+0000", 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9349999999999994e-06 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626091+0000", 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9787999999999997e-05 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.626100+0000", 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6150000000000003e-06 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.633572+0000", 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0074717230000001 2023-09-23T22:24:45.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.633653+0000", 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1277000000000001e-05 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.633745+0000", 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1607999999999994e-05 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638348+0000", 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046025989999999998 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.650976+0000", 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012628758 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651066+0000", 2023-09-23T22:24:45.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9389999999999996e-05 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651119+0000", 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3155999999999997e-05 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651146+0000", 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6772999999999999e-05 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.506 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59943 5.7 5:e02a7111:::benchmark_data_smithi026_286012_object59942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.635287+0000", 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.75837688900003, 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0159653, 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.507 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59943 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635287+0000", 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635287+0000", 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635285+0000", 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635349+0000", 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3595000000000004e-05 2023-09-23T22:24:45.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635365+0000", 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6218e-05 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635371+0000", 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1619999999999996e-06 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635435+0000", 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3384000000000002e-05 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.635446+0000", 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0754000000000001e-05 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.634774+0000", 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99932857799999997 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.634895+0000", 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012116999999999999 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.635052+0000", 2023-09-23T22:24:45.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000156162 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640622+0000", 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055706200000000001 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651186+0000", 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010564146999999999 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651210+0000", 2023-09-23T22:24:45.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3373000000000001e-05 2023-09-23T22:24:45.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651235+0000", 2023-09-23T22:24:45.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5253999999999999e-05 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651252+0000", 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7295999999999998e-05 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59963 5.7 5:ec56653b:::benchmark_data_smithi026_286012_object59962:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:24:45.515 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.657440+0000", 2023-09-23T22:24:45.516 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.73622418100001, 2023-09-23T22:24:45.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99390371200000005, 2023-09-23T22:24:45.516 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.516 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.516 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.516 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.516 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.516 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59963 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657440+0000", 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657440+0000", 2023-09-23T22:24:45.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657438+0000", 2023-09-23T22:24:45.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657481+0000", 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2404999999999998e-05 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657495+0000", 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4557e-05 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657502+0000", 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6989999999999997e-06 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657564+0000", 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1988000000000003e-05 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.657573+0000", 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4689999999999997e-06 2023-09-23T22:24:45.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636095+0000", 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97852269199999997 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636182+0000", 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6561e-05 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.636276+0000", 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4087999999999998e-05 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640638+0000", 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043621459999999999 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651280+0000", 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010641651 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651303+0000", 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2943999999999999e-05 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651326+0000", 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3354999999999999e-05 2023-09-23T22:24:45.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651343+0000", 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7377999999999999e-05 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:59987 5.7 5:ea604450:::benchmark_data_smithi026_286012_object59986:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.678386+0000", 2023-09-23T22:24:45.524 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.71527810199996, 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97304345299999995, 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 59987 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678386+0000", 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678386+0000", 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678384+0000", 2023-09-23T22:24:45.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678451+0000", 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6938000000000002e-05 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678463+0000", 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.146e-05 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678470+0000", 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5079999999999999e-06 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678519+0000", 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8720000000000001e-05 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.678530+0000", 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0726e-05 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.637191+0000", 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95866113100000006 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.637310+0000", 2023-09-23T22:24:45.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119374 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.637455+0000", 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014488599999999999 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.643133+0000", 2023-09-23T22:24:45.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056781310000000003 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651368+0000", 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082348999999999999 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651390+0000", 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2087999999999999e-05 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651413+0000", 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2354e-05 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651429+0000", 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6606999999999999e-05 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.532 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:60017 5.7 5:e7911ea8:::benchmark_data_smithi026_286012_object60016:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.703829+0000", 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.68983526500006, 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94768691699999996, 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.533 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60017 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703829+0000", 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703829+0000", 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703827+0000", 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703929+0000", 2023-09-23T22:24:45.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000102562 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703949+0000", 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9378999999999999e-05 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.703957+0000", 2023-09-23T22:24:45.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1659999999999994e-06 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.704024+0000", 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7266000000000003e-05 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.704036+0000", 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1711e-05 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638386+0000", 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93434993 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638486+0000", 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9834000000000007e-05 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.638579+0000", 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3580999999999997e-05 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.643145+0000", 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045657620000000001 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651454+0000", 2023-09-23T22:24:45.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083090219999999992 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651477+0000", 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2541999999999999e-05 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651499+0000", 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2277999999999999e-05 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651515+0000", 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6566e-05 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:60025 5.7 5:ee51d15d:::benchmark_data_smithi026_286012_object60024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.711079+0000", 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.68258471800004, 2023-09-23T22:24:45.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94052496900000004, 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60025 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711079+0000", 2023-09-23T22:24:45.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711079+0000", 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711078+0000", 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711136+0000", 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7855000000000001e-05 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711150+0000", 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4219000000000001e-05 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711156+0000", 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5379999999999999e-06 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711209+0000", 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2714999999999999e-05 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.711221+0000", 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1578e-05 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.639471+0000", 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92825038299999996 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.639589+0000", 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011770200000000001 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.639740+0000", 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015135499999999999 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.645463+0000", 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057232120000000001 2023-09-23T22:24:45.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651540+0000", 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060769939999999996 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651563+0000", 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3059000000000001e-05 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651586+0000", 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2154000000000002e-05 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651604+0000", 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8445999999999999e-05 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.550 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:60041 5.7 5:e6579469:::benchmark_data_smithi026_286012_object60040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T22:24:45.550 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:07.728814+0000", 2023-09-23T22:24:45.550 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 517.66484998999999, 2023-09-23T22:24:45.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92288071299999996, 2023-09-23T22:24:45.550 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.550 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.550 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.550 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60041 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728814+0000", 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728814+0000", 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728813+0000", 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728906+0000", 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3043999999999999e-05 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728918+0000", 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2201000000000001e-05 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728923+0000", 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1590000000000001e-06 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728988+0000", 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5283000000000005e-05 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:07.728999+0000", 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0393e-05 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640683+0000", 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.911684351 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640810+0000", 2023-09-23T22:24:45.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012665500000000001 2023-09-23T22:24:45.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:24:45.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.640977+0000", 2023-09-23T22:24:45.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016692500000000001 2023-09-23T22:24:45.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.645469+0000", 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044926890000000002 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651632+0000", 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061627959999999999 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651654+0000", 2023-09-23T22:24:45.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2266e-05 2023-09-23T22:24:45.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651677+0000", 2023-09-23T22:24:45.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2198999999999999e-05 2023-09-23T22:24:45.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:08.651695+0000", 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7923000000000001e-05 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:93693 5.f 5:f82a8543:::benchmark_data_smithi026_286012_object93692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:48.289235+0000", 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 477.10442845599999, 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0721697720000001, 2023-09-23T22:24:45.559 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 93693 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289235+0000", 2023-09-23T22:24:45.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289235+0000", 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289234+0000", 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289280+0000", 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6313000000000002e-05 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289294+0000", 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3896999999999999e-05 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289299+0000", 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4210000000000003e-06 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289365+0000", 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6038999999999996e-05 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.289378+0000", 2023-09-23T22:24:45.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2371000000000001e-05 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152914+0000", 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.863536533 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152926+0000", 2023-09-23T22:24:45.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1677e-05 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.346430+0000", 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1935035489999999 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.346618+0000", 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000188378 2023-09-23T22:24:45.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.346836+0000", 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00021775200000000001 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358797+0000", 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011961090000000001 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358858+0000", 2023-09-23T22:24:45.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1081999999999996e-05 2023-09-23T22:24:45.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358963+0000", 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105529 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.359038+0000", 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4793e-05 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.361405+0000", 2023-09-23T22:24:45.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002366908 2023-09-23T22:24:45.571 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.571 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.571 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.571 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:93720 5.f 5:f1e7c9aa:::benchmark_data_smithi026_286012_object93719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:24:45.571 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:16:48.318037+0000", 2023-09-23T22:24:45.571 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 477.07562692300002, 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0455209810000001, 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 93720 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:24:45.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318037+0000", 2023-09-23T22:24:45.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:24:45.573 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318037+0000", 2023-09-23T22:24:45.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:24:45.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.573 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.573 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318035+0000", 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318080+0000", 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4404999999999999e-05 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318094+0000", 2023-09-23T22:24:45.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4014e-05 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318100+0000", 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6380000000000001e-06 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318152+0000", 2023-09-23T22:24:45.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2349000000000003e-05 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:48.318161+0000", 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9260000000000006e-06 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152958+0000", 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.834797067 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:49.152965+0000", 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3849999999999996e-06 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.350144+0000", 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.197178563 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.350310+0000", 2023-09-23T22:24:45.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000166584 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.350530+0000", 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000219235 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358894+0000", 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083640529999999998 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.358901+0000", 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1829999999999997e-06 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.361482+0000", 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002581046 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:24:45.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.361552+0000", 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9646000000000004e-05 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:16:50.363558+0000", 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020062909999999999 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:24:45.580 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:24:45.581 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:45.582 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:45.609 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:45.755 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:45.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:45.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:45.859 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:45.885 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:45.980 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_2' created 2023-09-23T22:24:46.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/204882261' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T22:24:46.153 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:45 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T22:24:46.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/204882261' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T22:24:46.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:45 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T22:24:46.155 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:46.156 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:24:46.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:46.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:46.205 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:24:46.223 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/204882261' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T22:24:46.223 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:45 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T22:24:46.261 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:46.287 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:46.330 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:46.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:46.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:46.358 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:24:46.435 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:46.462 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:46.509 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:46.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:46.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:46.552 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:46.613 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:46.645 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:46.814 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:46.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:46.918 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:46.945 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:47.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:46 smithi070 ceph-mon[192652]: pgmap v656: 83 pgs: 83 active+clean; 4.4 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 40 KiB/s, 4 objects/s recovering 2023-09-23T22:24:47.066 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:46 smithi070 ceph-mon[192652]: 1.4 scrub starts 2023-09-23T22:24:47.067 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:46 smithi070 ceph-mon[192652]: 1.4 scrub ok 2023-09-23T22:24:47.067 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:46 smithi070 ceph-mon[192652]: failed to encode map e260 with expected crc 2023-09-23T22:24:47.067 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:46 smithi070 ceph-mon[192652]: failed to encode map e260 with expected crc 2023-09-23T22:24:47.067 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:46 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T22:24:47.067 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:46 smithi070 ceph-mon[192652]: osdmap e260: 8 total, 8 up, 6 in 2023-09-23T22:24:47.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[215775]: pgmap v656: 83 pgs: 83 active+clean; 4.4 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 40 KiB/s, 4 objects/s recovering 2023-09-23T22:24:47.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[215775]: 1.4 scrub starts 2023-09-23T22:24:47.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[215775]: 1.4 scrub ok 2023-09-23T22:24:47.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[215775]: failed to encode map e260 with expected crc 2023-09-23T22:24:47.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[215775]: failed to encode map e260 with expected crc 2023-09-23T22:24:47.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T22:24:47.225 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[215775]: osdmap e260: 8 total, 8 up, 6 in 2023-09-23T22:24:47.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[223014]: pgmap v656: 83 pgs: 83 active+clean; 4.4 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail; 40 KiB/s, 4 objects/s recovering 2023-09-23T22:24:47.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[223014]: 1.4 scrub starts 2023-09-23T22:24:47.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[223014]: 1.4 scrub ok 2023-09-23T22:24:47.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[223014]: failed to encode map e260 with expected crc 2023-09-23T22:24:47.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[223014]: failed to encode map e260 with expected crc 2023-09-23T22:24:47.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T22:24:47.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:46 smithi026 ceph-mon[223014]: osdmap e260: 8 total, 8 up, 6 in 2023-09-23T22:24:47.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T22:24:47.251 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:47.254 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:47.355 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:47.382 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:47.509 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:47.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:47.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:47.542 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:47.550 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:47.613 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:47.642 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:47.726 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:47.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:47.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:47.829 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:24:47.860 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:47.983 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:24:48.030 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:24:48.030 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:24:48.030 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:24:48.030 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:24:48.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[223014]: failed to encode map e261 with expected crc 2023-09-23T22:24:48.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[223014]: osdmap e261: 8 total, 8 up, 6 in 2023-09-23T22:24:48.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[223014]: failed to encode map e261 with expected crc 2023-09-23T22:24:48.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[223014]: failed to encode map e261 with expected crc 2023-09-23T22:24:48.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1259261285' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:24:48.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:24:48.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[215775]: failed to encode map e261 with expected crc 2023-09-23T22:24:48.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[215775]: osdmap e261: 8 total, 8 up, 6 in 2023-09-23T22:24:48.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[215775]: failed to encode map e261 with expected crc 2023-09-23T22:24:48.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[215775]: failed to encode map e261 with expected crc 2023-09-23T22:24:48.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1259261285' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:24:48.069 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:47 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:24:48.071 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:24:48.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:48.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:24:48.139 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:48.177 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:48.206 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:48.250 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:48.251 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:47 smithi070 ceph-mon[192652]: failed to encode map e261 with expected crc 2023-09-23T22:24:48.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:47 smithi070 ceph-mon[192652]: osdmap e261: 8 total, 8 up, 6 in 2023-09-23T22:24:48.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:47 smithi070 ceph-mon[192652]: failed to encode map e261 with expected crc 2023-09-23T22:24:48.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:47 smithi070 ceph-mon[192652]: failed to encode map e261 with expected crc 2023-09-23T22:24:48.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:47 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1259261285' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:24:48.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:47 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:24:48.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:48.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:48.354 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:48.380 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:48.423 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:48.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:48.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:48.527 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:48.557 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:48.730 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:24:49.073 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:49.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[215775]: pgmap v659: 99 pgs: 6 creating+peering, 10 unknown, 83 active+clean; 4.4 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:24:49.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[215775]: 1.7 scrub starts 2023-09-23T22:24:49.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[215775]: 1.7 scrub ok 2023-09-23T22:24:49.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:24:49.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:24:49.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[215775]: failed to encode map e262 with expected crc 2023-09-23T22:24:49.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:24:49.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[215775]: osdmap e262: 8 total, 8 up, 6 in 2023-09-23T22:24:49.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[223014]: pgmap v659: 99 pgs: 6 creating+peering, 10 unknown, 83 active+clean; 4.4 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:24:49.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[223014]: 1.7 scrub starts 2023-09-23T22:24:49.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[223014]: 1.7 scrub ok 2023-09-23T22:24:49.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:24:49.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:24:49.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[223014]: failed to encode map e262 with expected crc 2023-09-23T22:24:49.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:24:49.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:48 smithi026 ceph-mon[223014]: osdmap e262: 8 total, 8 up, 6 in 2023-09-23T22:24:49.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:48 smithi070 ceph-mon[192652]: pgmap v659: 99 pgs: 6 creating+peering, 10 unknown, 83 active+clean; 4.4 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:24:49.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:48 smithi070 ceph-mon[192652]: 1.7 scrub starts 2023-09-23T22:24:49.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:48 smithi070 ceph-mon[192652]: 1.7 scrub ok 2023-09-23T22:24:49.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:48 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:24:49.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:48 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:24:49.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:48 smithi070 ceph-mon[192652]: failed to encode map e262 with expected crc 2023-09-23T22:24:49.249 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:48 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:24:49.250 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:48 smithi070 ceph-mon[192652]: osdmap e262: 8 total, 8 up, 6 in 2023-09-23T22:24:49.509 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:49.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:49.613 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:49.642 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:49.684 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:49.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:49.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:49.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:49.809 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:24:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:24:49] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.43.0" 2023-09-23T22:24:49.819 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:49.924 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:49.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:49.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:50.028 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:50.059 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:50.073 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:49 smithi026 ceph-mon[215775]: failed to encode map e262 with expected crc 2023-09-23T22:24:50.073 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:49 smithi026 ceph-mon[215775]: failed to encode map e262 with expected crc 2023-09-23T22:24:50.073 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:49 smithi026 ceph-mon[215775]: failed to encode map e262 with expected crc 2023-09-23T22:24:50.074 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:49 smithi026 ceph-mon[223014]: failed to encode map e262 with expected crc 2023-09-23T22:24:50.074 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:49 smithi026 ceph-mon[223014]: failed to encode map e262 with expected crc 2023-09-23T22:24:50.074 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:49 smithi026 ceph-mon[223014]: failed to encode map e262 with expected crc 2023-09-23T22:24:50.313 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:50.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:50.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:50.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:49 smithi070 ceph-mon[192652]: failed to encode map e262 with expected crc 2023-09-23T22:24:50.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:49 smithi070 ceph-mon[192652]: failed to encode map e262 with expected crc 2023-09-23T22:24:50.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:49 smithi070 ceph-mon[192652]: failed to encode map e262 with expected crc 2023-09-23T22:24:50.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:50.448 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:50.568 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:24:50.568 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:24:50.568 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:24:50.568 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:24:50.569 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:24:50.587 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:24:50.589 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:24:50.872 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:24:51.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:50 smithi070 ceph-mon[192652]: failed to encode map e262 with expected crc 2023-09-23T22:24:51.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:50 smithi070 ceph-mon[192652]: pgmap v661: 99 pgs: 6 creating+peering, 8 unknown, 85 active+clean; 4.4 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:24:51.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:50 smithi026 ceph-mon[223014]: failed to encode map e262 with expected crc 2023-09-23T22:24:51.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:50 smithi026 ceph-mon[223014]: pgmap v661: 99 pgs: 6 creating+peering, 8 unknown, 85 active+clean; 4.4 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:24:51.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:50 smithi026 ceph-mon[215775]: failed to encode map e262 with expected crc 2023-09-23T22:24:51.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:50 smithi026 ceph-mon[215775]: pgmap v661: 99 pgs: 6 creating+peering, 8 unknown, 85 active+clean; 4.4 MiB data, 2.0 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:24:51.252 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:51.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:51.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:51.271 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:24:51.356 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:51.383 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:51.426 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:51.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:51.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:51.533 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:51.560 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:51.605 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:51.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:51.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:51.629 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:51.639 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:24:51.660 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:24:51.679 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:24:51.691 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":661,"stamp":"2023-09-23T22:24:49.302746+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4569855,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":6070,"num_read_kb":12350,"num_write":951,"num_write_kb":13165,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":9311471,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2802,"ondisk_log_size":2802,"up":273,"acting":273,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2050828,"kb_used_data":539932,"kb_used_omap":20,"kb_used_meta":1510827,"kb_avail":560420084,"statfs":{"total":575970213888,"available":573870166016,"internally_reserved":0,"allocated":552890368,"data_stored":545561273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1547087627},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001733"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":246,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952790+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:24:39.952790+0000","last_peered":"2023-09-23T22:24:39.952790+0000","last_clean":"2023-09-23T22:24:39.952790+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:24:39.952790+0000","last_undegraded":"2023-09-23T22:24:39.952790+0000","last_fullsized":"2023-09-23T22:24:39.952790+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":238,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947705+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:24:39.947705+0000","last_peered":"2023-09-23T22:24:39.947705+0000","last_clean":"2023-09-23T22:24:39.947705+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:24:39.947705+0000","last_undegraded":"2023-09-23T22:24:39.947705+0000","last_fullsized":"2023-09-23T22:24:39.947705+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":203,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.977880+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:24:46.977880+0000","last_peered":"2023-09-23T22:24:46.977880+0000","last_clean":"2023-09-23T22:24:46.977880+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:24:46.977880+0000","last_undegraded":"2023-09-23T22:24:46.977880+0000","last_fullsized":"2023-09-23T22:24:46.977880+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":117,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.573205+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:24:42.573205+0000","last_peered":"2023-09-23T22:24:42.573205+0000","last_clean":"2023-09-23T22:24:42.573205+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:24:42.573205+0000","last_undegraded":"2023-09-23T22:24:42.573205+0000","last_fullsized":"2023-09-23T22:24:42.573205+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":278,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947058+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:24:39.947058+0000","last_peered":"2023-09-23T22:24:39.947058+0000","last_clean":"2023-09-23T22:24:39.947058+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:24:39.947058+0000","last_undegraded":"2023-09-23T22:24:39.947058+0000","last_fullsized":"2023-09-23T22:24:39.947058+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":214,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950875+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:24:39.950875+0000","last_peered":"2023-09-23T22:24:39.950875+0000","last_clean":"2023-09-23T22:24:39.950875+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:24:39.950875+0000","last_undegraded":"2023-09-23T22:24:39.950875+0000","last_fullsized":"2023-09-23T22:24:39.950875+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":146,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.573372+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:24:42.573372+0000","last_peered":"2023-09-23T22:24:42.573372+0000","last_clean":"2023-09-23T22:24:42.573372+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:24:42.573372+0000","last_undegraded":"2023-09-23T22:24:42.573372+0000","last_fullsized":"2023-09-23T22:24:42.573372+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":235,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947121+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:24:39.947121+0000","last_peered":"2023-09-23T22:24:39.947121+0000","last_clean":"2023-09-23T22:24:39.947121+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:24:39.947121+0000","last_undegraded":"2023-09-23T22:24:39.947121+0000","last_fullsized":"2023-09-23T22:24:39.947121+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":219,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.951105+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:24:39.951105+0000","last_peered":"2023-09-23T22:24:39.951105+0000","last_clean":"2023-09-23T22:24:39.951105+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:24:39.951105+0000","last_undegraded":"2023-09-23T22:24:39.951105+0000","last_fullsized":"2023-09-23T22:24:39.951105+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":203,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947007+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:24:39.947007+0000","last_peered":"2023-09-23T22:24:39.947007+0000","last_clean":"2023-09-23T22:24:39.947007+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:24:39.947007+0000","last_undegraded":"2023-09-23T22:24:39.947007+0000","last_fullsized":"2023-09-23T22:24:39.947007+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":202,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947628+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:24:39.947628+0000","last_peered":"2023-09-23T22:24:39.947628+0000","last_clean":"2023-09-23T22:24:39.947628+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:24:39.947628+0000","last_undegraded":"2023-09-23T22:24:39.947628+0000","last_fullsized":"2023-09-23T22:24:39.947628+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":240,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.977687+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:24:46.977687+0000","last_peered":"2023-09-23T22:24:46.977687+0000","last_clean":"2023-09-23T22:24:46.977687+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:24:46.977687+0000","last_undegraded":"2023-09-23T22:24:46.977687+0000","last_fullsized":"2023-09-23T22:24:46.977687+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":263,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.953883+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:24:39.953883+0000","last_peered":"2023-09-23T22:24:39.953883+0000","last_clean":"2023-09-23T22:24:39.953883+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:24:39.953883+0000","last_undegraded":"2023-09-23T22:24:39.953883+0000","last_fullsized":"2023-09-23T22:24:39.953883+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":227,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952789+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:24:39.952789+0000","last_peered":"2023-09-23T22:24:39.952789+0000","last_clean":"2023-09-23T22:24:39.952789+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:24:39.952789+0000","last_undegraded":"2023-09-23T22:24:39.952789+0000","last_fullsized":"2023-09-23T22:24:39.952789+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":186,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952872+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:24:39.952872+0000","last_peered":"2023-09-23T22:24:39.952872+0000","last_clean":"2023-09-23T22:24:39.952872+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:24:39.952872+0000","last_undegraded":"2023-09-23T22:24:39.952872+0000","last_fullsized":"2023-09-23T22:24:39.952872+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":226,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947427+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:24:39.947427+0000","last_peered":"2023-09-23T22:24:39.947427+0000","last_clean":"2023-09-23T22:24:39.947427+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:24:39.947427+0000","last_undegraded":"2023-09-23T22:24:39.947427+0000","last_fullsized":"2023-09-23T22:24:39.947427+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":135,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572944+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:24:42.572944+0000","last_peered":"2023-09-23T22:24:42.572944+0000","last_clean":"2023-09-23T22:24:42.572944+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:24:42.572944+0000","last_undegraded":"2023-09-23T22:24:42.572944+0000","last_fullsized":"2023-09-23T22:24:42.572944+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":238,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950612+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:24:39.950612+0000","last_peered":"2023-09-23T22:24:39.950612+0000","last_clean":"2023-09-23T22:24:39.950612+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:24:39.950612+0000","last_undegraded":"2023-09-23T22:24:39.950612+0000","last_fullsized":"2023-09-23T22:24:39.950612+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":182,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952552+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:24:39.952552+0000","last_peered":"2023-09-23T22:24:39.952552+0000","last_clean":"2023-09-23T22:24:39.952552+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:24:39.952552+0000","last_undegraded":"2023-09-23T22:24:39.952552+0000","last_fullsized":"2023-09-23T22:24:39.952552+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":117,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.573303+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:24:42.573303+0000","last_peered":"2023-09-23T22:24:42.573303+0000","last_clean":"2023-09-23T22:24:42.573303+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:24:42.573303+0000","last_undegraded":"2023-09-23T22:24:42.573303+0000","last_fullsized":"2023-09-23T22:24:42.573303+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":278,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572773+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:24:42.572773+0000","last_peered":"2023-09-23T22:24:42.572773+0000","last_clean":"2023-09-23T22:24:42.572773+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:24:42.572773+0000","last_undegraded":"2023-09-23T22:24:42.572773+0000","last_fullsized":"2023-09-23T22:24:42.572773+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":48,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:43.226630+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:24:43.226630+0000","last_peered":"2023-09-23T22:24:43.226630+0000","last_clean":"2023-09-23T22:24:43.226630+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:24:43.226630+0000","last_undegraded":"2023-09-23T22:24:43.226630+0000","last_fullsized":"2023-09-23T22:24:43.226630+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":11,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:24:48.839410+0000","last_change":"2023-09-23T22:24:46.980027+0000","last_active":"2023-09-23T22:24:48.839410+0000","last_peered":"2023-09-23T22:24:48.839410+0000","last_clean":"2023-09-23T22:24:48.839410+0000","last_became_active":"2023-09-23T22:24:46.979731+0000","last_became_peered":"2023-09-23T22:24:46.979731+0000","last_unstale":"2023-09-23T22:24:48.839410+0000","last_undegraded":"2023-09-23T22:24:48.839410+0000","last_fullsized":"2023-09-23T22:24:48.839410+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:06.319642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":254,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.954130+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:24:39.954130+0000","last_peered":"2023-09-23T22:24:39.954130+0000","last_clean":"2023-09-23T22:24:39.954130+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:24:39.954130+0000","last_undegraded":"2023-09-23T22:24:39.954130+0000","last_fullsized":"2023-09-23T22:24:39.954130+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":241,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950627+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:24:39.950627+0000","last_peered":"2023-09-23T22:24:39.950627+0000","last_clean":"2023-09-23T22:24:39.950627+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:24:39.950627+0000","last_undegraded":"2023-09-23T22:24:39.950627+0000","last_fullsized":"2023-09-23T22:24:39.950627+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:24:45.961568+0000","last_change":"2023-09-23T22:24:45.961568+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.961568+0000","last_undegraded":"2023-09-23T22:24:45.961568+0000","last_fullsized":"2023-09-23T22:24:45.961568+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":168,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.954199+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:24:39.954199+0000","last_peered":"2023-09-23T22:24:39.954199+0000","last_clean":"2023-09-23T22:24:39.954199+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:24:39.954199+0000","last_undegraded":"2023-09-23T22:24:39.954199+0000","last_fullsized":"2023-09-23T22:24:39.954199+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":246,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952515+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:24:39.952515+0000","last_peered":"2023-09-23T22:24:39.952515+0000","last_clean":"2023-09-23T22:24:39.952515+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:24:39.952515+0000","last_undegraded":"2023-09-23T22:24:39.952515+0000","last_fullsized":"2023-09-23T22:24:39.952515+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":1,"reported_epoch":260,"state":"creating+peering","last_fresh":"2023-09-23T22:24:45.974371+0000","last_change":"2023-09-23T22:24:45.974371+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.974371+0000","last_undegraded":"2023-09-23T22:24:45.974371+0000","last_fullsized":"2023-09-23T22:24:45.974371+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:24.132475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":184,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950702+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:24:39.950702+0000","last_peered":"2023-09-23T22:24:39.950702+0000","last_clean":"2023-09-23T22:24:39.950702+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:24:39.950702+0000","last_undegraded":"2023-09-23T22:24:39.950702+0000","last_fullsized":"2023-09-23T22:24:39.950702+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.977782+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:24:46.977782+0000","last_peered":"2023-09-23T22:24:46.977782+0000","last_clean":"2023-09-23T22:24:46.977782+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:24:46.977782+0000","last_undegraded":"2023-09-23T22:24:46.977782+0000","last_fullsized":"2023-09-23T22:24:46.977782+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:24:45.961568+0000","last_change":"2023-09-23T22:24:45.961568+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.961568+0000","last_undegraded":"2023-09-23T22:24:45.961568+0000","last_fullsized":"2023-09-23T22:24:45.961568+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":249,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950745+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:24:39.950745+0000","last_peered":"2023-09-23T22:24:39.950745+0000","last_clean":"2023-09-23T22:24:39.950745+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:24:39.950745+0000","last_undegraded":"2023-09-23T22:24:39.950745+0000","last_fullsized":"2023-09-23T22:24:39.950745+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":115,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572823+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:24:42.572823+0000","last_peered":"2023-09-23T22:24:42.572823+0000","last_clean":"2023-09-23T22:24:42.572823+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:24:42.572823+0000","last_undegraded":"2023-09-23T22:24:42.572823+0000","last_fullsized":"2023-09-23T22:24:42.572823+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":1,"reported_epoch":260,"state":"creating+peering","last_fresh":"2023-09-23T22:24:45.975323+0000","last_change":"2023-09-23T22:24:45.975323+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.975323+0000","last_undegraded":"2023-09-23T22:24:45.975323+0000","last_fullsized":"2023-09-23T22:24:45.975323+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:44.762872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":563,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.946843+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:24:39.946843+0000","last_peered":"2023-09-23T22:24:39.946843+0000","last_clean":"2023-09-23T22:24:39.946843+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:24:39.946843+0000","last_undegraded":"2023-09-23T22:24:39.946843+0000","last_fullsized":"2023-09-23T22:24:39.946843+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":272,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947921+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:24:39.947921+0000","last_peered":"2023-09-23T22:24:39.947921+0000","last_clean":"2023-09-23T22:24:39.947921+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:24:39.947921+0000","last_undegraded":"2023-09-23T22:24:39.947921+0000","last_fullsized":"2023-09-23T22:24:39.947921+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":227,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.953747+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:24:39.953747+0000","last_peered":"2023-09-23T22:24:39.953747+0000","last_clean":"2023-09-23T22:24:39.953747+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:24:39.953747+0000","last_undegraded":"2023-09-23T22:24:39.953747+0000","last_fullsized":"2023-09-23T22:24:39.953747+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:24:45.961568+0000","last_change":"2023-09-23T22:24:45.961568+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.961568+0000","last_undegraded":"2023-09-23T22:24:45.961568+0000","last_fullsized":"2023-09-23T22:24:45.961568+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":571,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.955082+0000","last_change":"2023-09-23T22:24:39.954805+0000","last_active":"2023-09-23T22:24:39.955082+0000","last_peered":"2023-09-23T22:24:39.955082+0000","last_clean":"2023-09-23T22:24:39.955082+0000","last_became_active":"2023-09-23T22:24:39.954350+0000","last_became_peered":"2023-09-23T22:24:39.954350+0000","last_unstale":"2023-09-23T22:24:39.955082+0000","last_undegraded":"2023-09-23T22:24:39.955082+0000","last_fullsized":"2023-09-23T22:24:39.955082+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:47.774412+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":175,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952594+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:24:39.952594+0000","last_peered":"2023-09-23T22:24:39.952594+0000","last_clean":"2023-09-23T22:24:39.952594+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:24:39.952594+0000","last_undegraded":"2023-09-23T22:24:39.952594+0000","last_fullsized":"2023-09-23T22:24:39.952594+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":244,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.953490+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:24:39.953490+0000","last_peered":"2023-09-23T22:24:39.953490+0000","last_clean":"2023-09-23T22:24:39.953490+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:24:39.953490+0000","last_undegraded":"2023-09-23T22:24:39.953490+0000","last_fullsized":"2023-09-23T22:24:39.953490+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":11,"reported_epoch":262,"state":"active+clean","last_fresh":"2023-09-23T22:24:48.839497+0000","last_change":"2023-09-23T22:24:46.982376+0000","last_active":"2023-09-23T22:24:48.839497+0000","last_peered":"2023-09-23T22:24:48.839497+0000","last_clean":"2023-09-23T22:24:48.839497+0000","last_became_active":"2023-09-23T22:24:46.982171+0000","last_became_peered":"2023-09-23T22:24:46.982171+0000","last_unstale":"2023-09-23T22:24:48.839497+0000","last_undegraded":"2023-09-23T22:24:48.839497+0000","last_fullsized":"2023-09-23T22:24:48.839497+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:03.026919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":220,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952423+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:24:39.952423+0000","last_peered":"2023-09-23T22:24:39.952423+0000","last_clean":"2023-09-23T22:24:39.952423+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:24:39.952423+0000","last_undegraded":"2023-09-23T22:24:39.952423+0000","last_fullsized":"2023-09-23T22:24:39.952423+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":235,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.954022+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:24:39.954022+0000","last_peered":"2023-09-23T22:24:39.954022+0000","last_clean":"2023-09-23T22:24:39.954022+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:24:39.954022+0000","last_undegraded":"2023-09-23T22:24:39.954022+0000","last_fullsized":"2023-09-23T22:24:39.954022+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:24:45.961568+0000","last_change":"2023-09-23T22:24:45.961568+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.961568+0000","last_undegraded":"2023-09-23T22:24:45.961568+0000","last_fullsized":"2023-09-23T22:24:45.961568+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":563,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950760+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:24:39.950760+0000","last_peered":"2023-09-23T22:24:39.950760+0000","last_clean":"2023-09-23T22:24:39.950760+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:24:39.950760+0000","last_undegraded":"2023-09-23T22:24:39.950760+0000","last_fullsized":"2023-09-23T22:24:39.950760+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":39,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.946704+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:24:39.946704+0000","last_peered":"2023-09-23T22:24:39.946704+0000","last_clean":"2023-09-23T22:24:39.946704+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:24:39.946704+0000","last_undegraded":"2023-09-23T22:24:39.946704+0000","last_fullsized":"2023-09-23T22:24:39.946704+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":204,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.978437+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:24:46.978437+0000","last_peered":"2023-09-23T22:24:46.978437+0000","last_clean":"2023-09-23T22:24:46.978437+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:24:46.978437+0000","last_undegraded":"2023-09-23T22:24:46.978437+0000","last_fullsized":"2023-09-23T22:24:46.978437+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:24:45.961568+0000","last_change":"2023-09-23T22:24:45.961568+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.961568+0000","last_undegraded":"2023-09-23T22:24:45.961568+0000","last_fullsized":"2023-09-23T22:24:45.961568+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":555,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.948009+0000","last_change":"2023-09-23T22:22:47.349660+0000","last_active":"2023-09-23T22:24:39.948009+0000","last_peered":"2023-09-23T22:24:39.948009+0000","last_clean":"2023-09-23T22:24:39.948009+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:24:39.948009+0000","last_undegraded":"2023-09-23T22:24:39.948009+0000","last_fullsized":"2023-09-23T22:24:39.948009+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.790411+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.790411+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":811,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.978756+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:24:46.978756+0000","last_peered":"2023-09-23T22:24:46.978756+0000","last_clean":"2023-09-23T22:24:46.978756+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:24:46.978756+0000","last_undegraded":"2023-09-23T22:24:46.978756+0000","last_fullsized":"2023-09-23T22:24:46.978756+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":276,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950421+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:24:39.950421+0000","last_peered":"2023-09-23T22:24:39.950421+0000","last_clean":"2023-09-23T22:24:39.950421+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:24:39.950421+0000","last_undegraded":"2023-09-23T22:24:39.950421+0000","last_fullsized":"2023-09-23T22:24:39.950421+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.977515+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:24:46.977515+0000","last_peered":"2023-09-23T22:24:46.977515+0000","last_clean":"2023-09-23T22:24:46.977515+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:24:46.977515+0000","last_undegraded":"2023-09-23T22:24:46.977515+0000","last_fullsized":"2023-09-23T22:24:46.977515+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":1,"reported_epoch":260,"state":"creating+peering","last_fresh":"2023-09-23T22:24:45.975014+0000","last_change":"2023-09-23T22:24:45.975014+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.975014+0000","last_undegraded":"2023-09-23T22:24:45.975014+0000","last_fullsized":"2023-09-23T22:24:45.975014+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:01.377206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":563,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950359+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:24:39.950359+0000","last_peered":"2023-09-23T22:24:39.950359+0000","last_clean":"2023-09-23T22:24:39.950359+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:24:39.950359+0000","last_undegraded":"2023-09-23T22:24:39.950359+0000","last_fullsized":"2023-09-23T22:24:39.950359+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":991,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950907+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:24:39.950907+0000","last_peered":"2023-09-23T22:24:39.950907+0000","last_clean":"2023-09-23T22:24:39.950907+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:24:39.950907+0000","last_undegraded":"2023-09-23T22:24:39.950907+0000","last_fullsized":"2023-09-23T22:24:39.950907+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":268,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952719+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:24:39.952719+0000","last_peered":"2023-09-23T22:24:39.952719+0000","last_clean":"2023-09-23T22:24:39.952719+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:24:39.952719+0000","last_undegraded":"2023-09-23T22:24:39.952719+0000","last_fullsized":"2023-09-23T22:24:39.952719+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":223,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950877+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:24:39.950877+0000","last_peered":"2023-09-23T22:24:39.950877+0000","last_clean":"2023-09-23T22:24:39.950877+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:24:39.950877+0000","last_undegraded":"2023-09-23T22:24:39.950877+0000","last_fullsized":"2023-09-23T22:24:39.950877+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":1,"reported_epoch":260,"state":"creating+peering","last_fresh":"2023-09-23T22:24:45.974984+0000","last_change":"2023-09-23T22:24:45.974984+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.974984+0000","last_undegraded":"2023-09-23T22:24:45.974984+0000","last_fullsized":"2023-09-23T22:24:45.974984+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:58.452186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":589,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572821+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:24:42.572821+0000","last_peered":"2023-09-23T22:24:42.572821+0000","last_clean":"2023-09-23T22:24:42.572821+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:24:42.572821+0000","last_undegraded":"2023-09-23T22:24:42.572821+0000","last_fullsized":"2023-09-23T22:24:42.572821+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":394,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947156+0000","last_change":"2023-09-23T22:24:38.072053+0000","last_active":"2023-09-23T22:24:39.947156+0000","last_peered":"2023-09-23T22:24:39.947156+0000","last_clean":"2023-09-23T22:24:39.947156+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:24:39.947156+0000","last_undegraded":"2023-09-23T22:24:39.947156+0000","last_fullsized":"2023-09-23T22:24:39.947156+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":222,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.977473+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:24:46.977473+0000","last_peered":"2023-09-23T22:24:46.977473+0000","last_clean":"2023-09-23T22:24:46.977473+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:24:46.977473+0000","last_undegraded":"2023-09-23T22:24:46.977473+0000","last_fullsized":"2023-09-23T22:24:46.977473+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":269,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.978475+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:24:46.978475+0000","last_peered":"2023-09-23T22:24:46.978475+0000","last_clean":"2023-09-23T22:24:46.978475+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:24:46.978475+0000","last_undegraded":"2023-09-23T22:24:46.978475+0000","last_fullsized":"2023-09-23T22:24:46.978475+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":1,"reported_epoch":260,"state":"creating+peering","last_fresh":"2023-09-23T22:24:45.974127+0000","last_change":"2023-09-23T22:24:45.974127+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.974127+0000","last_undegraded":"2023-09-23T22:24:45.974127+0000","last_fullsized":"2023-09-23T22:24:45.974127+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:20.557721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":886,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.955065+0000","last_change":"2023-09-23T22:24:39.954768+0000","last_active":"2023-09-23T22:24:39.955065+0000","last_peered":"2023-09-23T22:24:39.955065+0000","last_clean":"2023-09-23T22:24:39.955065+0000","last_became_active":"2023-09-23T22:24:39.954171+0000","last_became_peered":"2023-09-23T22:24:39.954171+0000","last_unstale":"2023-09-23T22:24:39.955065+0000","last_undegraded":"2023-09-23T22:24:39.955065+0000","last_fullsized":"2023-09-23T22:24:39.955065+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:00.983002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":572,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572532+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:24:42.572532+0000","last_peered":"2023-09-23T22:24:42.572532+0000","last_clean":"2023-09-23T22:24:42.572532+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:24:42.572532+0000","last_undegraded":"2023-09-23T22:24:42.572532+0000","last_fullsized":"2023-09-23T22:24:42.572532+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":184,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947432+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:24:39.947432+0000","last_peered":"2023-09-23T22:24:39.947432+0000","last_clean":"2023-09-23T22:24:39.947432+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:24:39.947432+0000","last_undegraded":"2023-09-23T22:24:39.947432+0000","last_fullsized":"2023-09-23T22:24:39.947432+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":145,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.953635+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:24:39.953635+0000","last_peered":"2023-09-23T22:24:39.953635+0000","last_clean":"2023-09-23T22:24:39.953635+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:24:39.953635+0000","last_undegraded":"2023-09-23T22:24:39.953635+0000","last_fullsized":"2023-09-23T22:24:39.953635+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":1,"reported_epoch":260,"state":"creating+peering","last_fresh":"2023-09-23T22:24:45.975222+0000","last_change":"2023-09-23T22:24:45.975222+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.975222+0000","last_undegraded":"2023-09-23T22:24:45.975222+0000","last_fullsized":"2023-09-23T22:24:45.975222+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.224040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":594,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.977999+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:24:46.977999+0000","last_peered":"2023-09-23T22:24:46.977999+0000","last_clean":"2023-09-23T22:24:46.977999+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:24:46.977999+0000","last_undegraded":"2023-09-23T22:24:46.977999+0000","last_fullsized":"2023-09-23T22:24:46.977999+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":612,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.977628+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:24:46.977628+0000","last_peered":"2023-09-23T22:24:46.977628+0000","last_clean":"2023-09-23T22:24:46.977628+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:24:46.977628+0000","last_undegraded":"2023-09-23T22:24:46.977628+0000","last_fullsized":"2023-09-23T22:24:46.977628+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":142,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572651+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:24:42.572651+0000","last_peered":"2023-09-23T22:24:42.572651+0000","last_clean":"2023-09-23T22:24:42.572651+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:24:42.572651+0000","last_undegraded":"2023-09-23T22:24:42.572651+0000","last_fullsized":"2023-09-23T22:24:42.572651+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":99,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572946+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:24:42.572946+0000","last_peered":"2023-09-23T22:24:42.572946+0000","last_clean":"2023-09-23T22:24:42.572946+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:24:42.572946+0000","last_undegraded":"2023-09-23T22:24:42.572946+0000","last_fullsized":"2023-09-23T22:24:42.572946+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:24:45.961568+0000","last_change":"2023-09-23T22:24:45.961568+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.961568+0000","last_undegraded":"2023-09-23T22:24:45.961568+0000","last_fullsized":"2023-09-23T22:24:45.961568+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":581,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.573291+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:24:42.573291+0000","last_peered":"2023-09-23T22:24:42.573291+0000","last_clean":"2023-09-23T22:24:42.573291+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:24:42.573291+0000","last_undegraded":"2023-09-23T22:24:42.573291+0000","last_fullsized":"2023-09-23T22:24:42.573291+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":151,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572376+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:24:42.572376+0000","last_peered":"2023-09-23T22:24:42.572376+0000","last_clean":"2023-09-23T22:24:42.572376+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:24:42.572376+0000","last_undegraded":"2023-09-23T22:24:42.572376+0000","last_fullsized":"2023-09-23T22:24:42.572376+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":213,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.979043+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:24:46.979043+0000","last_peered":"2023-09-23T22:24:46.979043+0000","last_clean":"2023-09-23T22:24:46.979043+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:24:46.979043+0000","last_undegraded":"2023-09-23T22:24:46.979043+0000","last_fullsized":"2023-09-23T22:24:46.979043+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":115,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572822+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:24:42.572822+0000","last_peered":"2023-09-23T22:24:42.572822+0000","last_clean":"2023-09-23T22:24:42.572822+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:24:42.572822+0000","last_undegraded":"2023-09-23T22:24:42.572822+0000","last_fullsized":"2023-09-23T22:24:42.572822+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:24:45.961568+0000","last_change":"2023-09-23T22:24:45.961568+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.961568+0000","last_undegraded":"2023-09-23T22:24:45.961568+0000","last_fullsized":"2023-09-23T22:24:45.961568+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":563,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.953370+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:24:39.953370+0000","last_peered":"2023-09-23T22:24:39.953370+0000","last_clean":"2023-09-23T22:24:39.953370+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:24:39.953370+0000","last_undegraded":"2023-09-23T22:24:39.953370+0000","last_fullsized":"2023-09-23T22:24:39.953370+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":272,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.948188+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:24:39.948188+0000","last_peered":"2023-09-23T22:24:39.948188+0000","last_clean":"2023-09-23T22:24:39.948188+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:24:39.948188+0000","last_undegraded":"2023-09-23T22:24:39.948188+0000","last_fullsized":"2023-09-23T22:24:39.948188+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":904,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.978156+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:24:46.978156+0000","last_peered":"2023-09-23T22:24:46.978156+0000","last_clean":"2023-09-23T22:24:46.978156+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:24:46.978156+0000","last_undegraded":"2023-09-23T22:24:46.978156+0000","last_fullsized":"2023-09-23T22:24:46.978156+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":48,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.946540+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:24:39.946540+0000","last_peered":"2023-09-23T22:24:39.946540+0000","last_clean":"2023-09-23T22:24:39.946540+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:24:39.946540+0000","last_undegraded":"2023-09-23T22:24:39.946540+0000","last_fullsized":"2023-09-23T22:24:39.946540+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:24:45.961568+0000","last_change":"2023-09-23T22:24:45.961568+0000","last_active":"2023-09-23T22:24:45.961568+0000","last_peered":"2023-09-23T22:24:45.961568+0000","last_clean":"2023-09-23T22:24:45.961568+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:24:45.961568+0000","last_undegraded":"2023-09-23T22:24:45.961568+0000","last_fullsized":"2023-09-23T22:24:45.961568+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":547,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947589+0000","last_change":"2023-09-23T22:22:19.883431+0000","last_active":"2023-09-23T22:24:39.947589+0000","last_peered":"2023-09-23T22:24:39.947589+0000","last_clean":"2023-09-23T22:24:39.947589+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:24:39.947589+0000","last_undegraded":"2023-09-23T22:24:39.947589+0000","last_fullsized":"2023-09-23T22:24:39.947589+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:46.903071+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:46.903071+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":253,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.948272+0000","last_change":"2023-09-23T22:24:38.076201+0000","last_active":"2023-09-23T22:24:39.948272+0000","last_peered":"2023-09-23T22:24:39.948272+0000","last_clean":"2023-09-23T22:24:39.948272+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:24:39.948272+0000","last_undegraded":"2023-09-23T22:24:39.948272+0000","last_fullsized":"2023-09-23T22:24:39.948272+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":94,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.573214+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:24:42.573214+0000","last_peered":"2023-09-23T22:24:42.573214+0000","last_clean":"2023-09-23T22:24:42.573214+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:24:42.573214+0000","last_undegraded":"2023-09-23T22:24:42.573214+0000","last_fullsized":"2023-09-23T22:24:42.573214+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":222,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952439+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:24:39.952439+0000","last_peered":"2023-09-23T22:24:39.952439+0000","last_clean":"2023-09-23T22:24:39.952439+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:24:39.952439+0000","last_undegraded":"2023-09-23T22:24:39.952439+0000","last_fullsized":"2023-09-23T22:24:39.952439+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":90,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.572364+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:24:42.572364+0000","last_peered":"2023-09-23T22:24:42.572364+0000","last_clean":"2023-09-23T22:24:42.572364+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:24:42.572364+0000","last_undegraded":"2023-09-23T22:24:42.572364+0000","last_fullsized":"2023-09-23T22:24:42.572364+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":273,"reported_epoch":261,"state":"active+clean","last_fresh":"2023-09-23T22:24:46.978961+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:24:46.978961+0000","last_peered":"2023-09-23T22:24:46.978961+0000","last_clean":"2023-09-23T22:24:46.978961+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:24:46.978961+0000","last_undegraded":"2023-09-23T22:24:46.978961+0000","last_fullsized":"2023-09-23T22:24:46.978961+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":221,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950666+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:24:39.950666+0000","last_peered":"2023-09-23T22:24:39.950666+0000","last_clean":"2023-09-23T22:24:39.950666+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:24:39.950666+0000","last_undegraded":"2023-09-23T22:24:39.950666+0000","last_fullsized":"2023-09-23T22:24:39.950666+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":234,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952944+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:24:39.952944+0000","last_peered":"2023-09-23T22:24:39.952944+0000","last_clean":"2023-09-23T22:24:39.952944+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:24:39.952944+0000","last_undegraded":"2023-09-23T22:24:39.952944+0000","last_fullsized":"2023-09-23T22:24:39.952944+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":218,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.950523+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:24:39.950523+0000","last_peered":"2023-09-23T22:24:39.950523+0000","last_clean":"2023-09-23T22:24:39.950523+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:24:39.950523+0000","last_undegraded":"2023-09-23T22:24:39.950523+0000","last_fullsized":"2023-09-23T22:24:39.950523+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":93,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.952667+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:24:39.952667+0000","last_peered":"2023-09-23T22:24:39.952667+0000","last_clean":"2023-09-23T22:24:39.952667+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:24:39.952667+0000","last_undegraded":"2023-09-23T22:24:39.952667+0000","last_fullsized":"2023-09-23T22:24:39.952667+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":240,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.947944+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:24:39.947944+0000","last_peered":"2023-09-23T22:24:39.947944+0000","last_clean":"2023-09-23T22:24:39.947944+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:24:39.947944+0000","last_undegraded":"2023-09-23T22:24:39.947944+0000","last_fullsized":"2023-09-23T22:24:39.947944+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":281,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.946928+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:24:39.946928+0000","last_peered":"2023-09-23T22:24:39.946928+0000","last_clean":"2023-09-23T22:24:39.946928+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:24:39.946928+0000","last_undegraded":"2023-09-23T22:24:39.946928+0000","last_fullsized":"2023-09-23T22:24:39.946928+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":149,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T22:24:42.573376+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:24:42.573376+0000","last_peered":"2023-09-23T22:24:42.573376+0000","last_clean":"2023-09-23T22:24:42.573376+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:24:42.573376+0000","last_undegraded":"2023-09-23T22:24:42.573376+0000","last_fullsized":"2023-09-23T22:24:42.573376+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":267,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T22:24:39.953196+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:24:39.953196+0000","last_peered":"2023-09-23T22:24:39.953196+0000","last_clean":"2023-09-23T22:24:39.953196+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:24:39.953196+0000","last_undegraded":"2023-09-23T22:24:39.953196+0000","last_fullsized":"2023-09-23T22:24:39.953196+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":2},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":370028,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":88,"num_read_kb":166,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":912387,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":126,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234244,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522252,"kb_used_data":325252,"kb_used_omap":0,"kb_used_meta":196992,"kb_avail":93222900,"statfs":{"total":95995035648,"available":95460249600,"internally_reserved":0,"allocated":333058048,"data_stored":332029204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":201719808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":4.1870000000000003,"15min":2.3159999999999998},"min":{"1min":0.22600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.5499999999999998,"5min":158.042,"15min":158.042},"last":0.497},{"interface":"front","average":{"1min":2.4369999999999998,"5min":4.6920000000000002,"15min":2.6070000000000002},"min":{"1min":0.28199999999999997,"5min":0.217,"15min":0.217},"max":{"1min":6.6719999999999997,"5min":157.952,"15min":157.952},"last":0.371}]},{"osd":1,"last update":"Sat Sep 23 22:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":4.4050000000000002,"15min":3.0630000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":6.2839999999999998,"5min":101.145,"15min":123.79300000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.8759999999999999,"5min":4.4969999999999999,"15min":3.2170000000000001},"min":{"1min":0.33200000000000002,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":8.4149999999999991,"5min":101.13,"15min":123.66500000000001},"last":0.59799999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":2.589,"5min":4.1260000000000003,"15min":3.7650000000000001},"min":{"1min":0.307,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.7939999999999996,"5min":105.34399999999999,"15min":125.101},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.6440000000000001,"5min":4.0960000000000001,"15min":3.5110000000000001},"min":{"1min":0.30499999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.3769999999999998,"5min":105.63200000000001,"15min":126.021},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:24:16 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":3.9329999999999998,"15min":2.335},"min":{"1min":0.46800000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.6479999999999997,"5min":170.86099999999999,"15min":170.86099999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.0619999999999998,"5min":3.7410000000000001,"15min":2.4100000000000001},"min":{"1min":0.44800000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.7919999999999998,"5min":174.57499999999999,"15min":174.57499999999999},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":2.2120000000000002,"15min":1.3280000000000001},"min":{"1min":0.28899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.5169999999999999,"5min":88.543000000000006,"15min":88.543000000000006},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.29,"15min":1.3009999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.6719999999999997,"5min":88.495999999999995,"15min":88.495999999999995},"last":0.443}]},{"osd":5,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.76500000000000001,"5min":2.2530000000000001,"15min":1.2509999999999999},"min":{"1min":0.30299999999999999,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":3.21,"5min":93.816000000000003,"15min":93.816000000000003},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.81499999999999995,"5min":2.2970000000000002,"15min":1.466},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.544,"5min":95.579999999999998,"15min":95.579999999999998},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:23:46 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":0.68100000000000005,"15min":0.74099999999999999},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.7120000000000002,"5min":5.343,"15min":9.0139999999999993},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.84199999999999997,"5min":0.86299999999999999,"15min":0.93899999999999995},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.308},"max":{"1min":3.6339999999999999,"5min":6.3879999999999999,"15min":13.362},"last":0.51300000000000001}]}]},{"osd":6,"up_from":0,"seq":197568495880,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795153,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":332516,"kb_used_data":17436,"kb_used_omap":7,"kb_used_meta":315064,"kb_avail":93412636,"statfs":{"total":95995035648,"available":95654539264,"internally_reserved":0,"allocated":17854464,"data_stored":16466703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":322625908},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":2.2770000000000001,"15min":1.4750000000000001},"min":{"1min":0.22700000000000001,"5min":0.189,"15min":0.189},"max":{"1min":10.125999999999999,"5min":56.503,"15min":61.890000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":4.2060000000000004,"5min":3.1179999999999999,"15min":1.982},"min":{"1min":0.28199999999999997,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":15.965999999999999,"5min":56.560000000000002,"15min":62.689999999999998},"last":0.31}]},{"osd":2,"last update":"Sat Sep 23 22:24:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.0649999999999999,"15min":1.284},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.83899999999999997,"5min":11.179,"15min":33.902000000000001},"last":0.377},{"interface":"front","average":{"1min":3.101,"5min":2.2730000000000001,"15min":1.9059999999999999},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.053000000000001,"5min":16.053000000000001,"15min":34.781999999999996},"last":0.40699999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":5.5910000000000002,"5min":4.6269999999999998,"15min":2.4049999999999998},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":13.871,"5min":110.417,"15min":110.417},"last":0.34799999999999998},{"interface":"front","average":{"1min":5.29,"5min":4.6589999999999998,"15min":2.343},"min":{"1min":0.28299999999999997,"5min":0.223,"15min":0.223},"max":{"1min":16.346,"5min":110.721,"15min":110.721},"last":1.546}]},{"osd":4,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":4.5359999999999996,"15min":2.0779999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27000000000000002},"max":{"1min":5.0869999999999997,"5min":234.79900000000001,"15min":234.79900000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":2.153,"5min":5.1689999999999996,"15min":2.4510000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":6.7160000000000002,"5min":234.81700000000001,"15min":234.81700000000001},"last":0.56599999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":3.5310000000000001,"15min":1.889},"min":{"1min":0.34000000000000002,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":6.7480000000000002,"5min":196.79900000000001,"15min":196.79900000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":2.5190000000000001,"5min":3.8450000000000002,"15min":1.9239999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":16.207000000000001,"5min":196.68299999999999,"15min":196.68299999999999},"last":1.4379999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":2.1240000000000001,"15min":1.3759999999999999},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":16.314,"5min":16.314,"15min":16.314},"last":1.5149999999999999},{"interface":"front","average":{"1min":1.218,"5min":1.206,"15min":1.042},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.1310000000000002,"5min":11.385999999999999,"15min":11.385999999999999},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":2.2909999999999999,"15min":1.484},"min":{"1min":0.57699999999999996,"5min":0.33800000000000002,"15min":0.32900000000000001},"max":{"1min":16.265000000000001,"5min":16.265000000000001,"15min":16.265000000000001},"last":1.484},{"interface":"front","average":{"1min":1.087,"5min":1.21,"15min":1.0860000000000001},"min":{"1min":0.42199999999999999,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":5.194,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.61699999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958687,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":314372,"kb_used_data":7356,"kb_used_omap":8,"kb_used_meta":306999,"kb_avail":93430780,"statfs":{"total":95995035648,"available":95673118720,"internally_reserved":0,"allocated":7532544,"data_stored":6111031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":314367850},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0619999999999998,"5min":4.5759999999999996,"15min":2.4550000000000001},"min":{"1min":0.214,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":16.253,"5min":169.77199999999999,"15min":169.77199999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":3.089,"5min":4.7729999999999997,"15min":2.746},"min":{"1min":0.34999999999999998,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":13.108000000000001,"5min":169.52099999999999,"15min":169.52099999999999},"last":0.79100000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:23:50 2023","interfaces":[{"interface":"back","average":{"1min":1.806,"5min":6.1360000000000001,"15min":2.6800000000000002},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":16.097999999999999,"5min":206.495,"15min":206.495},"last":0.76600000000000001},{"interface":"front","average":{"1min":2.258,"5min":6.6130000000000004,"15min":3.3050000000000002},"min":{"1min":0.29299999999999998,"5min":0.28499999999999998,"15min":0.22800000000000001},"max":{"1min":11.138,"5min":207.34399999999999,"15min":207.34399999999999},"last":0.69999999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:24:23 2023","interfaces":[{"interface":"back","average":{"1min":11.73,"5min":5.4100000000000001,"15min":2.754},"min":{"1min":0.35899999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":146.87299999999999,"5min":146.87299999999999,"15min":146.87299999999999},"last":0.68400000000000005},{"interface":"front","average":{"1min":10.827,"5min":5.2699999999999996,"15min":2.7959999999999998},"min":{"1min":0.33600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":145.48099999999999,"5min":145.48099999999999,"15min":145.48099999999999},"last":0.63500000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":3.7839999999999998,"5min":3.1299999999999999,"15min":1.994},"min":{"1min":0.33500000000000002,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":16.347999999999999,"5min":82.825000000000003,"15min":82.825000000000003},"last":0.80800000000000005},{"interface":"front","average":{"1min":2.5910000000000002,"5min":2.2269999999999999,"15min":1.3799999999999999},"min":{"1min":0.377,"5min":0.313,"15min":0.308},"max":{"1min":16.184000000000001,"5min":75.930999999999997,"15min":75.930999999999997},"last":0.85799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":3.8180000000000001,"5min":4.2919999999999998,"15min":2.3809999999999998},"min":{"1min":0.44,"5min":0.34799999999999998,"15min":0.29499999999999998},"max":{"1min":16.323,"5min":93.685000000000002,"15min":93.685000000000002},"last":0.94399999999999995},{"interface":"front","average":{"1min":2.431,"5min":3.4980000000000002,"15min":1.819},"min":{"1min":0.38600000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":16.152000000000001,"5min":93.546000000000006,"15min":93.546000000000006},"last":0.82599999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":4.093,"5min":2.2130000000000001,"15min":1.6990000000000001},"min":{"1min":0.47799999999999998,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":16.366,"5min":16.366,"15min":16.366},"last":1.0660000000000001},{"interface":"front","average":{"1min":2.3519999999999999,"5min":1.661,"15min":1.224},"min":{"1min":0.439,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":16.225999999999999,"5min":16.225999999999999,"15min":22.263000000000002},"last":0.89600000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":2.21,"15min":1.746},"min":{"1min":0.44600000000000001,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.718,"5min":32.773000000000003,"15min":32.773000000000003},"last":1.2290000000000001},{"interface":"front","average":{"1min":4.1059999999999999,"5min":2.343,"15min":1.9079999999999999},"min":{"1min":0.50600000000000001,"5min":0.39900000000000002,"15min":0.38900000000000001},"max":{"1min":16.381,"5min":16.381,"15min":23.265999999999998},"last":1.0269999999999999}]}]},{"osd":2,"up_from":0,"seq":442381631626,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468099,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":341056,"kb_used_data":92344,"kb_used_omap":4,"kb_used_meta":248699,"kb_avail":93404096,"statfs":{"total":95995035648,"available":95645794304,"internally_reserved":0,"allocated":94560256,"data_stored":93129056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":254668333},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1020000000000003,"5min":5.7489999999999997,"15min":2.9550000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.19900000000000001},"max":{"1min":10.547000000000001,"5min":225.86699999999999,"15min":225.86699999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":4.3819999999999997,"5min":6.1420000000000003,"15min":3.202},"min":{"1min":0.39100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":10.976000000000001,"5min":225.84800000000001,"15min":225.84800000000001},"last":0.36299999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2080000000000002,"5min":3.2309999999999999,"15min":2.3740000000000001},"min":{"1min":0.371,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":13.374000000000001,"5min":37.002000000000002,"15min":82.951999999999998},"last":0.34599999999999997},{"interface":"front","average":{"1min":3.3439999999999999,"5min":3.2290000000000001,"15min":2.4129999999999998},"min":{"1min":0.25700000000000001,"5min":0.22800000000000001,"15min":0.189},"max":{"1min":13.723000000000001,"5min":33.119999999999997,"15min":86.001000000000005},"last":0.379}]},{"osd":2,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.806,"5min":2.5270000000000001,"15min":3.5459999999999998},"min":{"1min":0.40300000000000002,"5min":0.252,"15min":0.223},"max":{"1min":13.518000000000001,"5min":29.076000000000001,"15min":99.962999999999994},"last":0.39300000000000002},{"interface":"front","average":{"1min":3.6030000000000002,"5min":2.6840000000000002,"15min":3.4249999999999998},"min":{"1min":0.29199999999999998,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":13.692,"5min":13.692,"15min":100.429},"last":0.40600000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.903,"15min":1.282},"min":{"1min":0.314,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":7.2039999999999997,"5min":11.911,"15min":11.911},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.798,"5min":1.9179999999999999,"15min":1.3460000000000001},"min":{"1min":0.34399999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.2940000000000005,"5min":10.949999999999999,"15min":12.036},"last":0.57199999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8300000000000001,"5min":2.5920000000000001,"15min":1.538},"min":{"1min":0.41499999999999998,"5min":0.309,"15min":0.309},"max":{"1min":9.4179999999999993,"5min":49.695999999999998,"15min":49.695999999999998},"last":0.54300000000000004},{"interface":"front","average":{"1min":2.5070000000000001,"5min":2.383,"15min":1.506},"min":{"1min":0.376,"5min":0.31,"15min":0.31},"max":{"1min":13.494,"5min":49.668999999999997,"15min":49.668999999999997},"last":0.501}]},{"osd":6,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.9020000000000001,"5min":2.1829999999999998,"15min":1.482},"min":{"1min":0.436,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.3439999999999994,"5min":10.992000000000001,"15min":12.068},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.9650000000000001,"15min":1.373},"min":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.35399999999999998},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.1030000000000002,"15min":1.4470000000000001},"min":{"1min":0.434,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":13.467000000000001,"5min":16.849,"15min":16.849},"last":0.60699999999999998},{"interface":"front","average":{"1min":2.375,"5min":2.0649999999999999,"15min":1.385},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.36699999999999999},"max":{"1min":7.2480000000000002,"5min":11.945,"15min":12.382},"last":0.55300000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019149,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":219732,"kb_used_data":27532,"kb_used_omap":0,"kb_used_meta":192192,"kb_avail":93525420,"statfs":{"total":95995035648,"available":95770030080,"internally_reserved":0,"allocated":28192768,"data_stored":27157083,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":196804608},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7069999999999999,"5min":2.9369999999999998,"15min":1.8540000000000001},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.218},"max":{"1min":25.425000000000001,"5min":46.18,"15min":66.355000000000004},"last":0.32700000000000001},{"interface":"front","average":{"1min":3.444,"5min":2.9039999999999999,"15min":1.921},"min":{"1min":0.28799999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":25.451000000000001,"5min":46.207000000000001,"15min":66.332999999999998},"last":0.52600000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":5.4039999999999999,"5min":2.7160000000000002,"15min":1.6699999999999999},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":85.662000000000006,"5min":85.662000000000006,"15min":85.662000000000006},"last":0.46300000000000002},{"interface":"front","average":{"1min":5.5999999999999996,"5min":3.0899999999999999,"15min":2.0289999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":81.195999999999998,"5min":81.195999999999998,"15min":81.195999999999998},"last":0.442}]},{"osd":2,"last update":"Sat Sep 23 22:23:44 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":3.472,"15min":2.2970000000000002},"min":{"1min":0.36099999999999999,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":10.051,"5min":138.06100000000001,"15min":138.06100000000001},"last":0.51100000000000001},{"interface":"front","average":{"1min":2.1030000000000002,"5min":3.3380000000000001,"15min":2.4340000000000002},"min":{"1min":0.35499999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":5.2510000000000003,"5min":138.15199999999999,"15min":138.15199999999999},"last":0.34399999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.669,"15min":1.2709999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":5.0140000000000002,"5min":12.129,"15min":12.129},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.3599999999999999,"5min":1.8580000000000001,"15min":1.347},"min":{"1min":0.32300000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.5600000000000005,"5min":8.5600000000000005,"15min":9.8109999999999999},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.3069999999999999,"15min":0.91000000000000003},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.165,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.498},{"interface":"front","average":{"1min":1.26,"5min":1.4550000000000001,"15min":0.95499999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27900000000000003},"max":{"1min":5.601,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":0.98299999999999998,"15min":0.77600000000000002},"min":{"1min":0.33800000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":7.8550000000000004,"5min":7.8550000000000004,"15min":7.8550000000000004},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.81599999999999995,"15min":0.77500000000000002},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.802,"5min":3.9239999999999999,"15min":8.5210000000000008},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":0.92900000000000005,"15min":0.878},"min":{"1min":0.33300000000000002,"5min":0.313,"15min":0.307},"max":{"1min":3.347,"5min":6.7480000000000002,"15min":8.577},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.377,"5min":1.0149999999999999,"15min":0.83999999999999997},"min":{"1min":0.34200000000000003,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":7.8109999999999999,"5min":7.8109999999999999,"15min":7.8109999999999999},"last":0.55400000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757517,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":320900,"kb_used_data":70012,"kb_used_omap":0,"kb_used_meta":250880,"kb_avail":93424252,"statfs":{"total":95995035648,"available":95666434048,"internally_reserved":0,"allocated":71692288,"data_stored":70668196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":256901120},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:23:57 2023","interfaces":[{"interface":"back","average":{"1min":5.0540000000000003,"5min":2.7989999999999999,"15min":2.1579999999999999},"min":{"1min":0.23999999999999999,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":64.344999999999999,"5min":64.344999999999999,"15min":133.22499999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":4.891,"5min":2.528,"15min":2.181},"min":{"1min":0.247,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":64.022000000000006,"5min":64.022000000000006,"15min":132.78200000000001},"last":0.48499999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":3.1040000000000001,"15min":2.7589999999999999},"min":{"1min":0.27500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.7870000000000008,"5min":93.981999999999999,"15min":93.981999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.885,"5min":3.0310000000000001,"15min":2.7389999999999999},"min":{"1min":0.35499999999999998,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":10.949,"5min":93.909999999999997,"15min":93.909999999999997},"last":0.58099999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.621,"5min":3.9239999999999999,"15min":3.504},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.3079999999999998,"5min":95.790999999999997,"15min":104.254},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.355,"5min":4.0119999999999996,"15min":3.7549999999999999},"min":{"1min":0.32500000000000001,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":5.3479999999999999,"5min":95.366,"15min":104.217},"last":0.496}]},{"osd":3,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1499999999999999,"5min":3.786,"15min":2.335},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.246},"max":{"1min":5.0839999999999996,"5min":93.739999999999995,"15min":93.739999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.9199999999999999,"5min":3.3279999999999998,"15min":2.2589999999999999},"min":{"1min":0.38200000000000001,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":4.2569999999999997,"5min":93.197999999999993,"15min":93.197999999999993},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:24:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":1.0089999999999999,"15min":1.046},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.7160000000000002,"5min":10.163,"15min":82.530000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.02,"15min":1.0880000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.8900000000000001,"5min":7.2119999999999997,"15min":85.441000000000003},"last":0.29999999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:24:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":1.179},"min":{"1min":0.34699999999999998,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":2.6040000000000001,"5min":7.0289999999999999,"15min":114.83499999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.72499999999999998,"15min":1.0880000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7130000000000001,"5min":3.8690000000000002,"15min":110.905},"last":0.373}]},{"osd":7,"last update":"Sat Sep 23 22:23:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":0.86799999999999999,"15min":0.878},"min":{"1min":0.33300000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.2759999999999998,"5min":4.3760000000000003,"15min":8.8030000000000008},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.74099999999999999,"15min":0.76200000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":4.3479999999999999,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":0.40799999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:24:51.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:24:51.700 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:24:51.711 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:51.722 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:24:51.737 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:51.747 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:24:51.769 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:24:51.780 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:51.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:51.789 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:24:51.791 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:24:51.791 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:24:51.791 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:24:51.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:24:51.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:24:51.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:24:51.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:24:51.792 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:24:51.793 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:51.883 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:51.918 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:52.232 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:51 smithi070 ceph-mon[192652]: from='client.45314 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:52.232 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:51 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4139608941' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:52.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:51 smithi026 ceph-mon[223014]: from='client.45314 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:52.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:51 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4139608941' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:52.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:51 smithi026 ceph-mon[215775]: from='client.45314 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:24:52.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:51 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4139608941' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:24:52.556 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:52.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:52.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:52.660 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:52.688 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:52.795 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:52.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:52.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:52.899 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:52.927 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:52.997 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T22:24:53.005 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:53.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:53.007 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:53.069 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:53.108 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:53.139 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:52 smithi026 ceph-mon[223014]: pgmap v662: 99 pgs: 6 creating+peering, 93 active+clean; 4.6 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:24:53.140 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1449355237' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:24:53.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:52 smithi026 ceph-mon[215775]: pgmap v662: 99 pgs: 6 creating+peering, 93 active+clean; 4.6 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:24:53.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1449355237' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:24:53.143 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:53.321 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:24:53.321 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:24:53.350 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:52 smithi070 ceph-mon[192652]: pgmap v662: 99 pgs: 6 creating+peering, 93 active+clean; 4.6 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:24:53.350 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1449355237' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:24:53.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:24:54.096 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:54.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:54.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:54.200 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:54.226 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:53 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1449355237' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:24:54.226 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:53 smithi070 ceph-mon[192652]: osdmap e263: 8 total, 8 up, 6 in 2023-09-23T22:24:54.226 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:53 smithi070 ceph-mon[192652]: failed to encode map e263 with expected crc 2023-09-23T22:24:54.229 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:54.277 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:53 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1449355237' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:24:54.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:53 smithi026 ceph-mon[223014]: osdmap e263: 8 total, 8 up, 6 in 2023-09-23T22:24:54.278 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:53 smithi026 ceph-mon[223014]: failed to encode map e263 with expected crc 2023-09-23T22:24:54.278 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:53 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1449355237' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:24:54.278 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:53 smithi026 ceph-mon[215775]: osdmap e263: 8 total, 8 up, 6 in 2023-09-23T22:24:54.279 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:53 smithi026 ceph-mon[215775]: failed to encode map e263 with expected crc 2023-09-23T22:24:54.324 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:54.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:54.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:54.329 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:54.428 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:24:54.454 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:54.547 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:24:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:54.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:24:54.641 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:24:54.650 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:54.677 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:54.973 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:54.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:54.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:55.057 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:24:55.077 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:55.093 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T22:24:55.108 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:55.110 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:24:55.110 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:24:55.253 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T22:24:55.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:24:55.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_410749 2023-09-23T22:24:55.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:24:55.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:24:55.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[223014]: pgmap v664: 99 pgs: 99 active+clean; 4.6 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:24:55.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[223014]: failed to encode map e263 with expected crc 2023-09-23T22:24:55.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[223014]: failed to encode map e263 with expected crc 2023-09-23T22:24:55.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[223014]: failed to encode map e263 with expected crc 2023-09-23T22:24:55.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:55.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:55.281 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4125363422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:24:55.281 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[215775]: pgmap v664: 99 pgs: 99 active+clean; 4.6 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:24:55.281 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[215775]: failed to encode map e263 with expected crc 2023-09-23T22:24:55.281 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[215775]: failed to encode map e263 with expected crc 2023-09-23T22:24:55.281 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[215775]: failed to encode map e263 with expected crc 2023-09-23T22:24:55.281 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:55.281 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:55.281 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4125363422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:24:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:54 smithi070 ceph-mon[192652]: pgmap v664: 99 pgs: 99 active+clean; 4.6 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:24:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:54 smithi070 ceph-mon[192652]: failed to encode map e263 with expected crc 2023-09-23T22:24:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:54 smithi070 ceph-mon[192652]: failed to encode map e263 with expected crc 2023-09-23T22:24:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:54 smithi070 ceph-mon[192652]: failed to encode map e263 with expected crc 2023-09-23T22:24:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:24:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:24:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4125363422' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:24:55.862 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:55.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:55.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:55.978 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:24:55.984 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:24:55.985 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:24:55.985 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:24:56.045 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:56.161 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:24:56.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:56.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[215775]: failed to encode map e264 with expected crc 2023-09-23T22:24:56.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4125363422' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:24:56.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[215775]: osdmap e264: 8 total, 8 up, 6 in 2023-09-23T22:24:56.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[215775]: failed to encode map e264 with expected crc 2023-09-23T22:24:56.162 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:56.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[223014]: failed to encode map e264 with expected crc 2023-09-23T22:24:56.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4125363422' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:24:56.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[223014]: osdmap e264: 8 total, 8 up, 6 in 2023-09-23T22:24:56.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:56 smithi026 ceph-mon[223014]: failed to encode map e264 with expected crc 2023-09-23T22:24:56.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:56.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:24:56.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 15 2807 2792 174.476 174.5 0.00524928 0.00570609 2023-09-23T22:24:56.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:24:56.332 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:56.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:56 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:24:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:56 smithi070 ceph-mon[192652]: failed to encode map e264 with expected crc 2023-09-23T22:24:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4125363422' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:24:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:56 smithi070 ceph-mon[192652]: osdmap e264: 8 total, 8 up, 6 in 2023-09-23T22:24:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:56 smithi070 ceph-mon[192652]: failed to encode map e264 with expected crc 2023-09-23T22:24:56.385 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:24:56.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:56.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:24:56.489 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:56.519 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:56.562 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:56.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:56.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:24:56.699 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:56.777 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:24:56.777 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":880,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":264,"num_osds":8,"num_up_osds":8,"osd_up_since":1695507239,"num_in_osds":6,"osd_in_since":1695507716,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":99}],"num_pgs":99,"num_pools":5,"num_objects":28,"data_bytes":4790195,"bytes_used":1865019392,"bytes_avail":574105194496,"bytes_total":575970213888},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-23T22:24:41.300662+0000","services":{}},"progress_events":{"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:24:56.889 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:24:56.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:56.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:24:56.949 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:24:56.994 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:57.034 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:57.104 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:57.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:57.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:57.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[215775]: failed to encode map e264 with expected crc 2023-09-23T22:24:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[215775]: failed to encode map e264 with expected crc 2023-09-23T22:24:57.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[215775]: failed to encode map e264 with expected crc 2023-09-23T22:24:57.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[215775]: pgmap v666: 99 pgs: 99 active+clean; 4.6 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:24:57.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2826095127' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:24:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[223014]: failed to encode map e264 with expected crc 2023-09-23T22:24:57.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[223014]: failed to encode map e264 with expected crc 2023-09-23T22:24:57.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[223014]: failed to encode map e264 with expected crc 2023-09-23T22:24:57.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[223014]: pgmap v666: 99 pgs: 99 active+clean; 4.6 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:24:57.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2826095127' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:24:57.244 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:57 smithi070 ceph-mon[192652]: failed to encode map e264 with expected crc 2023-09-23T22:24:57.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:57 smithi070 ceph-mon[192652]: failed to encode map e264 with expected crc 2023-09-23T22:24:57.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:57 smithi070 ceph-mon[192652]: failed to encode map e264 with expected crc 2023-09-23T22:24:57.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:57 smithi070 ceph-mon[192652]: pgmap v666: 99 pgs: 99 active+clean; 4.6 MiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2023-09-23T22:24:57.245 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2826095127' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:24:57.248 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:57.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 4916 4900 153.108 131.75 0.00533676 0.00652378 2023-09-23T22:24:57.313 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:57.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:57.419 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:57.454 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:24:57.458 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:57.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:57.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:57.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:57.635 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:57.673 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:57.741 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:57.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:57.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:57.848 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:57.886 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:57.888 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:24:58.147 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:58.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:58.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:58.252 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:24:58.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 7567 7551 157.294 165.688 0.00889768 0.0063481 2023-09-23T22:24:58.286 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:58.344 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:24:58.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:58.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:24:58.448 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:58.508 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:58.626 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:58.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:58.733 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:58.784 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:24:58 smithi026 ceph-mon[223014]: pgmap v667: 99 pgs: 99 active+clean; 77 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 9.0 MiB/s wr, 138 op/s 2023-09-23T22:24:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:24:58 smithi026 ceph-mon[215775]: pgmap v667: 99 pgs: 99 active+clean; 77 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 9.0 MiB/s wr, 138 op/s 2023-09-23T22:24:59.254 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 10025 10009 156.373 153.625 0.00666841 0.00638608 2023-09-23T22:24:59.296 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:24:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:59.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:24:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:24:58 smithi070 ceph-mon[192652]: pgmap v667: 99 pgs: 99 active+clean; 77 MiB data, 1.8 GiB used, 535 GiB / 536 GiB avail; 9.0 MiB/s wr, 138 op/s 2023-09-23T22:24:59.406 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:24:59.448 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:59.566 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:24:59.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:24:59.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:24:59.661 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:24:59.662 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:24:59.671 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:24:59.736 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:24:59.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:24:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:24:59] "GET /metrics HTTP/1.1" 200 36286 "" "Prometheus/2.43.0" 2023-09-23T22:25:00.133 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:00.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:00.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:00.155 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T22:25:00.156 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:25:00.245 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:00.255 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 12450 12434 155.408 151.562 0.00284004 0.00640905 2023-09-23T22:25:00.265 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:00.274 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:00.315 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:00.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:00.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:00.418 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:00.452 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:00 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:00.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:00 smithi026 ceph-mon[215775]: pgmap v668: 99 pgs: 99 active+clean; 146 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 18 MiB/s wr, 282 op/s 2023-09-23T22:25:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:00 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:00 smithi026 ceph-mon[223014]: pgmap v668: 99 pgs: 99 active+clean; 146 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 18 MiB/s wr, 282 op/s 2023-09-23T22:25:00.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:00.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:00.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:00.612 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:00.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:00 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:00 smithi070 ceph-mon[192652]: pgmap v668: 99 pgs: 99 active+clean; 146 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 18 MiB/s wr, 282 op/s 2023-09-23T22:25:00.672 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:00.836 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:00.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:00.902 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:25:00.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:00.988 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:25:01.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:01.028 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:25:01.036 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:01.064 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:25:01.095 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:01.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:01.109 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:25:01.144 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:25:01.184 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:25:01.200 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:01.226 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:25:01.237 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:01.255 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 15 14097 14082 146.672 103 0.00905671 0.00680779 2023-09-23T22:25:01.266 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:25:01.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:25:01.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:25:01.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:25:01.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:25:01.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:25:01.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:25:01.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:25:01.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:25:01.301 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:01.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:01.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:01.371 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:25:01.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/493410570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:01.410 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:01.475 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:01.590 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:01.593 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/493410570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:01.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:01.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:01.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/493410570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:01.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:01.731 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:01.789 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:01.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:01.793 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:01.894 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:01.952 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:02.255 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 16405 16389 146.301 144.188 0.00622044 0.00682904 2023-09-23T22:25:02.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:02 smithi070 ceph-mon[192652]: pgmap v669: 99 pgs: 99 active+clean; 490 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 58 MiB/s wr, 933 op/s 2023-09-23T22:25:02.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:02 smithi026 ceph-mon[215775]: pgmap v669: 99 pgs: 99 active+clean; 490 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 58 MiB/s wr, 933 op/s 2023-09-23T22:25:02.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:02 smithi026 ceph-mon[223014]: pgmap v669: 99 pgs: 99 active+clean; 490 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 58 MiB/s wr, 933 op/s 2023-09-23T22:25:03.255 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 18363 18347 143.309 122.375 0.00519577 0.00697224 2023-09-23T22:25:03.422 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:03.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:03.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:03.440 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:25:03.440 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:25:03.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:25:03.440 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:14264 5.1 e115/111 5:85e28b51:::benchmark_data_smithi026_286012_object14263:head v 115'855, mlcod=115'853)", 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.815928+0000", 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 577.62160280099999, 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16326769999999999, 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815928+0000", 2023-09-23T22:25:03.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815928+0000", 2023-09-23T22:25:03.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815928+0000", 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815954+0000", 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6536000000000001e-05 2023-09-23T22:25:03.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815971+0000", 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6943999999999999e-05 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.815977+0000", 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5149999999999997e-06 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816016+0000", 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9079000000000003e-05 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977199+0000", 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.161183399 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979161+0000", 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001961646 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979196+0000", 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5267e-05 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.447 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:14265 5.1 e115/111 5:89dc4bcb:::benchmark_data_smithi026_286012_object14264:head v 115'856, mlcod=115'853)", 2023-09-23T22:25:03.447 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:15:25.816637+0000", 2023-09-23T22:25:03.447 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 577.620894475, 2023-09-23T22:25:03.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16258351400000001, 2023-09-23T22:25:03.447 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.447 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.447 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816637+0000", 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816637+0000", 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816636+0000", 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.816659+0000", 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2614999999999999e-05 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977207+0000", 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16054861300000001 2023-09-23T22:25:03.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977227+0000", 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9551999999999998e-05 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977927+0000", 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00069972700000000003 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.977953+0000", 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6245999999999999e-05 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979199+0000", 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012457709999999999 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:15:25.979220+0000", 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.158e-05 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101023 5.5 e161/152 5:a887f58c:::benchmark_data_smithi026_286012_object7215:head v 161'6460, mlcod=0'0)", 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.654324+0000", 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 462.78320724100001, 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.053302833000000001, 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.452 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654324+0000", 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654324+0000", 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654323+0000", 2023-09-23T22:25:03.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:25:03.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654333+0000", 2023-09-23T22:25:03.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8530000000000004e-06 2023-09-23T22:25:03.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654346+0000", 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3842000000000001e-05 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654353+0000", 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.489e-06 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.664025+0000", 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096725869999999999 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.664052+0000", 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6594999999999999e-05 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707599+0000", 2023-09-23T22:25:03.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.043547359000000001 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707627+0000", 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7282000000000001e-05 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101021 5.d e161/111 5:ba53bdac:::benchmark_data_smithi026_286012_object7213:head v 161'6371, mlcod=161'6368)", 2023-09-23T22:25:03.457 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.656838+0000", 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 462.78069302599999, 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050757414000000001, 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656838+0000", 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656838+0000", 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656836+0000", 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656849+0000", 2023-09-23T22:25:03.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2418e-05 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656861+0000", 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2558000000000001e-05 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656869+0000", 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3440000000000003e-06 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659763+0000", 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002893825 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659783+0000", 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0205999999999999e-05 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707552+0000", 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047768928000000002 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707595+0000", 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3696000000000003e-05 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.462 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101026 5.d e161/111 5:bd4efaeb:::benchmark_data_smithi026_286012_object7218:head v 161'6372, mlcod=161'6368)", 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.657139+0000", 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 462.780392248, 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050510059000000003, 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657139+0000", 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657139+0000", 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657138+0000", 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657144+0000", 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9499999999999998e-06 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657156+0000", 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2381999999999999e-05 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657162+0000", 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5629999999999999e-06 2023-09-23T22:25:03.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.662814+0000", 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005652341 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.662834+0000", 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9296999999999998e-05 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707629+0000", 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.044795436000000001 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707649+0000", 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9789999999999999e-05 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101024 5.5 e161/152 5:adf8d131:::benchmark_data_smithi026_286012_object7216:head v 161'6461, mlcod=0'0)", 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.659691+0000", 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 462.77784028100001, 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048001597999999999, 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659691+0000", 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659691+0000", 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659689+0000", 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659701+0000", 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2172e-05 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659726+0000", 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5106e-05 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659733+0000", 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1729999999999998e-06 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.673568+0000", 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013834377 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.673593+0000", 2023-09-23T22:25:03.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4805999999999999e-05 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707672+0000", 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034079600000000002 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707692+0000", 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0287000000000001e-05 2023-09-23T22:25:03.472 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.473 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.473 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.473 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101025 5.5 e161/152 5:ac250153:::benchmark_data_smithi026_286012_object7217:head v 161'6462, mlcod=0'0)", 2023-09-23T22:25:03.473 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.660139+0000", 2023-09-23T22:25:03.473 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 462.77739245200002, 2023-09-23T22:25:03.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047578966, 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660139+0000", 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660139+0000", 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660138+0000", 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660144+0000", 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8930000000000002e-06 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660153+0000", 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8540000000000003e-06 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660158+0000", 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8180000000000003e-06 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.680575+0000", 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020417219 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.680606+0000", 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0701000000000003e-05 2023-09-23T22:25:03.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707696+0000", 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.02709056 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707718+0000", 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1460000000000001e-05 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.478 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101029 5.d e161/111 5:b06f3af1:::benchmark_data_smithi026_286012_object7221:head v 161'6373, mlcod=161'6368)", 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.668426+0000", 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 462.76910532400001, 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039244399999999999, 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668426+0000", 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668426+0000", 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668424+0000", 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:03.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668437+0000", 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2584e-05 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668456+0000", 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9378e-05 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.668465+0000", 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3520000000000007e-06 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670857+0000", 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023919100000000001 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.670874+0000", 2023-09-23T22:25:03.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7413000000000001e-05 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707651+0000", 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036776674000000002 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707670+0000", 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.95e-05 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.483 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101956 5.d e161/111 5:b91a2013:::benchmark_data_smithi026_286012_object8148:head v 161'6423, mlcod=161'6418)", 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 461.46391288299998, 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.075111800000000006, 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:03.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973635+0000", 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6799e-05 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973642+0000", 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9199999999999998e-06 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973646+0000", 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1540000000000004e-06 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.979441+0000", 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057947739999999999 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.979459+0000", 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7938999999999999e-05 2023-09-23T22:25:03.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048696+0000", 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.069237189000000005 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048730+0000", 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4269e-05 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.488 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101969 5.4 e161/111 5:2c36eb9f:::benchmark_data_smithi026_286012_object8161:head v 161'6383, mlcod=161'6381)", 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:21.982992+0000", 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 461.45453956199998, 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.071878993000000002, 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982992+0000", 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982992+0000", 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982991+0000", 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:25:03.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982997+0000", 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1310000000000003e-06 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048677+0000", 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.065680429999999998 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048692+0000", 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4908e-05 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048737+0000", 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4496000000000002e-05 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048761+0000", 2023-09-23T22:25:03.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4675999999999999e-05 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.054828+0000", 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006067037 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.054871+0000", 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2276e-05 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.493 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:107865 5.d e162/111 5:b3790187:::benchmark_data_smithi026_286012_object14057:head v 162'6782, mlcod=162'6780)", 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:31.722221+0000", 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 451.715309945, 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045419582, 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722221+0000", 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722221+0000", 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722220+0000", 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:25:03.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722227+0000", 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0509999999999999e-06 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722236+0000", 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.149e-06 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722242+0000", 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2900000000000002e-06 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722299+0000", 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7377000000000003e-05 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.722316+0000", 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6634999999999999e-05 2023-09-23T22:25:03.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.767615+0000", 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045299530999999997 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.767641+0000", 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5423999999999999e-05 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.498 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:107867 5.b e162/155 5:df3eb44e:::benchmark_data_smithi026_286012_object14059:head v 162'6625, mlcod=162'6622)", 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:31.730105+0000", 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 451.70742588899998, 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.037833218000000002, 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.730105+0000", 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.730105+0000", 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.730104+0000", 2023-09-23T22:25:03.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.730115+0000", 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1071e-05 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755382+0000", 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025266956 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755388+0000", 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7529999999999998e-06 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755412+0000", 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4536e-05 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.755427+0000", 2023-09-23T22:25:03.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4117999999999999e-05 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.767913+0000", 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012486667999999999 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:31.767938+0000", 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.514e-05 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.503 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:109357 5.5 e163/152 5:a53d5020:::benchmark_data_smithi026_286012_object15549:head v 163'6993, mlcod=0'0)", 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:34.041874+0000", 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 449.39565700000003, 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.038666124000000003, 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041874+0000", 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041874+0000", 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041873+0000", 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:25:03.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041884+0000", 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.119e-05 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041899+0000", 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.446e-05 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.041905+0000", 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6900000000000003e-06 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.052998+0000", 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01109245 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.053028+0000", 2023-09-23T22:25:03.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0366e-05 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.080486+0000", 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027457314 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:34.080540+0000", 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4583000000000002e-05 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.508 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45002.0:104787 6.6 e243/231 6:61492a31:::benchmark_data_smithi026_355573_object23437:head v 243'6579, mlcod=218'4589)", 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:22:52.784533+0000", 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 130.65299819399999, 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.044209120999999997, 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.784533+0000", 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.784533+0000", 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.784532+0000", 2023-09-23T22:25:03.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.784541+0000", 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1689999999999998e-06 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.784552+0000", 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1138000000000001e-05 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.784558+0000", 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9859999999999999e-06 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.784603+0000", 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4824000000000003e-05 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.784626+0000", 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2884e-05 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.828697+0000", 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.044071460999999999 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.828742+0000", 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4858999999999999e-05 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45002.0:104782 6.8 e243/240 6:131c13f0:::benchmark_data_smithi026_355573_object23432:head v 243'6710, mlcod=208'1215)", 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:22:52.789793+0000", 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 130.64773843, 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.055001656000000003, 2023-09-23T22:25:03.514 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.789793+0000", 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.789793+0000", 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.789792+0000", 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.789806+0000", 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3899e-05 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.823581+0000", 2023-09-23T22:25:03.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033775740999999998 2023-09-23T22:25:03.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.823594+0000", 2023-09-23T22:25:03.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2738999999999999e-05 2023-09-23T22:25:03.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.823741+0000", 2023-09-23T22:25:03.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147243 2023-09-23T22:25:03.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.823770+0000", 2023-09-23T22:25:03.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8470999999999999e-05 2023-09-23T22:25:03.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.844747+0000", 2023-09-23T22:25:03.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020977182 2023-09-23T22:25:03.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.844794+0000", 2023-09-23T22:25:03.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7302999999999998e-05 2023-09-23T22:25:03.521 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.521 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.521 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.521 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16555 7.5 e264/260 7:a4423849:::benchmark_data_smithi026_410749_object16554:head v 264'1083, mlcod=264'1081)", 2023-09-23T22:25:03.522 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.326981+0000", 2023-09-23T22:25:03.522 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.110550529, 2023-09-23T22:25:03.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17186283799999999, 2023-09-23T22:25:03.522 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.522 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.522 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.326981+0000", 2023-09-23T22:25:03.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.326981+0000", 2023-09-23T22:25:03.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.326980+0000", 2023-09-23T22:25:03.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:03.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327020+0000", 2023-09-23T22:25:03.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0092999999999997e-05 2023-09-23T22:25:03.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327027+0000", 2023-09-23T22:25:03.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0530000000000001e-06 2023-09-23T22:25:03.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327031+0000", 2023-09-23T22:25:03.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3840000000000001e-06 2023-09-23T22:25:03.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327068+0000", 2023-09-23T22:25:03.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7005000000000002e-05 2023-09-23T22:25:03.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327085+0000", 2023-09-23T22:25:03.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7181e-05 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498806+0000", 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17172121400000001 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498843+0000", 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7351000000000002e-05 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.529 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16562 7.5 e264/260 7:ab68d64a:::benchmark_data_smithi026_410749_object16561:head v 264'1084, mlcod=264'1081)", 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.327040+0000", 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.110491525, 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.171824014, 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327040+0000", 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327040+0000", 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327039+0000", 2023-09-23T22:25:03.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967296 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327186+0000", 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014651900000000001 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327190+0000", 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1849999999999997e-06 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327193+0000", 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.497e-06 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327646+0000", 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045361700000000002 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327663+0000", 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6540000000000001e-05 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498849+0000", 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17118603199999999 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498864+0000", 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4829e-05 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.534 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16567 7.3 e264/260 7:cd2ae393:::benchmark_data_smithi026_410749_object16566:head v 264'1045, mlcod=264'1042)", 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.327226+0000", 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.110304677, 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17163550499999999, 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327226+0000", 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327226+0000", 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327226+0000", 2023-09-23T22:25:03.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327292+0000", 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5415999999999998e-05 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327296+0000", 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.977e-06 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327299+0000", 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5739999999999998e-06 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327323+0000", 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4108999999999999e-05 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327335+0000", 2023-09-23T22:25:03.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2143e-05 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498824+0000", 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.171488848 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498862+0000", 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7737999999999997e-05 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.539 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16570 7.5 e264/260 7:ab27bf89:::benchmark_data_smithi026_410749_object16569:head v 264'1085, mlcod=264'1081)", 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.330592+0000", 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.1069393080000001, 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17050346699999999, 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330592+0000", 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330592+0000", 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330591+0000", 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:03.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330624+0000", 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2951999999999999e-05 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498796+0000", 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.168172236 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498807+0000", 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0747999999999999e-05 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498888+0000", 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1130000000000004e-05 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498909+0000", 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0795e-05 2023-09-23T22:25:03.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.501072+0000", 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002162841 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.501095+0000", 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3391999999999999e-05 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.544 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16572 7.2 e264/260 7:4b8eea19:::benchmark_data_smithi026_410749_object16571:head v 264'1051, mlcod=264'1049)", 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.330913+0000", 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.1066182040000001, 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17039949600000001, 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330913+0000", 2023-09-23T22:25:03.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330913+0000", 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330912+0000", 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330974+0000", 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1835999999999995e-05 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498801+0000", 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16782661900000001 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498809+0000", 2023-09-23T22:25:03.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6200000000000005e-06 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.499112+0000", 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030310500000000002 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.499130+0000", 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8016000000000002e-05 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.501278+0000", 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021478320000000001 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.501312+0000", 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4060999999999998e-05 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:25:03.549 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:25:03.550 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:25:03.550 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:03.614 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:03.726 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:03.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:03.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:03.833 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:03.870 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:03.948 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:03.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:04.053 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:04.091 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:04.108 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:04.186 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:04.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:04.256 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 20007 19991 138.802 102.75 0.00530695 0.00719931 2023-09-23T22:25:04.291 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:04.327 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:04.567 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:25:04.575 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:04.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:04.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:04.681 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:04.824 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:04.926 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:04 smithi026 ceph-mon[215775]: pgmap v670: 99 pgs: 99 active+clean; 773 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T22:25:04.997 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:05.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:05.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:05.109 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:05.143 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:05.204 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:05.210 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:04 smithi070 ceph-mon[192652]: pgmap v670: 99 pgs: 99 active+clean; 773 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T22:25:05.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:05.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:05.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:04 smithi026 ceph-mon[223014]: pgmap v670: 99 pgs: 99 active+clean; 773 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 77 MiB/s wr, 1.23k op/s 2023-09-23T22:25:05.257 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 21668 21652 135.302 103.812 0.0109843 0.0073853 2023-09-23T22:25:05.313 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:05.392 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:05.963 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:25:05.978 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:05.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:05.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:06.037 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:25:06.087 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:06.116 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:06.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:06.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:06.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:06.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:05 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1782242451' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:06.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:05 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1782242451' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:06.257 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 23341 23325 132.507 104.562 0.00744721 0.00753915 2023-09-23T22:25:06.269 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:06.299 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:05 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1782242451' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:06.301 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:06.350 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:06.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:06.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:06.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:06.489 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:06.546 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:06.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:06.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:06.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:06.681 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:06.942 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: pgmap v671: 99 pgs: 99 active+clean; 1.1 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-09-23T22:25:06.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:06.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1782242451' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:25:06.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: osdmap e265: 8 total, 8 up, 6 in 2023-09-23T22:25:06.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: failed to encode map e265 with expected crc 2023-09-23T22:25:06.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: failed to encode map e265 with expected crc 2023-09-23T22:25:06.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: failed to encode map e265 with expected crc 2023-09-23T22:25:06.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: failed to encode map e265 with expected crc 2023-09-23T22:25:06.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: failed to encode map e265 with expected crc 2023-09-23T22:25:06.943 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[215775]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.160 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:07.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:07.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:07.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: pgmap v671: 99 pgs: 99 active+clean; 1.1 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-09-23T22:25:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1782242451' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:25:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: osdmap e265: 8 total, 8 up, 6 in 2023-09-23T22:25:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:06 smithi026 ceph-mon[223014]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.258 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 24771 24755 128.912 89.375 0.0212973 0.00774404 2023-09-23T22:25:07.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:07.295 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: pgmap v671: 99 pgs: 99 active+clean; 1.1 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-09-23T22:25:07.295 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:07.295 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1782242451' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:25:07.295 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: osdmap e265: 8 total, 8 up, 6 in 2023-09-23T22:25:07.296 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.296 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.296 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.296 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.296 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.296 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:06 smithi070 ceph-mon[192652]: failed to encode map e265 with expected crc 2023-09-23T22:25:07.297 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:07.350 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:07.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:07.456 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:07.530 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:07.660 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:07.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:07.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:07.683 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:07.684 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:07.763 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:07.792 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:07.845 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:07.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:07.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:07.951 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:08.043 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:08.161 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:25:08.171 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:25:08.257 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 26227 26211 125.984 91 0.00768459 0.00792998 2023-09-23T22:25:09.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:08 smithi026 ceph-mon[223014]: pgmap v673: 99 pgs: 99 active+clean; 1.3 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T22:25:09.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:08 smithi026 ceph-mon[215775]: pgmap v673: 99 pgs: 99 active+clean; 1.3 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T22:25:09.258 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 28076 28060 125.232 115.562 0.00783636 0.0079792 2023-09-23T22:25:09.298 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:25:09.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:08 smithi070 ceph-mon[192652]: pgmap v673: 99 pgs: 99 active+clean; 1.3 GiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T22:25:09.841 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:09.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:09.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:09.947 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:09.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:25:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:25:09] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.43.0" 2023-09-23T22:25:09.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:09 smithi026 ceph-mon[215775]: from='client.45359 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:25:09.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:09.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:09.982 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:10.043 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:10.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:10.148 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:10.174 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:10.219 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:10.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:10.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:10.224 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:09 smithi070 ceph-mon[192652]: from='client.45359 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:25:10.224 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:10.224 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:10.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:09 smithi026 ceph-mon[223014]: from='client.45359 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:25:10.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:10.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:10.258 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 29299 29283 121.979 76.4375 0.00325733 0.00816852 2023-09-23T22:25:10.324 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:10.439 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:10.921 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:10.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:10.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:11.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:11.092 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:11.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:10 smithi026 ceph-mon[215775]: pgmap v674: 99 pgs: 99 active+clean; 1.4 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T22:25:11.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:10 smithi026 ceph-mon[223014]: pgmap v674: 99 pgs: 99 active+clean; 1.4 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T22:25:11.258 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 30648 30632 119.625 84.3125 0.00420803 0.00835387 2023-09-23T22:25:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:10 smithi070 ceph-mon[192652]: pgmap v674: 99 pgs: 99 active+clean; 1.4 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T22:25:11.580 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":673,"stamp":"2023-09-23T22:25:07.307619+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1432950707,"num_objects":21820,"num_object_clones":0,"num_object_copies":65460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21820,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":22699,"num_write_kb":1407413,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":8661772,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":24594,"ondisk_log_size":24594,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5875280,"kb_used_data":4151264,"kb_used_omap":20,"kb_used_meta":1723947,"kb_avail":556595632,"statfs":{"total":575970213888,"available":569953927168,"internally_reserved":0,"allocated":4250894336,"data_stored":4243522260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1765322507},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,3,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1352531968,"num_objects":20638,"num_object_clones":0,"num_object_copies":61914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":20638,"num_write_kb":1320832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003011"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":251,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352134+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:24:53.352134+0000","last_peered":"2023-09-23T22:24:53.352134+0000","last_clean":"2023-09-23T22:24:53.352134+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:24:53.352134+0000","last_undegraded":"2023-09-23T22:24:53.352134+0000","last_fullsized":"2023-09-23T22:24:53.352134+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":243,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994264+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:24:52.994264+0000","last_peered":"2023-09-23T22:24:52.994264+0000","last_clean":"2023-09-23T22:24:52.994264+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:24:52.994264+0000","last_undegraded":"2023-09-23T22:24:52.994264+0000","last_fullsized":"2023-09-23T22:24:52.994264+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":206,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.258369+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:24:55.258369+0000","last_peered":"2023-09-23T22:24:55.258369+0000","last_clean":"2023-09-23T22:24:55.258369+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:24:55.258369+0000","last_undegraded":"2023-09-23T22:24:55.258369+0000","last_fullsized":"2023-09-23T22:24:55.258369+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":123,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:01.646241+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:25:01.646241+0000","last_peered":"2023-09-23T22:25:01.646241+0000","last_clean":"2023-09-23T22:25:01.646241+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:25:01.646241+0000","last_undegraded":"2023-09-23T22:25:01.646241+0000","last_fullsized":"2023-09-23T22:25:01.646241+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":283,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993993+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:24:52.993993+0000","last_peered":"2023-09-23T22:24:52.993993+0000","last_clean":"2023-09-23T22:24:52.993993+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:24:52.993993+0000","last_undegraded":"2023-09-23T22:24:52.993993+0000","last_fullsized":"2023-09-23T22:24:52.993993+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":219,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350563+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:24:53.350563+0000","last_peered":"2023-09-23T22:24:53.350563+0000","last_clean":"2023-09-23T22:24:53.350563+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:24:53.350563+0000","last_undegraded":"2023-09-23T22:24:53.350563+0000","last_fullsized":"2023-09-23T22:24:53.350563+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":151,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054252+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:24:55.054252+0000","last_peered":"2023-09-23T22:24:55.054252+0000","last_clean":"2023-09-23T22:24:55.054252+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:24:55.054252+0000","last_undegraded":"2023-09-23T22:24:55.054252+0000","last_fullsized":"2023-09-23T22:24:55.054252+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":240,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994069+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:24:52.994069+0000","last_peered":"2023-09-23T22:24:52.994069+0000","last_clean":"2023-09-23T22:24:52.994069+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:24:52.994069+0000","last_undegraded":"2023-09-23T22:24:52.994069+0000","last_fullsized":"2023-09-23T22:24:52.994069+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":224,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350422+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:24:53.350422+0000","last_peered":"2023-09-23T22:24:53.350422+0000","last_clean":"2023-09-23T22:24:53.350422+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:24:53.350422+0000","last_undegraded":"2023-09-23T22:24:53.350422+0000","last_fullsized":"2023-09-23T22:24:53.350422+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":208,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994316+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:24:52.994316+0000","last_peered":"2023-09-23T22:24:52.994316+0000","last_clean":"2023-09-23T22:24:52.994316+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:24:52.994316+0000","last_undegraded":"2023-09-23T22:24:52.994316+0000","last_fullsized":"2023-09-23T22:24:52.994316+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":208,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.258656+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:24:55.258656+0000","last_peered":"2023-09-23T22:24:55.258656+0000","last_clean":"2023-09-23T22:24:55.258656+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:24:55.258656+0000","last_undegraded":"2023-09-23T22:24:55.258656+0000","last_fullsized":"2023-09-23T22:24:55.258656+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":242,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993734+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:24:52.993734+0000","last_peered":"2023-09-23T22:24:52.993734+0000","last_clean":"2023-09-23T22:24:52.993734+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:24:52.993734+0000","last_undegraded":"2023-09-23T22:24:52.993734+0000","last_fullsized":"2023-09-23T22:24:52.993734+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":268,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352604+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:24:53.352604+0000","last_peered":"2023-09-23T22:24:53.352604+0000","last_clean":"2023-09-23T22:24:53.352604+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:24:53.352604+0000","last_undegraded":"2023-09-23T22:24:53.352604+0000","last_fullsized":"2023-09-23T22:24:53.352604+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":232,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352179+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:24:53.352179+0000","last_peered":"2023-09-23T22:24:53.352179+0000","last_clean":"2023-09-23T22:24:53.352179+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:24:53.352179+0000","last_undegraded":"2023-09-23T22:24:53.352179+0000","last_fullsized":"2023-09-23T22:24:53.352179+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":191,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352224+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:24:53.352224+0000","last_peered":"2023-09-23T22:24:53.352224+0000","last_clean":"2023-09-23T22:24:53.352224+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:24:53.352224+0000","last_undegraded":"2023-09-23T22:24:53.352224+0000","last_fullsized":"2023-09-23T22:24:53.352224+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":231,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994815+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:24:52.994815+0000","last_peered":"2023-09-23T22:24:52.994815+0000","last_clean":"2023-09-23T22:24:52.994815+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:24:52.994815+0000","last_undegraded":"2023-09-23T22:24:52.994815+0000","last_fullsized":"2023-09-23T22:24:52.994815+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":140,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053704+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:24:55.053704+0000","last_peered":"2023-09-23T22:24:55.053704+0000","last_clean":"2023-09-23T22:24:55.053704+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:24:55.053704+0000","last_undegraded":"2023-09-23T22:24:55.053704+0000","last_fullsized":"2023-09-23T22:24:55.053704+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":243,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350366+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:24:53.350366+0000","last_peered":"2023-09-23T22:24:53.350366+0000","last_clean":"2023-09-23T22:24:53.350366+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:24:53.350366+0000","last_undegraded":"2023-09-23T22:24:53.350366+0000","last_fullsized":"2023-09-23T22:24:53.350366+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":187,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352142+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:24:53.352142+0000","last_peered":"2023-09-23T22:24:53.352142+0000","last_clean":"2023-09-23T22:24:53.352142+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:24:53.352142+0000","last_undegraded":"2023-09-23T22:24:53.352142+0000","last_fullsized":"2023-09-23T22:24:53.352142+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":122,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054643+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:24:55.054643+0000","last_peered":"2023-09-23T22:24:55.054643+0000","last_clean":"2023-09-23T22:24:55.054643+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:24:55.054643+0000","last_undegraded":"2023-09-23T22:24:55.054643+0000","last_fullsized":"2023-09-23T22:24:55.054643+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":283,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.052809+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:24:55.052809+0000","last_peered":"2023-09-23T22:24:55.052809+0000","last_clean":"2023-09-23T22:24:55.052809+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:24:55.052809+0000","last_undegraded":"2023-09-23T22:24:55.052809+0000","last_fullsized":"2023-09-23T22:24:55.052809+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.995165+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:24:52.995165+0000","last_peered":"2023-09-23T22:24:52.995165+0000","last_clean":"2023-09-23T22:24:52.995165+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:24:52.995165+0000","last_undegraded":"2023-09-23T22:24:52.995165+0000","last_fullsized":"2023-09-23T22:24:52.995165+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"264'1214","reported_seq":1227,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:03.727468+0000","last_change":"2023-09-23T22:24:46.980027+0000","last_active":"2023-09-23T22:25:03.727468+0000","last_peered":"2023-09-23T22:25:03.727468+0000","last_clean":"2023-09-23T22:25:03.727468+0000","last_became_active":"2023-09-23T22:24:46.979731+0000","last_became_peered":"2023-09-23T22:24:46.979731+0000","last_unstale":"2023-09-23T22:25:03.727468+0000","last_undegraded":"2023-09-23T22:25:03.727468+0000","last_fullsized":"2023-09-23T22:25:03.727468+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1214,"log_dups_size":0,"ondisk_log_size":1214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:06.319642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79560704,"num_objects":1214,"num_object_clones":0,"num_object_copies":3642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1214,"num_write_kb":77696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":259,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352495+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:24:53.352495+0000","last_peered":"2023-09-23T22:24:53.352495+0000","last_clean":"2023-09-23T22:24:53.352495+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:24:53.352495+0000","last_undegraded":"2023-09-23T22:24:53.352495+0000","last_fullsized":"2023-09-23T22:24:53.352495+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":246,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.349299+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:24:53.349299+0000","last_peered":"2023-09-23T22:24:53.349299+0000","last_clean":"2023-09-23T22:24:53.349299+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:24:53.349299+0000","last_undegraded":"2023-09-23T22:24:53.349299+0000","last_fullsized":"2023-09-23T22:24:53.349299+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"264'1290","reported_seq":1303,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:04.391436+0000","last_change":"2023-09-23T22:24:46.984090+0000","last_active":"2023-09-23T22:25:04.391436+0000","last_peered":"2023-09-23T22:25:04.391436+0000","last_clean":"2023-09-23T22:25:04.391436+0000","last_became_active":"2023-09-23T22:24:46.983784+0000","last_became_peered":"2023-09-23T22:24:46.983784+0000","last_unstale":"2023-09-23T22:25:04.391436+0000","last_undegraded":"2023-09-23T22:25:04.391436+0000","last_fullsized":"2023-09-23T22:25:04.391436+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1290,"log_dups_size":0,"ondisk_log_size":1290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:14.302288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84541440,"num_objects":1290,"num_object_clones":0,"num_object_copies":3870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1290,"num_write_kb":82560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":173,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352499+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:24:53.352499+0000","last_peered":"2023-09-23T22:24:53.352499+0000","last_clean":"2023-09-23T22:24:53.352499+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:24:53.352499+0000","last_undegraded":"2023-09-23T22:24:53.352499+0000","last_fullsized":"2023-09-23T22:24:53.352499+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":251,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352107+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:24:53.352107+0000","last_peered":"2023-09-23T22:24:53.352107+0000","last_clean":"2023-09-23T22:24:53.352107+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:24:53.352107+0000","last_undegraded":"2023-09-23T22:24:53.352107+0000","last_fullsized":"2023-09-23T22:24:53.352107+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"265'1505","reported_seq":1519,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:06.612882+0000","last_change":"2023-09-23T22:24:46.981119+0000","last_active":"2023-09-23T22:25:06.612882+0000","last_peered":"2023-09-23T22:25:06.612882+0000","last_clean":"2023-09-23T22:25:06.612882+0000","last_became_active":"2023-09-23T22:24:46.980795+0000","last_became_peered":"2023-09-23T22:24:46.980795+0000","last_unstale":"2023-09-23T22:25:06.612882+0000","last_undegraded":"2023-09-23T22:25:06.612882+0000","last_fullsized":"2023-09-23T22:25:06.612882+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1505,"log_dups_size":0,"ondisk_log_size":1505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:24.132475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98631680,"num_objects":1505,"num_object_clones":0,"num_object_copies":4515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1505,"num_write_kb":96320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":189,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350453+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:24:53.350453+0000","last_peered":"2023-09-23T22:24:53.350453+0000","last_clean":"2023-09-23T22:24:53.350453+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:24:53.350453+0000","last_undegraded":"2023-09-23T22:24:53.350453+0000","last_fullsized":"2023-09-23T22:24:53.350453+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":199,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993783+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:24:52.993783+0000","last_peered":"2023-09-23T22:24:52.993783+0000","last_clean":"2023-09-23T22:24:52.993783+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:24:52.993783+0000","last_undegraded":"2023-09-23T22:24:52.993783+0000","last_fullsized":"2023-09-23T22:24:52.993783+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"264'1291","reported_seq":1303,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:04.911781+0000","last_change":"2023-09-23T22:24:47.009845+0000","last_active":"2023-09-23T22:25:04.911781+0000","last_peered":"2023-09-23T22:25:04.911781+0000","last_clean":"2023-09-23T22:25:04.911781+0000","last_became_active":"2023-09-23T22:24:47.009559+0000","last_became_peered":"2023-09-23T22:24:47.009559+0000","last_unstale":"2023-09-23T22:25:04.911781+0000","last_undegraded":"2023-09-23T22:25:04.911781+0000","last_fullsized":"2023-09-23T22:25:04.911781+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1291,"log_dups_size":0,"ondisk_log_size":1291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:19.040683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84606976,"num_objects":1291,"num_object_clones":0,"num_object_copies":3873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1291,"num_write_kb":82624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":254,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350556+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:24:53.350556+0000","last_peered":"2023-09-23T22:24:53.350556+0000","last_clean":"2023-09-23T22:24:53.350556+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:24:53.350556+0000","last_undegraded":"2023-09-23T22:24:53.350556+0000","last_fullsized":"2023-09-23T22:24:53.350556+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":120,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053143+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:24:55.053143+0000","last_peered":"2023-09-23T22:24:55.053143+0000","last_clean":"2023-09-23T22:24:55.053143+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:24:55.053143+0000","last_undegraded":"2023-09-23T22:24:55.053143+0000","last_fullsized":"2023-09-23T22:24:55.053143+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"265'1444","reported_seq":1458,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:06.863906+0000","last_change":"2023-09-23T22:24:47.009804+0000","last_active":"2023-09-23T22:25:06.863906+0000","last_peered":"2023-09-23T22:25:06.863906+0000","last_clean":"2023-09-23T22:25:06.863906+0000","last_became_active":"2023-09-23T22:24:47.009551+0000","last_became_peered":"2023-09-23T22:24:47.009551+0000","last_unstale":"2023-09-23T22:25:06.863906+0000","last_undegraded":"2023-09-23T22:25:06.863906+0000","last_fullsized":"2023-09-23T22:25:06.863906+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1444,"log_dups_size":0,"ondisk_log_size":1444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:44.762872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94633984,"num_objects":1444,"num_object_clones":0,"num_object_copies":4332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1444,"num_write_kb":92416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":568,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994160+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:24:52.994160+0000","last_peered":"2023-09-23T22:24:52.994160+0000","last_clean":"2023-09-23T22:24:52.994160+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:24:52.994160+0000","last_undegraded":"2023-09-23T22:24:52.994160+0000","last_fullsized":"2023-09-23T22:24:52.994160+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":277,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993984+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:24:52.993984+0000","last_peered":"2023-09-23T22:24:52.993984+0000","last_clean":"2023-09-23T22:24:52.993984+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:24:52.993984+0000","last_undegraded":"2023-09-23T22:24:52.993984+0000","last_fullsized":"2023-09-23T22:24:52.993984+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":232,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352515+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:24:53.352515+0000","last_peered":"2023-09-23T22:24:53.352515+0000","last_clean":"2023-09-23T22:24:53.352515+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:24:53.352515+0000","last_undegraded":"2023-09-23T22:24:53.352515+0000","last_fullsized":"2023-09-23T22:24:53.352515+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"264'1304","reported_seq":1316,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:04.897056+0000","last_change":"2023-09-23T22:24:46.981702+0000","last_active":"2023-09-23T22:25:04.897056+0000","last_peered":"2023-09-23T22:25:04.897056+0000","last_clean":"2023-09-23T22:25:04.897056+0000","last_became_active":"2023-09-23T22:24:46.981422+0000","last_became_peered":"2023-09-23T22:24:46.981422+0000","last_unstale":"2023-09-23T22:25:04.897056+0000","last_undegraded":"2023-09-23T22:25:04.897056+0000","last_fullsized":"2023-09-23T22:25:04.897056+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1304,"log_dups_size":0,"ondisk_log_size":1304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:08.014616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85458944,"num_objects":1304,"num_object_clones":0,"num_object_copies":3912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1304,"num_write_kb":83456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":576,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.349213+0000","last_change":"2023-09-23T22:24:39.954805+0000","last_active":"2023-09-23T22:24:53.349213+0000","last_peered":"2023-09-23T22:24:53.349213+0000","last_clean":"2023-09-23T22:24:53.349213+0000","last_became_active":"2023-09-23T22:24:39.954350+0000","last_became_peered":"2023-09-23T22:24:39.954350+0000","last_unstale":"2023-09-23T22:24:53.349213+0000","last_undegraded":"2023-09-23T22:24:53.349213+0000","last_fullsized":"2023-09-23T22:24:53.349213+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:47.774412+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":180,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352040+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:24:53.352040+0000","last_peered":"2023-09-23T22:24:53.352040+0000","last_clean":"2023-09-23T22:24:53.352040+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:24:53.352040+0000","last_undegraded":"2023-09-23T22:24:53.352040+0000","last_fullsized":"2023-09-23T22:24:53.352040+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":249,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352245+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:24:53.352245+0000","last_peered":"2023-09-23T22:24:53.352245+0000","last_clean":"2023-09-23T22:24:53.352245+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:24:53.352245+0000","last_undegraded":"2023-09-23T22:24:53.352245+0000","last_fullsized":"2023-09-23T22:24:53.352245+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"264'1264","reported_seq":1277,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:03.850679+0000","last_change":"2023-09-23T22:24:46.982376+0000","last_active":"2023-09-23T22:25:03.850679+0000","last_peered":"2023-09-23T22:25:03.850679+0000","last_clean":"2023-09-23T22:25:03.850679+0000","last_became_active":"2023-09-23T22:24:46.982171+0000","last_became_peered":"2023-09-23T22:24:46.982171+0000","last_unstale":"2023-09-23T22:25:03.850679+0000","last_undegraded":"2023-09-23T22:25:03.850679+0000","last_fullsized":"2023-09-23T22:25:03.850679+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1264,"log_dups_size":0,"ondisk_log_size":1264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:03.026919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82837504,"num_objects":1264,"num_object_clones":0,"num_object_copies":3792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1264,"num_write_kb":80896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":225,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.351980+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:24:53.351980+0000","last_peered":"2023-09-23T22:24:53.351980+0000","last_clean":"2023-09-23T22:24:53.351980+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:24:53.351980+0000","last_undegraded":"2023-09-23T22:24:53.351980+0000","last_fullsized":"2023-09-23T22:24:53.351980+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":240,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352409+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:24:53.352409+0000","last_peered":"2023-09-23T22:24:53.352409+0000","last_clean":"2023-09-23T22:24:53.352409+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:24:53.352409+0000","last_undegraded":"2023-09-23T22:24:53.352409+0000","last_fullsized":"2023-09-23T22:24:53.352409+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"264'1294","reported_seq":1306,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:04.387038+0000","last_change":"2023-09-23T22:24:47.009404+0000","last_active":"2023-09-23T22:25:04.387038+0000","last_peered":"2023-09-23T22:25:04.387038+0000","last_clean":"2023-09-23T22:25:04.387038+0000","last_became_active":"2023-09-23T22:24:47.009063+0000","last_became_peered":"2023-09-23T22:24:47.009063+0000","last_unstale":"2023-09-23T22:25:04.387038+0000","last_undegraded":"2023-09-23T22:25:04.387038+0000","last_fullsized":"2023-09-23T22:25:04.387038+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1294,"log_dups_size":0,"ondisk_log_size":1294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:52.237759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84803584,"num_objects":1294,"num_object_clones":0,"num_object_copies":3882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1294,"num_write_kb":82816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":568,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.349377+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:24:53.349377+0000","last_peered":"2023-09-23T22:24:53.349377+0000","last_clean":"2023-09-23T22:24:53.349377+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:24:53.349377+0000","last_undegraded":"2023-09-23T22:24:53.349377+0000","last_fullsized":"2023-09-23T22:24:53.349377+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":44,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993783+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:24:52.993783+0000","last_peered":"2023-09-23T22:24:52.993783+0000","last_clean":"2023-09-23T22:24:52.993783+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:24:52.993783+0000","last_undegraded":"2023-09-23T22:24:52.993783+0000","last_fullsized":"2023-09-23T22:24:52.993783+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":206,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994741+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:24:52.994741+0000","last_peered":"2023-09-23T22:24:52.994741+0000","last_clean":"2023-09-23T22:24:52.994741+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:24:52.994741+0000","last_undegraded":"2023-09-23T22:24:52.994741+0000","last_fullsized":"2023-09-23T22:24:52.994741+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"264'1282","reported_seq":1294,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:04.926231+0000","last_change":"2023-09-23T22:24:46.980482+0000","last_active":"2023-09-23T22:25:04.926231+0000","last_peered":"2023-09-23T22:25:04.926231+0000","last_clean":"2023-09-23T22:25:04.926231+0000","last_became_active":"2023-09-23T22:24:46.980314+0000","last_became_peered":"2023-09-23T22:24:46.980314+0000","last_unstale":"2023-09-23T22:25:04.926231+0000","last_undegraded":"2023-09-23T22:25:04.926231+0000","last_fullsized":"2023-09-23T22:25:04.926231+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1282,"log_dups_size":0,"ondisk_log_size":1282,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.107504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84017152,"num_objects":1282,"num_object_clones":0,"num_object_copies":3846,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1282,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1282,"num_write_kb":82048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":572,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.995566+0000","last_change":"2023-09-23T22:24:47.781592+0000","last_active":"2023-09-23T22:24:52.995566+0000","last_peered":"2023-09-23T22:24:52.995566+0000","last_clean":"2023-09-23T22:24:52.995566+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:24:52.995566+0000","last_undegraded":"2023-09-23T22:24:52.995566+0000","last_fullsized":"2023-09-23T22:24:52.995566+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:47.781486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:47.781486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:01.152802+0000","scrub_duration":0.0020463370000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":813,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.995418+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:24:52.995418+0000","last_peered":"2023-09-23T22:24:52.995418+0000","last_clean":"2023-09-23T22:24:52.995418+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:24:52.995418+0000","last_undegraded":"2023-09-23T22:24:52.995418+0000","last_fullsized":"2023-09-23T22:24:52.995418+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":281,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350193+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:24:53.350193+0000","last_peered":"2023-09-23T22:24:53.350193+0000","last_clean":"2023-09-23T22:24:53.350193+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:24:53.350193+0000","last_undegraded":"2023-09-23T22:24:53.350193+0000","last_fullsized":"2023-09-23T22:24:53.350193+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":50,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993554+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:24:52.993554+0000","last_peered":"2023-09-23T22:24:52.993554+0000","last_clean":"2023-09-23T22:24:52.993554+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:24:52.993554+0000","last_undegraded":"2023-09-23T22:24:52.993554+0000","last_fullsized":"2023-09-23T22:24:52.993554+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"265'1505","reported_seq":1517,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:06.863812+0000","last_change":"2023-09-23T22:24:46.981073+0000","last_active":"2023-09-23T22:25:06.863812+0000","last_peered":"2023-09-23T22:25:06.863812+0000","last_clean":"2023-09-23T22:25:06.863812+0000","last_became_active":"2023-09-23T22:24:46.980758+0000","last_became_peered":"2023-09-23T22:24:46.980758+0000","last_unstale":"2023-09-23T22:25:06.863812+0000","last_undegraded":"2023-09-23T22:25:06.863812+0000","last_fullsized":"2023-09-23T22:25:06.863812+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1505,"log_dups_size":0,"ondisk_log_size":1505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:01.377206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98631680,"num_objects":1505,"num_object_clones":0,"num_object_copies":4515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1505,"num_write_kb":96320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":568,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.349382+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:24:53.349382+0000","last_peered":"2023-09-23T22:24:53.349382+0000","last_clean":"2023-09-23T22:24:53.349382+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:24:53.349382+0000","last_undegraded":"2023-09-23T22:24:53.349382+0000","last_fullsized":"2023-09-23T22:24:53.349382+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":996,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.349521+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:24:53.349521+0000","last_peered":"2023-09-23T22:24:53.349521+0000","last_clean":"2023-09-23T22:24:53.349521+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:24:53.349521+0000","last_undegraded":"2023-09-23T22:24:53.349521+0000","last_fullsized":"2023-09-23T22:24:53.349521+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":273,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352163+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:24:53.352163+0000","last_peered":"2023-09-23T22:24:53.352163+0000","last_clean":"2023-09-23T22:24:53.352163+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:24:53.352163+0000","last_undegraded":"2023-09-23T22:24:53.352163+0000","last_fullsized":"2023-09-23T22:24:53.352163+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":228,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350631+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:24:53.350631+0000","last_peered":"2023-09-23T22:24:53.350631+0000","last_clean":"2023-09-23T22:24:53.350631+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:24:53.350631+0000","last_undegraded":"2023-09-23T22:24:53.350631+0000","last_fullsized":"2023-09-23T22:24:53.350631+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"265'1509","reported_seq":1523,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:06.621295+0000","last_change":"2023-09-23T22:24:46.980328+0000","last_active":"2023-09-23T22:25:06.621295+0000","last_peered":"2023-09-23T22:25:06.621295+0000","last_clean":"2023-09-23T22:25:06.621295+0000","last_became_active":"2023-09-23T22:24:46.980033+0000","last_became_peered":"2023-09-23T22:24:46.980033+0000","last_unstale":"2023-09-23T22:25:06.621295+0000","last_undegraded":"2023-09-23T22:25:06.621295+0000","last_fullsized":"2023-09-23T22:25:06.621295+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1509,"log_dups_size":0,"ondisk_log_size":1509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:58.452186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98893824,"num_objects":1509,"num_object_clones":0,"num_object_copies":4527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1509,"num_write_kb":96576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":594,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053540+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:24:55.053540+0000","last_peered":"2023-09-23T22:24:55.053540+0000","last_clean":"2023-09-23T22:24:55.053540+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:24:55.053540+0000","last_undegraded":"2023-09-23T22:24:55.053540+0000","last_fullsized":"2023-09-23T22:24:55.053540+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":399,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994463+0000","last_change":"2023-09-23T22:24:38.072053+0000","last_active":"2023-09-23T22:24:52.994463+0000","last_peered":"2023-09-23T22:24:52.994463+0000","last_clean":"2023-09-23T22:24:52.994463+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:24:52.994463+0000","last_undegraded":"2023-09-23T22:24:52.994463+0000","last_fullsized":"2023-09-23T22:24:52.994463+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":225,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.856366+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:24:53.856366+0000","last_peered":"2023-09-23T22:24:53.856366+0000","last_clean":"2023-09-23T22:24:53.856366+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:24:53.856366+0000","last_undegraded":"2023-09-23T22:24:53.856366+0000","last_fullsized":"2023-09-23T22:24:53.856366+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":271,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994827+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:24:52.994827+0000","last_peered":"2023-09-23T22:24:52.994827+0000","last_clean":"2023-09-23T22:24:52.994827+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:24:52.994827+0000","last_undegraded":"2023-09-23T22:24:52.994827+0000","last_fullsized":"2023-09-23T22:24:52.994827+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"265'1517","reported_seq":1531,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:06.612731+0000","last_change":"2023-09-23T22:24:47.009793+0000","last_active":"2023-09-23T22:25:06.612731+0000","last_peered":"2023-09-23T22:25:06.612731+0000","last_clean":"2023-09-23T22:25:06.612731+0000","last_became_active":"2023-09-23T22:24:47.009468+0000","last_became_peered":"2023-09-23T22:24:47.009468+0000","last_unstale":"2023-09-23T22:25:06.612731+0000","last_undegraded":"2023-09-23T22:25:06.612731+0000","last_fullsized":"2023-09-23T22:25:06.612731+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1517,"log_dups_size":0,"ondisk_log_size":1517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:20.557721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99418112,"num_objects":1517,"num_object_clones":0,"num_object_copies":4551,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1517,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1517,"num_write_kb":97088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":891,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350064+0000","last_change":"2023-09-23T22:24:39.954768+0000","last_active":"2023-09-23T22:24:53.350064+0000","last_peered":"2023-09-23T22:24:53.350064+0000","last_clean":"2023-09-23T22:24:53.350064+0000","last_became_active":"2023-09-23T22:24:39.954171+0000","last_became_peered":"2023-09-23T22:24:39.954171+0000","last_unstale":"2023-09-23T22:24:53.350064+0000","last_undegraded":"2023-09-23T22:24:53.350064+0000","last_fullsized":"2023-09-23T22:24:53.350064+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:00.983002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":578,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:01.646069+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:25:01.646069+0000","last_peered":"2023-09-23T22:25:01.646069+0000","last_clean":"2023-09-23T22:25:01.646069+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:25:01.646069+0000","last_undegraded":"2023-09-23T22:25:01.646069+0000","last_fullsized":"2023-09-23T22:25:01.646069+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":189,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993479+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:24:52.993479+0000","last_peered":"2023-09-23T22:24:52.993479+0000","last_clean":"2023-09-23T22:24:52.993479+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:24:52.993479+0000","last_undegraded":"2023-09-23T22:24:52.993479+0000","last_fullsized":"2023-09-23T22:24:52.993479+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":150,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352445+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:24:53.352445+0000","last_peered":"2023-09-23T22:24:53.352445+0000","last_clean":"2023-09-23T22:24:53.352445+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:24:53.352445+0000","last_undegraded":"2023-09-23T22:24:53.352445+0000","last_fullsized":"2023-09-23T22:24:53.352445+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"265'1540","reported_seq":1554,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:06.620960+0000","last_change":"2023-09-23T22:24:47.009893+0000","last_active":"2023-09-23T22:25:06.620960+0000","last_peered":"2023-09-23T22:25:06.620960+0000","last_clean":"2023-09-23T22:25:06.620960+0000","last_became_active":"2023-09-23T22:24:47.009599+0000","last_became_peered":"2023-09-23T22:24:47.009599+0000","last_unstale":"2023-09-23T22:25:06.620960+0000","last_undegraded":"2023-09-23T22:25:06.620960+0000","last_fullsized":"2023-09-23T22:25:06.620960+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1540,"log_dups_size":0,"ondisk_log_size":1540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.224040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100925440,"num_objects":1540,"num_object_clones":0,"num_object_copies":4620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1540,"num_write_kb":98560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":596,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993416+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:24:52.993416+0000","last_peered":"2023-09-23T22:24:52.993416+0000","last_clean":"2023-09-23T22:24:52.993416+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:24:52.993416+0000","last_undegraded":"2023-09-23T22:24:52.993416+0000","last_fullsized":"2023-09-23T22:24:52.993416+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":614,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993225+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:24:52.993225+0000","last_peered":"2023-09-23T22:24:52.993225+0000","last_clean":"2023-09-23T22:24:52.993225+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:24:52.993225+0000","last_undegraded":"2023-09-23T22:24:52.993225+0000","last_fullsized":"2023-09-23T22:24:52.993225+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":147,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.052869+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:24:55.052869+0000","last_peered":"2023-09-23T22:24:55.052869+0000","last_clean":"2023-09-23T22:24:55.052869+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:24:55.052869+0000","last_undegraded":"2023-09-23T22:24:55.052869+0000","last_fullsized":"2023-09-23T22:24:55.052869+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":104,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054654+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:24:55.054654+0000","last_peered":"2023-09-23T22:24:55.054654+0000","last_clean":"2023-09-23T22:24:55.054654+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:24:55.054654+0000","last_undegraded":"2023-09-23T22:24:55.054654+0000","last_fullsized":"2023-09-23T22:24:55.054654+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"264'1281","reported_seq":1292,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:04.937345+0000","last_change":"2023-09-23T22:24:46.984221+0000","last_active":"2023-09-23T22:25:04.937345+0000","last_peered":"2023-09-23T22:25:04.937345+0000","last_clean":"2023-09-23T22:25:04.937345+0000","last_became_active":"2023-09-23T22:24:46.983941+0000","last_became_peered":"2023-09-23T22:24:46.983941+0000","last_unstale":"2023-09-23T22:25:04.937345+0000","last_undegraded":"2023-09-23T22:25:04.937345+0000","last_fullsized":"2023-09-23T22:25:04.937345+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1281,"log_dups_size":0,"ondisk_log_size":1281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:31.961352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83951616,"num_objects":1281,"num_object_clones":0,"num_object_copies":3843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1281,"num_write_kb":81984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":586,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054363+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:24:55.054363+0000","last_peered":"2023-09-23T22:24:55.054363+0000","last_clean":"2023-09-23T22:24:55.054363+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:24:55.054363+0000","last_undegraded":"2023-09-23T22:24:55.054363+0000","last_fullsized":"2023-09-23T22:24:55.054363+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":156,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053982+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:24:55.053982+0000","last_peered":"2023-09-23T22:24:55.053982+0000","last_clean":"2023-09-23T22:24:55.053982+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:24:55.053982+0000","last_undegraded":"2023-09-23T22:24:55.053982+0000","last_fullsized":"2023-09-23T22:24:55.053982+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":215,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.995699+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:24:52.995699+0000","last_peered":"2023-09-23T22:24:52.995699+0000","last_clean":"2023-09-23T22:24:52.995699+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:24:52.995699+0000","last_undegraded":"2023-09-23T22:24:52.995699+0000","last_fullsized":"2023-09-23T22:24:52.995699+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":120,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053077+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:24:55.053077+0000","last_peered":"2023-09-23T22:24:55.053077+0000","last_clean":"2023-09-23T22:24:55.053077+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:24:55.053077+0000","last_undegraded":"2023-09-23T22:24:55.053077+0000","last_fullsized":"2023-09-23T22:24:55.053077+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"264'1271","reported_seq":1284,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:04.399033+0000","last_change":"2023-09-23T22:24:47.010826+0000","last_active":"2023-09-23T22:25:04.399033+0000","last_peered":"2023-09-23T22:25:04.399033+0000","last_clean":"2023-09-23T22:25:04.399033+0000","last_became_active":"2023-09-23T22:24:47.010507+0000","last_became_peered":"2023-09-23T22:24:47.010507+0000","last_unstale":"2023-09-23T22:25:04.399033+0000","last_undegraded":"2023-09-23T22:25:04.399033+0000","last_fullsized":"2023-09-23T22:25:04.399033+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1271,"log_dups_size":0,"ondisk_log_size":1271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:22.053927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83296256,"num_objects":1271,"num_object_clones":0,"num_object_copies":3813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1271,"num_write_kb":81344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":568,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352162+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:24:53.352162+0000","last_peered":"2023-09-23T22:24:53.352162+0000","last_clean":"2023-09-23T22:24:53.352162+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:24:53.352162+0000","last_undegraded":"2023-09-23T22:24:53.352162+0000","last_fullsized":"2023-09-23T22:24:53.352162+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":278,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.995405+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:24:52.995405+0000","last_peered":"2023-09-23T22:24:52.995405+0000","last_clean":"2023-09-23T22:24:52.995405+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:24:52.995405+0000","last_undegraded":"2023-09-23T22:24:52.995405+0000","last_fullsized":"2023-09-23T22:24:52.995405+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":906,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994217+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:24:52.994217+0000","last_peered":"2023-09-23T22:24:52.994217+0000","last_clean":"2023-09-23T22:24:52.994217+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:24:52.994217+0000","last_undegraded":"2023-09-23T22:24:52.994217+0000","last_fullsized":"2023-09-23T22:24:52.994217+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":53,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.993628+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:24:52.993628+0000","last_peered":"2023-09-23T22:24:52.993628+0000","last_clean":"2023-09-23T22:24:52.993628+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:24:52.993628+0000","last_undegraded":"2023-09-23T22:24:52.993628+0000","last_fullsized":"2023-09-23T22:24:52.993628+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"264'1281","reported_seq":1294,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:04.391303+0000","last_change":"2023-09-23T22:24:47.009023+0000","last_active":"2023-09-23T22:25:04.391303+0000","last_peered":"2023-09-23T22:25:04.391303+0000","last_clean":"2023-09-23T22:25:04.391303+0000","last_became_active":"2023-09-23T22:24:47.008704+0000","last_became_peered":"2023-09-23T22:24:47.008704+0000","last_unstale":"2023-09-23T22:25:04.391303+0000","last_undegraded":"2023-09-23T22:25:04.391303+0000","last_fullsized":"2023-09-23T22:25:04.391303+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":1281,"log_dups_size":0,"ondisk_log_size":1281,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:28.997990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83951616,"num_objects":1281,"num_object_clones":0,"num_object_copies":3843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1281,"num_write_kb":81984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":564,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.995068+0000","last_change":"2023-09-23T22:24:45.734796+0000","last_active":"2023-09-23T22:24:52.995068+0000","last_peered":"2023-09-23T22:24:52.995068+0000","last_clean":"2023-09-23T22:24:52.995068+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:24:52.995068+0000","last_undegraded":"2023-09-23T22:24:52.995068+0000","last_fullsized":"2023-09-23T22:24:52.995068+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:45.734686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.734686+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.132151+0000","scrub_duration":0.0022175960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":258,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.995491+0000","last_change":"2023-09-23T22:24:38.076201+0000","last_active":"2023-09-23T22:24:52.995491+0000","last_peered":"2023-09-23T22:24:52.995491+0000","last_clean":"2023-09-23T22:24:52.995491+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:24:52.995491+0000","last_undegraded":"2023-09-23T22:24:52.995491+0000","last_fullsized":"2023-09-23T22:24:52.995491+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":99,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054436+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:24:55.054436+0000","last_peered":"2023-09-23T22:24:55.054436+0000","last_clean":"2023-09-23T22:24:55.054436+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:24:55.054436+0000","last_undegraded":"2023-09-23T22:24:55.054436+0000","last_fullsized":"2023-09-23T22:24:55.054436+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":227,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352024+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:24:53.352024+0000","last_peered":"2023-09-23T22:24:53.352024+0000","last_clean":"2023-09-23T22:24:53.352024+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:24:53.352024+0000","last_undegraded":"2023-09-23T22:24:53.352024+0000","last_fullsized":"2023-09-23T22:24:53.352024+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":95,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053003+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:24:55.053003+0000","last_peered":"2023-09-23T22:24:55.053003+0000","last_clean":"2023-09-23T22:24:55.053003+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:24:55.053003+0000","last_undegraded":"2023-09-23T22:24:55.053003+0000","last_fullsized":"2023-09-23T22:24:55.053003+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":275,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.995632+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:24:52.995632+0000","last_peered":"2023-09-23T22:24:52.995632+0000","last_clean":"2023-09-23T22:24:52.995632+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:24:52.995632+0000","last_undegraded":"2023-09-23T22:24:52.995632+0000","last_fullsized":"2023-09-23T22:24:52.995632+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":226,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350407+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:24:53.350407+0000","last_peered":"2023-09-23T22:24:53.350407+0000","last_clean":"2023-09-23T22:24:53.350407+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:24:53.350407+0000","last_undegraded":"2023-09-23T22:24:53.350407+0000","last_fullsized":"2023-09-23T22:24:53.350407+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":239,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352251+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:24:53.352251+0000","last_peered":"2023-09-23T22:24:53.352251+0000","last_clean":"2023-09-23T22:24:53.352251+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:24:53.352251+0000","last_undegraded":"2023-09-23T22:24:53.352251+0000","last_fullsized":"2023-09-23T22:24:53.352251+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":223,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.350284+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:24:53.350284+0000","last_peered":"2023-09-23T22:24:53.350284+0000","last_clean":"2023-09-23T22:24:53.350284+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:24:53.350284+0000","last_undegraded":"2023-09-23T22:24:53.350284+0000","last_fullsized":"2023-09-23T22:24:53.350284+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.352055+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:24:53.352055+0000","last_peered":"2023-09-23T22:24:53.352055+0000","last_clean":"2023-09-23T22:24:53.352055+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:24:53.352055+0000","last_undegraded":"2023-09-23T22:24:53.352055+0000","last_fullsized":"2023-09-23T22:24:53.352055+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":245,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994794+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:24:52.994794+0000","last_peered":"2023-09-23T22:24:52.994794+0000","last_clean":"2023-09-23T22:24:52.994794+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:24:52.994794+0000","last_undegraded":"2023-09-23T22:24:52.994794+0000","last_fullsized":"2023-09-23T22:24:52.994794+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":286,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:52.994243+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:24:52.994243+0000","last_peered":"2023-09-23T22:24:52.994243+0000","last_clean":"2023-09-23T22:24:52.994243+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:24:52.994243+0000","last_undegraded":"2023-09-23T22:24:52.994243+0000","last_fullsized":"2023-09-23T22:24:52.994243+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":154,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054211+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:24:55.054211+0000","last_peered":"2023-09-23T22:24:55.054211+0000","last_clean":"2023-09-23T22:24:55.054211+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:24:55.054211+0000","last_undegraded":"2023-09-23T22:24:55.054211+0000","last_fullsized":"2023-09-23T22:24:55.054211+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":272,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T22:24:53.351888+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:24:53.351888+0000","last_peered":"2023-09-23T22:24:53.351888+0000","last_clean":"2023-09-23T22:24:53.351888+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:24:53.351888+0000","last_undegraded":"2023-09-23T22:24:53.351888+0000","last_fullsized":"2023-09-23T22:24:53.351888+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1428160512,"num_objects":21792,"num_object_clones":0,"num_object_copies":65376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":21792,"num_write_kb":1394688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4155703296,"data_stored":4155703296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":21792,"ondisk_log_size":21792,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":126,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234248,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":981812,"kb_used_data":771884,"kb_used_omap":0,"kb_used_meta":209920,"kb_avail":92763340,"statfs":{"total":95995035648,"available":94989660160,"internally_reserved":0,"allocated":790409216,"data_stored":789379758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":214958080},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:05 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":4.1870000000000003,"15min":2.3159999999999998},"min":{"1min":0.22600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":4.5499999999999998,"5min":158.042,"15min":158.042},"last":1.52},{"interface":"front","average":{"1min":2.4369999999999998,"5min":4.6920000000000002,"15min":2.6070000000000002},"min":{"1min":0.28199999999999997,"5min":0.217,"15min":0.217},"max":{"1min":6.6719999999999997,"5min":157.952,"15min":157.952},"last":1.7609999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":4.4050000000000002,"15min":3.0630000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":6.2839999999999998,"5min":101.145,"15min":123.79300000000001},"last":0.83699999999999997},{"interface":"front","average":{"1min":1.8759999999999999,"5min":4.4969999999999999,"15min":3.2170000000000001},"min":{"1min":0.33200000000000002,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":8.4149999999999991,"5min":101.13,"15min":123.66500000000001},"last":5.077}]},{"osd":2,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.776,"15min":3.7850000000000001},"min":{"1min":0.29999999999999999,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":6.0810000000000004,"5min":19.044,"15min":125.101},"last":2.4780000000000002},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.819,"15min":3.548},"min":{"1min":0.311,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":4.9829999999999997,"5min":19.445,"15min":126.021},"last":4.8109999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:24:16 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":3.9329999999999998,"15min":2.335},"min":{"1min":0.46800000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.6479999999999997,"5min":170.86099999999999,"15min":170.86099999999999},"last":2.3839999999999999},{"interface":"front","average":{"1min":2.0619999999999998,"5min":3.7410000000000001,"15min":2.4100000000000001},"min":{"1min":0.44800000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.7919999999999998,"5min":174.57499999999999,"15min":174.57499999999999},"last":1.4299999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.0589999999999999,"15min":1.3700000000000001},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.0419999999999998,"5min":27.922000000000001,"15min":88.543000000000006},"last":0.746},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.143,"15min":1.3120000000000001},"min":{"1min":0.28100000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.0600000000000001,"5min":29.920000000000002,"15min":88.495999999999995},"last":0.32200000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0629999999999999,"15min":1.2849999999999999},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":5.7960000000000003,"5min":24.965,"15min":93.816000000000003},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.2330000000000001,"15min":1.5429999999999999},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":9.0760000000000005,"5min":24.712,"15min":95.579999999999998},"last":0.78800000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.73299999999999998,"15min":0.753},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.181,"5min":5.343,"15min":9.0139999999999993},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.351,"5min":0.95899999999999996,"15min":0.99199999999999999},"min":{"1min":0.35899999999999999,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":10.967000000000001,"5min":10.967000000000001,"15min":13.362},"last":0.81999999999999995}]}]},{"osd":6,"up_from":0,"seq":197568495884,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795156,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1154632,"kb_used_data":785408,"kb_used_omap":7,"kb_used_meta":369208,"kb_avail":92590520,"statfs":{"total":95995035648,"available":94812692480,"internally_reserved":0,"allocated":804257792,"data_stored":802858946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":378069364},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":2.2770000000000001,"15min":1.4750000000000001},"min":{"1min":0.22700000000000001,"5min":0.189,"15min":0.189},"max":{"1min":10.125999999999999,"5min":56.503,"15min":61.890000000000001},"last":2.7400000000000002},{"interface":"front","average":{"1min":4.2060000000000004,"5min":3.1179999999999999,"15min":1.982},"min":{"1min":0.28199999999999997,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":15.965999999999999,"5min":56.560000000000002,"15min":62.689999999999998},"last":1.004}]},{"osd":2,"last update":"Sat Sep 23 22:24:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.0649999999999999,"15min":1.284},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.83899999999999997,"5min":11.179,"15min":33.902000000000001},"last":2.7130000000000001},{"interface":"front","average":{"1min":3.101,"5min":2.2730000000000001,"15min":1.9059999999999999},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.053000000000001,"5min":16.053000000000001,"15min":34.781999999999996},"last":1.032}]},{"osd":3,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":5.5910000000000002,"5min":4.6269999999999998,"15min":2.4049999999999998},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":13.871,"5min":110.417,"15min":110.417},"last":0.60099999999999998},{"interface":"front","average":{"1min":5.29,"5min":4.6589999999999998,"15min":2.343},"min":{"1min":0.28299999999999997,"5min":0.223,"15min":0.223},"max":{"1min":16.346,"5min":110.721,"15min":110.721},"last":1.7310000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":4.5359999999999996,"15min":2.0779999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27000000000000002},"max":{"1min":5.0869999999999997,"5min":234.79900000000001,"15min":234.79900000000001},"last":2.6019999999999999},{"interface":"front","average":{"1min":2.153,"5min":5.1689999999999996,"15min":2.4510000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":6.7160000000000002,"5min":234.81700000000001,"15min":234.81700000000001},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":3.5310000000000001,"15min":1.889},"min":{"1min":0.34000000000000002,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":6.7480000000000002,"5min":196.79900000000001,"15min":196.79900000000001},"last":0.97299999999999998},{"interface":"front","average":{"1min":2.5190000000000001,"5min":3.8450000000000002,"15min":1.9239999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":16.207000000000001,"5min":196.68299999999999,"15min":196.68299999999999},"last":1.675}]},{"osd":6,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":2.1240000000000001,"15min":1.3759999999999999},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":16.314,"5min":16.314,"15min":16.314},"last":1.7050000000000001},{"interface":"front","average":{"1min":1.218,"5min":1.206,"15min":1.042},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.1310000000000002,"5min":11.385999999999999,"15min":11.385999999999999},"last":2.6840000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":2.2909999999999999,"15min":1.484},"min":{"1min":0.57699999999999996,"5min":0.33800000000000002,"15min":0.32900000000000001},"max":{"1min":16.265000000000001,"5min":16.265000000000001,"15min":16.265000000000001},"last":1.6479999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.21,"15min":1.0860000000000001},"min":{"1min":0.42199999999999999,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":5.194,"5min":12.045999999999999,"15min":12.045999999999999},"last":2.6509999999999998}]}]},{"osd":0,"up_from":93,"seq":399431958691,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1187316,"kb_used_data":826156,"kb_used_omap":8,"kb_used_meta":361143,"kb_avail":92557836,"statfs":{"total":95995035648,"available":94779224064,"internally_reserved":0,"allocated":845983744,"data_stored":844547260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":369811306},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0619999999999998,"5min":4.5759999999999996,"15min":2.4550000000000001},"min":{"1min":0.214,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":16.253,"5min":169.77199999999999,"15min":169.77199999999999},"last":4.6070000000000002},{"interface":"front","average":{"1min":3.089,"5min":4.7729999999999997,"15min":2.746},"min":{"1min":0.34999999999999998,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":13.108000000000001,"5min":169.52099999999999,"15min":169.52099999999999},"last":1.9419999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:24:54 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":5.6950000000000003,"15min":2.7200000000000002},"min":{"1min":0.26600000000000001,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":5.6390000000000002,"5min":206.495,"15min":206.495},"last":4.5739999999999998},{"interface":"front","average":{"1min":1.7330000000000001,"5min":6.101,"15min":3.3860000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22800000000000001},"max":{"1min":9.0920000000000005,"5min":207.34399999999999,"15min":207.34399999999999},"last":1.8300000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:24:23 2023","interfaces":[{"interface":"back","average":{"1min":11.73,"5min":5.4100000000000001,"15min":2.754},"min":{"1min":0.35899999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":146.87299999999999,"5min":146.87299999999999,"15min":146.87299999999999},"last":4.4539999999999997},{"interface":"front","average":{"1min":10.827,"5min":5.2699999999999996,"15min":2.7959999999999998},"min":{"1min":0.33600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":145.48099999999999,"5min":145.48099999999999,"15min":145.48099999999999},"last":1.9139999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":2.722,"15min":2.0089999999999999},"min":{"1min":0.40699999999999997,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":3.3679999999999999,"5min":82.825000000000003,"15min":82.825000000000003},"last":0.622},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.302,"15min":1.409},"min":{"1min":0.39700000000000002,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":4.798,"5min":75.930999999999997,"15min":75.930999999999997},"last":4.5}]},{"osd":5,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":3.8849999999999998,"15min":2.3959999999999999},"min":{"1min":0.378,"5min":0.36899999999999999,"15min":0.29499999999999998},"max":{"1min":3.4049999999999998,"5min":93.685000000000002,"15min":93.685000000000002},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.1180000000000001,"5min":3.5739999999999998,"15min":1.8480000000000001},"min":{"1min":0.45300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.8310000000000004,"5min":93.546000000000006,"15min":93.546000000000006},"last":4.5229999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.798,"15min":1.718},"min":{"1min":0.432,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":3.4529999999999998,"5min":16.366,"15min":16.366},"last":0.65800000000000003},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.6759999999999999,"15min":1.256},"min":{"1min":0.47399999999999998,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":4.883,"5min":16.225999999999999,"15min":22.263000000000002},"last":4.5439999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":2.0470000000000002,"15min":1.768},"min":{"1min":0.39800000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.1440000000000001,"5min":32.773000000000003,"15min":32.773000000000003},"last":1.8740000000000001},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.929,"15min":1.9199999999999999},"min":{"1min":0.437,"5min":0.39900000000000002,"15min":0.38900000000000001},"max":{"1min":3.4279999999999999,"5min":16.381,"15min":23.265999999999998},"last":0.71899999999999997}]}]},{"osd":2,"up_from":0,"seq":442381631630,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468103,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266168,"kb_used_data":923120,"kb_used_omap":4,"kb_used_meta":343035,"kb_avail":92478984,"statfs":{"total":95995035648,"available":94698479616,"internally_reserved":0,"allocated":945274880,"data_stored":943828597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":351268397},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":4.1020000000000003,"5min":5.7489999999999997,"15min":2.9550000000000001},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.19900000000000001},"max":{"1min":10.547000000000001,"5min":225.86699999999999,"15min":225.86699999999999},"last":0.308},{"interface":"front","average":{"1min":4.3819999999999997,"5min":6.1420000000000003,"15min":3.202},"min":{"1min":0.39100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":10.976000000000001,"5min":225.84800000000001,"15min":225.84800000000001},"last":3.8010000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":3.2080000000000002,"5min":3.2309999999999999,"15min":2.3740000000000001},"min":{"1min":0.371,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":13.374000000000001,"5min":37.002000000000002,"15min":82.951999999999998},"last":3.6379999999999999},{"interface":"front","average":{"1min":3.3439999999999999,"5min":3.2290000000000001,"15min":2.4129999999999998},"min":{"1min":0.25700000000000001,"5min":0.22800000000000001,"15min":0.189},"max":{"1min":13.723000000000001,"5min":33.119999999999997,"15min":86.001000000000005},"last":1.2010000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.806,"5min":2.5270000000000001,"15min":3.5459999999999998},"min":{"1min":0.40300000000000002,"5min":0.252,"15min":0.223},"max":{"1min":13.518000000000001,"5min":29.076000000000001,"15min":99.962999999999994},"last":3.3340000000000001},{"interface":"front","average":{"1min":3.6030000000000002,"5min":2.6840000000000002,"15min":3.4249999999999998},"min":{"1min":0.29199999999999998,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":13.692,"5min":13.692,"15min":100.429},"last":1.157}]},{"osd":4,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.903,"15min":1.282},"min":{"1min":0.314,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":7.2039999999999997,"5min":11.911,"15min":11.911},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.798,"5min":1.9179999999999999,"15min":1.3460000000000001},"min":{"1min":0.34399999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.2940000000000005,"5min":10.949999999999999,"15min":12.036},"last":3.9039999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.8300000000000001,"5min":2.5920000000000001,"15min":1.538},"min":{"1min":0.41499999999999998,"5min":0.309,"15min":0.309},"max":{"1min":9.4179999999999993,"5min":49.695999999999998,"15min":49.695999999999998},"last":3.9529999999999998},{"interface":"front","average":{"1min":2.5070000000000001,"5min":2.383,"15min":1.506},"min":{"1min":0.376,"5min":0.31,"15min":0.31},"max":{"1min":13.494,"5min":49.668999999999997,"15min":49.668999999999997},"last":3.7530000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.9020000000000001,"5min":2.1829999999999998,"15min":1.482},"min":{"1min":0.436,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":9.3439999999999994,"5min":10.992000000000001,"15min":12.068},"last":4.0309999999999997},{"interface":"front","average":{"1min":2.6110000000000002,"5min":1.9650000000000001,"15min":1.373},"min":{"1min":0.40899999999999997,"5min":0.39700000000000002,"15min":0.35399999999999998},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":3.7189999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:24:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.1030000000000002,"15min":1.4470000000000001},"min":{"1min":0.434,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":13.467000000000001,"5min":16.849,"15min":16.849},"last":3.7799999999999998},{"interface":"front","average":{"1min":2.375,"5min":2.0649999999999999,"15min":1.385},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.36699999999999999},"max":{"1min":7.2480000000000002,"5min":11.945,"15min":12.382},"last":0.53500000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019153,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":700220,"kb_used_data":504116,"kb_used_omap":0,"kb_used_meta":196096,"kb_avail":93044932,"statfs":{"total":95995035648,"available":95278010368,"internally_reserved":0,"allocated":516214784,"data_stored":515178485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":200802304},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":3.7069999999999999,"5min":2.9369999999999998,"15min":1.8540000000000001},"min":{"1min":0.25800000000000001,"5min":0.23699999999999999,"15min":0.218},"max":{"1min":25.425000000000001,"5min":46.18,"15min":66.355000000000004},"last":0.36399999999999999},{"interface":"front","average":{"1min":3.444,"5min":2.9039999999999999,"15min":1.921},"min":{"1min":0.28799999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":25.451000000000001,"5min":46.207000000000001,"15min":66.332999999999998},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":5.4039999999999999,"5min":2.7160000000000002,"15min":1.6699999999999999},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":85.662000000000006,"5min":85.662000000000006,"15min":85.662000000000006},"last":1.2969999999999999},{"interface":"front","average":{"1min":5.5999999999999996,"5min":3.0899999999999999,"15min":2.0289999999999999},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":81.195999999999998,"5min":81.195999999999998,"15min":81.195999999999998},"last":0.63400000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.8069999999999999,"15min":2.351},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":6.0170000000000003,"5min":12.397,"15min":138.06100000000001},"last":0.495},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.5409999999999999,"15min":2.5209999999999999},"min":{"1min":0.34399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.8639999999999999,"5min":6.9580000000000002,"15min":138.15199999999999},"last":0.54900000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.669,"15min":1.2709999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":5.0140000000000002,"5min":12.129,"15min":12.129},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.3599999999999999,"5min":1.8580000000000001,"15min":1.347},"min":{"1min":0.32300000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.5600000000000005,"5min":8.5600000000000005,"15min":9.8109999999999999},"last":1.78}]},{"osd":5,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.3069999999999999,"15min":0.91000000000000003},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.165,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.23999999999999999},{"interface":"front","average":{"1min":1.26,"5min":1.4550000000000001,"15min":0.95499999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27900000000000003},"max":{"1min":5.601,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.318}]},{"osd":6,"last update":"Sat Sep 23 22:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":0.98299999999999998,"15min":0.77600000000000002},"min":{"1min":0.33800000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":7.8550000000000004,"5min":7.8550000000000004,"15min":7.8550000000000004},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.81599999999999995,"15min":0.77500000000000002},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.802,"5min":3.9239999999999999,"15min":8.5210000000000008},"last":0.30099999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:24:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0820000000000001,"5min":0.92900000000000005,"15min":0.878},"min":{"1min":0.33300000000000002,"5min":0.313,"15min":0.307},"max":{"1min":3.347,"5min":6.7480000000000002,"15min":8.577},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.377,"5min":1.0149999999999999,"15min":0.83999999999999997},"min":{"1min":0.34200000000000003,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":7.8109999999999999,"5min":7.8109999999999999,"15min":7.8109999999999999},"last":0.41399999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757521,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":585132,"kb_used_data":340580,"kb_used_omap":0,"kb_used_meta":244544,"kb_avail":93160020,"statfs":{"total":95995035648,"available":95395860480,"internally_reserved":0,"allocated":348753920,"data_stored":347729214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":250413056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:59 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":2.2269999999999999,"15min":2.1859999999999999},"min":{"1min":0.23400000000000001,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":3.306,"5min":64.344999999999999,"15min":133.22499999999999},"last":1.149},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.1659999999999999,"15min":2.242},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.141,"5min":64.022000000000006,"15min":132.78200000000001},"last":0.29899999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":3.1040000000000001,"15min":2.7589999999999999},"min":{"1min":0.27500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.7870000000000008,"5min":93.981999999999999,"15min":93.981999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.885,"5min":3.0310000000000001,"15min":2.7389999999999999},"min":{"1min":0.35499999999999998,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":10.949,"5min":93.909999999999997,"15min":93.909999999999997},"last":0.55600000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":2.4260000000000002,"15min":3.536},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.702,"5min":95.790999999999997,"15min":104.254},"last":0.91500000000000004},{"interface":"front","average":{"1min":1.365,"5min":2.25,"15min":3.806},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":8.1010000000000009,"5min":95.366,"15min":104.217},"last":1.8520000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1499999999999999,"5min":3.786,"15min":2.335},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.246},"max":{"1min":5.0839999999999996,"5min":93.739999999999995,"15min":93.739999999999995},"last":1.575},{"interface":"front","average":{"1min":1.9199999999999999,"5min":3.3279999999999998,"15min":2.2589999999999999},"min":{"1min":0.38200000000000001,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":4.2569999999999997,"5min":93.197999999999993,"15min":93.197999999999993},"last":0.84299999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:24:15 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":1.0089999999999999,"15min":1.046},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.7160000000000002,"5min":10.163,"15min":82.530000000000001},"last":0.313},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.02,"15min":1.0880000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":2.8900000000000001,"5min":7.2119999999999997,"15min":85.441000000000003},"last":0.79300000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:24:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":1.179},"min":{"1min":0.34699999999999998,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":2.6040000000000001,"5min":7.0289999999999999,"15min":114.83499999999999},"last":0.88300000000000001},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.72499999999999998,"15min":1.0880000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7130000000000001,"5min":3.8690000000000002,"15min":110.905},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:24:59 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.82799999999999996,"15min":0.90700000000000003},"min":{"1min":0.36399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.0819999999999999,"5min":5.0819999999999999,"15min":8.8030000000000008},"last":1.048},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.77600000000000002,"15min":0.77800000000000002},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":3.6299999999999999,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":0.34899999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":890699776,"data_stored":890699776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":802029568,"data_stored":802029568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":994443264,"data_stored":994443264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":514588672,"data_stored":514588672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":343146496,"data_stored":343146496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":610795520,"data_stored":610795520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:25:11.760 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:11.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:11.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:11.872 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:11.957 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:12.156 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:12.239 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:12.259 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 32148 32132 118.099 93.75 0.0183401 0.0084604 2023-09-23T22:25:12.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:12.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:12.286 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:25:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:25:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.286 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:32174 7.6 7:6f33c030:::benchmark_data_smithi026_410749_object32173:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:12.278288+0000", 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001088016, 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011233199999999999, 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 32174 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.287 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278288+0000", 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278288+0000", 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278286+0000", 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278340+0000", 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3816000000000001e-05 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278353+0000", 2023-09-23T22:25:12.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3001e-05 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278361+0000", 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8879999999999997e-06 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278585+0000", 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000223833 2023-09-23T22:25:12.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278772+0000", 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018731499999999999 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278923+0000", 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151096 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:12.291 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:32176 7.4 7:2bd24bde:::benchmark_data_smithi026_410749_object32175:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e265)", 2023-09-23T22:25:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:12.278564+0000", 2023-09-23T22:25:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00081198100000000003, 2023-09-23T22:25:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00090948000000000005, 2023-09-23T22:25:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:25:12.292 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 32176 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278564+0000", 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278564+0000", 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278563+0000", 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:12.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278602+0000", 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8578999999999998e-05 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278649+0000", 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6958e-05 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278671+0000", 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2435000000000001e-05 2023-09-23T22:25:12.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278797+0000", 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012544499999999999 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278964+0000", 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016761200000000001 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.279103+0000", 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138641 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:32179 7.6 7.14718906 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e265)", 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:12.278694+0000", 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00068199899999999997, 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00082271600000000003, 2023-09-23T22:25:12.297 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 32179 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278694+0000", 2023-09-23T22:25:12.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278694+0000", 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278694+0000", 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278750+0000", 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5928999999999999e-05 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278775+0000", 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5964000000000001e-05 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:12.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:12.278783+0000", 2023-09-23T22:25:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5059999999999998e-06 2023-09-23T22:25:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:25:12.301 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 3 2023-09-23T22:25:12.301 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:25:12.361 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:12.407 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:12.464 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:12.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:12.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:12.570 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:12.601 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:12.602 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:25:12.638 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:25:12.657 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:12.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:12.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:12.677 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:25:12.720 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:25:12.756 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:25:12.763 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:12.791 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:25:12.826 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:25:12.828 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:12.864 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:25:12.878 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:25:12.878 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:25:12.878 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:25:12.879 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:25:12.879 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:25:12.879 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:25:12.879 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:25:12.879 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:25:12.935 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:12.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:12.939 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:13.040 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:13.066 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:13.107 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:13.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:13.134 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:12 smithi026 ceph-mon[215775]: pgmap v675: 99 pgs: 99 active+clean; 1.6 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 118 MiB/s wr, 1.90k op/s 2023-09-23T22:25:13.135 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1526446727' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:13.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:12 smithi026 ceph-mon[223014]: pgmap v675: 99 pgs: 99 active+clean; 1.6 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 118 MiB/s wr, 1.90k op/s 2023-09-23T22:25:13.135 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1526446727' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:13.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:13.241 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:13.259 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 33143 33127 114.99 62.1875 0.106656 0.00869164 2023-09-23T22:25:13.359 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:13.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:13.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:13.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:12 smithi070 ceph-mon[192652]: pgmap v675: 99 pgs: 99 active+clean; 1.6 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 118 MiB/s wr, 1.90k op/s 2023-09-23T22:25:13.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1526446727' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:13.469 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:13.535 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:14.260 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 34486 34470 113.356 83.9375 0.00313555 0.00879427 2023-09-23T22:25:14.445 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:25:14.548 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:14.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:14.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:14.588 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:25:14.657 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:14.685 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:14.734 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:14.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:14.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:14.841 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:14.872 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:14.926 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:14.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:14.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:14 smithi026 ceph-mon[223014]: pgmap v676: 99 pgs: 99 active+clean; 1.8 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T22:25:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:14 smithi026 ceph-mon[215775]: pgmap v676: 99 pgs: 99 active+clean; 1.8 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T22:25:15.030 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:15.061 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:15.117 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:15.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:15.223 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:15.250 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:14 smithi070 ceph-mon[192652]: pgmap v676: 99 pgs: 99 active+clean; 1.8 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T22:25:15.253 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:15.259 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:25:15.260238+0000 min lat: 0.00118556 max lat: 0.332866 avg lat: 0.00893098 2023-09-23T22:25:15.260 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:25:15.260 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 35833 35817 111.897 84.1875 0.00586286 0.00893098 2023-09-23T22:25:15.311 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:15.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:15.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:15.417 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:15.443 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:15.483 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:15.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:15.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:15.588 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:15.620 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:15.677 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:15.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:15.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:15.783 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:15.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:15.872 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:15.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:15.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:15.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:16.008 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:16.061 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:16.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:16.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:16.126 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:16.138 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:16.165 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:16.260 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 36978 36962 109.976 71.5625 0.00678378 0.00908839 2023-09-23T22:25:16.303 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:16.596 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:16.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:16.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:16.705 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:16.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:16.786 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:16.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:16.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:16.792 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:25:16.890 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:16.917 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:16.965 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:16.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:16.970 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:17.071 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:17.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:16 smithi070 ceph-mon[192652]: pgmap v677: 99 pgs: 99 active+clean; 2.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T22:25:17.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:16 smithi026 ceph-mon[223014]: pgmap v677: 99 pgs: 99 active+clean; 2.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T22:25:17.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:16 smithi026 ceph-mon[215775]: pgmap v677: 99 pgs: 99 active+clean; 2.1 GiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 95 MiB/s wr, 1.52k op/s 2023-09-23T22:25:17.141 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:17.262 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 38162 38146 108.34 74 0.00713332 0.00922459 2023-09-23T22:25:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:17 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1032143231' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:25:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:17 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:25:18.019 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:25:18.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:17 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1032143231' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:25:18.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:17 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:25:18.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 39616 39600 107.566 90.875 0.00795413 0.00929173 2023-09-23T22:25:18.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:17 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1032143231' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:25:18.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:17 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:25:18.824 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:18.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:18.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:18.966 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:19.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[215775]: pgmap v678: 99 pgs: 99 active+clean; 2.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T22:25:19.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[215775]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:25:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[215775]: osdmap e266: 8 total, 8 up, 6 in 2023-09-23T22:25:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[215775]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[215775]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[215775]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[223014]: pgmap v678: 99 pgs: 99 active+clean; 2.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T22:25:19.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[223014]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:25:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[223014]: osdmap e266: 8 total, 8 up, 6 in 2023-09-23T22:25:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[223014]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[223014]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:18 smithi026 ceph-mon[223014]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 40651 40635 105.78 64.6875 0.00747944 0.00942727 2023-09-23T22:25:19.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:18 smithi070 ceph-mon[192652]: pgmap v678: 99 pgs: 99 active+clean; 2.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T22:25:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:18 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:18 smithi070 ceph-mon[192652]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:18 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:25:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:18 smithi070 ceph-mon[192652]: osdmap e266: 8 total, 8 up, 6 in 2023-09-23T22:25:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:18 smithi070 ceph-mon[192652]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:18 smithi070 ceph-mon[192652]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:18 smithi070 ceph-mon[192652]: failed to encode map e266 with expected crc 2023-09-23T22:25:19.743 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:19.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:19.855 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:19.881 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:19.932 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:19.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:19.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:25:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:25:19] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.43.0" 2023-09-23T22:25:20.037 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:20.115 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:20.237 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:20.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:20.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:20.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 41674 41658 104.106 63.9375 0.00784573 0.00959944 2023-09-23T22:25:20.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:20.429 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:20.714 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:20.742 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:20.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:20.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:20.802 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:25:20.802 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:25:20.850 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:20.939 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:21.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:20 smithi026 ceph-mon[223014]: pgmap v680: 99 pgs: 99 active+clean; 2.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T22:25:21.041 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:20 smithi026 ceph-mon[215775]: pgmap v680: 99 pgs: 99 active+clean; 2.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T22:25:21.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 42844 42828 102.889 73.125 0.0169264 0.00971185 2023-09-23T22:25:21.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:20 smithi070 ceph-mon[192652]: pgmap v680: 99 pgs: 99 active+clean; 2.2 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T22:25:21.526 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:21.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:21.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:21.632 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:21.719 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:21.734 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:25:21.740 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:25:22.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 43922 43906 101.574 67.375 0.00220469 0.00978521 2023-09-23T22:25:22.490 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:25:23.192 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:23.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:23.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:23.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:25:23.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 45087 45071 100.547 72.8125 0.023499 0.00994048 2023-09-23T22:25:23.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:22 smithi026 ceph-mon[223014]: pgmap v681: 99 pgs: 99 active+clean; 2.4 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T22:25:23.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:22 smithi026 ceph-mon[223014]: from='client.45368 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:25:23.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:22 smithi026 ceph-mon[215775]: pgmap v681: 99 pgs: 99 active+clean; 2.4 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T22:25:23.287 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:22 smithi026 ceph-mon[215775]: from='client.45368 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:25:23.303 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:23.329 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:22 smithi070 ceph-mon[192652]: pgmap v681: 99 pgs: 99 active+clean; 2.4 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T22:25:23.330 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:22 smithi070 ceph-mon[192652]: from='client.45368 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:25:23.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:23.380 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:23.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:23.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:23.486 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:23.533 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:23.942 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:23.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:23.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:24.058 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:24.090 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":681,"stamp":"2023-09-23T22:25:21.311608+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2604537779,"num_objects":39697,"num_object_clones":0,"num_object_copies":119091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39697,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":40576,"num_write_kb":2551541,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":8661772,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42471,"ondisk_log_size":42471,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9581432,"kb_used_data":7762952,"kb_used_omap":20,"kb_used_meta":1818411,"kb_avail":552889480,"statfs":{"total":575970213888,"available":566158827520,"internally_reserved":0,"allocated":7949262848,"data_stored":7941874287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1862053643},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":848101376,"num_objects":12941,"num_object_clones":0,"num_object_copies":38823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12941,"num_write_kb":828224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003060"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":253,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954607+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:25:05.954607+0000","last_peered":"2023-09-23T22:25:05.954607+0000","last_clean":"2023-09-23T22:25:05.954607+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:25:05.954607+0000","last_undegraded":"2023-09-23T22:25:05.954607+0000","last_fullsized":"2023-09-23T22:25:05.954607+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":245,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.950434+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:25:05.950434+0000","last_peered":"2023-09-23T22:25:05.950434+0000","last_clean":"2023-09-23T22:25:05.950434+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:25:05.950434+0000","last_undegraded":"2023-09-23T22:25:05.950434+0000","last_fullsized":"2023-09-23T22:25:05.950434+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":208,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953028+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:25:05.953028+0000","last_peered":"2023-09-23T22:25:05.953028+0000","last_clean":"2023-09-23T22:25:05.953028+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:25:05.953028+0000","last_undegraded":"2023-09-23T22:25:05.953028+0000","last_fullsized":"2023-09-23T22:25:05.953028+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":123,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:01.646241+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:25:01.646241+0000","last_peered":"2023-09-23T22:25:01.646241+0000","last_clean":"2023-09-23T22:25:01.646241+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:25:01.646241+0000","last_undegraded":"2023-09-23T22:25:01.646241+0000","last_fullsized":"2023-09-23T22:25:01.646241+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":285,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954888+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:25:05.954888+0000","last_peered":"2023-09-23T22:25:05.954888+0000","last_clean":"2023-09-23T22:25:05.954888+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:25:05.954888+0000","last_undegraded":"2023-09-23T22:25:05.954888+0000","last_fullsized":"2023-09-23T22:25:05.954888+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":221,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957660+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:25:05.957660+0000","last_peered":"2023-09-23T22:25:05.957660+0000","last_clean":"2023-09-23T22:25:05.957660+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:25:05.957660+0000","last_undegraded":"2023-09-23T22:25:05.957660+0000","last_fullsized":"2023-09-23T22:25:05.957660+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":151,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054252+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:24:55.054252+0000","last_peered":"2023-09-23T22:24:55.054252+0000","last_clean":"2023-09-23T22:24:55.054252+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:24:55.054252+0000","last_undegraded":"2023-09-23T22:24:55.054252+0000","last_fullsized":"2023-09-23T22:24:55.054252+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":242,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954970+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:25:05.954970+0000","last_peered":"2023-09-23T22:25:05.954970+0000","last_clean":"2023-09-23T22:25:05.954970+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:25:05.954970+0000","last_undegraded":"2023-09-23T22:25:05.954970+0000","last_fullsized":"2023-09-23T22:25:05.954970+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":226,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957524+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:25:05.957524+0000","last_peered":"2023-09-23T22:25:05.957524+0000","last_clean":"2023-09-23T22:25:05.957524+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:25:05.957524+0000","last_undegraded":"2023-09-23T22:25:05.957524+0000","last_fullsized":"2023-09-23T22:25:05.957524+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":210,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949232+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:25:05.949232+0000","last_peered":"2023-09-23T22:25:05.949232+0000","last_clean":"2023-09-23T22:25:05.949232+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:25:05.949232+0000","last_undegraded":"2023-09-23T22:25:05.949232+0000","last_fullsized":"2023-09-23T22:25:05.949232+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":210,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949403+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:25:05.949403+0000","last_peered":"2023-09-23T22:25:05.949403+0000","last_clean":"2023-09-23T22:25:05.949403+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:25:05.949403+0000","last_undegraded":"2023-09-23T22:25:05.949403+0000","last_fullsized":"2023-09-23T22:25:05.949403+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":244,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954433+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:25:05.954433+0000","last_peered":"2023-09-23T22:25:05.954433+0000","last_clean":"2023-09-23T22:25:05.954433+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:25:05.954433+0000","last_undegraded":"2023-09-23T22:25:05.954433+0000","last_fullsized":"2023-09-23T22:25:05.954433+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":270,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954555+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:25:05.954555+0000","last_peered":"2023-09-23T22:25:05.954555+0000","last_clean":"2023-09-23T22:25:05.954555+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:25:05.954555+0000","last_undegraded":"2023-09-23T22:25:05.954555+0000","last_fullsized":"2023-09-23T22:25:05.954555+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":234,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954436+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:25:05.954436+0000","last_peered":"2023-09-23T22:25:05.954436+0000","last_clean":"2023-09-23T22:25:05.954436+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:25:05.954436+0000","last_undegraded":"2023-09-23T22:25:05.954436+0000","last_fullsized":"2023-09-23T22:25:05.954436+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":193,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954486+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:25:05.954486+0000","last_peered":"2023-09-23T22:25:05.954486+0000","last_clean":"2023-09-23T22:25:05.954486+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:25:05.954486+0000","last_undegraded":"2023-09-23T22:25:05.954486+0000","last_fullsized":"2023-09-23T22:25:05.954486+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":233,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949265+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:25:05.949265+0000","last_peered":"2023-09-23T22:25:05.949265+0000","last_clean":"2023-09-23T22:25:05.949265+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:25:05.949265+0000","last_undegraded":"2023-09-23T22:25:05.949265+0000","last_fullsized":"2023-09-23T22:25:05.949265+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":140,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053704+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:24:55.053704+0000","last_peered":"2023-09-23T22:24:55.053704+0000","last_clean":"2023-09-23T22:24:55.053704+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:24:55.053704+0000","last_undegraded":"2023-09-23T22:24:55.053704+0000","last_fullsized":"2023-09-23T22:24:55.053704+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":245,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957457+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:25:05.957457+0000","last_peered":"2023-09-23T22:25:05.957457+0000","last_clean":"2023-09-23T22:25:05.957457+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:25:05.957457+0000","last_undegraded":"2023-09-23T22:25:05.957457+0000","last_fullsized":"2023-09-23T22:25:05.957457+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":189,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954232+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:25:05.954232+0000","last_peered":"2023-09-23T22:25:05.954232+0000","last_clean":"2023-09-23T22:25:05.954232+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:25:05.954232+0000","last_undegraded":"2023-09-23T22:25:05.954232+0000","last_fullsized":"2023-09-23T22:25:05.954232+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":122,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054643+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:24:55.054643+0000","last_peered":"2023-09-23T22:24:55.054643+0000","last_clean":"2023-09-23T22:24:55.054643+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:24:55.054643+0000","last_undegraded":"2023-09-23T22:24:55.054643+0000","last_fullsized":"2023-09-23T22:24:55.054643+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":283,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.052809+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:24:55.052809+0000","last_peered":"2023-09-23T22:24:55.052809+0000","last_clean":"2023-09-23T22:24:55.052809+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:24:55.052809+0000","last_undegraded":"2023-09-23T22:24:55.052809+0000","last_fullsized":"2023-09-23T22:24:55.052809+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":55,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953240+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:25:05.953240+0000","last_peered":"2023-09-23T22:25:05.953240+0000","last_clean":"2023-09-23T22:25:05.953240+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:25:05.953240+0000","last_undegraded":"2023-09-23T22:25:05.953240+0000","last_fullsized":"2023-09-23T22:25:05.953240+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"266'2546","reported_seq":2561,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:18.839274+0000","last_change":"2023-09-23T22:24:46.980027+0000","last_active":"2023-09-23T22:25:18.839274+0000","last_peered":"2023-09-23T22:25:18.839274+0000","last_clean":"2023-09-23T22:25:18.839274+0000","last_became_active":"2023-09-23T22:24:46.979731+0000","last_became_peered":"2023-09-23T22:24:46.979731+0000","last_unstale":"2023-09-23T22:25:18.839274+0000","last_undegraded":"2023-09-23T22:25:18.839274+0000","last_fullsized":"2023-09-23T22:25:18.839274+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2546,"log_dups_size":0,"ondisk_log_size":2546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:06.319642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166854656,"num_objects":2546,"num_object_clones":0,"num_object_copies":7638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2546,"num_write_kb":162944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":261,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954441+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:25:05.954441+0000","last_peered":"2023-09-23T22:25:05.954441+0000","last_clean":"2023-09-23T22:25:05.954441+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:25:05.954441+0000","last_undegraded":"2023-09-23T22:25:05.954441+0000","last_fullsized":"2023-09-23T22:25:05.954441+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":248,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.956671+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:25:05.956671+0000","last_peered":"2023-09-23T22:25:05.956671+0000","last_clean":"2023-09-23T22:25:05.956671+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:25:05.956671+0000","last_undegraded":"2023-09-23T22:25:05.956671+0000","last_fullsized":"2023-09-23T22:25:05.956671+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"266'2592","reported_seq":2607,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:19.406535+0000","last_change":"2023-09-23T22:24:46.984090+0000","last_active":"2023-09-23T22:25:19.406535+0000","last_peered":"2023-09-23T22:25:19.406535+0000","last_clean":"2023-09-23T22:25:19.406535+0000","last_became_active":"2023-09-23T22:24:46.983784+0000","last_became_peered":"2023-09-23T22:24:46.983784+0000","last_unstale":"2023-09-23T22:25:19.406535+0000","last_undegraded":"2023-09-23T22:25:19.406535+0000","last_fullsized":"2023-09-23T22:25:19.406535+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2592,"log_dups_size":0,"ondisk_log_size":2592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:14.302288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169869312,"num_objects":2592,"num_object_clones":0,"num_object_copies":7776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2592,"num_write_kb":165888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":175,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954687+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:25:05.954687+0000","last_peered":"2023-09-23T22:25:05.954687+0000","last_clean":"2023-09-23T22:25:05.954687+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:25:05.954687+0000","last_undegraded":"2023-09-23T22:25:05.954687+0000","last_fullsized":"2023-09-23T22:25:05.954687+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":253,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954195+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:25:05.954195+0000","last_peered":"2023-09-23T22:25:05.954195+0000","last_clean":"2023-09-23T22:25:05.954195+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:25:05.954195+0000","last_undegraded":"2023-09-23T22:25:05.954195+0000","last_fullsized":"2023-09-23T22:25:05.954195+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"265'2404","reported_seq":2418,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:16.840741+0000","last_change":"2023-09-23T22:24:46.981119+0000","last_active":"2023-09-23T22:25:16.840741+0000","last_peered":"2023-09-23T22:25:16.840741+0000","last_clean":"2023-09-23T22:25:16.840741+0000","last_became_active":"2023-09-23T22:24:46.980795+0000","last_became_peered":"2023-09-23T22:24:46.980795+0000","last_unstale":"2023-09-23T22:25:16.840741+0000","last_undegraded":"2023-09-23T22:25:16.840741+0000","last_fullsized":"2023-09-23T22:25:16.840741+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2404,"log_dups_size":0,"ondisk_log_size":2404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:24.132475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157548544,"num_objects":2404,"num_object_clones":0,"num_object_copies":7212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2404,"num_write_kb":153856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":191,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957539+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:25:05.957539+0000","last_peered":"2023-09-23T22:25:05.957539+0000","last_clean":"2023-09-23T22:25:05.957539+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:25:05.957539+0000","last_undegraded":"2023-09-23T22:25:05.957539+0000","last_fullsized":"2023-09-23T22:25:05.957539+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":201,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954482+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:25:05.954482+0000","last_peered":"2023-09-23T22:25:05.954482+0000","last_clean":"2023-09-23T22:25:05.954482+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:25:05.954482+0000","last_undegraded":"2023-09-23T22:25:05.954482+0000","last_fullsized":"2023-09-23T22:25:05.954482+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"266'2488","reported_seq":2502,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:19.929686+0000","last_change":"2023-09-23T22:24:47.009845+0000","last_active":"2023-09-23T22:25:19.929686+0000","last_peered":"2023-09-23T22:25:19.929686+0000","last_clean":"2023-09-23T22:25:19.929686+0000","last_became_active":"2023-09-23T22:24:47.009559+0000","last_became_peered":"2023-09-23T22:24:47.009559+0000","last_unstale":"2023-09-23T22:25:19.929686+0000","last_undegraded":"2023-09-23T22:25:19.929686+0000","last_fullsized":"2023-09-23T22:25:19.929686+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2488,"log_dups_size":0,"ondisk_log_size":2488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:19.040683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163053568,"num_objects":2488,"num_object_clones":0,"num_object_copies":7464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2488,"num_write_kb":159232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":256,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957094+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:25:05.957094+0000","last_peered":"2023-09-23T22:25:05.957094+0000","last_clean":"2023-09-23T22:25:05.957094+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:25:05.957094+0000","last_undegraded":"2023-09-23T22:25:05.957094+0000","last_fullsized":"2023-09-23T22:25:05.957094+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":120,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053143+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:24:55.053143+0000","last_peered":"2023-09-23T22:24:55.053143+0000","last_clean":"2023-09-23T22:24:55.053143+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:24:55.053143+0000","last_undegraded":"2023-09-23T22:24:55.053143+0000","last_fullsized":"2023-09-23T22:24:55.053143+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"265'2296","reported_seq":2309,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:16.848774+0000","last_change":"2023-09-23T22:24:47.009804+0000","last_active":"2023-09-23T22:25:16.848774+0000","last_peered":"2023-09-23T22:25:16.848774+0000","last_clean":"2023-09-23T22:25:16.848774+0000","last_became_active":"2023-09-23T22:24:47.009551+0000","last_became_peered":"2023-09-23T22:24:47.009551+0000","last_unstale":"2023-09-23T22:25:16.848774+0000","last_undegraded":"2023-09-23T22:25:16.848774+0000","last_fullsized":"2023-09-23T22:25:16.848774+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2296,"log_dups_size":0,"ondisk_log_size":2296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:44.762872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150470656,"num_objects":2296,"num_object_clones":0,"num_object_copies":6888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2296,"num_write_kb":146944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":570,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949066+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:25:05.949066+0000","last_peered":"2023-09-23T22:25:05.949066+0000","last_clean":"2023-09-23T22:25:05.949066+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:25:05.949066+0000","last_undegraded":"2023-09-23T22:25:05.949066+0000","last_fullsized":"2023-09-23T22:25:05.949066+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":280,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:14.406314+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:25:14.406314+0000","last_peered":"2023-09-23T22:25:14.406314+0000","last_clean":"2023-09-23T22:25:14.406314+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:25:14.406314+0000","last_undegraded":"2023-09-23T22:25:14.406314+0000","last_fullsized":"2023-09-23T22:25:14.406314+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":234,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954460+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:25:05.954460+0000","last_peered":"2023-09-23T22:25:05.954460+0000","last_clean":"2023-09-23T22:25:05.954460+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:25:05.954460+0000","last_undegraded":"2023-09-23T22:25:05.954460+0000","last_fullsized":"2023-09-23T22:25:05.954460+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"266'2508","reported_seq":2522,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:19.925908+0000","last_change":"2023-09-23T22:24:46.981702+0000","last_active":"2023-09-23T22:25:19.925908+0000","last_peered":"2023-09-23T22:25:19.925908+0000","last_clean":"2023-09-23T22:25:19.925908+0000","last_became_active":"2023-09-23T22:24:46.981422+0000","last_became_peered":"2023-09-23T22:24:46.981422+0000","last_unstale":"2023-09-23T22:25:19.925908+0000","last_undegraded":"2023-09-23T22:25:19.925908+0000","last_fullsized":"2023-09-23T22:25:19.925908+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2508,"log_dups_size":0,"ondisk_log_size":2508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:08.014616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164364288,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2508,"num_write_kb":160512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":578,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.956607+0000","last_change":"2023-09-23T22:24:39.954805+0000","last_active":"2023-09-23T22:25:05.956607+0000","last_peered":"2023-09-23T22:25:05.956607+0000","last_clean":"2023-09-23T22:25:05.956607+0000","last_became_active":"2023-09-23T22:24:39.954350+0000","last_became_peered":"2023-09-23T22:24:39.954350+0000","last_unstale":"2023-09-23T22:25:05.956607+0000","last_undegraded":"2023-09-23T22:25:05.956607+0000","last_fullsized":"2023-09-23T22:25:05.956607+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:47.774412+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":182,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954341+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:25:05.954341+0000","last_peered":"2023-09-23T22:25:05.954341+0000","last_clean":"2023-09-23T22:25:05.954341+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:25:05.954341+0000","last_undegraded":"2023-09-23T22:25:05.954341+0000","last_fullsized":"2023-09-23T22:25:05.954341+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":251,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954464+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:25:05.954464+0000","last_peered":"2023-09-23T22:25:05.954464+0000","last_clean":"2023-09-23T22:25:05.954464+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:25:05.954464+0000","last_undegraded":"2023-09-23T22:25:05.954464+0000","last_fullsized":"2023-09-23T22:25:05.954464+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"266'2567","reported_seq":2582,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:18.856168+0000","last_change":"2023-09-23T22:24:46.982376+0000","last_active":"2023-09-23T22:25:18.856168+0000","last_peered":"2023-09-23T22:25:18.856168+0000","last_clean":"2023-09-23T22:25:18.856168+0000","last_became_active":"2023-09-23T22:24:46.982171+0000","last_became_peered":"2023-09-23T22:24:46.982171+0000","last_unstale":"2023-09-23T22:25:18.856168+0000","last_undegraded":"2023-09-23T22:25:18.856168+0000","last_fullsized":"2023-09-23T22:25:18.856168+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2567,"log_dups_size":0,"ondisk_log_size":2567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:03.026919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168230912,"num_objects":2567,"num_object_clones":0,"num_object_copies":7701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2567,"num_write_kb":164288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":227,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954101+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:25:05.954101+0000","last_peered":"2023-09-23T22:25:05.954101+0000","last_clean":"2023-09-23T22:25:05.954101+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:25:05.954101+0000","last_undegraded":"2023-09-23T22:25:05.954101+0000","last_fullsized":"2023-09-23T22:25:05.954101+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":242,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954527+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:25:05.954527+0000","last_peered":"2023-09-23T22:25:05.954527+0000","last_clean":"2023-09-23T22:25:05.954527+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:25:05.954527+0000","last_undegraded":"2023-09-23T22:25:05.954527+0000","last_fullsized":"2023-09-23T22:25:05.954527+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"266'2559","reported_seq":2574,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:19.412452+0000","last_change":"2023-09-23T22:24:47.009404+0000","last_active":"2023-09-23T22:25:19.412452+0000","last_peered":"2023-09-23T22:25:19.412452+0000","last_clean":"2023-09-23T22:25:19.412452+0000","last_became_active":"2023-09-23T22:24:47.009063+0000","last_became_peered":"2023-09-23T22:24:47.009063+0000","last_unstale":"2023-09-23T22:25:19.412452+0000","last_undegraded":"2023-09-23T22:25:19.412452+0000","last_fullsized":"2023-09-23T22:25:19.412452+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2559,"log_dups_size":0,"ondisk_log_size":2559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:52.237759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167706624,"num_objects":2559,"num_object_clones":0,"num_object_copies":7677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2559,"num_write_kb":163776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":570,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957039+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:25:05.957039+0000","last_peered":"2023-09-23T22:25:05.957039+0000","last_clean":"2023-09-23T22:25:05.957039+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:25:05.957039+0000","last_undegraded":"2023-09-23T22:25:05.957039+0000","last_fullsized":"2023-09-23T22:25:05.957039+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":46,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954661+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:25:05.954661+0000","last_peered":"2023-09-23T22:25:05.954661+0000","last_clean":"2023-09-23T22:25:05.954661+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:25:05.954661+0000","last_undegraded":"2023-09-23T22:25:05.954661+0000","last_fullsized":"2023-09-23T22:25:05.954661+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":208,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953809+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:25:05.953809+0000","last_peered":"2023-09-23T22:25:05.953809+0000","last_clean":"2023-09-23T22:25:05.953809+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:25:05.953809+0000","last_undegraded":"2023-09-23T22:25:05.953809+0000","last_fullsized":"2023-09-23T22:25:05.953809+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"266'2489","reported_seq":2503,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:19.786711+0000","last_change":"2023-09-23T22:24:46.980482+0000","last_active":"2023-09-23T22:25:19.786711+0000","last_peered":"2023-09-23T22:25:19.786711+0000","last_clean":"2023-09-23T22:25:19.786711+0000","last_became_active":"2023-09-23T22:24:46.980314+0000","last_became_peered":"2023-09-23T22:24:46.980314+0000","last_unstale":"2023-09-23T22:25:19.786711+0000","last_undegraded":"2023-09-23T22:25:19.786711+0000","last_fullsized":"2023-09-23T22:25:19.786711+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2489,"log_dups_size":0,"ondisk_log_size":2489,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.107504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163119104,"num_objects":2489,"num_object_clones":0,"num_object_copies":7467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2489,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2489,"num_write_kb":159296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":574,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.951243+0000","last_change":"2023-09-23T22:24:47.781592+0000","last_active":"2023-09-23T22:25:05.951243+0000","last_peered":"2023-09-23T22:25:05.951243+0000","last_clean":"2023-09-23T22:25:05.951243+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:25:05.951243+0000","last_undegraded":"2023-09-23T22:25:05.951243+0000","last_fullsized":"2023-09-23T22:25:05.951243+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:47.781486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:47.781486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:01.152802+0000","scrub_duration":0.0020463370000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":815,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952296+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:25:05.952296+0000","last_peered":"2023-09-23T22:25:05.952296+0000","last_clean":"2023-09-23T22:25:05.952296+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:25:05.952296+0000","last_undegraded":"2023-09-23T22:25:05.952296+0000","last_fullsized":"2023-09-23T22:25:05.952296+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":283,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957090+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:25:05.957090+0000","last_peered":"2023-09-23T22:25:05.957090+0000","last_clean":"2023-09-23T22:25:05.957090+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:25:05.957090+0000","last_undegraded":"2023-09-23T22:25:05.957090+0000","last_fullsized":"2023-09-23T22:25:05.957090+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954290+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:25:05.954290+0000","last_peered":"2023-09-23T22:25:05.954290+0000","last_clean":"2023-09-23T22:25:05.954290+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:25:05.954290+0000","last_undegraded":"2023-09-23T22:25:05.954290+0000","last_fullsized":"2023-09-23T22:25:05.954290+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"265'2386","reported_seq":2400,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:16.829886+0000","last_change":"2023-09-23T22:24:46.981073+0000","last_active":"2023-09-23T22:25:16.829886+0000","last_peered":"2023-09-23T22:25:16.829886+0000","last_clean":"2023-09-23T22:25:16.829886+0000","last_became_active":"2023-09-23T22:24:46.980758+0000","last_became_peered":"2023-09-23T22:24:46.980758+0000","last_unstale":"2023-09-23T22:25:16.829886+0000","last_undegraded":"2023-09-23T22:25:16.829886+0000","last_fullsized":"2023-09-23T22:25:16.829886+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2386,"log_dups_size":0,"ondisk_log_size":2386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:01.377206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156368896,"num_objects":2386,"num_object_clones":0,"num_object_copies":7158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2386,"num_write_kb":152704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":570,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.956784+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:25:05.956784+0000","last_peered":"2023-09-23T22:25:05.956784+0000","last_clean":"2023-09-23T22:25:05.956784+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:25:05.956784+0000","last_undegraded":"2023-09-23T22:25:05.956784+0000","last_fullsized":"2023-09-23T22:25:05.956784+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":998,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957177+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:25:05.957177+0000","last_peered":"2023-09-23T22:25:05.957177+0000","last_clean":"2023-09-23T22:25:05.957177+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:25:05.957177+0000","last_undegraded":"2023-09-23T22:25:05.957177+0000","last_fullsized":"2023-09-23T22:25:05.957177+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":275,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954391+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:25:05.954391+0000","last_peered":"2023-09-23T22:25:05.954391+0000","last_clean":"2023-09-23T22:25:05.954391+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:25:05.954391+0000","last_undegraded":"2023-09-23T22:25:05.954391+0000","last_fullsized":"2023-09-23T22:25:05.954391+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":230,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957221+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:25:05.957221+0000","last_peered":"2023-09-23T22:25:05.957221+0000","last_clean":"2023-09-23T22:25:05.957221+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:25:05.957221+0000","last_undegraded":"2023-09-23T22:25:05.957221+0000","last_fullsized":"2023-09-23T22:25:05.957221+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"265'2377","reported_seq":2391,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:16.852632+0000","last_change":"2023-09-23T22:24:46.980328+0000","last_active":"2023-09-23T22:25:16.852632+0000","last_peered":"2023-09-23T22:25:16.852632+0000","last_clean":"2023-09-23T22:25:16.852632+0000","last_became_active":"2023-09-23T22:24:46.980033+0000","last_became_peered":"2023-09-23T22:24:46.980033+0000","last_unstale":"2023-09-23T22:25:16.852632+0000","last_undegraded":"2023-09-23T22:25:16.852632+0000","last_fullsized":"2023-09-23T22:25:16.852632+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2377,"log_dups_size":0,"ondisk_log_size":2377,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:58.452186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155779072,"num_objects":2377,"num_object_clones":0,"num_object_copies":7131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2377,"num_write_kb":152128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":595,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.945554+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:25:05.945554+0000","last_peered":"2023-09-23T22:25:05.945554+0000","last_clean":"2023-09-23T22:25:05.945554+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:25:05.945554+0000","last_undegraded":"2023-09-23T22:25:05.945554+0000","last_fullsized":"2023-09-23T22:25:05.945554+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":401,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949393+0000","last_change":"2023-09-23T22:24:38.072053+0000","last_active":"2023-09-23T22:25:05.949393+0000","last_peered":"2023-09-23T22:25:05.949393+0000","last_clean":"2023-09-23T22:25:05.949393+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:25:05.949393+0000","last_undegraded":"2023-09-23T22:25:05.949393+0000","last_fullsized":"2023-09-23T22:25:05.949393+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":227,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952353+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:25:05.952353+0000","last_peered":"2023-09-23T22:25:05.952353+0000","last_clean":"2023-09-23T22:25:05.952353+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:25:05.952353+0000","last_undegraded":"2023-09-23T22:25:05.952353+0000","last_fullsized":"2023-09-23T22:25:05.952353+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":273,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953882+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:25:05.953882+0000","last_peered":"2023-09-23T22:25:05.953882+0000","last_clean":"2023-09-23T22:25:05.953882+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:25:05.953882+0000","last_undegraded":"2023-09-23T22:25:05.953882+0000","last_fullsized":"2023-09-23T22:25:05.953882+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"265'2372","reported_seq":2386,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:16.863247+0000","last_change":"2023-09-23T22:24:47.009793+0000","last_active":"2023-09-23T22:25:16.863247+0000","last_peered":"2023-09-23T22:25:16.863247+0000","last_clean":"2023-09-23T22:25:16.863247+0000","last_became_active":"2023-09-23T22:24:47.009468+0000","last_became_peered":"2023-09-23T22:24:47.009468+0000","last_unstale":"2023-09-23T22:25:16.863247+0000","last_undegraded":"2023-09-23T22:25:16.863247+0000","last_fullsized":"2023-09-23T22:25:16.863247+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2372,"log_dups_size":0,"ondisk_log_size":2372,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:20.557721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155451392,"num_objects":2372,"num_object_clones":0,"num_object_copies":7116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2372,"num_write_kb":151808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":893,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957296+0000","last_change":"2023-09-23T22:24:39.954768+0000","last_active":"2023-09-23T22:25:05.957296+0000","last_peered":"2023-09-23T22:25:05.957296+0000","last_clean":"2023-09-23T22:25:05.957296+0000","last_became_active":"2023-09-23T22:24:39.954171+0000","last_became_peered":"2023-09-23T22:24:39.954171+0000","last_unstale":"2023-09-23T22:25:05.957296+0000","last_undegraded":"2023-09-23T22:25:05.957296+0000","last_fullsized":"2023-09-23T22:25:05.957296+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:00.983002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":578,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:25:01.646069+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:25:01.646069+0000","last_peered":"2023-09-23T22:25:01.646069+0000","last_clean":"2023-09-23T22:25:01.646069+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:25:01.646069+0000","last_undegraded":"2023-09-23T22:25:01.646069+0000","last_fullsized":"2023-09-23T22:25:01.646069+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":191,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949322+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:25:05.949322+0000","last_peered":"2023-09-23T22:25:05.949322+0000","last_clean":"2023-09-23T22:25:05.949322+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:25:05.949322+0000","last_undegraded":"2023-09-23T22:25:05.949322+0000","last_fullsized":"2023-09-23T22:25:05.949322+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954683+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:25:05.954683+0000","last_peered":"2023-09-23T22:25:05.954683+0000","last_clean":"2023-09-23T22:25:05.954683+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:25:05.954683+0000","last_undegraded":"2023-09-23T22:25:05.954683+0000","last_fullsized":"2023-09-23T22:25:05.954683+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"265'2421","reported_seq":2435,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:16.846382+0000","last_change":"2023-09-23T22:24:47.009893+0000","last_active":"2023-09-23T22:25:16.846382+0000","last_peered":"2023-09-23T22:25:16.846382+0000","last_clean":"2023-09-23T22:25:16.846382+0000","last_became_active":"2023-09-23T22:24:47.009599+0000","last_became_peered":"2023-09-23T22:24:47.009599+0000","last_unstale":"2023-09-23T22:25:16.846382+0000","last_undegraded":"2023-09-23T22:25:16.846382+0000","last_fullsized":"2023-09-23T22:25:16.846382+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2421,"log_dups_size":0,"ondisk_log_size":2421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.224040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158662656,"num_objects":2421,"num_object_clones":0,"num_object_copies":7263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2421,"num_write_kb":154944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":598,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952319+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:25:05.952319+0000","last_peered":"2023-09-23T22:25:05.952319+0000","last_clean":"2023-09-23T22:25:05.952319+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:25:05.952319+0000","last_undegraded":"2023-09-23T22:25:05.952319+0000","last_fullsized":"2023-09-23T22:25:05.952319+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":617,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:17.997617+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:25:17.997617+0000","last_peered":"2023-09-23T22:25:17.997617+0000","last_clean":"2023-09-23T22:25:17.997617+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:25:17.997617+0000","last_undegraded":"2023-09-23T22:25:17.997617+0000","last_fullsized":"2023-09-23T22:25:17.997617+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":147,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.052869+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:24:55.052869+0000","last_peered":"2023-09-23T22:24:55.052869+0000","last_clean":"2023-09-23T22:24:55.052869+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:24:55.052869+0000","last_undegraded":"2023-09-23T22:24:55.052869+0000","last_fullsized":"2023-09-23T22:24:55.052869+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":104,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054654+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:24:55.054654+0000","last_peered":"2023-09-23T22:24:55.054654+0000","last_clean":"2023-09-23T22:24:55.054654+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:24:55.054654+0000","last_undegraded":"2023-09-23T22:24:55.054654+0000","last_fullsized":"2023-09-23T22:24:55.054654+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"266'2582","reported_seq":2596,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:19.939193+0000","last_change":"2023-09-23T22:24:46.984221+0000","last_active":"2023-09-23T22:25:19.939193+0000","last_peered":"2023-09-23T22:25:19.939193+0000","last_clean":"2023-09-23T22:25:19.939193+0000","last_became_active":"2023-09-23T22:24:46.983941+0000","last_became_peered":"2023-09-23T22:24:46.983941+0000","last_unstale":"2023-09-23T22:25:19.939193+0000","last_undegraded":"2023-09-23T22:25:19.939193+0000","last_fullsized":"2023-09-23T22:25:19.939193+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2582,"log_dups_size":0,"ondisk_log_size":2582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:31.961352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169213952,"num_objects":2582,"num_object_clones":0,"num_object_copies":7746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2582,"num_write_kb":165248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":586,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054363+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:24:55.054363+0000","last_peered":"2023-09-23T22:24:55.054363+0000","last_clean":"2023-09-23T22:24:55.054363+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:24:55.054363+0000","last_undegraded":"2023-09-23T22:24:55.054363+0000","last_fullsized":"2023-09-23T22:24:55.054363+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":157,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.946176+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:25:05.946176+0000","last_peered":"2023-09-23T22:25:05.946176+0000","last_clean":"2023-09-23T22:25:05.946176+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:25:05.946176+0000","last_undegraded":"2023-09-23T22:25:05.946176+0000","last_fullsized":"2023-09-23T22:25:05.946176+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":217,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952611+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:25:05.952611+0000","last_peered":"2023-09-23T22:25:05.952611+0000","last_clean":"2023-09-23T22:25:05.952611+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:25:05.952611+0000","last_undegraded":"2023-09-23T22:25:05.952611+0000","last_fullsized":"2023-09-23T22:25:05.952611+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":120,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053077+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:24:55.053077+0000","last_peered":"2023-09-23T22:24:55.053077+0000","last_clean":"2023-09-23T22:24:55.053077+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:24:55.053077+0000","last_undegraded":"2023-09-23T22:24:55.053077+0000","last_fullsized":"2023-09-23T22:24:55.053077+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"266'2549","reported_seq":2564,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:19.402737+0000","last_change":"2023-09-23T22:24:47.010826+0000","last_active":"2023-09-23T22:25:19.402737+0000","last_peered":"2023-09-23T22:25:19.402737+0000","last_clean":"2023-09-23T22:25:19.402737+0000","last_became_active":"2023-09-23T22:24:47.010507+0000","last_became_peered":"2023-09-23T22:24:47.010507+0000","last_unstale":"2023-09-23T22:25:19.402737+0000","last_undegraded":"2023-09-23T22:25:19.402737+0000","last_fullsized":"2023-09-23T22:25:19.402737+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2549,"log_dups_size":0,"ondisk_log_size":2549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:22.053927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167051264,"num_objects":2549,"num_object_clones":0,"num_object_copies":7647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2549,"num_write_kb":163136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":570,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954363+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:25:05.954363+0000","last_peered":"2023-09-23T22:25:05.954363+0000","last_clean":"2023-09-23T22:25:05.954363+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:25:05.954363+0000","last_undegraded":"2023-09-23T22:25:05.954363+0000","last_fullsized":"2023-09-23T22:25:05.954363+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":280,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954150+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:25:05.954150+0000","last_peered":"2023-09-23T22:25:05.954150+0000","last_clean":"2023-09-23T22:25:05.954150+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:25:05.954150+0000","last_undegraded":"2023-09-23T22:25:05.954150+0000","last_fullsized":"2023-09-23T22:25:05.954150+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":908,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953347+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:25:05.953347+0000","last_peered":"2023-09-23T22:25:05.953347+0000","last_clean":"2023-09-23T22:25:05.953347+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:25:05.953347+0000","last_undegraded":"2023-09-23T22:25:05.953347+0000","last_fullsized":"2023-09-23T22:25:05.953347+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":55,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954467+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:25:05.954467+0000","last_peered":"2023-09-23T22:25:05.954467+0000","last_clean":"2023-09-23T22:25:05.954467+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:25:05.954467+0000","last_undegraded":"2023-09-23T22:25:05.954467+0000","last_fullsized":"2023-09-23T22:25:05.954467+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"266'2533","reported_seq":2548,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:19.189860+0000","last_change":"2023-09-23T22:24:47.009023+0000","last_active":"2023-09-23T22:25:19.189860+0000","last_peered":"2023-09-23T22:25:19.189860+0000","last_clean":"2023-09-23T22:25:19.189860+0000","last_became_active":"2023-09-23T22:24:47.008704+0000","last_became_peered":"2023-09-23T22:24:47.008704+0000","last_unstale":"2023-09-23T22:25:19.189860+0000","last_undegraded":"2023-09-23T22:25:19.189860+0000","last_fullsized":"2023-09-23T22:25:19.189860+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2533,"log_dups_size":0,"ondisk_log_size":2533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:28.997990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166002688,"num_objects":2533,"num_object_clones":0,"num_object_copies":7599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2533,"num_write_kb":162112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":566,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949495+0000","last_change":"2023-09-23T22:24:45.734796+0000","last_active":"2023-09-23T22:25:05.949495+0000","last_peered":"2023-09-23T22:25:05.949495+0000","last_clean":"2023-09-23T22:25:05.949495+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:25:05.949495+0000","last_undegraded":"2023-09-23T22:25:05.949495+0000","last_fullsized":"2023-09-23T22:25:05.949495+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:45.734686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.734686+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.132151+0000","scrub_duration":0.0022175960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":260,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954234+0000","last_change":"2023-09-23T22:24:38.076201+0000","last_active":"2023-09-23T22:25:05.954234+0000","last_peered":"2023-09-23T22:25:05.954234+0000","last_clean":"2023-09-23T22:25:05.954234+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:25:05.954234+0000","last_undegraded":"2023-09-23T22:25:05.954234+0000","last_fullsized":"2023-09-23T22:25:05.954234+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":99,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.054436+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:24:55.054436+0000","last_peered":"2023-09-23T22:24:55.054436+0000","last_clean":"2023-09-23T22:24:55.054436+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:24:55.054436+0000","last_undegraded":"2023-09-23T22:24:55.054436+0000","last_fullsized":"2023-09-23T22:24:55.054436+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":229,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954118+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:25:05.954118+0000","last_peered":"2023-09-23T22:25:05.954118+0000","last_clean":"2023-09-23T22:25:05.954118+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:25:05.954118+0000","last_undegraded":"2023-09-23T22:25:05.954118+0000","last_fullsized":"2023-09-23T22:25:05.954118+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":95,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T22:24:55.053003+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:24:55.053003+0000","last_peered":"2023-09-23T22:24:55.053003+0000","last_clean":"2023-09-23T22:24:55.053003+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:24:55.053003+0000","last_undegraded":"2023-09-23T22:24:55.053003+0000","last_fullsized":"2023-09-23T22:24:55.053003+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":277,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952517+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:25:05.952517+0000","last_peered":"2023-09-23T22:25:05.952517+0000","last_clean":"2023-09-23T22:25:05.952517+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:25:05.952517+0000","last_undegraded":"2023-09-23T22:25:05.952517+0000","last_fullsized":"2023-09-23T22:25:05.952517+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":228,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957346+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:25:05.957346+0000","last_peered":"2023-09-23T22:25:05.957346+0000","last_clean":"2023-09-23T22:25:05.957346+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:25:05.957346+0000","last_undegraded":"2023-09-23T22:25:05.957346+0000","last_fullsized":"2023-09-23T22:25:05.957346+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":241,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954525+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:25:05.954525+0000","last_peered":"2023-09-23T22:25:05.954525+0000","last_clean":"2023-09-23T22:25:05.954525+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:25:05.954525+0000","last_undegraded":"2023-09-23T22:25:05.954525+0000","last_fullsized":"2023-09-23T22:25:05.954525+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":225,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957461+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:25:05.957461+0000","last_peered":"2023-09-23T22:25:05.957461+0000","last_clean":"2023-09-23T22:25:05.957461+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:25:05.957461+0000","last_undegraded":"2023-09-23T22:25:05.957461+0000","last_fullsized":"2023-09-23T22:25:05.957461+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":100,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954508+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:25:05.954508+0000","last_peered":"2023-09-23T22:25:05.954508+0000","last_clean":"2023-09-23T22:25:05.954508+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:25:05.954508+0000","last_undegraded":"2023-09-23T22:25:05.954508+0000","last_fullsized":"2023-09-23T22:25:05.954508+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":247,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953495+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:25:05.953495+0000","last_peered":"2023-09-23T22:25:05.953495+0000","last_clean":"2023-09-23T22:25:05.953495+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:25:05.953495+0000","last_undegraded":"2023-09-23T22:25:05.953495+0000","last_fullsized":"2023-09-23T22:25:05.953495+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":288,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949150+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:25:05.949150+0000","last_peered":"2023-09-23T22:25:05.949150+0000","last_clean":"2023-09-23T22:25:05.949150+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:25:05.949150+0000","last_undegraded":"2023-09-23T22:25:05.949150+0000","last_fullsized":"2023-09-23T22:25:05.949150+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":155,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952210+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:25:05.952210+0000","last_peered":"2023-09-23T22:25:05.952210+0000","last_clean":"2023-09-23T22:25:05.952210+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:25:05.952210+0000","last_undegraded":"2023-09-23T22:25:05.952210+0000","last_fullsized":"2023-09-23T22:25:05.952210+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":274,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954000+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:25:05.954000+0000","last_peered":"2023-09-23T22:25:05.954000+0000","last_clean":"2023-09-23T22:25:05.954000+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:25:05.954000+0000","last_undegraded":"2023-09-23T22:25:05.954000+0000","last_fullsized":"2023-09-23T22:25:05.954000+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2599747584,"num_objects":39669,"num_object_clones":0,"num_object_copies":119007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":39669,"num_write_kb":2538816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7914782720,"data_stored":7914782720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39669,"ondisk_log_size":39669,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":126,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234251,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487748,"kb_used_data":1268476,"kb_used_omap":0,"kb_used_meta":219264,"kb_avail":92257404,"statfs":{"total":95995035648,"available":94471581696,"internally_reserved":0,"allocated":1298919424,"data_stored":1297889728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":224526336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:25:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":3.9340000000000002,"15min":2.3239999999999998},"min":{"1min":0.28000000000000003,"5min":0.22,"15min":0.22},"max":{"1min":3.6989999999999998,"5min":158.042,"15min":158.042},"last":0.94899999999999995},{"interface":"front","average":{"1min":1.131,"5min":4.5679999999999996,"15min":2.6429999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.217},"max":{"1min":5.9189999999999996,"5min":157.952,"15min":157.952},"last":1.075}]},{"osd":1,"last update":"Sat Sep 23 22:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.8039999999999998,"15min":3.1030000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.9370000000000003,"5min":99.385999999999996,"15min":123.79300000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.784,"5min":2.9540000000000002,"15min":3.2970000000000002},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":6.1820000000000004,"5min":95.441999999999993,"15min":123.66500000000001},"last":1.8899999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.776,"15min":3.7850000000000001},"min":{"1min":0.29999999999999999,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":6.0810000000000004,"5min":19.044,"15min":125.101},"last":1.7989999999999999},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.819,"15min":3.548},"min":{"1min":0.311,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":4.9829999999999997,"5min":19.445,"15min":126.021},"last":0.35899999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":3.9870000000000001,"15min":2.383},"min":{"1min":0.39600000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.1010000000000009,"5min":170.86099999999999,"15min":170.86099999999999},"last":1.5800000000000001},{"interface":"front","average":{"1min":1.2649999999999999,"5min":3.746,"15min":2.4489999999999998},"min":{"1min":0.38300000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.8689999999999998,"5min":174.57499999999999,"15min":174.57499999999999},"last":0.48699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.0589999999999999,"15min":1.3700000000000001},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.0419999999999998,"5min":27.922000000000001,"15min":88.543000000000006},"last":0.379},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.143,"15min":1.3120000000000001},"min":{"1min":0.28100000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.0600000000000001,"5min":29.920000000000002,"15min":88.495999999999995},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0629999999999999,"15min":1.2849999999999999},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":5.7960000000000003,"5min":24.965,"15min":93.816000000000003},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.2330000000000001,"15min":1.5429999999999999},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":9.0760000000000005,"5min":24.712,"15min":95.579999999999998},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.73299999999999998,"15min":0.753},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.181,"5min":5.343,"15min":9.0139999999999993},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.351,"5min":0.95899999999999996,"15min":0.99199999999999999},"min":{"1min":0.35899999999999999,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":10.967000000000001,"5min":10.967000000000001,"15min":13.362},"last":0.56499999999999995}]}]},{"osd":6,"up_from":0,"seq":197568495887,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795159,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2008796,"kb_used_data":1617428,"kb_used_omap":7,"kb_used_meta":391352,"kb_avail":91736356,"statfs":{"total":95995035648,"available":93938028544,"internally_reserved":0,"allocated":1656246272,"data_stored":1654843394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":400744820},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7709999999999999,"5min":2.2770000000000001,"15min":1.4750000000000001},"min":{"1min":0.22700000000000001,"5min":0.189,"15min":0.189},"max":{"1min":10.125999999999999,"5min":56.503,"15min":61.890000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":4.2060000000000004,"5min":3.1179999999999999,"15min":1.982},"min":{"1min":0.28199999999999997,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":15.965999999999999,"5min":56.560000000000002,"15min":62.689999999999998},"last":0.64300000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:24:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.0649999999999999,"15min":1.284},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.83899999999999997,"5min":11.179,"15min":33.902000000000001},"last":1.5720000000000001},{"interface":"front","average":{"1min":3.101,"5min":2.2730000000000001,"15min":1.9059999999999999},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.053000000000001,"5min":16.053000000000001,"15min":34.781999999999996},"last":0.46500000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":5.5910000000000002,"5min":4.6269999999999998,"15min":2.4049999999999998},"min":{"1min":0.29599999999999999,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":13.871,"5min":110.417,"15min":110.417},"last":0.36799999999999999},{"interface":"front","average":{"1min":5.29,"5min":4.6589999999999998,"15min":2.343},"min":{"1min":0.28299999999999997,"5min":0.223,"15min":0.223},"max":{"1min":16.346,"5min":110.721,"15min":110.721},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":0.99299999999999999,"5min":4.5359999999999996,"15min":2.0779999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27000000000000002},"max":{"1min":5.0869999999999997,"5min":234.79900000000001,"15min":234.79900000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.153,"5min":5.1689999999999996,"15min":2.4510000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":6.7160000000000002,"5min":234.81700000000001,"15min":234.81700000000001},"last":0.40600000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":3.5310000000000001,"15min":1.889},"min":{"1min":0.34000000000000002,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":6.7480000000000002,"5min":196.79900000000001,"15min":196.79900000000001},"last":0.436},{"interface":"front","average":{"1min":2.5190000000000001,"5min":3.8450000000000002,"15min":1.9239999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":16.207000000000001,"5min":196.68299999999999,"15min":196.68299999999999},"last":1.661}]},{"osd":6,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.5489999999999999,"5min":2.1240000000000001,"15min":1.3759999999999999},"min":{"1min":0.373,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":16.314,"5min":16.314,"15min":16.314},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.218,"5min":1.206,"15min":1.042},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.1310000000000002,"5min":11.385999999999999,"15min":11.385999999999999},"last":0.51900000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":2.2909999999999999,"15min":1.484},"min":{"1min":0.57699999999999996,"5min":0.33800000000000002,"15min":0.32900000000000001},"max":{"1min":16.265000000000001,"5min":16.265000000000001,"15min":16.265000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.087,"5min":1.21,"15min":1.0860000000000001},"min":{"1min":0.42199999999999999,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":5.194,"5min":12.045999999999999,"15min":12.045999999999999},"last":0.496}]}]},{"osd":0,"up_from":93,"seq":399431958693,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1699904,"kb_used_data":1330552,"kb_used_omap":8,"kb_used_meta":369335,"kb_avail":92045248,"statfs":{"total":95995035648,"available":94254333952,"internally_reserved":0,"allocated":1362485248,"data_stored":1361044851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":378199914},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":3.0619999999999998,"5min":4.5759999999999996,"15min":2.4550000000000001},"min":{"1min":0.214,"5min":0.20399999999999999,"15min":0.20200000000000001},"max":{"1min":16.253,"5min":169.77199999999999,"15min":169.77199999999999},"last":1.9259999999999999},{"interface":"front","average":{"1min":3.089,"5min":4.7729999999999997,"15min":2.746},"min":{"1min":0.34999999999999998,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":13.108000000000001,"5min":169.52099999999999,"15min":169.52099999999999},"last":0.38100000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:24:54 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":5.6950000000000003,"15min":2.7200000000000002},"min":{"1min":0.26600000000000001,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":5.6390000000000002,"5min":206.495,"15min":206.495},"last":0.309},{"interface":"front","average":{"1min":1.7330000000000001,"5min":6.101,"15min":3.3860000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22800000000000001},"max":{"1min":9.0920000000000005,"5min":207.34399999999999,"15min":207.34399999999999},"last":1.97}]},{"osd":3,"last update":"Sat Sep 23 22:24:23 2023","interfaces":[{"interface":"back","average":{"1min":11.73,"5min":5.4100000000000001,"15min":2.754},"min":{"1min":0.35899999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":146.87299999999999,"5min":146.87299999999999,"15min":146.87299999999999},"last":1.8819999999999999},{"interface":"front","average":{"1min":10.827,"5min":5.2699999999999996,"15min":2.7959999999999998},"min":{"1min":0.33600000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":145.48099999999999,"5min":145.48099999999999,"15min":145.48099999999999},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":2.722,"15min":2.0089999999999999},"min":{"1min":0.40699999999999997,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":3.3679999999999999,"5min":82.825000000000003,"15min":82.825000000000003},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.302,"15min":1.409},"min":{"1min":0.39700000000000002,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":4.798,"5min":75.930999999999997,"15min":75.930999999999997},"last":0.27300000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":3.8849999999999998,"15min":2.3959999999999999},"min":{"1min":0.378,"5min":0.36899999999999999,"15min":0.29499999999999998},"max":{"1min":3.4049999999999998,"5min":93.685000000000002,"15min":93.685000000000002},"last":0.74299999999999999},{"interface":"front","average":{"1min":1.1180000000000001,"5min":3.5739999999999998,"15min":1.8480000000000001},"min":{"1min":0.45300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.8310000000000004,"5min":93.546000000000006,"15min":93.546000000000006},"last":0.39700000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.798,"15min":1.718},"min":{"1min":0.432,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":3.4529999999999998,"5min":16.366,"15min":16.366},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.6759999999999999,"15min":1.256},"min":{"1min":0.47399999999999998,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":4.883,"5min":16.225999999999999,"15min":22.263000000000002},"last":0.44500000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":2.0470000000000002,"15min":1.768},"min":{"1min":0.39800000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.1440000000000001,"5min":32.773000000000003,"15min":32.773000000000003},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.929,"15min":1.9199999999999999},"min":{"1min":0.437,"5min":0.39900000000000002,"15min":0.38900000000000001},"max":{"1min":3.4279999999999999,"5min":16.381,"15min":23.265999999999998},"last":0.68999999999999995}]}]},{"osd":2,"up_from":0,"seq":442381631632,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468106,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2283216,"kb_used_data":1913928,"kb_used_omap":4,"kb_used_meta":369275,"kb_avail":91461936,"statfs":{"total":95995035648,"available":93657022464,"internally_reserved":0,"allocated":1959862272,"data_stored":1958408163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":378138157},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":5.0609999999999999,"15min":3.0230000000000001},"min":{"1min":0.22900000000000001,"5min":0.20200000000000001,"15min":0.19900000000000001},"max":{"1min":11.484999999999999,"5min":225.86699999999999,"15min":225.86699999999999},"last":2.0459999999999998},{"interface":"front","average":{"1min":1.1080000000000001,"5min":5.3840000000000003,"15min":3.2519999999999998},"min":{"1min":0.29399999999999998,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":5.7590000000000003,"5min":225.84800000000001,"15min":225.84800000000001},"last":1.196}]},{"osd":1,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.835,"15min":2.464},"min":{"1min":0.247,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.5069999999999997,"5min":37.002000000000002,"15min":82.951999999999998},"last":5.1029999999999998},{"interface":"front","average":{"1min":1.3999999999999999,"5min":3.0270000000000001,"15min":2.4790000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.189},"max":{"1min":11.397,"5min":33.119999999999997,"15min":86.001000000000005},"last":2.0840000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":2.1549999999999998,"15min":3.609},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.223},"max":{"1min":5.9740000000000002,"5min":29.076000000000001,"15min":99.962999999999994},"last":4.343},{"interface":"front","average":{"1min":1.2709999999999999,"5min":2.2909999999999999,"15min":3.4820000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.251},"max":{"1min":11.362,"5min":13.692,"15min":100.429},"last":1.9890000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0990000000000002,"5min":1.903,"15min":1.282},"min":{"1min":0.314,"5min":0.29699999999999999,"15min":0.28499999999999998},"max":{"1min":7.2039999999999997,"5min":11.911,"15min":11.911},"last":2.109},{"interface":"front","average":{"1min":2.798,"5min":1.9179999999999999,"15min":1.3460000000000001},"min":{"1min":0.34399999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":9.2940000000000005,"5min":10.949999999999999,"15min":12.036},"last":1.266}]},{"osd":5,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.861,"15min":1.548},"min":{"1min":0.315,"5min":0.309,"15min":0.309},"max":{"1min":3.9529999999999998,"5min":11.028,"15min":49.695999999999998},"last":1.2410000000000001},{"interface":"front","average":{"1min":1.3779999999999999,"5min":1.7130000000000001,"15min":1.5569999999999999},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":6.0179999999999998,"5min":13.494,"15min":49.668999999999997},"last":4.2400000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":1.98,"15min":1.492},"min":{"1min":0.38800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.0309999999999997,"5min":10.992000000000001,"15min":12.068},"last":1.29},{"interface":"front","average":{"1min":1.444,"5min":1.8320000000000001,"15min":1.427},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.35399999999999998},"max":{"1min":6.0919999999999996,"5min":13.425000000000001,"15min":13.425000000000001},"last":4.3789999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.97,"15min":1.502},"min":{"1min":0.41699999999999998,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":6.0549999999999997,"5min":16.849,"15min":16.849},"last":4.3040000000000003},{"interface":"front","average":{"1min":0.878,"5min":1.7569999999999999,"15min":1.3979999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.36699999999999999},"max":{"1min":3.2429999999999999,"5min":8.1400000000000006,"15min":12.382},"last":2.2029999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019156,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1182284,"kb_used_data":966724,"kb_used_omap":0,"kb_used_meta":215552,"kb_avail":92562868,"statfs":{"total":95995035648,"available":94784376832,"internally_reserved":0,"allocated":989925376,"data_stored":988888839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":220725248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:25:11 2023","interfaces":[{"interface":"back","average":{"1min":8.3260000000000005,"5min":3.992,"15min":2.367},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.218},"max":{"1min":150.63499999999999,"5min":150.63499999999999,"15min":150.63499999999999},"last":2.2810000000000001},{"interface":"front","average":{"1min":8.0739999999999998,"5min":3.8919999999999999,"15min":2.4159999999999999},"min":{"1min":0.32000000000000001,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":150.672,"5min":150.672,"15min":150.672},"last":2.2669999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:25:11 2023","interfaces":[{"interface":"back","average":{"1min":6.2930000000000001,"5min":3.4239999999999999,"15min":2.0529999999999999},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":107.346,"5min":107.346,"15min":107.346},"last":1.6919999999999999},{"interface":"front","average":{"1min":6.2409999999999997,"5min":3.7330000000000001,"15min":2.4060000000000001},"min":{"1min":0.27200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":107.33499999999999,"5min":107.33499999999999,"15min":107.33499999999999},"last":1.1890000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.8069999999999999,"15min":2.351},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":6.0170000000000003,"5min":12.397,"15min":138.06100000000001},"last":1.7370000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.5409999999999999,"15min":2.5209999999999999},"min":{"1min":0.34399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.8639999999999999,"5min":6.9580000000000002,"15min":138.15199999999999},"last":1.776}]},{"osd":3,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.669,"15min":1.2709999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.30399999999999999},"max":{"1min":5.0140000000000002,"5min":12.129,"15min":12.129},"last":4.0780000000000003},{"interface":"front","average":{"1min":2.3599999999999999,"5min":1.8580000000000001,"15min":1.347},"min":{"1min":0.32300000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.5600000000000005,"5min":8.5600000000000005,"15min":9.8109999999999999},"last":0.372}]},{"osd":5,"last update":"Sat Sep 23 22:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":1.3069999999999999,"15min":0.91000000000000003},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.165,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.26,"5min":1.4550000000000001,"15min":0.95499999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.27900000000000003},"max":{"1min":5.601,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.34499999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":0.98299999999999998,"15min":0.77600000000000002},"min":{"1min":0.33800000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":7.8550000000000004,"5min":7.8550000000000004,"15min":7.8550000000000004},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.81599999999999995,"15min":0.77500000000000002},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.802,"5min":3.9239999999999999,"15min":8.5210000000000008},"last":0.35799999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.83399999999999996,"15min":0.88800000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.827,"5min":5.2590000000000003,"15min":8.577},"last":0.312},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.01,"15min":0.86099999999999999},"min":{"1min":0.41299999999999998,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":4.9740000000000002,"5min":7.8109999999999999,"15min":7.8109999999999999},"last":0.33000000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757524,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919484,"kb_used_data":665844,"kb_used_omap":0,"kb_used_meta":253632,"kb_avail":92825668,"statfs":{"total":95995035648,"available":95053484032,"internally_reserved":0,"allocated":681824256,"data_stored":680799312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":259719168},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:59 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":2.2269999999999999,"15min":2.1859999999999999},"min":{"1min":0.23400000000000001,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":3.306,"5min":64.344999999999999,"15min":133.22499999999999},"last":5.3810000000000002},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.1659999999999999,"15min":2.242},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.141,"5min":64.022000000000006,"15min":132.78200000000001},"last":2.8620000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":3.1040000000000001,"15min":2.7589999999999999},"min":{"1min":0.27500000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.7870000000000008,"5min":93.981999999999999,"15min":93.981999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.885,"5min":3.0310000000000001,"15min":2.7389999999999999},"min":{"1min":0.35499999999999998,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":10.949,"5min":93.909999999999997,"15min":93.909999999999997},"last":0.47199999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":2.4260000000000002,"15min":3.536},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.702,"5min":95.790999999999997,"15min":104.254},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.365,"5min":2.25,"15min":3.806},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":8.1010000000000009,"5min":95.366,"15min":104.217},"last":0.443}]},{"osd":3,"last update":"Sat Sep 23 22:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1499999999999999,"5min":3.786,"15min":2.335},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.246},"max":{"1min":5.0839999999999996,"5min":93.739999999999995,"15min":93.739999999999995},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.9199999999999999,"5min":3.3279999999999998,"15min":2.2589999999999999},"min":{"1min":0.38200000000000001,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":4.2569999999999997,"5min":93.197999999999993,"15min":93.197999999999993},"last":0.376}]},{"osd":4,"last update":"Sat Sep 23 22:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.97699999999999998,"15min":1.0409999999999999},"min":{"1min":0.28299999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.54200000000000004,"5min":10.163,"15min":82.530000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.91700000000000004,"15min":1.0840000000000001},"min":{"1min":0.28799999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.79300000000000004,"5min":7.2119999999999997,"15min":85.441000000000003},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:24:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":1.179},"min":{"1min":0.34699999999999998,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":2.6040000000000001,"5min":7.0289999999999999,"15min":114.83499999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.72499999999999998,"15min":1.0880000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7130000000000001,"5min":3.8690000000000002,"15min":110.905},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:24:59 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.82799999999999996,"15min":0.90700000000000003},"min":{"1min":0.36399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.0819999999999999,"5min":5.0819999999999999,"15min":8.8030000000000008},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.77600000000000002,"15min":0.77800000000000002},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":3.6299999999999999,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":0.627}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1398079488,"data_stored":1398079488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1659961344,"data_stored":1659961344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1989607424,"data_stored":1989607424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1008467968,"data_stored":1008467968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":676921344,"data_stored":676921344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1181745152,"data_stored":1181745152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:25:24.097 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T22:25:24.097 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:25:24.098 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T22:25:24.098 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:25:24.099 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:24.141 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:24.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:24.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:24.246 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:24.270 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 46086 46070 99.2333 62.4375 0.00252645 0.0100714 2023-09-23T22:25:24.363 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:24.422 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:24.523 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:24.555 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:25:24.556 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:24.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:24.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:24.603 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:25:24.644 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:25:24.673 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:24.686 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:25:24.724 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:25:24.737 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:24.759 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:25:24.799 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:25:24.842 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:25:24.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:25:24.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:25:24.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:25:24.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:25:24.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:25:24.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:25:24.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:25:24.857 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:25:25.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:25.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:25.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:25.193 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:25.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:25 smithi026 ceph-mon[215775]: pgmap v682: 99 pgs: 99 active+clean; 2.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T22:25:25.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:25.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2760078196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:25.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:25 smithi026 ceph-mon[223014]: pgmap v682: 99 pgs: 99 active+clean; 2.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T22:25:25.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:25.262 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2760078196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:25.270 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:25.271 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 47098 47082 98.0318 63.25 0.0140689 0.0101933 2023-09-23T22:25:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:25 smithi070 ceph-mon[192652]: pgmap v682: 99 pgs: 99 active+clean; 2.6 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T22:25:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2760078196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:26.272 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 48152 48136 96.9923 65.875 0.0124457 0.0103047 2023-09-23T22:25:26.404 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:26.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:26.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:26.515 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:26.587 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:27.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:27 smithi026 ceph-mon[215775]: pgmap v683: 99 pgs: 99 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T22:25:27.219 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:27 smithi026 ceph-mon[223014]: pgmap v683: 99 pgs: 99 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T22:25:27.244 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:27.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:27.272 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 49334 49318 96.2701 73.875 0.00809949 0.0103824 2023-09-23T22:25:27.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:27.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:27 smithi070 ceph-mon[192652]: pgmap v683: 99 pgs: 99 active+clean; 2.8 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T22:25:27.380 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:27.433 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:27.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:27.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:27.542 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:27.611 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:27.887 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:27.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:27.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:27.977 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:27.993 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:28.096 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:28.272 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 50431 50415 95.4307 68.5625 0.0137186 0.0104728 2023-09-23T22:25:28.395 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:28.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:28.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:28.502 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:28.596 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:28.678 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:25:28.678 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:25:28.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93682 5.3 5:c6581ca7:::benchmark_data_smithi026_286012_object93681:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.274322+0000", 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 520.39539094700001, 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1735642620000002, 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.679 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93682 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.274322+0000", 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.274322+0000", 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.274320+0000", 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.274347+0000", 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6979999999999999e-05 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.274365+0000", 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8309e-05 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.274377+0000", 2023-09-23T22:25:28.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1781e-05 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.274443+0000", 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6291999999999995e-05 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.274455+0000", 2023-09-23T22:25:28.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1807000000000001e-05 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140115+0000", 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.86566033899999995 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140125+0000", 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4239999999999999e-06 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.438534+0000", 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.298409438 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.438697+0000", 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016233999999999999 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.438904+0000", 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020729200000000001 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.442284+0000", 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033798880000000002 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.442306+0000", 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2489000000000001e-05 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.447748+0000", 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054414299999999997 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.447785+0000", 2023-09-23T22:25:28.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7279e-05 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.447820+0000", 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4968999999999997e-05 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.447886+0000", 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6311999999999998e-05 2023-09-23T22:25:28.688 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93692 5.2 5:465c1fa5:::benchmark_data_smithi026_286012_object93691:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.291181+0000", 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 520.37853181100002, 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0101498200000001, 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.689 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93692 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.291181+0000", 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.291181+0000", 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.291179+0000", 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.291208+0000", 2023-09-23T22:25:28.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8784000000000001e-05 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.291224+0000", 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5773999999999998e-05 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.291232+0000", 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4160000000000004e-06 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.291339+0000", 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106329 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.291350+0000", 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1535999999999999e-05 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.141227+0000", 2023-09-23T22:25:28.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84987641999999997 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.141236+0000", 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6730000000000004e-06 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.290870+0000", 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.149633997 2023-09-23T22:25:28.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.291098+0000", 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022770000000000001 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.291357+0000", 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025895300000000003 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.298268+0000", 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069110769999999998 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.298330+0000", 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1848999999999997e-05 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.298345+0000", 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5207e-05 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.298415+0000", 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0017000000000004e-05 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.301331+0000", 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002915749 2023-09-23T22:25:28.697 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93705 5.2 5:4d2696e9:::benchmark_data_smithi026_286012_object93704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.301195+0000", 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 520.36851818699995, 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.015020496, 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.698 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93705 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.301195+0000", 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.301195+0000", 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.301194+0000", 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.301214+0000", 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9680000000000001e-05 2023-09-23T22:25:28.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.301221+0000", 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2230000000000001e-06 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.301227+0000", 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5670000000000002e-06 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.301329+0000", 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000102755 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.301340+0000", 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0264e-05 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.141261+0000", 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83992116800000005 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.141268+0000", 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6720000000000004e-06 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.301395+0000", 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1601266889999999 2023-09-23T22:25:28.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.301534+0000", 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138961 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.301655+0000", 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012099099999999999 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.307841+0000", 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061854889999999997 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.307907+0000", 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6241999999999994e-05 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.313066+0000", 2023-09-23T22:25:28.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051589679999999999 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.313129+0000", 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3070000000000004e-05 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.316215+0000", 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003086311 2023-09-23T22:25:28.706 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93714 5.3 5:c7e21259:::benchmark_data_smithi026_286012_object93713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.312085+0000", 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 520.35762826200005, 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1408043079999999, 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.707 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93714 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.312085+0000", 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.312085+0000", 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.312084+0000", 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.312105+0000", 2023-09-23T22:25:28.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.071e-05 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.312112+0000", 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7449999999999998e-06 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.312117+0000", 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4380000000000003e-06 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.312165+0000", 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8170999999999997e-05 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.312175+0000", 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9469999999999998e-06 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140163+0000", 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.827987311 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140172+0000", 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6649999999999998e-06 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.442320+0000", 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3021480400000001 2023-09-23T22:25:28.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.442416+0000", 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5937e-05 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.442539+0000", 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012270900000000001 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.447778+0000", 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005238932 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.447897+0000", 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118794 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452689+0000", 2023-09-23T22:25:28.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047924739999999997 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452790+0000", 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100721 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452824+0000", 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3804999999999999e-05 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452889+0000", 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5371999999999997e-05 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93722 5.3 5:ce24ade9:::benchmark_data_smithi026_286012_object93721:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.318777+0000", 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 520.35093603600001, 2023-09-23T22:25:28.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1342403600000002, 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93722 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.318777+0000", 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.318777+0000", 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.318776+0000", 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.318797+0000", 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0905999999999999e-05 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.318806+0000", 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8260000000000005e-06 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.318812+0000", 2023-09-23T22:25:28.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.818e-06 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.318867+0000", 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5287000000000001e-05 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.318879+0000", 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1022000000000001e-05 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140201+0000", 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82132234900000001 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140206+0000", 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7940000000000002e-06 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.444803+0000", 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.304597365 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.444901+0000", 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8152999999999994e-05 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.445003+0000", 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101618 2023-09-23T22:25:28.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.447923+0000", 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002920532 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452661+0000", 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047371959999999999 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452726+0000", 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5946999999999997e-05 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452924+0000", 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019796899999999999 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452954+0000", 2023-09-23T22:25:28.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9621e-05 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.453017+0000", 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3176999999999995e-05 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93723 5.2 5:499fc8ac:::benchmark_data_smithi026_286012_object93722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:25:28.725 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.319390+0000", 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 520.35032312800001, 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0002244980000001, 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93723 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.319390+0000", 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.319390+0000", 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.319389+0000", 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.319409+0000", 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0163e-05 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.319418+0000", 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5569999999999996e-06 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.319423+0000", 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1499999999999998e-06 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.319483+0000", 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9882e-05 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.319494+0000", 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1423000000000001e-05 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.141292+0000", 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82179816299999997 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.141300+0000", 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4780000000000002e-06 2023-09-23T22:25:28.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.304629+0000", 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.163329566 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.304841+0000", 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021113699999999999 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.305055+0000", 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021460899999999999 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.307894+0000", 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028385770000000001 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.310779+0000", 2023-09-23T22:25:28.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002884944 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.316662+0000", 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005883765 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.316752+0000", 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9310999999999999e-05 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.319614+0000", 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002862514 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93731 5.3 5:c49fed5b:::benchmark_data_smithi026_286012_object93730:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.339762+0000", 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 520.32995046200006, 2023-09-23T22:25:28.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1253518250000001, 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93731 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.339762+0000", 2023-09-23T22:25:28.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.339762+0000", 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.339760+0000", 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.339781+0000", 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0562000000000001e-05 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.339794+0000", 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3148e-05 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.339801+0000", 2023-09-23T22:25:28.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5220000000000004e-06 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.339843+0000", 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2330999999999998e-05 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.339850+0000", 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9110000000000004e-06 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140247+0000", 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80039689400000003 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140257+0000", 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0156999999999999e-05 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.447938+0000", 2023-09-23T22:25:28.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.307681423 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.448062+0000", 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012313600000000001 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.448161+0000", 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9308000000000005e-05 2023-09-23T22:25:28.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452735+0000", 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045742680000000003 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.452762+0000", 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6902000000000001e-05 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.464803+0000", 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012040508 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.464928+0000", 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012551700000000001 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.464988+0000", 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9806000000000003e-05 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.465114+0000", 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000126434 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.743 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93733 5.3 5:ce59b14e:::benchmark_data_smithi026_286012_object93732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.345262+0000", 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 520.32445110699996, 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1201006759999999, 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93733 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.345262+0000", 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.345262+0000", 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.345259+0000", 2023-09-23T22:25:28.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.345290+0000", 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0402000000000001e-05 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.345307+0000", 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7071999999999998e-05 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.345316+0000", 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8650000000000007e-06 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.345382+0000", 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6081000000000006e-05 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.345396+0000", 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3957999999999999e-05 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140290+0000", 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.79489391499999995 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.140299+0000", 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1509999999999993e-06 2023-09-23T22:25:28.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.450344+0000", 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3100455580000001 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.450431+0000", 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6897000000000005e-05 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,5,6", 2023-09-23T22:25:28.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.450531+0000", 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9160000000000006e-05 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.453072+0000", 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00254156 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.453083+0000", 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1049999999999999e-05 2023-09-23T22:25:28.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.464863+0000", 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011780159 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.465199+0000", 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033551700000000002 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.465249+0000", 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0618999999999998e-05 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.465362+0000", 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113045 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.752 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:93734 5.2 5:474de225:::benchmark_data_smithi026_286012_object93733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:48.346626+0000", 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 520.32308726999997, 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9769583210000001, 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93734 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.346626+0000", 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.346626+0000", 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.346624+0000", 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.346646+0000", 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1888000000000001e-05 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.346658+0000", 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1348999999999999e-05 2023-09-23T22:25:28.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.346663+0000", 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4029999999999998e-06 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.346721+0000", 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7819999999999999e-05 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:48.346729+0000", 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5920000000000003e-06 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.141350+0000", 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.79462160400000004 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:49.141359+0000", 2023-09-23T22:25:28.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5769999999999994e-06 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.307936+0000", 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1665771519999999 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.308112+0000", 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017631100000000001 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.308331+0000", 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021844200000000001 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.310760+0000", 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002429016 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.310883+0000", 2023-09-23T22:25:28.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000123242 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.320894+0000", 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010011544000000001 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.320973+0000", 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9005000000000007e-05 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:50.323584+0000", 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002610533 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160066 5.3 5:cf864644:::benchmark_data_smithi026_286012_object66258:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724400+0000", 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.94531283100002, 2023-09-23T22:25:28.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5659008060000001, 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160066 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724400+0000", 2023-09-23T22:25:28.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724400+0000", 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724400+0000", 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724402+0000", 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1710000000000001e-06 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724409+0000", 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3950000000000003e-06 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724415+0000", 2023-09-23T22:25:28.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3349999999999997e-06 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725871+0000", 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014559379999999999 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725875+0000", 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5360000000000003e-06 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.035756+0000", 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3098814270000001 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.036147+0000", 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000390276 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.053539+0000", 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017392409000000001 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.053674+0000", 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135306 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.054299+0000", 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00062442400000000005 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.070040+0000", 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015741027000000001 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.070085+0000", 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4923999999999999e-05 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.083065+0000", 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012980047999999999 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.288967+0000", 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20590188000000001 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.289024+0000", 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6818999999999999e-05 2023-09-23T22:25:28.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.290301+0000", 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001277296 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160067 5.3 5:c73fe88c:::benchmark_data_smithi026_286012_object66259:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724420+0000", 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.94529320499998, 2023-09-23T22:25:28.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5671979359999999, 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160067 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724420+0000", 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724420+0000", 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724419+0000", 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724421+0000", 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.984e-06 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724426+0000", 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0690000000000001e-06 2023-09-23T22:25:28.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724431+0000", 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9110000000000001e-06 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725889+0000", 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014581290000000001 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725894+0000", 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0189999999999997e-06 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.036192+0000", 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3102978089999999 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.036767+0000", 2023-09-23T22:25:28.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057512800000000005 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.073186+0000", 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.036419066 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.073316+0000", 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129686 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.073488+0000", 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017221599999999999 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.089235+0000", 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015746687999999998 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.119472+0000", 2023-09-23T22:25:28.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030236613999999998 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.119541+0000", 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9079999999999996e-05 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.290378+0000", 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.170836919 2023-09-23T22:25:28.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.290448+0000", 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0296000000000006e-05 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.291618+0000", 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011696880000000001 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160081 5.3 5:cfd739d8:::benchmark_data_smithi026_286012_object66273:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724435+0000", 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.94527743399999, 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.568329584, 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.781 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160081 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724435+0000", 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724435+0000", 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724435+0000", 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724437+0000", 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9420000000000002e-06 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724442+0000", 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9760000000000003e-06 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724447+0000", 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9049999999999996e-06 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725908+0000", 2023-09-23T22:25:28.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014606 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725912+0000", 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3100000000000002e-06 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.036814+0000", 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3109024380000001 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037101+0000", 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028662199999999999 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.079431+0000", 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.042330206000000002 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.079562+0000", 2023-09-23T22:25:28.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130839 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.079752+0000", 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000189469 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.095018+0000", 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01526634 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.119583+0000", 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024565441 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125033+0000", 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054497169999999998 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.291683+0000", 2023-09-23T22:25:28.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.166650353 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.291749+0000", 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5491e-05 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.292765+0000", 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010162089999999999 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.789 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160090 5.3 5:c5c01081:::benchmark_data_smithi026_286012_object66282:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724451+0000", 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.94526207899997, 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5694129160000001, 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.790 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160090 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724451+0000", 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724451+0000", 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724451+0000", 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724456+0000", 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6899999999999997e-06 2023-09-23T22:25:28.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724461+0000", 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0520000000000001e-06 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724466+0000", 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8890000000000001e-06 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725924+0000", 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014574060000000001 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725928+0000", 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.33e-06 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037149+0000", 2023-09-23T22:25:28.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3112209639999999 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037542+0000", 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039318900000000001 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.085738+0000", 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048196211000000003 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.085840+0000", 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101345 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.085972+0000", 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132791 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.097699+0000", 2023-09-23T22:25:28.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011726221 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125076+0000", 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027376948000000002 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125123+0000", 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7067999999999997e-05 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.292841+0000", 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16771852700000001 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.292904+0000", 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2728e-05 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.293864+0000", 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000959826 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160098 5.3 5:cfe0d09f:::benchmark_data_smithi026_286012_object66290:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724470+0000", 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.94524281600002, 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.570344789, 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.799 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160098 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724470+0000", 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724470+0000", 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724470+0000", 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724472+0000", 2023-09-23T22:25:28.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.323e-06 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724477+0000", 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0780000000000003e-06 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724482+0000", 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6009999999999997e-06 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725940+0000", 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001457898 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725944+0000", 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2869999999999998e-06 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037585+0000", 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3116414009999999 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037953+0000", 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036729699999999999 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.091574+0000", 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.053620959000000003 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.091698+0000", 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012485 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.091877+0000", 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017887099999999999 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.101104+0000", 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092271279999999994 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125165+0000", 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024060350000000001 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125208+0000", 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3374000000000002e-05 2023-09-23T22:25:28.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.293938+0000", 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.168729515 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.293999+0000", 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1302000000000004e-05 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.294815+0000", 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081585700000000002 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160108 5.3 5:c2b2bc92:::benchmark_data_smithi026_286012_object66300:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724485+0000", 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.94522747799999, 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5712676249999999, 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.808 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160108 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724485+0000", 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724485+0000", 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724485+0000", 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724487+0000", 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8530000000000001e-06 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724492+0000", 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1819999999999997e-06 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724497+0000", 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6179999999999997e-06 2023-09-23T22:25:28.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725956+0000", 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014589029999999999 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725960+0000", 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2069999999999997e-06 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038012+0000", 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3120517139999999 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038386+0000", 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037387599999999998 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.097736+0000", 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059349986 2023-09-23T22:25:28.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.097802+0000", 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6389000000000004e-05 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.097927+0000", 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012475100000000001 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.109875+0000", 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011948215999999999 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.130691+0000", 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020815904 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.130756+0000", 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5492000000000002e-05 2023-09-23T22:25:28.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.294890+0000", 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16413367700000001 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.294945+0000", 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4957999999999998e-05 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.295753+0000", 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00080815300000000004 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160125 5.3 5:c0b4c552:::benchmark_data_smithi026_286012_object66317:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724501+0000", 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.94521220899998, 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.572192045, 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.817 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160125 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724501+0000", 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724501+0000", 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724500+0000", 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724502+0000", 2023-09-23T22:25:28.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.891e-06 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724507+0000", 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.062e-06 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724512+0000", 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2100000000000003e-06 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725972+0000", 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001460097 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725976+0000", 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3669999999999999e-06 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038446+0000", 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.312469847 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038851+0000", 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040529699999999999 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.104126+0000", 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065275193999999995 2023-09-23T22:25:28.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.104252+0000", 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012552799999999999 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.104352+0000", 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010051399999999999 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.116000+0000", 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011647210999999999 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135693+0000", 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019693161000000001 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135744+0000", 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1597000000000001e-05 2023-09-23T22:25:28.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.295827+0000", 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16008249399999999 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.295880+0000", 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3418999999999998e-05 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.296693+0000", 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081249700000000002 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160141 5.3 5:c5e24ba2:::benchmark_data_smithi026_286012_object66333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.948336+0000", 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.72137676400001, 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3492636490000001, 2023-09-23T22:25:28.826 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160141 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948336+0000", 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948336+0000", 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948335+0000", 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948341+0000", 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6309999999999997e-06 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948355+0000", 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3841e-05 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948362+0000", 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0090000000000001e-06 2023-09-23T22:25:28.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948422+0000", 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9821000000000002e-05 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948431+0000", 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5830000000000007e-06 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038909+0000", 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0904784590000001 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.039292+0000", 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038332500000000001 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.109910+0000", 2023-09-23T22:25:28.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.070617271999999995 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.109988+0000", 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8589000000000003e-05 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.110092+0000", 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010372 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.119489+0000", 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093965960000000001 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135777+0000", 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016287978000000002 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135823+0000", 2023-09-23T22:25:28.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6122000000000003e-05 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.296763+0000", 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16094037 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.296819+0000", 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6005999999999999e-05 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.297600+0000", 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00078078999999999996 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160147 5.3 5:c3a1d842:::benchmark_data_smithi026_286012_object66339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.966597+0000", 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.70311578600001, 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3318658239999999, 2023-09-23T22:25:28.835 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160147 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966597+0000", 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966597+0000", 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966595+0000", 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966603+0000", 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8990000000000001e-06 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966623+0000", 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9882e-05 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966632+0000", 2023-09-23T22:25:28.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0129999999999999e-06 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966699+0000", 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6631999999999995e-05 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966720+0000", 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1373e-05 2023-09-23T22:25:28.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.039353+0000", 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0726333219999999 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.039739+0000", 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038586399999999998 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.116044+0000", 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.076304935000000004 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.116162+0000", 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118059 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.116276+0000", 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011349399999999999 2023-09-23T22:25:28.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125088+0000", 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088123579999999993 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135904+0000", 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010815791 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135983+0000", 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9462999999999995e-05 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.297682+0000", 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16169909599999999 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.297747+0000", 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4618999999999999e-05 2023-09-23T22:25:28.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.298463+0000", 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00071606700000000003 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160148 5.3 5:c44bd4f2:::benchmark_data_smithi026_286012_object66340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.968612+0000", 2023-09-23T22:25:28.844 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.701100838, 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.330931401, 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160148 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.968612+0000", 2023-09-23T22:25:28.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.968612+0000", 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.968611+0000", 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:28.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.968616+0000", 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5670000000000002e-06 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.968629+0000", 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2438e-05 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.968638+0000", 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8969999999999997e-06 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.968690+0000", 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2522000000000003e-05 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.968700+0000", 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9010000000000006e-06 2023-09-23T22:25:28.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.039792+0000", 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.07109181 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.040150+0000", 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035834799999999998 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.122083+0000", 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.081933259999999994 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.122154+0000", 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0771000000000004e-05 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.122255+0000", 2023-09-23T22:25:28.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100716 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125175+0000", 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029201190000000001 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.136025+0000", 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010849510999999999 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.136083+0000", 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8149000000000002e-05 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.298526+0000", 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16244294500000001 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.298571+0000", 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5745000000000003e-05 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.299543+0000", 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00097208400000000001 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160157 5.3 5:cccc247d:::benchmark_data_smithi026_286012_object66349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:25:28.853 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.985833+0000", 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 390.68387963200001, 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3148116459999999, 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160157 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.985833+0000", 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.985833+0000", 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.985832+0000", 2023-09-23T22:25:28.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.985839+0000", 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4880000000000001e-06 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.985857+0000", 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7955999999999999e-05 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.985866+0000", 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2180000000000005e-06 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.985932+0000", 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5284999999999996e-05 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:28.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.985940+0000", 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2369999999999992e-06 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.040204+0000", 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0542639600000001 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.040561+0000", 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035729400000000003 2023-09-23T22:25:28.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.127617+0000", 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.087056376000000005 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.127690+0000", 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2243999999999995e-05 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.127797+0000", 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106951 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135703+0000", 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079068009999999998 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.136061+0000", 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035802400000000003 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.136114+0000", 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2064000000000002e-05 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.299612+0000", 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.163498327 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.299659+0000", 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7120000000000003e-05 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.300645+0000", 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098596800000000004 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.862 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.863 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:28.863 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:28.863 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:25:28.919 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:28.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:28.938 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:25:28.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:28 smithi026 ceph-mon[223014]: pgmap v684: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T22:25:28.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:28 smithi026 ceph-mon[215775]: pgmap v684: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T22:25:29.027 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:29.058 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:29.108 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:29.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:29.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:29.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:29.240 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:28 smithi070 ceph-mon[192652]: pgmap v684: 99 pgs: 99 active+clean; 2.9 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T22:25:29.242 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:29.272 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 51457 51441 94.5102 64.125 0.0155015 0.0105746 2023-09-23T22:25:29.293 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:29.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:29.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:29.468 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:29.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:25:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:25:29] "GET /metrics HTTP/1.1" 200 36352 "" "Prometheus/2.43.0" 2023-09-23T22:25:29.983 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:25:30.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:29 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/429435850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:30.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:29 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:29 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/429435850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:30.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:29 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:30.272 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 52361 52345 93.4245 56.5 0.00907502 0.0106891 2023-09-23T22:25:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:29 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/429435850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:29 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:30.740 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:30.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:30.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:30.868 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:30.925 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:31.228 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:31.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:31.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[215775]: pgmap v685: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T22:25:31.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:25:31.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:25:31.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[215775]: failed to encode map e267 with expected crc 2023-09-23T22:25:31.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:25:31.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[215775]: osdmap e267: 8 total, 8 up, 6 in 2023-09-23T22:25:31.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[215775]: failed to encode map e267 with expected crc 2023-09-23T22:25:31.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:31.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[223014]: pgmap v685: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T22:25:31.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:25:31.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:25:31.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[223014]: failed to encode map e267 with expected crc 2023-09-23T22:25:31.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:25:31.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[223014]: osdmap e267: 8 total, 8 up, 6 in 2023-09-23T22:25:31.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:30 smithi026 ceph-mon[223014]: failed to encode map e267 with expected crc 2023-09-23T22:25:31.275 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 53310 53294 92.4702 59.3125 0.00869524 0.010809 2023-09-23T22:25:31.340 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:31.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:30 smithi070 ceph-mon[192652]: pgmap v685: 99 pgs: 99 active+clean; 2.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T22:25:31.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:30 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:25:31.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:30 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:25:31.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:30 smithi070 ceph-mon[192652]: failed to encode map e267 with expected crc 2023-09-23T22:25:31.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:30 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:25:31.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:30 smithi070 ceph-mon[192652]: osdmap e267: 8 total, 8 up, 6 in 2023-09-23T22:25:31.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:30 smithi070 ceph-mon[192652]: failed to encode map e267 with expected crc 2023-09-23T22:25:31.371 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:31.414 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:25:31.418 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:31.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:31.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:31.524 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:31.623 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:31.772 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:31.855 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:31.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:31.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:31.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:32.071 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:32.136 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:31 smithi026 ceph-mon[215775]: failed to encode map e267 with expected crc 2023-09-23T22:25:32.138 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:31 smithi026 ceph-mon[223014]: failed to encode map e267 with expected crc 2023-09-23T22:25:32.193 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:32.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:32.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:32.237 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:25:32.239 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:25:32.277 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 54243 54227 91.5435 58.3125 0.0274275 0.0109186 2023-09-23T22:25:32.299 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:32.328 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:31 smithi070 ceph-mon[192652]: failed to encode map e267 with expected crc 2023-09-23T22:25:32.331 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:32.384 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:32.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:32.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:32.488 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:32.520 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:32.573 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:32.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:32.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:32.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:32.756 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:33.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:32 smithi026 ceph-mon[223014]: pgmap v687: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:25:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:32 smithi026 ceph-mon[215775]: pgmap v687: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:25:33.278 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 55205 55189 90.714 60.125 0.00722159 0.0110185 2023-09-23T22:25:33.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:32 smithi070 ceph-mon[192652]: pgmap v687: 99 pgs: 99 active+clean; 3.1 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T22:25:33.500 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:33.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:33.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:33.554 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:25:33.610 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:33.638 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:33.694 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:33.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:33.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:33.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:33.829 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:33.880 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:33.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:33.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:33.983 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:33 smithi070 ceph-mon[192652]: from='client.55213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:25:33.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:33 smithi026 ceph-mon[223014]: from='client.55213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:25:33.987 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:33.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:33 smithi026 ceph-mon[215775]: from='client.55213 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:25:34.054 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:34.111 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":687,"stamp":"2023-09-23T22:25:31.316375+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3322615731,"num_objects":50654,"num_object_clones":0,"num_object_copies":151962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50654,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":51533,"num_write_kb":3252789,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":172,"num_bytes_recovered":8661772,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":53428,"ondisk_log_size":53428,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":11703812,"kb_used_data":9851860,"kb_used_omap":20,"kb_used_meta":1851883,"kb_avail":550767100,"statfs":{"total":575970213888,"available":563985510400,"internally_reserved":0,"allocated":10088304640,"data_stored":10080912166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20725,"internal_metadata":1896328971},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,1,7,0,0,0,0,11],"upper_bound":512},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":718077952,"num_objects":10957,"num_object_clones":0,"num_object_copies":32871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10957,"num_write_kb":701248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004767"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":253,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954607+0000","last_change":"2023-09-23T22:21:57.866776+0000","last_active":"2023-09-23T22:25:05.954607+0000","last_peered":"2023-09-23T22:25:05.954607+0000","last_clean":"2023-09-23T22:25:05.954607+0000","last_became_active":"2023-09-23T22:21:57.866457+0000","last_became_peered":"2023-09-23T22:21:57.866457+0000","last_unstale":"2023-09-23T22:25:05.954607+0000","last_undegraded":"2023-09-23T22:25:05.954607+0000","last_fullsized":"2023-09-23T22:25:05.954607+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:57.604795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":245,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.950434+0000","last_change":"2023-09-23T22:22:18.894767+0000","last_active":"2023-09-23T22:25:05.950434+0000","last_peered":"2023-09-23T22:25:05.950434+0000","last_clean":"2023-09-23T22:25:05.950434+0000","last_became_active":"2023-09-23T22:22:18.894134+0000","last_became_peered":"2023-09-23T22:22:18.894134+0000","last_unstale":"2023-09-23T22:25:05.950434+0000","last_undegraded":"2023-09-23T22:25:05.950434+0000","last_fullsized":"2023-09-23T22:25:05.950434+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.454710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":208,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953028+0000","last_change":"2023-09-23T22:14:00.791655+0000","last_active":"2023-09-23T22:25:05.953028+0000","last_peered":"2023-09-23T22:25:05.953028+0000","last_clean":"2023-09-23T22:25:05.953028+0000","last_became_active":"2023-09-23T22:14:00.791133+0000","last_became_peered":"2023-09-23T22:14:00.791133+0000","last_unstale":"2023-09-23T22:25:05.953028+0000","last_undegraded":"2023-09-23T22:25:05.953028+0000","last_fullsized":"2023-09-23T22:25:05.953028+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:31.483337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":124,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952077+0000","last_change":"2023-09-23T22:22:44.334482+0000","last_active":"2023-09-23T22:25:05.952077+0000","last_peered":"2023-09-23T22:25:05.952077+0000","last_clean":"2023-09-23T22:25:05.952077+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:25:05.952077+0000","last_undegraded":"2023-09-23T22:25:05.952077+0000","last_fullsized":"2023-09-23T22:25:05.952077+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:36.699003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":285,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954888+0000","last_change":"2023-09-23T22:22:18.914332+0000","last_active":"2023-09-23T22:25:05.954888+0000","last_peered":"2023-09-23T22:25:05.954888+0000","last_clean":"2023-09-23T22:25:05.954888+0000","last_became_active":"2023-09-23T22:22:18.913829+0000","last_became_peered":"2023-09-23T22:22:18.913829+0000","last_unstale":"2023-09-23T22:25:05.954888+0000","last_undegraded":"2023-09-23T22:25:05.954888+0000","last_fullsized":"2023-09-23T22:25:05.954888+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:41.861798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":221,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957660+0000","last_change":"2023-09-23T22:20:12.581018+0000","last_active":"2023-09-23T22:25:05.957660+0000","last_peered":"2023-09-23T22:25:05.957660+0000","last_clean":"2023-09-23T22:25:05.957660+0000","last_became_active":"2023-09-23T22:20:12.580783+0000","last_became_peered":"2023-09-23T22:20:12.580783+0000","last_unstale":"2023-09-23T22:25:05.957660+0000","last_undegraded":"2023-09-23T22:25:05.957660+0000","last_fullsized":"2023-09-23T22:25:05.957660+0000","mapping_epoch":197,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:29.038836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.951435+0000","last_change":"2023-09-23T22:22:18.888525+0000","last_active":"2023-09-23T22:25:05.951435+0000","last_peered":"2023-09-23T22:25:05.951435+0000","last_clean":"2023-09-23T22:25:05.951435+0000","last_became_active":"2023-09-23T22:22:18.888170+0000","last_became_peered":"2023-09-23T22:22:18.888170+0000","last_unstale":"2023-09-23T22:25:05.951435+0000","last_undegraded":"2023-09-23T22:25:05.951435+0000","last_fullsized":"2023-09-23T22:25:05.951435+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:40.510573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":242,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954970+0000","last_change":"2023-09-23T22:16:49.161973+0000","last_active":"2023-09-23T22:25:05.954970+0000","last_peered":"2023-09-23T22:25:05.954970+0000","last_clean":"2023-09-23T22:25:05.954970+0000","last_became_active":"2023-09-23T22:16:49.161652+0000","last_became_peered":"2023-09-23T22:16:49.161652+0000","last_unstale":"2023-09-23T22:25:05.954970+0000","last_undegraded":"2023-09-23T22:25:05.954970+0000","last_fullsized":"2023-09-23T22:25:05.954970+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:22.761165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":226,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957524+0000","last_change":"2023-09-23T22:12:54.875510+0000","last_active":"2023-09-23T22:25:05.957524+0000","last_peered":"2023-09-23T22:25:05.957524+0000","last_clean":"2023-09-23T22:25:05.957524+0000","last_became_active":"2023-09-23T22:12:54.875178+0000","last_became_peered":"2023-09-23T22:12:54.875178+0000","last_unstale":"2023-09-23T22:25:05.957524+0000","last_undegraded":"2023-09-23T22:25:05.957524+0000","last_fullsized":"2023-09-23T22:25:05.957524+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:30.061097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":210,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949232+0000","last_change":"2023-09-23T22:22:44.343459+0000","last_active":"2023-09-23T22:25:05.949232+0000","last_peered":"2023-09-23T22:25:05.949232+0000","last_clean":"2023-09-23T22:25:05.949232+0000","last_became_active":"2023-09-23T22:22:44.343213+0000","last_became_peered":"2023-09-23T22:22:44.343213+0000","last_unstale":"2023-09-23T22:25:05.949232+0000","last_undegraded":"2023-09-23T22:25:05.949232+0000","last_fullsized":"2023-09-23T22:25:05.949232+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:31.526946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":210,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949403+0000","last_change":"2023-09-23T22:22:44.352455+0000","last_active":"2023-09-23T22:25:05.949403+0000","last_peered":"2023-09-23T22:25:05.949403+0000","last_clean":"2023-09-23T22:25:05.949403+0000","last_became_active":"2023-09-23T22:22:44.340287+0000","last_became_peered":"2023-09-23T22:22:44.340287+0000","last_unstale":"2023-09-23T22:25:05.949403+0000","last_undegraded":"2023-09-23T22:25:05.949403+0000","last_fullsized":"2023-09-23T22:25:05.949403+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:05.919053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":244,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954433+0000","last_change":"2023-09-23T22:21:57.825287+0000","last_active":"2023-09-23T22:25:05.954433+0000","last_peered":"2023-09-23T22:25:05.954433+0000","last_clean":"2023-09-23T22:25:05.954433+0000","last_became_active":"2023-09-23T22:21:57.824925+0000","last_became_peered":"2023-09-23T22:21:57.824925+0000","last_unstale":"2023-09-23T22:25:05.954433+0000","last_undegraded":"2023-09-23T22:25:05.954433+0000","last_fullsized":"2023-09-23T22:25:05.954433+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.557461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":270,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954555+0000","last_change":"2023-09-23T22:22:17.985145+0000","last_active":"2023-09-23T22:25:05.954555+0000","last_peered":"2023-09-23T22:25:05.954555+0000","last_clean":"2023-09-23T22:25:05.954555+0000","last_became_active":"2023-09-23T22:21:57.866485+0000","last_became_peered":"2023-09-23T22:21:57.866485+0000","last_unstale":"2023-09-23T22:25:05.954555+0000","last_undegraded":"2023-09-23T22:25:05.954555+0000","last_fullsized":"2023-09-23T22:25:05.954555+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:42.883605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":234,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954436+0000","last_change":"2023-09-23T22:16:49.163461+0000","last_active":"2023-09-23T22:25:05.954436+0000","last_peered":"2023-09-23T22:25:05.954436+0000","last_clean":"2023-09-23T22:25:05.954436+0000","last_became_active":"2023-09-23T22:16:49.163140+0000","last_became_peered":"2023-09-23T22:16:49.163140+0000","last_unstale":"2023-09-23T22:25:05.954436+0000","last_undegraded":"2023-09-23T22:25:05.954436+0000","last_fullsized":"2023-09-23T22:25:05.954436+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:50.682163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":193,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954486+0000","last_change":"2023-09-23T22:22:17.939155+0000","last_active":"2023-09-23T22:25:05.954486+0000","last_peered":"2023-09-23T22:25:05.954486+0000","last_clean":"2023-09-23T22:25:05.954486+0000","last_became_active":"2023-09-23T22:21:57.858467+0000","last_became_peered":"2023-09-23T22:21:57.858467+0000","last_unstale":"2023-09-23T22:25:05.954486+0000","last_undegraded":"2023-09-23T22:25:05.954486+0000","last_fullsized":"2023-09-23T22:25:05.954486+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:47.285888+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":233,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949265+0000","last_change":"2023-09-23T22:14:00.794351+0000","last_active":"2023-09-23T22:25:05.949265+0000","last_peered":"2023-09-23T22:25:05.949265+0000","last_clean":"2023-09-23T22:25:05.949265+0000","last_became_active":"2023-09-23T22:14:00.794064+0000","last_became_peered":"2023-09-23T22:14:00.794064+0000","last_unstale":"2023-09-23T22:25:05.949265+0000","last_undegraded":"2023-09-23T22:25:05.949265+0000","last_fullsized":"2023-09-23T22:25:05.949265+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:47.656016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":141,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.945679+0000","last_change":"2023-09-23T22:22:44.334349+0000","last_active":"2023-09-23T22:25:05.945679+0000","last_peered":"2023-09-23T22:25:05.945679+0000","last_clean":"2023-09-23T22:25:05.945679+0000","last_became_active":"2023-09-23T22:22:44.333939+0000","last_became_peered":"2023-09-23T22:22:44.333939+0000","last_unstale":"2023-09-23T22:25:05.945679+0000","last_undegraded":"2023-09-23T22:25:05.945679+0000","last_fullsized":"2023-09-23T22:25:05.945679+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.828694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":245,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957457+0000","last_change":"2023-09-23T22:21:57.824270+0000","last_active":"2023-09-23T22:25:05.957457+0000","last_peered":"2023-09-23T22:25:05.957457+0000","last_clean":"2023-09-23T22:25:05.957457+0000","last_became_active":"2023-09-23T22:21:57.824037+0000","last_became_peered":"2023-09-23T22:21:57.824037+0000","last_unstale":"2023-09-23T22:25:05.957457+0000","last_undegraded":"2023-09-23T22:25:05.957457+0000","last_fullsized":"2023-09-23T22:25:05.957457+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:52.035859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":189,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954232+0000","last_change":"2023-09-23T22:17:15.720568+0000","last_active":"2023-09-23T22:25:05.954232+0000","last_peered":"2023-09-23T22:25:05.954232+0000","last_clean":"2023-09-23T22:25:05.954232+0000","last_became_active":"2023-09-23T22:17:15.719884+0000","last_became_peered":"2023-09-23T22:17:15.719884+0000","last_unstale":"2023-09-23T22:25:05.954232+0000","last_undegraded":"2023-09-23T22:25:05.954232+0000","last_fullsized":"2023-09-23T22:25:05.954232+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:59.071427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":123,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952240+0000","last_change":"2023-09-23T22:22:44.334296+0000","last_active":"2023-09-23T22:25:05.952240+0000","last_peered":"2023-09-23T22:25:05.952240+0000","last_clean":"2023-09-23T22:25:05.952240+0000","last_became_active":"2023-09-23T22:22:44.333973+0000","last_became_peered":"2023-09-23T22:22:44.333973+0000","last_unstale":"2023-09-23T22:25:05.952240+0000","last_undegraded":"2023-09-23T22:25:05.952240+0000","last_fullsized":"2023-09-23T22:25:05.952240+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:10.895867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":284,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.944559+0000","last_change":"2023-09-23T22:22:18.889655+0000","last_active":"2023-09-23T22:25:05.944559+0000","last_peered":"2023-09-23T22:25:05.944559+0000","last_clean":"2023-09-23T22:25:05.944559+0000","last_became_active":"2023-09-23T22:22:18.889324+0000","last_became_peered":"2023-09-23T22:22:18.889324+0000","last_unstale":"2023-09-23T22:25:05.944559+0000","last_undegraded":"2023-09-23T22:25:05.944559+0000","last_fullsized":"2023-09-23T22:25:05.944559+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:41.674877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":55,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953240+0000","last_change":"2023-09-23T22:22:44.343051+0000","last_active":"2023-09-23T22:25:05.953240+0000","last_peered":"2023-09-23T22:25:05.953240+0000","last_clean":"2023-09-23T22:25:05.953240+0000","last_became_active":"2023-09-23T22:22:44.342689+0000","last_became_peered":"2023-09-23T22:22:44.342689+0000","last_unstale":"2023-09-23T22:25:05.953240+0000","last_undegraded":"2023-09-23T22:25:05.953240+0000","last_fullsized":"2023-09-23T22:25:05.953240+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:31.704647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"266'3213","reported_seq":3228,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:28.836265+0000","last_change":"2023-09-23T22:24:46.980027+0000","last_active":"2023-09-23T22:25:28.836265+0000","last_peered":"2023-09-23T22:25:28.836265+0000","last_clean":"2023-09-23T22:25:28.836265+0000","last_became_active":"2023-09-23T22:24:46.979731+0000","last_became_peered":"2023-09-23T22:24:46.979731+0000","last_unstale":"2023-09-23T22:25:28.836265+0000","last_undegraded":"2023-09-23T22:25:28.836265+0000","last_fullsized":"2023-09-23T22:25:28.836265+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3213,"log_dups_size":0,"ondisk_log_size":3213,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:06.319642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210567168,"num_objects":3213,"num_object_clones":0,"num_object_copies":9639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3213,"num_write_kb":205632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":261,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954441+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:25:05.954441+0000","last_peered":"2023-09-23T22:25:05.954441+0000","last_clean":"2023-09-23T22:25:05.954441+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:25:05.954441+0000","last_undegraded":"2023-09-23T22:25:05.954441+0000","last_fullsized":"2023-09-23T22:25:05.954441+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":248,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.956671+0000","last_change":"2023-09-23T22:22:44.330150+0000","last_active":"2023-09-23T22:25:05.956671+0000","last_peered":"2023-09-23T22:25:05.956671+0000","last_clean":"2023-09-23T22:25:05.956671+0000","last_became_active":"2023-09-23T22:22:44.329835+0000","last_became_peered":"2023-09-23T22:22:44.329835+0000","last_unstale":"2023-09-23T22:25:05.956671+0000","last_undegraded":"2023-09-23T22:25:05.956671+0000","last_fullsized":"2023-09-23T22:25:05.956671+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:05.304044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"266'3287","reported_seq":3302,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:29.422887+0000","last_change":"2023-09-23T22:24:46.984090+0000","last_active":"2023-09-23T22:25:29.422887+0000","last_peered":"2023-09-23T22:25:29.422887+0000","last_clean":"2023-09-23T22:25:29.422887+0000","last_became_active":"2023-09-23T22:24:46.983784+0000","last_became_peered":"2023-09-23T22:24:46.983784+0000","last_unstale":"2023-09-23T22:25:29.422887+0000","last_undegraded":"2023-09-23T22:25:29.422887+0000","last_fullsized":"2023-09-23T22:25:29.422887+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3287,"log_dups_size":0,"ondisk_log_size":3287,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:14.302288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215416832,"num_objects":3287,"num_object_clones":0,"num_object_copies":9861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3287,"num_write_kb":210368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":175,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954687+0000","last_change":"2023-09-23T22:15:14.833298+0000","last_active":"2023-09-23T22:25:05.954687+0000","last_peered":"2023-09-23T22:25:05.954687+0000","last_clean":"2023-09-23T22:25:05.954687+0000","last_became_active":"2023-09-23T22:15:14.832937+0000","last_became_peered":"2023-09-23T22:15:14.832937+0000","last_unstale":"2023-09-23T22:25:05.954687+0000","last_undegraded":"2023-09-23T22:25:05.954687+0000","last_fullsized":"2023-09-23T22:25:05.954687+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:32.464785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":253,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954195+0000","last_change":"2023-09-23T22:22:18.917562+0000","last_active":"2023-09-23T22:25:05.954195+0000","last_peered":"2023-09-23T22:25:05.954195+0000","last_clean":"2023-09-23T22:25:05.954195+0000","last_became_active":"2023-09-23T22:22:18.917240+0000","last_became_peered":"2023-09-23T22:22:18.917240+0000","last_unstale":"2023-09-23T22:25:05.954195+0000","last_undegraded":"2023-09-23T22:25:05.954195+0000","last_fullsized":"2023-09-23T22:25:05.954195+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:50.054090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"266'3077","reported_seq":3092,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:26.866301+0000","last_change":"2023-09-23T22:24:46.981119+0000","last_active":"2023-09-23T22:25:26.866301+0000","last_peered":"2023-09-23T22:25:26.866301+0000","last_clean":"2023-09-23T22:25:26.866301+0000","last_became_active":"2023-09-23T22:24:46.980795+0000","last_became_peered":"2023-09-23T22:24:46.980795+0000","last_unstale":"2023-09-23T22:25:26.866301+0000","last_undegraded":"2023-09-23T22:25:26.866301+0000","last_fullsized":"2023-09-23T22:25:26.866301+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3077,"log_dups_size":0,"ondisk_log_size":3077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:24.132475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201654272,"num_objects":3077,"num_object_clones":0,"num_object_copies":9231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3077,"num_write_kb":196928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":191,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957539+0000","last_change":"2023-09-23T22:16:49.161661+0000","last_active":"2023-09-23T22:25:05.957539+0000","last_peered":"2023-09-23T22:25:05.957539+0000","last_clean":"2023-09-23T22:25:05.957539+0000","last_became_active":"2023-09-23T22:16:49.158260+0000","last_became_peered":"2023-09-23T22:16:49.158260+0000","last_unstale":"2023-09-23T22:25:05.957539+0000","last_undegraded":"2023-09-23T22:25:05.957539+0000","last_fullsized":"2023-09-23T22:25:05.957539+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:05.396861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":201,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954482+0000","last_change":"2023-09-23T22:22:44.323603+0000","last_active":"2023-09-23T22:25:05.954482+0000","last_peered":"2023-09-23T22:25:05.954482+0000","last_clean":"2023-09-23T22:25:05.954482+0000","last_became_active":"2023-09-23T22:22:44.323261+0000","last_became_peered":"2023-09-23T22:22:44.323261+0000","last_unstale":"2023-09-23T22:25:05.954482+0000","last_undegraded":"2023-09-23T22:25:05.954482+0000","last_fullsized":"2023-09-23T22:25:05.954482+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:40.656406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.8","version":"266'3137","reported_seq":3151,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:29.917969+0000","last_change":"2023-09-23T22:24:47.009845+0000","last_active":"2023-09-23T22:25:29.917969+0000","last_peered":"2023-09-23T22:25:29.917969+0000","last_clean":"2023-09-23T22:25:29.917969+0000","last_became_active":"2023-09-23T22:24:47.009559+0000","last_became_peered":"2023-09-23T22:24:47.009559+0000","last_unstale":"2023-09-23T22:25:29.917969+0000","last_undegraded":"2023-09-23T22:25:29.917969+0000","last_fullsized":"2023-09-23T22:25:29.917969+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3137,"log_dups_size":0,"ondisk_log_size":3137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:19.040683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205586432,"num_objects":3137,"num_object_clones":0,"num_object_copies":9411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3137,"num_write_kb":200768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":256,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957094+0000","last_change":"2023-09-23T22:22:18.909693+0000","last_active":"2023-09-23T22:25:05.957094+0000","last_peered":"2023-09-23T22:25:05.957094+0000","last_clean":"2023-09-23T22:25:05.957094+0000","last_became_active":"2023-09-23T22:22:18.909301+0000","last_became_peered":"2023-09-23T22:22:18.909301+0000","last_unstale":"2023-09-23T22:25:05.957094+0000","last_undegraded":"2023-09-23T22:25:05.957094+0000","last_fullsized":"2023-09-23T22:25:05.957094+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:39.140817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":121,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.944989+0000","last_change":"2023-09-23T22:17:15.724085+0000","last_active":"2023-09-23T22:25:05.944989+0000","last_peered":"2023-09-23T22:25:05.944989+0000","last_clean":"2023-09-23T22:25:05.944989+0000","last_became_active":"2023-09-23T22:17:15.723694+0000","last_became_peered":"2023-09-23T22:17:15.723694+0000","last_unstale":"2023-09-23T22:25:05.944989+0000","last_undegraded":"2023-09-23T22:25:05.944989+0000","last_fullsized":"2023-09-23T22:25:05.944989+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:37:07.147171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"266'2969","reported_seq":2984,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:26.866105+0000","last_change":"2023-09-23T22:24:47.009804+0000","last_active":"2023-09-23T22:25:26.866105+0000","last_peered":"2023-09-23T22:25:26.866105+0000","last_clean":"2023-09-23T22:25:26.866105+0000","last_became_active":"2023-09-23T22:24:47.009551+0000","last_became_peered":"2023-09-23T22:24:47.009551+0000","last_unstale":"2023-09-23T22:25:26.866105+0000","last_undegraded":"2023-09-23T22:25:26.866105+0000","last_fullsized":"2023-09-23T22:25:26.866105+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":2969,"log_dups_size":0,"ondisk_log_size":2969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:44.762872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194576384,"num_objects":2969,"num_object_clones":0,"num_object_copies":8907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2969,"num_write_kb":190016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":570,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949066+0000","last_change":"2023-09-23T22:21:36.904704+0000","last_active":"2023-09-23T22:25:05.949066+0000","last_peered":"2023-09-23T22:25:05.949066+0000","last_clean":"2023-09-23T22:25:05.949066+0000","last_became_active":"2023-09-23T22:21:36.904336+0000","last_became_peered":"2023-09-23T22:21:36.904336+0000","last_unstale":"2023-09-23T22:25:05.949066+0000","last_undegraded":"2023-09-23T22:25:05.949066+0000","last_fullsized":"2023-09-23T22:25:05.949066+0000","mapping_epoch":222,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":223,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:38.045766+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":280,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:14.406314+0000","last_change":"2023-09-23T22:18:57.722202+0000","last_active":"2023-09-23T22:25:14.406314+0000","last_peered":"2023-09-23T22:25:14.406314+0000","last_clean":"2023-09-23T22:25:14.406314+0000","last_became_active":"2023-09-23T22:18:57.721789+0000","last_became_peered":"2023-09-23T22:18:57.721789+0000","last_unstale":"2023-09-23T22:25:14.406314+0000","last_undegraded":"2023-09-23T22:25:14.406314+0000","last_fullsized":"2023-09-23T22:25:14.406314+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:24.910551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":234,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954460+0000","last_change":"2023-09-23T22:16:49.164663+0000","last_active":"2023-09-23T22:25:05.954460+0000","last_peered":"2023-09-23T22:25:05.954460+0000","last_clean":"2023-09-23T22:25:05.954460+0000","last_became_active":"2023-09-23T22:16:49.164336+0000","last_became_peered":"2023-09-23T22:16:49.164336+0000","last_unstale":"2023-09-23T22:25:05.954460+0000","last_undegraded":"2023-09-23T22:25:05.954460+0000","last_fullsized":"2023-09-23T22:25:05.954460+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:41.787625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"266'3195","reported_seq":3209,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:29.932878+0000","last_change":"2023-09-23T22:24:46.981702+0000","last_active":"2023-09-23T22:25:29.932878+0000","last_peered":"2023-09-23T22:25:29.932878+0000","last_clean":"2023-09-23T22:25:29.932878+0000","last_became_active":"2023-09-23T22:24:46.981422+0000","last_became_peered":"2023-09-23T22:24:46.981422+0000","last_unstale":"2023-09-23T22:25:29.932878+0000","last_undegraded":"2023-09-23T22:25:29.932878+0000","last_fullsized":"2023-09-23T22:25:29.932878+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3195,"log_dups_size":0,"ondisk_log_size":3195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:40:08.014616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209387520,"num_objects":3195,"num_object_clones":0,"num_object_copies":9585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3195,"num_write_kb":204480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":578,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.956607+0000","last_change":"2023-09-23T22:24:39.954805+0000","last_active":"2023-09-23T22:25:05.956607+0000","last_peered":"2023-09-23T22:25:05.956607+0000","last_clean":"2023-09-23T22:25:05.956607+0000","last_became_active":"2023-09-23T22:24:39.954350+0000","last_became_peered":"2023-09-23T22:24:39.954350+0000","last_unstale":"2023-09-23T22:25:05.956607+0000","last_undegraded":"2023-09-23T22:25:05.956607+0000","last_fullsized":"2023-09-23T22:25:05.956607+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:47.774412+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":182,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954341+0000","last_change":"2023-09-23T22:16:49.164602+0000","last_active":"2023-09-23T22:25:05.954341+0000","last_peered":"2023-09-23T22:25:05.954341+0000","last_clean":"2023-09-23T22:25:05.954341+0000","last_became_active":"2023-09-23T22:16:49.164288+0000","last_became_peered":"2023-09-23T22:16:49.164288+0000","last_unstale":"2023-09-23T22:25:05.954341+0000","last_undegraded":"2023-09-23T22:25:05.954341+0000","last_fullsized":"2023-09-23T22:25:05.954341+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.220172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":251,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954464+0000","last_change":"2023-09-23T22:21:33.899326+0000","last_active":"2023-09-23T22:25:05.954464+0000","last_peered":"2023-09-23T22:25:05.954464+0000","last_clean":"2023-09-23T22:25:05.954464+0000","last_became_active":"2023-09-23T22:21:33.899071+0000","last_became_peered":"2023-09-23T22:21:33.899071+0000","last_unstale":"2023-09-23T22:25:05.954464+0000","last_undegraded":"2023-09-23T22:25:05.954464+0000","last_fullsized":"2023-09-23T22:25:05.954464+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:07.541034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"266'3212","reported_seq":3227,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:28.823839+0000","last_change":"2023-09-23T22:24:46.982376+0000","last_active":"2023-09-23T22:25:28.823839+0000","last_peered":"2023-09-23T22:25:28.823839+0000","last_clean":"2023-09-23T22:25:28.823839+0000","last_became_active":"2023-09-23T22:24:46.982171+0000","last_became_peered":"2023-09-23T22:24:46.982171+0000","last_unstale":"2023-09-23T22:25:28.823839+0000","last_undegraded":"2023-09-23T22:25:28.823839+0000","last_fullsized":"2023-09-23T22:25:28.823839+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3212,"log_dups_size":0,"ondisk_log_size":3212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:03.026919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210501632,"num_objects":3212,"num_object_clones":0,"num_object_copies":9636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3212,"num_write_kb":205568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":227,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954101+0000","last_change":"2023-09-23T22:21:57.844475+0000","last_active":"2023-09-23T22:25:05.954101+0000","last_peered":"2023-09-23T22:25:05.954101+0000","last_clean":"2023-09-23T22:25:05.954101+0000","last_became_active":"2023-09-23T22:21:57.844162+0000","last_became_peered":"2023-09-23T22:21:57.844162+0000","last_unstale":"2023-09-23T22:25:05.954101+0000","last_undegraded":"2023-09-23T22:25:05.954101+0000","last_fullsized":"2023-09-23T22:25:05.954101+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.359047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":242,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954527+0000","last_change":"2023-09-23T22:20:40.853041+0000","last_active":"2023-09-23T22:25:05.954527+0000","last_peered":"2023-09-23T22:25:05.954527+0000","last_clean":"2023-09-23T22:25:05.954527+0000","last_became_active":"2023-09-23T22:20:40.852728+0000","last_became_peered":"2023-09-23T22:20:40.852728+0000","last_unstale":"2023-09-23T22:25:05.954527+0000","last_undegraded":"2023-09-23T22:25:05.954527+0000","last_fullsized":"2023-09-23T22:25:05.954527+0000","mapping_epoch":209,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":210,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:56.562802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"266'3200","reported_seq":3215,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:29.362804+0000","last_change":"2023-09-23T22:24:47.009404+0000","last_active":"2023-09-23T22:25:29.362804+0000","last_peered":"2023-09-23T22:25:29.362804+0000","last_clean":"2023-09-23T22:25:29.362804+0000","last_became_active":"2023-09-23T22:24:47.009063+0000","last_became_peered":"2023-09-23T22:24:47.009063+0000","last_unstale":"2023-09-23T22:25:29.362804+0000","last_undegraded":"2023-09-23T22:25:29.362804+0000","last_fullsized":"2023-09-23T22:25:29.362804+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3200,"log_dups_size":0,"ondisk_log_size":3200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:52.237759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209715200,"num_objects":3200,"num_object_clones":0,"num_object_copies":9600,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3200,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3200,"num_write_kb":204800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":570,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957039+0000","last_change":"2023-09-23T22:22:00.992618+0000","last_active":"2023-09-23T22:25:05.957039+0000","last_peered":"2023-09-23T22:25:05.957039+0000","last_clean":"2023-09-23T22:25:05.957039+0000","last_became_active":"2023-09-23T22:22:00.992272+0000","last_became_peered":"2023-09-23T22:22:00.992272+0000","last_unstale":"2023-09-23T22:25:05.957039+0000","last_undegraded":"2023-09-23T22:25:05.957039+0000","last_fullsized":"2023-09-23T22:25:05.957039+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:18:29.055216+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":46,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954661+0000","last_change":"2023-09-23T22:22:18.914736+0000","last_active":"2023-09-23T22:25:05.954661+0000","last_peered":"2023-09-23T22:25:05.954661+0000","last_clean":"2023-09-23T22:25:05.954661+0000","last_became_active":"2023-09-23T22:22:18.913999+0000","last_became_peered":"2023-09-23T22:22:18.913999+0000","last_unstale":"2023-09-23T22:25:05.954661+0000","last_undegraded":"2023-09-23T22:25:05.954661+0000","last_fullsized":"2023-09-23T22:25:05.954661+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:50.563747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":208,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953809+0000","last_change":"2023-09-23T22:14:00.791734+0000","last_active":"2023-09-23T22:25:05.953809+0000","last_peered":"2023-09-23T22:25:05.953809+0000","last_clean":"2023-09-23T22:25:05.953809+0000","last_became_active":"2023-09-23T22:14:00.791388+0000","last_became_peered":"2023-09-23T22:14:00.791388+0000","last_unstale":"2023-09-23T22:25:05.953809+0000","last_undegraded":"2023-09-23T22:25:05.953809+0000","last_fullsized":"2023-09-23T22:25:05.953809+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.567503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"266'3209","reported_seq":3223,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:29.923680+0000","last_change":"2023-09-23T22:24:46.980482+0000","last_active":"2023-09-23T22:25:29.923680+0000","last_peered":"2023-09-23T22:25:29.923680+0000","last_clean":"2023-09-23T22:25:29.923680+0000","last_became_active":"2023-09-23T22:24:46.980314+0000","last_became_peered":"2023-09-23T22:24:46.980314+0000","last_unstale":"2023-09-23T22:25:29.923680+0000","last_undegraded":"2023-09-23T22:25:29.923680+0000","last_fullsized":"2023-09-23T22:25:29.923680+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3209,"log_dups_size":0,"ondisk_log_size":3209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:08.107504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210305024,"num_objects":3209,"num_object_clones":0,"num_object_copies":9627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3209,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3209,"num_write_kb":205376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":574,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.951243+0000","last_change":"2023-09-23T22:24:47.781592+0000","last_active":"2023-09-23T22:25:05.951243+0000","last_peered":"2023-09-23T22:25:05.951243+0000","last_clean":"2023-09-23T22:25:05.951243+0000","last_became_active":"2023-09-23T22:22:47.349209+0000","last_became_peered":"2023-09-23T22:22:47.349209+0000","last_unstale":"2023-09-23T22:25:05.951243+0000","last_undegraded":"2023-09-23T22:25:05.951243+0000","last_fullsized":"2023-09-23T22:25:05.951243+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:47.781486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:47.781486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:01.152802+0000","scrub_duration":0.0020463370000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":816,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:17.999238+0000","last_change":"2023-09-23T22:16:51.164817+0000","last_active":"2023-09-23T22:25:17.999238+0000","last_peered":"2023-09-23T22:25:17.999238+0000","last_clean":"2023-09-23T22:25:17.999238+0000","last_became_active":"2023-09-23T22:16:51.164015+0000","last_became_peered":"2023-09-23T22:16:51.164015+0000","last_unstale":"2023-09-23T22:25:17.999238+0000","last_undegraded":"2023-09-23T22:25:17.999238+0000","last_fullsized":"2023-09-23T22:25:17.999238+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":149,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:21.118714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":283,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957090+0000","last_change":"2023-09-23T22:22:44.348418+0000","last_active":"2023-09-23T22:25:05.957090+0000","last_peered":"2023-09-23T22:25:05.957090+0000","last_clean":"2023-09-23T22:25:05.957090+0000","last_became_active":"2023-09-23T22:22:44.348067+0000","last_became_peered":"2023-09-23T22:22:44.348067+0000","last_unstale":"2023-09-23T22:25:05.957090+0000","last_undegraded":"2023-09-23T22:25:05.957090+0000","last_fullsized":"2023-09-23T22:25:05.957090+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:47.789777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":52,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954290+0000","last_change":"2023-09-23T22:21:57.844330+0000","last_active":"2023-09-23T22:25:05.954290+0000","last_peered":"2023-09-23T22:25:05.954290+0000","last_clean":"2023-09-23T22:25:05.954290+0000","last_became_active":"2023-09-23T22:21:57.844063+0000","last_became_peered":"2023-09-23T22:21:57.844063+0000","last_unstale":"2023-09-23T22:25:05.954290+0000","last_undegraded":"2023-09-23T22:25:05.954290+0000","last_fullsized":"2023-09-23T22:25:05.954290+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.201060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"266'3093","reported_seq":3108,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:26.859340+0000","last_change":"2023-09-23T22:24:46.981073+0000","last_active":"2023-09-23T22:25:26.859340+0000","last_peered":"2023-09-23T22:25:26.859340+0000","last_clean":"2023-09-23T22:25:26.859340+0000","last_became_active":"2023-09-23T22:24:46.980758+0000","last_became_peered":"2023-09-23T22:24:46.980758+0000","last_unstale":"2023-09-23T22:25:26.859340+0000","last_undegraded":"2023-09-23T22:25:26.859340+0000","last_fullsized":"2023-09-23T22:25:26.859340+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3093,"log_dups_size":0,"ondisk_log_size":3093,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:01.377206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202702848,"num_objects":3093,"num_object_clones":0,"num_object_copies":9279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3093,"num_write_kb":197952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":570,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.956784+0000","last_change":"2023-09-23T22:22:00.973119+0000","last_active":"2023-09-23T22:25:05.956784+0000","last_peered":"2023-09-23T22:25:05.956784+0000","last_clean":"2023-09-23T22:25:05.956784+0000","last_became_active":"2023-09-23T22:22:00.972754+0000","last_became_peered":"2023-09-23T22:22:00.972754+0000","last_unstale":"2023-09-23T22:25:05.956784+0000","last_undegraded":"2023-09-23T22:25:05.956784+0000","last_fullsized":"2023-09-23T22:25:05.956784+0000","mapping_epoch":228,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":229,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:00.431170+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":998,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957177+0000","last_change":"2023-09-23T22:19:00.893402+0000","last_active":"2023-09-23T22:25:05.957177+0000","last_peered":"2023-09-23T22:25:05.957177+0000","last_clean":"2023-09-23T22:25:05.957177+0000","last_became_active":"2023-09-23T22:17:15.722275+0000","last_became_peered":"2023-09-23T22:17:15.722275+0000","last_unstale":"2023-09-23T22:25:05.957177+0000","last_undegraded":"2023-09-23T22:25:05.957177+0000","last_fullsized":"2023-09-23T22:25:05.957177+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:58.390600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":275,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954391+0000","last_change":"2023-09-23T22:17:15.724787+0000","last_active":"2023-09-23T22:25:05.954391+0000","last_peered":"2023-09-23T22:25:05.954391+0000","last_clean":"2023-09-23T22:25:05.954391+0000","last_became_active":"2023-09-23T22:17:15.724306+0000","last_became_peered":"2023-09-23T22:17:15.724306+0000","last_unstale":"2023-09-23T22:25:05.954391+0000","last_undegraded":"2023-09-23T22:25:05.954391+0000","last_fullsized":"2023-09-23T22:25:05.954391+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:24.895616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":230,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957221+0000","last_change":"2023-09-23T22:22:44.333349+0000","last_active":"2023-09-23T22:25:05.957221+0000","last_peered":"2023-09-23T22:25:05.957221+0000","last_clean":"2023-09-23T22:25:05.957221+0000","last_became_active":"2023-09-23T22:22:44.333092+0000","last_became_peered":"2023-09-23T22:22:44.333092+0000","last_unstale":"2023-09-23T22:25:05.957221+0000","last_undegraded":"2023-09-23T22:25:05.957221+0000","last_fullsized":"2023-09-23T22:25:05.957221+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:06.829726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.4","version":"266'3113","reported_seq":3126,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:26.874719+0000","last_change":"2023-09-23T22:24:46.980328+0000","last_active":"2023-09-23T22:25:26.874719+0000","last_peered":"2023-09-23T22:25:26.874719+0000","last_clean":"2023-09-23T22:25:26.874719+0000","last_became_active":"2023-09-23T22:24:46.980033+0000","last_became_peered":"2023-09-23T22:24:46.980033+0000","last_unstale":"2023-09-23T22:25:26.874719+0000","last_undegraded":"2023-09-23T22:25:26.874719+0000","last_fullsized":"2023-09-23T22:25:26.874719+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3113,"log_dups_size":0,"ondisk_log_size":3113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:58.452186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204013568,"num_objects":3113,"num_object_clones":0,"num_object_copies":9339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3113,"num_write_kb":199232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":596,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:16.869827+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:25:16.869827+0000","last_peered":"2023-09-23T22:25:16.869827+0000","last_clean":"2023-09-23T22:25:16.869827+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:25:16.869827+0000","last_undegraded":"2023-09-23T22:25:16.869827+0000","last_fullsized":"2023-09-23T22:25:16.869827+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":401,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949393+0000","last_change":"2023-09-23T22:24:38.072053+0000","last_active":"2023-09-23T22:25:05.949393+0000","last_peered":"2023-09-23T22:25:05.949393+0000","last_clean":"2023-09-23T22:25:05.949393+0000","last_became_active":"2023-09-23T22:22:18.917198+0000","last_became_peered":"2023-09-23T22:22:18.917198+0000","last_unstale":"2023-09-23T22:25:05.949393+0000","last_undegraded":"2023-09-23T22:25:05.949393+0000","last_fullsized":"2023-09-23T22:25:05.949393+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:06.239709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":227,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952353+0000","last_change":"2023-09-23T22:21:57.830754+0000","last_active":"2023-09-23T22:25:05.952353+0000","last_peered":"2023-09-23T22:25:05.952353+0000","last_clean":"2023-09-23T22:25:05.952353+0000","last_became_active":"2023-09-23T22:21:57.830434+0000","last_became_peered":"2023-09-23T22:21:57.830434+0000","last_unstale":"2023-09-23T22:25:05.952353+0000","last_undegraded":"2023-09-23T22:25:05.952353+0000","last_fullsized":"2023-09-23T22:25:05.952353+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:47.899759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":273,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953882+0000","last_change":"2023-09-23T22:18:57.727927+0000","last_active":"2023-09-23T22:25:05.953882+0000","last_peered":"2023-09-23T22:25:05.953882+0000","last_clean":"2023-09-23T22:25:05.953882+0000","last_became_active":"2023-09-23T22:18:57.727619+0000","last_became_peered":"2023-09-23T22:18:57.727619+0000","last_unstale":"2023-09-23T22:25:05.953882+0000","last_undegraded":"2023-09-23T22:25:05.953882+0000","last_fullsized":"2023-09-23T22:25:05.953882+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:22.020921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"266'3076","reported_seq":3091,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:26.853102+0000","last_change":"2023-09-23T22:24:47.009793+0000","last_active":"2023-09-23T22:25:26.853102+0000","last_peered":"2023-09-23T22:25:26.853102+0000","last_clean":"2023-09-23T22:25:26.853102+0000","last_became_active":"2023-09-23T22:24:47.009468+0000","last_became_peered":"2023-09-23T22:24:47.009468+0000","last_unstale":"2023-09-23T22:25:26.853102+0000","last_undegraded":"2023-09-23T22:25:26.853102+0000","last_fullsized":"2023-09-23T22:25:26.853102+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3076,"log_dups_size":0,"ondisk_log_size":3076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:20.557721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201588736,"num_objects":3076,"num_object_clones":0,"num_object_copies":9228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3076,"num_write_kb":196864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":893,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957296+0000","last_change":"2023-09-23T22:24:39.954768+0000","last_active":"2023-09-23T22:25:05.957296+0000","last_peered":"2023-09-23T22:25:05.957296+0000","last_clean":"2023-09-23T22:25:05.957296+0000","last_became_active":"2023-09-23T22:24:39.954171+0000","last_became_peered":"2023-09-23T22:24:39.954171+0000","last_unstale":"2023-09-23T22:25:05.957296+0000","last_undegraded":"2023-09-23T22:25:05.957296+0000","last_fullsized":"2023-09-23T22:25:05.957296+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:00.983002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":579,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.944415+0000","last_change":"2023-09-23T22:22:47.454331+0000","last_active":"2023-09-23T22:25:05.944415+0000","last_peered":"2023-09-23T22:25:05.944415+0000","last_clean":"2023-09-23T22:25:05.944415+0000","last_became_active":"2023-09-23T22:22:47.453948+0000","last_became_peered":"2023-09-23T22:22:47.453948+0000","last_unstale":"2023-09-23T22:25:05.944415+0000","last_undegraded":"2023-09-23T22:25:05.944415+0000","last_fullsized":"2023-09-23T22:25:05.944415+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:07.711837+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":191,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949322+0000","last_change":"2023-09-23T22:22:18.986626+0000","last_active":"2023-09-23T22:25:05.949322+0000","last_peered":"2023-09-23T22:25:05.949322+0000","last_clean":"2023-09-23T22:25:05.949322+0000","last_became_active":"2023-09-23T22:22:18.986230+0000","last_became_peered":"2023-09-23T22:22:18.986230+0000","last_unstale":"2023-09-23T22:25:05.949322+0000","last_undegraded":"2023-09-23T22:25:05.949322+0000","last_fullsized":"2023-09-23T22:25:05.949322+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:07.528769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954683+0000","last_change":"2023-09-23T22:17:15.720321+0000","last_active":"2023-09-23T22:25:05.954683+0000","last_peered":"2023-09-23T22:25:05.954683+0000","last_clean":"2023-09-23T22:25:05.954683+0000","last_became_active":"2023-09-23T22:17:15.720002+0000","last_became_peered":"2023-09-23T22:17:15.720002+0000","last_unstale":"2023-09-23T22:25:05.954683+0000","last_undegraded":"2023-09-23T22:25:05.954683+0000","last_fullsized":"2023-09-23T22:25:05.954683+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:18.357393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.5","version":"266'3097","reported_seq":3112,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:26.859586+0000","last_change":"2023-09-23T22:24:47.009893+0000","last_active":"2023-09-23T22:25:26.859586+0000","last_peered":"2023-09-23T22:25:26.859586+0000","last_clean":"2023-09-23T22:25:26.859586+0000","last_became_active":"2023-09-23T22:24:47.009599+0000","last_became_peered":"2023-09-23T22:24:47.009599+0000","last_unstale":"2023-09-23T22:25:26.859586+0000","last_undegraded":"2023-09-23T22:25:26.859586+0000","last_fullsized":"2023-09-23T22:25:26.859586+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3097,"log_dups_size":0,"ondisk_log_size":3097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:03.224040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202964992,"num_objects":3097,"num_object_clones":0,"num_object_copies":9291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3097,"num_write_kb":198208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":598,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952319+0000","last_change":"2023-09-23T22:20:39.683988+0000","last_active":"2023-09-23T22:25:05.952319+0000","last_peered":"2023-09-23T22:25:05.952319+0000","last_clean":"2023-09-23T22:25:05.952319+0000","last_became_active":"2023-09-23T22:20:18.463781+0000","last_became_peered":"2023-09-23T22:20:18.463781+0000","last_unstale":"2023-09-23T22:25:05.952319+0000","last_undegraded":"2023-09-23T22:25:05.952319+0000","last_fullsized":"2023-09-23T22:25:05.952319+0000","mapping_epoch":202,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":203,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:13:11.746597+0000","scrub_duration":0.001758368,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":617,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:17.997617+0000","last_change":"2023-09-23T22:15:51.430184+0000","last_active":"2023-09-23T22:25:17.997617+0000","last_peered":"2023-09-23T22:25:17.997617+0000","last_clean":"2023-09-23T22:25:17.997617+0000","last_became_active":"2023-09-23T22:15:43.936376+0000","last_became_peered":"2023-09-23T22:15:43.936376+0000","last_unstale":"2023-09-23T22:25:17.997617+0000","last_undegraded":"2023-09-23T22:25:17.997617+0000","last_fullsized":"2023-09-23T22:25:17.997617+0000","mapping_epoch":124,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:20.734737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":148,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.944600+0000","last_change":"2023-09-23T22:22:18.888497+0000","last_active":"2023-09-23T22:25:05.944600+0000","last_peered":"2023-09-23T22:25:05.944600+0000","last_clean":"2023-09-23T22:25:05.944600+0000","last_became_active":"2023-09-23T22:22:18.888145+0000","last_became_peered":"2023-09-23T22:22:18.888145+0000","last_unstale":"2023-09-23T22:25:05.944600+0000","last_undegraded":"2023-09-23T22:25:05.944600+0000","last_fullsized":"2023-09-23T22:25:05.944600+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:16.043894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":105,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952158+0000","last_change":"2023-09-23T22:21:12.802088+0000","last_active":"2023-09-23T22:25:05.952158+0000","last_peered":"2023-09-23T22:25:05.952158+0000","last_clean":"2023-09-23T22:25:05.952158+0000","last_became_active":"2023-09-23T22:21:12.801184+0000","last_became_peered":"2023-09-23T22:21:12.801184+0000","last_unstale":"2023-09-23T22:25:05.952158+0000","last_undegraded":"2023-09-23T22:25:05.952158+0000","last_fullsized":"2023-09-23T22:25:05.952158+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:06.438402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"266'3264","reported_seq":3278,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:29.927147+0000","last_change":"2023-09-23T22:24:46.984221+0000","last_active":"2023-09-23T22:25:29.927147+0000","last_peered":"2023-09-23T22:25:29.927147+0000","last_clean":"2023-09-23T22:25:29.927147+0000","last_became_active":"2023-09-23T22:24:46.983941+0000","last_became_peered":"2023-09-23T22:24:46.983941+0000","last_unstale":"2023-09-23T22:25:29.927147+0000","last_undegraded":"2023-09-23T22:25:29.927147+0000","last_fullsized":"2023-09-23T22:25:29.927147+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3264,"log_dups_size":0,"ondisk_log_size":3264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:31.961352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213909504,"num_objects":3264,"num_object_clones":0,"num_object_copies":9792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3264,"num_write_kb":208896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":587,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.950863+0000","last_change":"2023-09-23T22:22:18.888704+0000","last_active":"2023-09-23T22:25:05.950863+0000","last_peered":"2023-09-23T22:25:05.950863+0000","last_clean":"2023-09-23T22:25:05.950863+0000","last_became_active":"2023-09-23T22:22:18.888225+0000","last_became_peered":"2023-09-23T22:22:18.888225+0000","last_unstale":"2023-09-23T22:25:05.950863+0000","last_undegraded":"2023-09-23T22:25:05.950863+0000","last_fullsized":"2023-09-23T22:25:05.950863+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:09:22.593360+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":158,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:16.869900+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:25:16.869900+0000","last_peered":"2023-09-23T22:25:16.869900+0000","last_clean":"2023-09-23T22:25:16.869900+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:25:16.869900+0000","last_undegraded":"2023-09-23T22:25:16.869900+0000","last_fullsized":"2023-09-23T22:25:16.869900+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":218,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:17.999397+0000","last_change":"2023-09-23T22:21:12.801451+0000","last_active":"2023-09-23T22:25:17.999397+0000","last_peered":"2023-09-23T22:25:17.999397+0000","last_clean":"2023-09-23T22:25:17.999397+0000","last_became_active":"2023-09-23T22:21:12.800689+0000","last_became_peered":"2023-09-23T22:21:12.800689+0000","last_unstale":"2023-09-23T22:25:17.999397+0000","last_undegraded":"2023-09-23T22:25:17.999397+0000","last_fullsized":"2023-09-23T22:25:17.999397+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:31.292165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":121,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.944915+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:25:05.944915+0000","last_peered":"2023-09-23T22:25:05.944915+0000","last_clean":"2023-09-23T22:25:05.944915+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:25:05.944915+0000","last_undegraded":"2023-09-23T22:25:05.944915+0000","last_fullsized":"2023-09-23T22:25:05.944915+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"266'3236","reported_seq":3251,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:29.400476+0000","last_change":"2023-09-23T22:24:47.010826+0000","last_active":"2023-09-23T22:25:29.400476+0000","last_peered":"2023-09-23T22:25:29.400476+0000","last_clean":"2023-09-23T22:25:29.400476+0000","last_became_active":"2023-09-23T22:24:47.010507+0000","last_became_peered":"2023-09-23T22:24:47.010507+0000","last_unstale":"2023-09-23T22:25:29.400476+0000","last_undegraded":"2023-09-23T22:25:29.400476+0000","last_fullsized":"2023-09-23T22:25:29.400476+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3236,"log_dups_size":0,"ondisk_log_size":3236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:22.053927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212074496,"num_objects":3236,"num_object_clones":0,"num_object_copies":9708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3236,"num_write_kb":207104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":570,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954363+0000","last_change":"2023-09-23T22:21:15.831144+0000","last_active":"2023-09-23T22:25:05.954363+0000","last_peered":"2023-09-23T22:25:05.954363+0000","last_clean":"2023-09-23T22:25:05.954363+0000","last_became_active":"2023-09-23T22:21:15.830812+0000","last_became_peered":"2023-09-23T22:21:15.830812+0000","last_unstale":"2023-09-23T22:25:05.954363+0000","last_undegraded":"2023-09-23T22:25:05.954363+0000","last_fullsized":"2023-09-23T22:25:05.954363+0000","mapping_epoch":216,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":217,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:55.327449+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":280,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954150+0000","last_change":"2023-09-23T22:17:15.717752+0000","last_active":"2023-09-23T22:25:05.954150+0000","last_peered":"2023-09-23T22:25:05.954150+0000","last_clean":"2023-09-23T22:25:05.954150+0000","last_became_active":"2023-09-23T22:17:15.717389+0000","last_became_peered":"2023-09-23T22:17:15.717389+0000","last_unstale":"2023-09-23T22:25:05.954150+0000","last_undegraded":"2023-09-23T22:25:05.954150+0000","last_fullsized":"2023-09-23T22:25:05.954150+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:24.543885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":908,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953347+0000","last_change":"2023-09-23T22:16:52.162978+0000","last_active":"2023-09-23T22:25:05.953347+0000","last_peered":"2023-09-23T22:25:05.953347+0000","last_clean":"2023-09-23T22:25:05.953347+0000","last_became_active":"2023-09-23T22:16:52.162465+0000","last_became_peered":"2023-09-23T22:16:52.162465+0000","last_unstale":"2023-09-23T22:25:05.953347+0000","last_undegraded":"2023-09-23T22:25:05.953347+0000","last_fullsized":"2023-09-23T22:25:05.953347+0000","mapping_epoch":149,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":150,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:48.586130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":55,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954467+0000","last_change":"2023-09-23T22:22:44.348090+0000","last_active":"2023-09-23T22:25:05.954467+0000","last_peered":"2023-09-23T22:25:05.954467+0000","last_clean":"2023-09-23T22:25:05.954467+0000","last_became_active":"2023-09-23T22:22:44.347751+0000","last_became_peered":"2023-09-23T22:22:44.347751+0000","last_unstale":"2023-09-23T22:25:05.954467+0000","last_undegraded":"2023-09-23T22:25:05.954467+0000","last_fullsized":"2023-09-23T22:25:05.954467+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:53.637619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"266'3248","reported_seq":3263,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T22:25:29.384426+0000","last_change":"2023-09-23T22:24:47.009023+0000","last_active":"2023-09-23T22:25:29.384426+0000","last_peered":"2023-09-23T22:25:29.384426+0000","last_clean":"2023-09-23T22:25:29.384426+0000","last_became_active":"2023-09-23T22:24:47.008704+0000","last_became_peered":"2023-09-23T22:24:47.008704+0000","last_unstale":"2023-09-23T22:25:29.384426+0000","last_undegraded":"2023-09-23T22:25:29.384426+0000","last_fullsized":"2023-09-23T22:25:29.384426+0000","mapping_epoch":260,"log_start":"0'0","ondisk_log_start":"0'0","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":3248,"log_dups_size":0,"ondisk_log_size":3248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:28.997990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212860928,"num_objects":3248,"num_object_clones":0,"num_object_copies":9744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3248,"num_write_kb":207872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":566,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949495+0000","last_change":"2023-09-23T22:24:45.734796+0000","last_active":"2023-09-23T22:25:05.949495+0000","last_peered":"2023-09-23T22:25:05.949495+0000","last_clean":"2023-09-23T22:25:05.949495+0000","last_became_active":"2023-09-23T22:22:19.882990+0000","last_became_peered":"2023-09-23T22:22:19.882990+0000","last_unstale":"2023-09-23T22:25:05.949495+0000","last_undegraded":"2023-09-23T22:25:05.949495+0000","last_fullsized":"2023-09-23T22:25:05.949495+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:45.734686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.734686+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:42.132151+0000","scrub_duration":0.0022175960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":260,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954234+0000","last_change":"2023-09-23T22:24:38.076201+0000","last_active":"2023-09-23T22:25:05.954234+0000","last_peered":"2023-09-23T22:25:05.954234+0000","last_clean":"2023-09-23T22:25:05.954234+0000","last_became_active":"2023-09-23T22:22:44.341945+0000","last_became_peered":"2023-09-23T22:22:44.341945+0000","last_unstale":"2023-09-23T22:25:05.954234+0000","last_undegraded":"2023-09-23T22:25:05.954234+0000","last_fullsized":"2023-09-23T22:25:05.954234+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":256,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:52.655759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":100,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.951437+0000","last_change":"2023-09-23T22:19:00.703581+0000","last_active":"2023-09-23T22:25:05.951437+0000","last_peered":"2023-09-23T22:25:05.951437+0000","last_clean":"2023-09-23T22:25:05.951437+0000","last_became_active":"2023-09-23T22:18:57.735892+0000","last_became_peered":"2023-09-23T22:18:57.735892+0000","last_unstale":"2023-09-23T22:25:05.951437+0000","last_undegraded":"2023-09-23T22:25:05.951437+0000","last_fullsized":"2023-09-23T22:25:05.951437+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.068084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":229,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954118+0000","last_change":"2023-09-23T22:17:15.720746+0000","last_active":"2023-09-23T22:25:05.954118+0000","last_peered":"2023-09-23T22:25:05.954118+0000","last_clean":"2023-09-23T22:25:05.954118+0000","last_became_active":"2023-09-23T22:17:15.720277+0000","last_became_peered":"2023-09-23T22:17:15.720277+0000","last_unstale":"2023-09-23T22:25:05.954118+0000","last_undegraded":"2023-09-23T22:25:05.954118+0000","last_fullsized":"2023-09-23T22:25:05.954118+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:53.749945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":96,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.944794+0000","last_change":"2023-09-23T22:18:57.717664+0000","last_active":"2023-09-23T22:25:05.944794+0000","last_peered":"2023-09-23T22:25:05.944794+0000","last_clean":"2023-09-23T22:25:05.944794+0000","last_became_active":"2023-09-23T22:18:57.717275+0000","last_became_peered":"2023-09-23T22:18:57.717275+0000","last_unstale":"2023-09-23T22:25:05.944794+0000","last_undegraded":"2023-09-23T22:25:05.944794+0000","last_fullsized":"2023-09-23T22:25:05.944794+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:23.981684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":277,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.952517+0000","last_change":"2023-09-23T22:22:18.935483+0000","last_active":"2023-09-23T22:25:05.952517+0000","last_peered":"2023-09-23T22:25:05.952517+0000","last_clean":"2023-09-23T22:25:05.952517+0000","last_became_active":"2023-09-23T22:22:18.935151+0000","last_became_peered":"2023-09-23T22:22:18.935151+0000","last_unstale":"2023-09-23T22:25:05.952517+0000","last_undegraded":"2023-09-23T22:25:05.952517+0000","last_fullsized":"2023-09-23T22:25:05.952517+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:36.823230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":228,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957346+0000","last_change":"2023-09-23T22:12:54.876938+0000","last_active":"2023-09-23T22:25:05.957346+0000","last_peered":"2023-09-23T22:25:05.957346+0000","last_clean":"2023-09-23T22:25:05.957346+0000","last_became_active":"2023-09-23T22:12:54.876576+0000","last_became_peered":"2023-09-23T22:12:54.876576+0000","last_unstale":"2023-09-23T22:25:05.957346+0000","last_undegraded":"2023-09-23T22:25:05.957346+0000","last_fullsized":"2023-09-23T22:25:05.957346+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:01.080430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":241,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954525+0000","last_change":"2023-09-23T22:14:00.794701+0000","last_active":"2023-09-23T22:25:05.954525+0000","last_peered":"2023-09-23T22:25:05.954525+0000","last_clean":"2023-09-23T22:25:05.954525+0000","last_became_active":"2023-09-23T22:14:00.794356+0000","last_became_peered":"2023-09-23T22:14:00.794356+0000","last_unstale":"2023-09-23T22:25:05.954525+0000","last_undegraded":"2023-09-23T22:25:05.954525+0000","last_fullsized":"2023-09-23T22:25:05.954525+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:06.525686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":225,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.957461+0000","last_change":"2023-09-23T22:16:49.168531+0000","last_active":"2023-09-23T22:25:05.957461+0000","last_peered":"2023-09-23T22:25:05.957461+0000","last_clean":"2023-09-23T22:25:05.957461+0000","last_became_active":"2023-09-23T22:16:49.163889+0000","last_became_peered":"2023-09-23T22:16:49.163889+0000","last_unstale":"2023-09-23T22:25:05.957461+0000","last_undegraded":"2023-09-23T22:25:05.957461+0000","last_fullsized":"2023-09-23T22:25:05.957461+0000","mapping_epoch":146,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":147,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:15.699886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":100,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954508+0000","last_change":"2023-09-23T22:18:31.797623+0000","last_active":"2023-09-23T22:25:05.954508+0000","last_peered":"2023-09-23T22:25:05.954508+0000","last_clean":"2023-09-23T22:25:05.954508+0000","last_became_active":"2023-09-23T22:18:31.797278+0000","last_became_peered":"2023-09-23T22:18:31.797278+0000","last_unstale":"2023-09-23T22:25:05.954508+0000","last_undegraded":"2023-09-23T22:25:05.954508+0000","last_fullsized":"2023-09-23T22:25:05.954508+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:25.225858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":247,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.953495+0000","last_change":"2023-09-23T22:22:18.885266+0000","last_active":"2023-09-23T22:25:05.953495+0000","last_peered":"2023-09-23T22:25:05.953495+0000","last_clean":"2023-09-23T22:25:05.953495+0000","last_became_active":"2023-09-23T22:22:18.884936+0000","last_became_peered":"2023-09-23T22:22:18.884936+0000","last_unstale":"2023-09-23T22:25:05.953495+0000","last_undegraded":"2023-09-23T22:25:05.953495+0000","last_fullsized":"2023-09-23T22:25:05.953495+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:03:09.257094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":288,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.949150+0000","last_change":"2023-09-23T22:22:18.891801+0000","last_active":"2023-09-23T22:25:05.949150+0000","last_peered":"2023-09-23T22:25:05.949150+0000","last_clean":"2023-09-23T22:25:05.949150+0000","last_became_active":"2023-09-23T22:22:18.891314+0000","last_became_peered":"2023-09-23T22:22:18.891314+0000","last_unstale":"2023-09-23T22:25:05.949150+0000","last_undegraded":"2023-09-23T22:25:05.949150+0000","last_fullsized":"2023-09-23T22:25:05.949150+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:04.038535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":156,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:16.869982+0000","last_change":"2023-09-23T22:21:12.813407+0000","last_active":"2023-09-23T22:25:16.869982+0000","last_peered":"2023-09-23T22:25:16.869982+0000","last_clean":"2023-09-23T22:25:16.869982+0000","last_became_active":"2023-09-23T22:21:12.807027+0000","last_became_peered":"2023-09-23T22:21:12.807027+0000","last_unstale":"2023-09-23T22:25:16.869982+0000","last_undegraded":"2023-09-23T22:25:16.869982+0000","last_fullsized":"2023-09-23T22:25:16.869982+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":214,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:06.552245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":274,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T22:25:05.954000+0000","last_change":"2023-09-23T22:22:18.876366+0000","last_active":"2023-09-23T22:25:05.954000+0000","last_peered":"2023-09-23T22:25:05.954000+0000","last_clean":"2023-09-23T22:25:05.954000+0000","last_became_active":"2023-09-23T22:22:18.876056+0000","last_became_peered":"2023-09-23T22:22:18.876056+0000","last_unstale":"2023-09-23T22:25:05.954000+0000","last_undegraded":"2023-09-23T22:25:05.954000+0000","last_fullsized":"2023-09-23T22:25:05.954000+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:58.457694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":3317825536,"num_objects":50626,"num_object_clones":0,"num_object_copies":151878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":50626,"num_write_kb":3240064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10056237056,"data_stored":10056237056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":50626,"ondisk_log_size":50626,"up":48,"acting":48,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":10438,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":126,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234253,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1776836,"kb_used_data":1548796,"kb_used_omap":0,"kb_used_meta":228032,"kb_avail":91968316,"statfs":{"total":95995035648,"available":94175555584,"internally_reserved":0,"allocated":1585967104,"data_stored":1584937408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":233504768},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:25:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":3.9340000000000002,"15min":2.3239999999999998},"min":{"1min":0.28000000000000003,"5min":0.22,"15min":0.22},"max":{"1min":3.6989999999999998,"5min":158.042,"15min":158.042},"last":0.23000000000000001},{"interface":"front","average":{"1min":1.131,"5min":4.5679999999999996,"15min":2.6429999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.217},"max":{"1min":5.9189999999999996,"5min":157.952,"15min":157.952},"last":0.33500000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.8039999999999998,"15min":3.1030000000000002},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.9370000000000003,"5min":99.385999999999996,"15min":123.79300000000001},"last":0.254},{"interface":"front","average":{"1min":1.784,"5min":2.9540000000000002,"15min":3.2970000000000002},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.24299999999999999},"max":{"1min":6.1820000000000004,"5min":95.441999999999993,"15min":123.66500000000001},"last":0.76500000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.776,"15min":3.7850000000000001},"min":{"1min":0.29999999999999999,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":6.0810000000000004,"5min":19.044,"15min":125.101},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.819,"15min":3.548},"min":{"1min":0.311,"5min":0.29199999999999998,"15min":0.28999999999999998},"max":{"1min":4.9829999999999997,"5min":19.445,"15min":126.021},"last":0.27500000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:25:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":3.9870000000000001,"15min":2.383},"min":{"1min":0.39600000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.1010000000000009,"5min":170.86099999999999,"15min":170.86099999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.2649999999999999,"5min":3.746,"15min":2.4489999999999998},"min":{"1min":0.38300000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.8689999999999998,"5min":174.57499999999999,"15min":174.57499999999999},"last":0.47899999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0509999999999999,"5min":1.0589999999999999,"15min":1.3700000000000001},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.0419999999999998,"5min":27.922000000000001,"15min":88.543000000000006},"last":0.438},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.143,"15min":1.3120000000000001},"min":{"1min":0.28100000000000003,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.0600000000000001,"5min":29.920000000000002,"15min":88.495999999999995},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0629999999999999,"15min":1.2849999999999999},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.24299999999999999},"max":{"1min":5.7960000000000003,"5min":24.965,"15min":93.816000000000003},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.2330000000000001,"15min":1.5429999999999999},"min":{"1min":0.32900000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":9.0760000000000005,"5min":24.712,"15min":95.579999999999998},"last":0.379}]},{"osd":6,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":0.73299999999999998,"15min":0.753},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.181,"5min":5.343,"15min":9.0139999999999993},"last":0.37},{"interface":"front","average":{"1min":1.351,"5min":0.95899999999999996,"15min":0.99199999999999999},"min":{"1min":0.35899999999999999,"5min":0.33700000000000002,"15min":0.308},"max":{"1min":10.967000000000001,"5min":10.967000000000001,"15min":13.362},"last":0.40899999999999997}]}]},{"osd":6,"up_from":0,"seq":197568495889,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":98,"seq":420906795161,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2468700,"kb_used_data":2059284,"kb_used_omap":7,"kb_used_meta":409400,"kb_avail":91276452,"statfs":{"total":95995035648,"available":93467086848,"internally_reserved":0,"allocated":2108706816,"data_stored":2107303938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":419225972},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:25:21 2023","interfaces":[{"interface":"back","average":{"1min":13.973000000000001,"5min":4.867,"15min":2.383},"min":{"1min":0.20300000000000001,"5min":0.189,"15min":0.189},"max":{"1min":196.095,"5min":196.095,"15min":196.095},"last":2.6000000000000001},{"interface":"front","average":{"1min":14.218999999999999,"5min":5.5,"15min":2.903},"min":{"1min":0.24099999999999999,"5min":0.20999999999999999,"15min":0.20499999999999999},"max":{"1min":196.85300000000001,"5min":196.85300000000001,"15min":196.85300000000001},"last":6.1840000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:24:47 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":1.0649999999999999,"15min":1.284},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.83899999999999997,"5min":11.179,"15min":33.902000000000001},"last":2.5699999999999998},{"interface":"front","average":{"1min":3.101,"5min":2.2730000000000001,"15min":1.9059999999999999},"min":{"1min":0.27700000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.053000000000001,"5min":16.053000000000001,"15min":34.781999999999996},"last":6.1159999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:25:21 2023","interfaces":[{"interface":"back","average":{"1min":11.787000000000001,"5min":6.6379999999999999,"15min":3.1640000000000001},"min":{"1min":0.28100000000000003,"5min":0.27900000000000003,"15min":0.252},"max":{"1min":160.499,"5min":160.499,"15min":160.499},"last":6.1580000000000004},{"interface":"front","average":{"1min":11.837,"5min":6.5819999999999999,"15min":3.105},"min":{"1min":0.26200000000000001,"5min":0.223,"15min":0.223},"max":{"1min":160.465,"5min":160.465,"15min":160.465},"last":3.105}]},{"osd":4,"last update":"Sat Sep 23 22:25:21 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":4.484,"15min":2.0750000000000002},"min":{"1min":0.35099999999999998,"5min":0.27500000000000002,"15min":0.27000000000000002},"max":{"1min":2.6019999999999999,"5min":234.79900000000001,"15min":234.79900000000001},"last":2.3380000000000001},{"interface":"front","average":{"1min":0.81799999999999995,"5min":5.0229999999999997,"15min":2.4609999999999999},"min":{"1min":0.40600000000000003,"5min":0.35399999999999998,"15min":0.318},"max":{"1min":2.4249999999999998,"5min":234.81700000000001,"15min":234.81700000000001},"last":1.236}]},{"osd":5,"last update":"Sat Sep 23 22:25:21 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":3.3809999999999998,"15min":1.9059999999999999},"min":{"1min":0.39900000000000002,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":2.649,"5min":196.79900000000001,"15min":196.79900000000001},"last":1.2729999999999999},{"interface":"front","average":{"1min":0.96499999999999997,"5min":3.6120000000000001,"15min":1.946},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.29599999999999999},"max":{"1min":2.8690000000000002,"5min":196.68299999999999,"15min":196.68299999999999},"last":2.8919999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:25:21 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.8999999999999999,"15min":1.401},"min":{"1min":0.45700000000000002,"5min":0.36499999999999999,"15min":0.34100000000000003},"max":{"1min":2.9369999999999998,"5min":16.314,"15min":16.314},"last":3.1339999999999999},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.1779999999999999,"15min":1.0700000000000001},"min":{"1min":0.40899999999999997,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.8529999999999998,"5min":11.385999999999999,"15min":11.385999999999999},"last":2.4119999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:25:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":2.0910000000000002,"15min":1.508},"min":{"1min":0.44,"5min":0.35699999999999998,"15min":0.32900000000000001},"max":{"1min":2.9049999999999998,"5min":16.265000000000001,"15min":16.265000000000001},"last":2.9409999999999998},{"interface":"front","average":{"1min":0.69699999999999995,"5min":1.163,"15min":1.087},"min":{"1min":0.437,"5min":0.38300000000000001,"15min":0.35799999999999998},"max":{"1min":2.6509999999999998,"5min":12.045999999999999,"15min":12.045999999999999},"last":2.3820000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958695,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2107340,"kb_used_data":1737988,"kb_used_omap":8,"kb_used_meta":369335,"kb_avail":91637812,"statfs":{"total":95995035648,"available":93837119488,"internally_reserved":0,"allocated":1779699712,"data_stored":1778255402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":378199914},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,11],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:25:23 2023","interfaces":[{"interface":"back","average":{"1min":2.6499999999999999,"5min":4.907,"15min":2.6099999999999999},"min":{"1min":0.217,"5min":0.21299999999999999,"15min":0.20200000000000001},"max":{"1min":21.934000000000001,"5min":169.77199999999999,"15min":169.77199999999999},"last":2.6619999999999999},{"interface":"front","average":{"1min":2.1240000000000001,"5min":4.6920000000000002,"15min":2.863},"min":{"1min":0.24299999999999999,"5min":0.22,"15min":0.20300000000000001},"max":{"1min":21.516999999999999,"5min":169.52099999999999,"15min":169.52099999999999},"last":1.8620000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:24:54 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":5.6950000000000003,"15min":2.7200000000000002},"min":{"1min":0.26600000000000001,"5min":0.249,"15min":0.20799999999999999},"max":{"1min":5.6390000000000002,"5min":206.495,"15min":206.495},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.7330000000000001,"5min":6.101,"15min":3.3860000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.22800000000000001},"max":{"1min":9.0920000000000005,"5min":207.34399999999999,"15min":207.34399999999999},"last":4.9020000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:25:23 2023","interfaces":[{"interface":"back","average":{"1min":4.734,"5min":6.1070000000000002,"15min":3.04},"min":{"1min":0.314,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":68.638000000000005,"5min":146.87299999999999,"15min":146.87299999999999},"last":2.403},{"interface":"front","average":{"1min":5.5810000000000004,"5min":6.0419999999999998,"15min":3.1389999999999998},"min":{"1min":0.33500000000000002,"5min":0.247,"15min":0.23799999999999999},"max":{"1min":67.753,"5min":145.48099999999999,"15min":145.48099999999999},"last":2.774}]},{"osd":4,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":2.722,"15min":2.0089999999999999},"min":{"1min":0.40699999999999997,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":3.3679999999999999,"5min":82.825000000000003,"15min":82.825000000000003},"last":4.04},{"interface":"front","average":{"1min":1.0940000000000001,"5min":2.302,"15min":1.409},"min":{"1min":0.39700000000000002,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":4.798,"5min":75.930999999999997,"15min":75.930999999999997},"last":2.351}]},{"osd":5,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":3.8849999999999998,"15min":2.3959999999999999},"min":{"1min":0.378,"5min":0.36899999999999999,"15min":0.29499999999999998},"max":{"1min":3.4049999999999998,"5min":93.685000000000002,"15min":93.685000000000002},"last":4.1340000000000003},{"interface":"front","average":{"1min":1.1180000000000001,"5min":3.5739999999999998,"15min":1.8480000000000001},"min":{"1min":0.45300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.8310000000000004,"5min":93.546000000000006,"15min":93.546000000000006},"last":2.379}]},{"osd":6,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":0.95399999999999996,"5min":1.798,"15min":1.718},"min":{"1min":0.432,"5min":0.38400000000000001,"15min":0.379},"max":{"1min":3.4529999999999998,"5min":16.366,"15min":16.366},"last":4.1059999999999999},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.6759999999999999,"15min":1.256},"min":{"1min":0.47399999999999998,"5min":0.34999999999999998,"15min":0.33900000000000002},"max":{"1min":4.883,"5min":16.225999999999999,"15min":22.263000000000002},"last":2.4249999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:25:05 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":2.0470000000000002,"15min":1.768},"min":{"1min":0.39800000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.1440000000000001,"5min":32.773000000000003,"15min":32.773000000000003},"last":2.1219999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.929,"15min":1.9199999999999999},"min":{"1min":0.437,"5min":0.39900000000000002,"15min":0.38900000000000001},"max":{"1min":3.4279999999999999,"5min":16.381,"15min":23.265999999999998},"last":4.0750000000000002}]}]},{"osd":2,"up_from":0,"seq":442381631634,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":108,"seq":463856468108,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2796816,"kb_used_data":2439816,"kb_used_omap":4,"kb_used_meta":356987,"kb_avail":90948336,"statfs":{"total":95995035648,"available":93131096064,"internally_reserved":0,"allocated":2498371584,"data_stored":2496917475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":365555245},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,5],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":5.0609999999999999,"15min":3.0230000000000001},"min":{"1min":0.22900000000000001,"5min":0.20200000000000001,"15min":0.19900000000000001},"max":{"1min":11.484999999999999,"5min":225.86699999999999,"15min":225.86699999999999},"last":8.0830000000000002},{"interface":"front","average":{"1min":1.1080000000000001,"5min":5.3840000000000003,"15min":3.2519999999999998},"min":{"1min":0.29399999999999998,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":5.7590000000000003,"5min":225.84800000000001,"15min":225.84800000000001},"last":8.3900000000000006}]},{"osd":1,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":2.835,"15min":2.464},"min":{"1min":0.247,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":6.5069999999999997,"5min":37.002000000000002,"15min":82.951999999999998},"last":6.3250000000000002},{"interface":"front","average":{"1min":1.3999999999999999,"5min":3.0270000000000001,"15min":2.4790000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.189},"max":{"1min":11.397,"5min":33.119999999999997,"15min":86.001000000000005},"last":8.032}]},{"osd":2,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":2.1549999999999998,"15min":3.609},"min":{"1min":0.28699999999999998,"5min":0.252,"15min":0.223},"max":{"1min":5.9740000000000002,"5min":29.076000000000001,"15min":99.962999999999994},"last":3.8719999999999999},{"interface":"front","average":{"1min":1.2709999999999999,"5min":2.2909999999999999,"15min":3.4820000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.251},"max":{"1min":11.362,"5min":13.692,"15min":100.429},"last":4.6639999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:25:23 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.617,"15min":1.2809999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.28499999999999998},"max":{"1min":2.109,"5min":7.5620000000000003,"15min":11.911},"last":4.6479999999999997},{"interface":"front","average":{"1min":0.77700000000000002,"5min":1.6970000000000001,"15min":1.3540000000000001},"min":{"1min":0.35799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.9039999999999999,"5min":10.949999999999999,"15min":12.036},"last":5.1589999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.861,"15min":1.548},"min":{"1min":0.315,"5min":0.309,"15min":0.309},"max":{"1min":3.9529999999999998,"5min":11.028,"15min":49.695999999999998},"last":5.2000000000000002},{"interface":"front","average":{"1min":1.3779999999999999,"5min":1.7130000000000001,"15min":1.5569999999999999},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":6.0179999999999998,"5min":13.494,"15min":49.668999999999997},"last":3.6160000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":0.79900000000000004,"5min":1.98,"15min":1.492},"min":{"1min":0.38800000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.0309999999999997,"5min":10.992000000000001,"15min":12.068},"last":5.2489999999999997},{"interface":"front","average":{"1min":1.444,"5min":1.8320000000000001,"15min":1.427},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.35399999999999998},"max":{"1min":6.0919999999999996,"5min":13.425000000000001,"15min":13.425000000000001},"last":3.6619999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:25:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":1.97,"15min":1.502},"min":{"1min":0.41699999999999998,"5min":0.38500000000000001,"15min":0.38100000000000001},"max":{"1min":6.0549999999999997,"5min":16.849,"15min":16.849},"last":3.6949999999999998},{"interface":"front","average":{"1min":0.878,"5min":1.7569999999999999,"15min":1.3979999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.36699999999999999},"max":{"1min":3.2429999999999999,"5min":8.1400000000000006,"15min":12.382},"last":4.6189999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019158,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1447116,"kb_used_data":1227588,"kb_used_omap":0,"kb_used_meta":219520,"kb_avail":92298036,"statfs":{"total":95995035648,"available":94513188864,"internally_reserved":0,"allocated":1257050112,"data_stored":1256013575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":224788480},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:25:11 2023","interfaces":[{"interface":"back","average":{"1min":8.3260000000000005,"5min":3.992,"15min":2.367},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.218},"max":{"1min":150.63499999999999,"5min":150.63499999999999,"15min":150.63499999999999},"last":0.218},{"interface":"front","average":{"1min":8.0739999999999998,"5min":3.8919999999999999,"15min":2.4159999999999999},"min":{"1min":0.32000000000000001,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":150.672,"5min":150.672,"15min":150.672},"last":0.25700000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:25:11 2023","interfaces":[{"interface":"back","average":{"1min":6.2930000000000001,"5min":3.4239999999999999,"15min":2.0529999999999999},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.21299999999999999},"max":{"1min":107.346,"5min":107.346,"15min":107.346},"last":0.76300000000000001},{"interface":"front","average":{"1min":6.2409999999999997,"5min":3.7330000000000001,"15min":2.4060000000000001},"min":{"1min":0.27200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":107.33499999999999,"5min":107.33499999999999,"15min":107.33499999999999},"last":2.794}]},{"osd":2,"last update":"Sat Sep 23 22:24:47 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.8069999999999999,"15min":2.351},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.255},"max":{"1min":6.0170000000000003,"5min":12.397,"15min":138.06100000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.8300000000000001,"5min":1.5409999999999999,"15min":2.5209999999999999},"min":{"1min":0.34399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.8639999999999999,"5min":6.9580000000000002,"15min":138.15199999999999},"last":0.49399999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.5269999999999999,"15min":1.306},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30399999999999999},"max":{"1min":4.0780000000000003,"5min":12.129,"15min":12.129},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.755,"5min":1.714,"15min":1.359},"min":{"1min":0.28899999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.78,"5min":8.5600000000000005,"15min":9.8109999999999999},"last":2.073}]},{"osd":5,"last update":"Sat Sep 23 22:25:22 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.1659999999999999,"15min":0.91300000000000003},"min":{"1min":0.23999999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.946,"5min":28.399000000000001,"15min":28.399000000000001},"last":0.312},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.3939999999999999,"15min":0.95499999999999996},"min":{"1min":0.318,"5min":0.29799999999999999,"15min":0.27900000000000003},"max":{"1min":1.5860000000000001,"5min":28.350999999999999,"15min":28.350999999999999},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:24:37 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":0.98299999999999998,"15min":0.77600000000000002},"min":{"1min":0.33800000000000002,"5min":0.27500000000000002,"15min":0.251},"max":{"1min":7.8550000000000004,"5min":7.8550000000000004,"15min":7.8550000000000004},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.81599999999999995,"15min":0.77500000000000002},"min":{"1min":0.318,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.802,"5min":3.9239999999999999,"15min":8.5210000000000008},"last":0.34699999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:25:11 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.83399999999999996,"15min":0.88800000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":2.827,"5min":5.2590000000000003,"15min":8.577},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.91300000000000003,"5min":1.01,"15min":0.86099999999999999},"min":{"1min":0.41299999999999998,"5min":0.312,"15min":0.28699999999999998},"max":{"1min":4.9740000000000002,"5min":7.8109999999999999,"15min":7.8109999999999999},"last":0.72899999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757526,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1107004,"kb_used_data":838388,"kb_used_omap":0,"kb_used_meta":268608,"kb_avail":92638148,"statfs":{"total":95995035648,"available":94861463552,"internally_reserved":0,"allocated":858509312,"data_stored":857484368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":275054592},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:24:59 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":2.2269999999999999,"15min":2.1859999999999999},"min":{"1min":0.23400000000000001,"5min":0.20799999999999999,"15min":0.20499999999999999},"max":{"1min":3.306,"5min":64.344999999999999,"15min":133.22499999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.5609999999999999,"5min":2.1659999999999999,"15min":2.242},"min":{"1min":0.248,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.141,"5min":64.022000000000006,"15min":132.78200000000001},"last":0.32800000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:25:22 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":2.867,"15min":2.758},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":2.153,"5min":93.981999999999999,"15min":93.981999999999999},"last":1.708},{"interface":"front","average":{"1min":0.64000000000000001,"5min":2.8879999999999999,"15min":2.7429999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.252},"max":{"1min":2.1669999999999998,"5min":93.909999999999997,"15min":93.909999999999997},"last":1.72}]},{"osd":2,"last update":"Sat Sep 23 22:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":2.4260000000000002,"15min":3.536},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.702,"5min":95.790999999999997,"15min":104.254},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.365,"5min":2.25,"15min":3.806},"min":{"1min":0.33600000000000002,"5min":0.32000000000000001,"15min":0.26900000000000002},"max":{"1min":8.1010000000000009,"5min":95.366,"15min":104.217},"last":2.4449999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:25:22 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":3.4649999999999999,"15min":2.3359999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.246},"max":{"1min":1.7549999999999999,"5min":93.739999999999995,"15min":93.739999999999995},"last":1.1799999999999999},{"interface":"front","average":{"1min":0.73199999999999998,"5min":3.044,"15min":2.2690000000000001},"min":{"1min":0.376,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":1.8380000000000001,"5min":93.197999999999993,"15min":93.197999999999993},"last":0.89600000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:25:19 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":0.97699999999999998,"15min":1.0409999999999999},"min":{"1min":0.28299999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.54200000000000004,"5min":10.163,"15min":82.530000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.45100000000000001,"5min":0.91700000000000004,"15min":1.0840000000000001},"min":{"1min":0.28799999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.79300000000000004,"5min":7.2119999999999997,"15min":85.441000000000003},"last":0.36099999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:24:40 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":1.179},"min":{"1min":0.34699999999999998,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":2.6040000000000001,"5min":7.0289999999999999,"15min":114.83499999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.83599999999999997,"5min":0.72499999999999998,"15min":1.0880000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.7130000000000001,"5min":3.8690000000000002,"15min":110.905},"last":0.41199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:24:59 2023","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.82799999999999996,"15min":0.90700000000000003},"min":{"1min":0.36399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.0819999999999999,"5min":5.0819999999999999,"15min":8.8030000000000008},"last":0.38},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.77600000000000002,"15min":0.77800000000000002},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":3.6299999999999999,"5min":6.4550000000000001,"15min":7.6609999999999996},"last":0.505}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1804533760,"data_stored":1804533760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2106064896,"data_stored":2106064896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2528378880,"data_stored":2528378880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1274740736,"data_stored":1274740736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":855310336,"data_stored":855310336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1487208448,"data_stored":1487208448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:25:34.118 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 11 2023-09-23T22:25:34.122 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:34.174 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:34.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:34.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:34.278 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 56244 56228 90.0532 64.9375 0.0180284 0.0110982 2023-09-23T22:25:34.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:34.317 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:34.368 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:34.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:34.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:34.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:34.575 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:35.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:34 smithi026 ceph-mon[215775]: pgmap v688: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T22:25:35.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:34 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2070330760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:34 smithi026 ceph-mon[223014]: pgmap v688: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T22:25:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:34 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2070330760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:35.278 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:25:35.278906+0000 min lat: 0.00118556 max lat: 0.332866 avg lat: 0.0111903 2023-09-23T22:25:35.278 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:25:35.278 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 57213 57197 89.3162 60.5625 0.00935924 0.0111903 2023-09-23T22:25:35.294 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:25:35.294 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:25:35.294 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:25:35.294 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:25:35.295 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:25:35.295 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:25:35.295 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:25:35.295 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:25:35.295 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:25:35.295 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:25:35.295 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:25:35.295 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:25:35.295 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:25:35.296 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:25:35.296 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:25:35.296 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:25:35.306 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:35.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:35.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:34 smithi070 ceph-mon[192652]: pgmap v688: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T22:25:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:34 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2070330760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:35.413 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:35.444 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:35.496 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:35.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:35.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:35.601 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:35.671 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:35.947 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:35.977 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:35.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:35.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:36.091 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:36.169 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:36.279 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 58317 58301 88.8208 69 0.0143179 0.011253 2023-09-23T22:25:36.567 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:36.741 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:36.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:36.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:36.848 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:36.937 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:37.039 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:36 smithi026 ceph-mon[215775]: pgmap v689: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:25:37.040 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:36 smithi026 ceph-mon[223014]: pgmap v689: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:25:37.052 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:37.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:37.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:37.158 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:37.187 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:37.241 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:37.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:37.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:37.279 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 59432 59416 88.3652 69.6875 0.0189781 0.0113119 2023-09-23T22:25:37.346 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:37.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:36 smithi070 ceph-mon[192652]: pgmap v689: 99 pgs: 99 active+clean; 3.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:25:37.424 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:37.567 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:25:37.567 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:25:37.846 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:37.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:37.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:37.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:38.051 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:38.280 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 60336 60320 87.6243 56.5 0.00539601 0.0113864 2023-09-23T22:25:38.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:25:39.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:39 smithi026 ceph-mon[223014]: pgmap v690: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:25:39.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:39 smithi026 ceph-mon[215775]: pgmap v690: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:25:39.279 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 61215 61199 86.8817 54.9375 0.0119969 0.0115037 2023-09-23T22:25:39.317 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:39.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:39.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:39 smithi070 ceph-mon[192652]: pgmap v690: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T22:25:39.425 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:39.498 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:39.624 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:25:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:25:39] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-09-23T22:25:40.007 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:40.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:40.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:40.112 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:40.145 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:40.252 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:40.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:40.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:40.279 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 61994 61978 86.0331 48.6875 0.00766831 0.011582 2023-09-23T22:25:40.355 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:40 smithi070 ceph-mon[192652]: pgmap v691: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T22:25:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:40.414 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:40 smithi026 ceph-mon[215775]: pgmap v691: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T22:25:40.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:40.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:40.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:40 smithi026 ceph-mon[223014]: pgmap v691: 99 pgs: 99 active+clean; 3.5 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T22:25:40.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:25:40.423 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:40.479 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:25:40.744 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:40.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:40.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:40.851 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:40.885 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:40.943 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:40.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:41.047 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:41.070 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:41.166 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:41.284 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 62749 62733 85.1786 47.1875 0.0203481 0.0117327 2023-09-23T22:25:41.328 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:41.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:41.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:41.434 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:41.729 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:41.833 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:41.881 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:41.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:41.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:42.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:42.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:42.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 63621 63605 84.5261 54.5 0.0309865 0.0118246 2023-09-23T22:25:42.311 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:42.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:42.317 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:42.415 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:25:42.416 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pgp_num to 11 2023-09-23T22:25:42.418 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:42.486 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:42.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:42 smithi026 ceph-mon[223014]: pgmap v692: 99 pgs: 99 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 874 op/s 2023-09-23T22:25:42.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3369913495' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:25:42.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:42 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:25:42.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3494452938' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T22:25:42.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:42 smithi026 ceph-mon[215775]: pgmap v692: 99 pgs: 99 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 874 op/s 2023-09-23T22:25:42.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3369913495' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:25:42.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:42 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:25:42.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3494452938' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T22:25:42.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:42 smithi070 ceph-mon[192652]: pgmap v692: 99 pgs: 99 active+clean; 3.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 874 op/s 2023-09-23T22:25:42.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3369913495' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:25:42.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:42 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:25:42.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3494452938' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]: dispatch 2023-09-23T22:25:43.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 64467 64451 83.867 52.875 0.0050927 0.0118862 2023-09-23T22:25:43.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:43.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[215775]: failed to encode map e268 with expected crc 2023-09-23T22:25:43.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:25:43.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3494452938' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T22:25:43.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[215775]: osdmap e268: 8 total, 8 up, 6 in 2023-09-23T22:25:43.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[215775]: failed to encode map e268 with expected crc 2023-09-23T22:25:43.701 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[215775]: failed to encode map e268 with expected crc 2023-09-23T22:25:43.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:43.702 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:43.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:43.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[223014]: failed to encode map e268 with expected crc 2023-09-23T22:25:43.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:25:43.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3494452938' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T22:25:43.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[223014]: osdmap e268: 8 total, 8 up, 6 in 2023-09-23T22:25:43.702 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[223014]: failed to encode map e268 with expected crc 2023-09-23T22:25:43.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[223014]: failed to encode map e268 with expected crc 2023-09-23T22:25:43.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:43.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:43 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:43 smithi070 ceph-mon[192652]: failed to encode map e268 with expected crc 2023-09-23T22:25:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:43 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:25:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3494452938' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2023-09-23T22:25:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:43 smithi070 ceph-mon[192652]: osdmap e268: 8 total, 8 up, 6 in 2023-09-23T22:25:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:43 smithi070 ceph-mon[192652]: failed to encode map e268 with expected crc 2023-09-23T22:25:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:43 smithi070 ceph-mon[192652]: failed to encode map e268 with expected crc 2023-09-23T22:25:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:44.286 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 65264 65248 83.1723 49.8125 0.0256891 0.0120176 2023-09-23T22:25:44.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:44 smithi026 ceph-mon[215775]: pgmap v694: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T22:25:44.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:44 smithi026 ceph-mon[223014]: pgmap v694: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T22:25:44.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:44 smithi070 ceph-mon[192652]: pgmap v694: 99 pgs: 99 active+clean; 3.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 60 MiB/s wr, 966 op/s 2023-09-23T22:25:45.206 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:45.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:45.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:45.285 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 66184 66168 82.659 57.5 0.0128209 0.0120928 2023-09-23T22:25:45.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:45.402 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:45.759 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:45.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:45.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:45.864 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:45.894 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:45.941 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:45.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:45.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:45.970 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:25:45.970 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:25:46.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:46.136 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:46.190 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:25:46.231 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:25:46.260 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:46.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:46.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:46.279 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:25:46.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 67020 67004 82.0506 52.25 0.00901922 0.0121817 2023-09-23T22:25:46.320 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:25:46.355 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:25:46.369 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:46.389 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:25:46.431 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:25:46.439 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:46.471 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:25:46.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:25:46.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:25:46.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:25:46.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:25:46.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:25:46.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:25:46.483 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:25:46.484 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:25:46.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:46 smithi026 ceph-mon[215775]: pgmap v695: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T22:25:46.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2296317847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:46.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:46 smithi026 ceph-mon[223014]: pgmap v695: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T22:25:46.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2296317847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:46 smithi070 ceph-mon[192652]: pgmap v695: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T22:25:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2296317847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:47.175 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:47.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:47.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:47.287 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:47.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 67936 67920 81.5738 57.25 0.018814 0.0122534 2023-09-23T22:25:47.317 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:47.363 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:47.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:47.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:47.415 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:25:47.467 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:47.497 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:47.553 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:47.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:47.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:47.663 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:47.690 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:47.745 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:47.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:47.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:47.908 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:25:47.949 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:48.180 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:48.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:48.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:48.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 68732 68716 80.9737 49.75 0.0253813 0.0123427 2023-09-23T22:25:48.296 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:48.325 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:48.326 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:48.373 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:48.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:48.479 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:48.507 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:48.559 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:48.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:48.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:48.698 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:48.753 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:48.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:48.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:48.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:48.932 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:48.994 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:48 smithi026 ceph-mon[215775]: pgmap v696: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-09-23T22:25:48.995 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:48 smithi026 ceph-mon[223014]: pgmap v696: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-09-23T22:25:49.202 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:49.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 69618 69602 80.4999 55.375 0.0173423 0.012417 2023-09-23T22:25:49.307 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:49.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:49.317 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:49.354 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 7, 0] out_osds: [6, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:25:49.355 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:25:49.355 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T22:25:49.355 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:25:49.355 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:25:49.355 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T22:25:49.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:48 smithi070 ceph-mon[192652]: pgmap v696: 99 pgs: 99 active+clean; 4.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 55 MiB/s wr, 878 op/s 2023-09-23T22:25:49.418 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:49.515 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:49.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:25:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:25:49] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-09-23T22:25:49.789 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:25:49.963 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:25:50.161 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:50.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:50.281 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:50.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 70566 70550 80.1136 59.25 0.0170287 0.0124776 2023-09-23T22:25:50.319 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:50.423 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:50.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:50.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:50.527 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:50.627 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:50.950 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:50.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:50.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:51.060 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:51.093 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:51.151 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:51.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:51.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:51.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:50 smithi026 ceph-mon[215775]: pgmap v697: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 873 op/s 2023-09-23T22:25:51.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:50 smithi026 ceph-mon[223014]: pgmap v697: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 873 op/s 2023-09-23T22:25:51.254 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:51.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 71428 71412 79.6453 53.875 0.0203929 0.0125499 2023-09-23T22:25:51.336 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:50 smithi070 ceph-mon[192652]: pgmap v697: 99 pgs: 99 active+clean; 4.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 55 MiB/s wr, 873 op/s 2023-09-23T22:25:52.070 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:52.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:52.177 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:25:52.271 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:52.285 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:52.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 72305 72289 79.2098 54.8125 0.00831924 0.0126106 2023-09-23T22:25:52.554 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:25:52.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:52.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:25:52.682 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:52.769 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:52.911 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:25:52.911 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79470 5.c 5:3839dd7e:::benchmark_data_smithi026_286012_object79469:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.138645+0000", 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.76881233100005, 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0226527860000001, 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.912 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79470 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138645+0000", 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138645+0000", 2023-09-23T22:25:52.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138643+0000", 2023-09-23T22:25:52.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:52.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138676+0000", 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3077000000000002e-05 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138690+0000", 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4126e-05 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138699+0000", 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3089999999999996e-06 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138791+0000", 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2932999999999999e-05 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.138804+0000", 2023-09-23T22:25:52.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2682e-05 2023-09-23T22:25:52.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.154254+0000", 2023-09-23T22:25:52.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0154501869999999 2023-09-23T22:25:52.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.154453+0000", 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000198934 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.154665+0000", 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000211843 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.157899+0000", 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032335440000000001 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.157942+0000", 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3003999999999997e-05 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161177+0000", 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032349140000000002 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161245+0000", 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8717e-05 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:52.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161298+0000", 2023-09-23T22:25:52.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2527e-05 2023-09-23T22:25:52.921 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.921 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:52.921 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.921 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79490 5.c 5:31f4c6c3:::benchmark_data_smithi026_286012_object79489:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.166333+0000", 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.74112391100005, 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.00561488, 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79490 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.922 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166333+0000", 2023-09-23T22:25:52.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166333+0000", 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166328+0000", 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999948 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166359+0000", 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1356999999999998e-05 2023-09-23T22:25:52.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166372+0000", 2023-09-23T22:25:52.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.243e-05 2023-09-23T22:25:52.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166380+0000", 2023-09-23T22:25:52.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9060000000000002e-06 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166455+0000", 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4969000000000007e-05 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166467+0000", 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2673e-05 2023-09-23T22:25:52.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.158005+0000", 2023-09-23T22:25:52.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99153777799999998 2023-09-23T22:25:52.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:52.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.158162+0000", 2023-09-23T22:25:52.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015700600000000001 2023-09-23T22:25:52.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:52.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.158350+0000", 2023-09-23T22:25:52.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018749700000000001 2023-09-23T22:25:52.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166611+0000", 2023-09-23T22:25:52.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082610119999999999 2023-09-23T22:25:52.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166652+0000", 2023-09-23T22:25:52.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1117e-05 2023-09-23T22:25:52.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:52.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.171851+0000", 2023-09-23T22:25:52.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051991600000000004 2023-09-23T22:25:52.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:52.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.171904+0000", 2023-09-23T22:25:52.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3298999999999999e-05 2023-09-23T22:25:52.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:52.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.171948+0000", 2023-09-23T22:25:52.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4152e-05 2023-09-23T22:25:52.931 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.931 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:52.932 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.932 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79491 5.c 5:39058f9e:::benchmark_data_smithi026_286012_object79490:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:25:52.932 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.166398+0000", 2023-09-23T22:25:52.932 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.74105884300002, 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.005660617, 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79491 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166398+0000", 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166398+0000", 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166398+0000", 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166429+0000", 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0815000000000002e-05 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166436+0000", 2023-09-23T22:25:52.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6059999999999997e-06 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166443+0000", 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3400000000000003e-06 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166497+0000", 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.393e-05 2023-09-23T22:25:52.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166505+0000", 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2509999999999994e-06 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161362+0000", 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99485717100000004 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161527+0000", 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000164599 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.161687+0000", 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016049200000000001 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166685+0000", 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004998382 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166760+0000", 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4945999999999997e-05 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.171964+0000", 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005203648 2023-09-23T22:25:52.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172020+0000", 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5829999999999999e-05 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172059+0000", 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9208999999999997e-05 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.940 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79492 5.c 5:39c85251:::benchmark_data_smithi026_286012_object79491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.166450+0000", 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.74100778900004, 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0058361490000001, 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.941 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79492 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166450+0000", 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166450+0000", 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166449+0000", 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166471+0000", 2023-09-23T22:25:52.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1767999999999999e-05 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166479+0000", 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1340000000000004e-06 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166484+0000", 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9679999999999997e-06 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166528+0000", 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3801000000000003e-05 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.166535+0000", 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3189999999999997e-06 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.164034+0000", 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99749939399999998 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.164170+0000", 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013584400000000001 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.164260+0000", 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9506999999999995e-05 2023-09-23T22:25:52.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172071+0000", 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078108199999999996 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172142+0000", 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1128000000000001e-05 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172198+0000", 2023-09-23T22:25:52.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6042999999999999e-05 2023-09-23T22:25:52.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:52.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172247+0000", 2023-09-23T22:25:52.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8841000000000002e-05 2023-09-23T22:25:52.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172286+0000", 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9174999999999997e-05 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79500 5.c 5:3178a3e0:::benchmark_data_smithi026_286012_object79499:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.178188+0000", 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.729269026, 2023-09-23T22:25:52.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.994386043, 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79500 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178188+0000", 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178188+0000", 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178188+0000", 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178210+0000", 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2399e-05 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178223+0000", 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3098999999999999e-05 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178230+0000", 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7700000000000004e-06 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178288+0000", 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8130000000000001e-05 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.178298+0000", 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0142999999999999e-05 2023-09-23T22:25:52.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166799+0000", 2023-09-23T22:25:52.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98850079499999999 2023-09-23T22:25:52.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:52.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.166918+0000", 2023-09-23T22:25:52.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011872200000000001 2023-09-23T22:25:52.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:52.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.167008+0000", 2023-09-23T22:25:52.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0263e-05 2023-09-23T22:25:52.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172304+0000", 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052957619999999999 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172371+0000", 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6679000000000003e-05 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172472+0000", 2023-09-23T22:25:52.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101407 2023-09-23T22:25:52.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:52.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172530+0000", 2023-09-23T22:25:52.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8408e-05 2023-09-23T22:25:52.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172574+0000", 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4017999999999998e-05 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79525 5.c 5:3c92c974:::benchmark_data_smithi026_286012_object79524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.238208+0000", 2023-09-23T22:25:52.959 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.66924926199999, 2023-09-23T22:25:52.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93452411499999999, 2023-09-23T22:25:52.960 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.960 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.960 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.960 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.960 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.960 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79525 2023-09-23T22:25:52.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.960 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238208+0000", 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238208+0000", 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238206+0000", 2023-09-23T22:25:52.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:52.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238237+0000", 2023-09-23T22:25:52.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1111000000000001e-05 2023-09-23T22:25:52.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238253+0000", 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5276000000000001e-05 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238262+0000", 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5750000000000007e-06 2023-09-23T22:25:52.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238329+0000", 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6920000000000003e-05 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238340+0000", 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0890000000000001e-05 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.169053+0000", 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93071300800000001 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.169154+0000", 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100717 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.169255+0000", 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101178 2023-09-23T22:25:52.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172312+0000", 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003057073 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172422+0000", 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109694 2023-09-23T22:25:52.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172634+0000", 2023-09-23T22:25:52.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021268400000000001 2023-09-23T22:25:52.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:52.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172681+0000", 2023-09-23T22:25:52.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6448e-05 2023-09-23T22:25:52.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:52.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172732+0000", 2023-09-23T22:25:52.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1388000000000003e-05 2023-09-23T22:25:52.968 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.968 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:52.968 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.969 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79531 5.c 5:32793f2d:::benchmark_data_smithi026_286012_object79530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:25:52.969 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.238283+0000", 2023-09-23T22:25:52.969 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.66917397300006, 2023-09-23T22:25:52.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95936797200000001, 2023-09-23T22:25:52.969 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.969 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.970 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79531 2023-09-23T22:25:52.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.970 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238283+0000", 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238283+0000", 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238283+0000", 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238306+0000", 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3264999999999998e-05 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238314+0000", 2023-09-23T22:25:52.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0399999999999993e-06 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238320+0000", 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0499999999999997e-06 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238371+0000", 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0946000000000003e-05 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.238381+0000", 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4390000000000006e-06 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172794+0000", 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93441326899999999 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.172986+0000", 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019185200000000001 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.173147+0000", 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016168399999999999 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.181599+0000", 2023-09-23T22:25:52.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084517519999999999 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.186964+0000", 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053651810000000001 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197540+0000", 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010576189999999999 2023-09-23T22:25:52.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197597+0000", 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6412000000000001e-05 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197651+0000", 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4431000000000001e-05 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:52.977 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79556 5.c 5:3b073673:::benchmark_data_smithi026_286012_object79555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.268850+0000", 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.63860700199996, 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.92890088599999998, 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.978 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79556 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268850+0000", 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268850+0000", 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268849+0000", 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268877+0000", 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7911000000000001e-05 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268892+0000", 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5311999999999998e-05 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268901+0000", 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5820000000000008e-06 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268966+0000", 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4175000000000001e-05 2023-09-23T22:25:52.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.268977+0000", 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1469e-05 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.175810+0000", 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90683305700000005 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.175941+0000", 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013063699999999999 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.176087+0000", 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014580499999999999 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.186996+0000", 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010908951 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197532+0000", 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010536538999999999 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197660+0000", 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012830099999999999 2023-09-23T22:25:52.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197703+0000", 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2586000000000002e-05 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197751+0000", 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8229999999999997e-05 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.985 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79578 5.c 5:30fa6cc1:::benchmark_data_smithi026_286012_object79577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.304544+0000", 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.60291354100002, 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95892923500000005, 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.986 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79578 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304544+0000", 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304544+0000", 2023-09-23T22:25:52.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304542+0000", 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304573+0000", 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1109000000000003e-05 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304595+0000", 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1950000000000002e-05 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304607+0000", 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1562e-05 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304702+0000", 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5725999999999998e-05 2023-09-23T22:25:52.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.304734+0000", 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1959999999999999e-05 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.184170+0000", 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87943575799999996 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.184302+0000", 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013180000000000001 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.184454+0000", 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000152077 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:52.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197767+0000", 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01331267 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.198028+0000", 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026184100000000002 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:52.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263385+0000", 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065356763999999998 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263432+0000", 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7209000000000002e-05 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263473+0000", 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0596999999999998e-05 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.993 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:52.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:52.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.994 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:79595 5.c 5:344ddd26:::benchmark_data_smithi026_286012_object79594:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T22:25:52.994 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:16:29.326572+0000", 2023-09-23T22:25:52.994 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 563.58088497000006, 2023-09-23T22:25:52.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.936993611, 2023-09-23T22:25:52.994 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79595 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326572+0000", 2023-09-23T22:25:52.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326572+0000", 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326570+0000", 2023-09-23T22:25:52.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326595+0000", 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4989000000000001e-05 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326616+0000", 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1123000000000001e-05 2023-09-23T22:25:52.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:52.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326625+0000", 2023-09-23T22:25:52.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7860000000000002e-06 2023-09-23T22:25:52.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326693+0000", 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8065000000000006e-05 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:29.326704+0000", 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0934000000000001e-05 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:52.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.187046+0000", 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.86034200699999996 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.187172+0000", 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125888 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:25:53.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.187317+0000", 2023-09-23T22:25:53.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014485999999999999 2023-09-23T22:25:53.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.197772+0000", 2023-09-23T22:25:53.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01045455 2023-09-23T22:25:53.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.198056+0000", 2023-09-23T22:25:53.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028478999999999999 2023-09-23T22:25:53.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263505+0000", 2023-09-23T22:25:53.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065448610000000004 2023-09-23T22:25:53.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263538+0000", 2023-09-23T22:25:53.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3037000000000002e-05 2023-09-23T22:25:53.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:16:30.263566+0000", 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7962000000000001e-05 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:51746 7.d 7:bedd80da:::benchmark_data_smithi026_410749_object51745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2023-09-23T22:25:53.004 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:29.622129+0000", 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 23.285327978000002, 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12497224899999999, 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 51746 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.622129+0000", 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.622129+0000", 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.622127+0000", 2023-09-23T22:25:53.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.622156+0000", 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9369999999999998e-05 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.622176+0000", 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9131e-05 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.622186+0000", 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0134e-05 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.622231+0000", 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4749000000000001e-05 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.622343+0000", 2023-09-23T22:25:53.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011230099999999999 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.622439+0000", 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6515999999999995e-05 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.652238+0000", 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029798195 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.652385+0000", 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147517 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.746998+0000", 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.094612552000000003 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.747060+0000", 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2261999999999994e-05 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:29.747102+0000", 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1749999999999998e-05 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.011 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:52357 7.d 7:b96dd69f:::benchmark_data_smithi026_410749_object52356:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:30.236098+0000", 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.67135914, 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12816582600000001, 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52357 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.236098+0000", 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.236098+0000", 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.236097+0000", 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:53.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.236121+0000", 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4278999999999999e-05 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.236134+0000", 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2734999999999999e-05 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.236142+0000", 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5459999999999992e-06 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.236184+0000", 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2092000000000003e-05 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.236309+0000", 2023-09-23T22:25:53.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012440800000000001 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.236421+0000", 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112716 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240093+0000", 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036718760000000001 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240113+0000", 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9650999999999999e-05 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.364173+0000", 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.124059711 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.364224+0000", 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1576999999999998e-05 2023-09-23T22:25:53.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.364264+0000", 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9768000000000003e-05 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:52358 7.d 7:be4a1048:::benchmark_data_smithi026_410749_object52357:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-09-23T22:25:53.018 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:30.238346+0000", 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.669111058999999, 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12950782399999999, 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52358 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.238346+0000", 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.238346+0000", 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.238343+0000", 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999971 2023-09-23T22:25:53.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.238372+0000", 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8104000000000001e-05 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.238384+0000", 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2303999999999999e-05 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.238392+0000", 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5739999999999996e-06 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.238443+0000", 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0136000000000002e-05 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.238593+0000", 2023-09-23T22:25:53.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015020699999999999 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.238723+0000", 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012975000000000001 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240231+0000", 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001508702 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.365485+0000", 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12525360499999999 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.367767+0000", 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00228246 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.367813+0000", 2023-09-23T22:25:53.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6019000000000001e-05 2023-09-23T22:25:53.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.367854+0000", 2023-09-23T22:25:53.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0745999999999999e-05 2023-09-23T22:25:53.025 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.025 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:52361 7.d 7:be557e24:::benchmark_data_smithi026_410749_object52360:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:30.240212+0000", 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.667245433000001, 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.156844607, 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.026 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.027 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 52361 2023-09-23T22:25:53.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.027 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240212+0000", 2023-09-23T22:25:53.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240212+0000", 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240211+0000", 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240232+0000", 2023-09-23T22:25:53.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1596000000000001e-05 2023-09-23T22:25:53.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240241+0000", 2023-09-23T22:25:53.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2940000000000006e-06 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240246+0000", 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9699999999999998e-06 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240283+0000", 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7058999999999999e-05 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240364+0000", 2023-09-23T22:25:53.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1754000000000004e-05 2023-09-23T22:25:53.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:25:53.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.240454+0000", 2023-09-23T22:25:53.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9970999999999996e-05 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.241630+0000", 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011756830000000001 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.367740+0000", 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12610959899999999 2023-09-23T22:25:53.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.396965+0000", 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029224868000000001 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.397018+0000", 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3081999999999998e-05 2023-09-23T22:25:53.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:30.397057+0000", 2023-09-23T22:25:53.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8816999999999998e-05 2023-09-23T22:25:53.034 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.034 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.034 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:55122 7.d 7:b09aa95f:::benchmark_data_smithi026_410749_object55121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:33.001045+0000", 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 19.906412641999999, 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.184411096, 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 55122 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.035 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.001045+0000", 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.001045+0000", 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.001043+0000", 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.001063+0000", 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0142000000000001e-05 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.001080+0000", 2023-09-23T22:25:53.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6399e-05 2023-09-23T22:25:53.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.001090+0000", 2023-09-23T22:25:53.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0637e-05 2023-09-23T22:25:53.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.001130+0000", 2023-09-23T22:25:53.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9592000000000003e-05 2023-09-23T22:25:53.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.001247+0000", 2023-09-23T22:25:53.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116725 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.001348+0000", 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101665 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.003522+0000", 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021734689999999999 2023-09-23T22:25:53.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.003640+0000", 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011843799999999999 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.185366+0000", 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.18172523199999999 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.185419+0000", 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3239000000000003e-05 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.185456+0000", 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6940000000000002e-05 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:55125 7.d 7:be65b5ac:::benchmark_data_smithi026_410749_object55124:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:33.004655+0000", 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 19.902802003000001, 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.18102042400000001, 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 55125 2023-09-23T22:25:53.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.004655+0000", 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.004655+0000", 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.004654+0000", 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.004673+0000", 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8700000000000001e-05 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.004685+0000", 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2288e-05 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.004691+0000", 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0000000000000002e-06 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.004738+0000", 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6879999999999998e-05 2023-09-23T22:25:53.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.004919+0000", 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018107900000000001 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.005103+0000", 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018415700000000001 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.006684+0000", 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001580758 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.006866+0000", 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018252700000000001 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.185579+0000", 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.17871295800000001 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.185633+0000", 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3612000000000001e-05 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:33.185676+0000", 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2740000000000001e-05 2023-09-23T22:25:53.049 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:67477 7.d 7:b87dff6d:::benchmark_data_smithi026_410749_object67476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:46.795735+0000", 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 6.1117221089999996, 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.081681278999999996, 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.050 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 67477 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.795735+0000", 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.795735+0000", 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.795733+0000", 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.795756+0000", 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3023e-05 2023-09-23T22:25:53.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.795775+0000", 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8680000000000001e-05 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.795785+0000", 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6229999999999992e-06 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.795829+0000", 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4635999999999998e-05 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.796016+0000", 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018649899999999999 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.796171+0000", 2023-09-23T22:25:53.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015555899999999999 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.801203+0000", 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050317920000000002 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.859950+0000", 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.058746843 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.877331+0000", 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017380896999999999 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.877379+0000", 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7793999999999997e-05 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:46.877417+0000", 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.773e-05 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:67971 7.d 7:b12cf0c7:::benchmark_data_smithi026_410749_object67970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:47.326539+0000", 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.5809178560000001, 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.091848382000000006, 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.057 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 67971 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326539+0000", 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326539+0000", 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326538+0000", 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326580+0000", 2023-09-23T22:25:53.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2478000000000002e-05 2023-09-23T22:25:53.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326593+0000", 2023-09-23T22:25:53.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2551000000000001e-05 2023-09-23T22:25:53.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326601+0000", 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1159999999999999e-06 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326645+0000", 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3470999999999998e-05 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326783+0000", 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013877599999999999 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326922+0000", 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013889800000000001 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.329700+0000", 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027777449999999999 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.414301+0000", 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.084601098 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.418297+0000", 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039960059999999999 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.418351+0000", 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3859000000000001e-05 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.418388+0000", 2023-09-23T22:25:53.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6962999999999998e-05 2023-09-23T22:25:53.065 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.065 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.065 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.065 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:67982 7.d 7:b2a17684:::benchmark_data_smithi026_410749_object67981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T22:25:53.065 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:47.326617+0000", 2023-09-23T22:25:53.065 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.5808400369999998, 2023-09-23T22:25:53.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.091870846000000006, 2023-09-23T22:25:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 67982 2023-09-23T22:25:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326617+0000", 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326617+0000", 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326617+0000", 2023-09-23T22:25:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:25:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326640+0000", 2023-09-23T22:25:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3317999999999999e-05 2023-09-23T22:25:53.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326648+0000", 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4009999999999997e-06 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.326655+0000", 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5409999999999997e-06 2023-09-23T22:25:53.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.330752+0000", 2023-09-23T22:25:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040970650000000004 2023-09-23T22:25:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.330877+0000", 2023-09-23T22:25:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012543600000000001 2023-09-23T22:25:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:25:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.331017+0000", 2023-09-23T22:25:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013939000000000001 2023-09-23T22:25:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.333082+0000", 2023-09-23T22:25:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020657010000000001 2023-09-23T22:25:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.415795+0000", 2023-09-23T22:25:53.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082712383 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.418423+0000", 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026283069999999999 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.418459+0000", 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5797999999999998e-05 2023-09-23T22:25:53.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:47.418488+0000", 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9164999999999999e-05 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:72381 7.b 7:d0b6aedf:::benchmark_data_smithi026_410749_object72380:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:25:52.425424+0000", 2023-09-23T22:25:53.074 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.48203284600000001, 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.105110588, 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 72381 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:25:53.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.425424+0000", 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.425424+0000", 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.425423+0000", 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.425443+0000", 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9843e-05 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.425455+0000", 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1996e-05 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:25:53.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.425461+0000", 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8900000000000004e-06 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.425516+0000", 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4707999999999999e-05 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.425684+0000", 2023-09-23T22:25:53.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000168367 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,5", 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.425857+0000", 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017299899999999999 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.428151+0000", 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022939140000000002 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.487863+0000", 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059712089000000003 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.530407+0000", 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.042543586000000001 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.530476+0000", 2023-09-23T22:25:53.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9451999999999998e-05 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:25:52.530535+0000", 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9095999999999998e-05 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:25:53.081 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:25:53.082 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:25:53.172 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:52 smithi070 ceph-mon[192652]: pgmap v698: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 52 MiB/s wr, 837 op/s 2023-09-23T22:25:53.172 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4199623374' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:53.172 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:52 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:53.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:52 smithi026 ceph-mon[223014]: pgmap v698: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 52 MiB/s wr, 837 op/s 2023-09-23T22:25:53.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4199623374' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:53.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:52 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:52 smithi026 ceph-mon[215775]: pgmap v698: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 52 MiB/s wr, 837 op/s 2023-09-23T22:25:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4199623374' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:53.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:52 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:25:53.293 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 72976 72960 78.5675 41.9375 0.0176451 0.0127218 2023-09-23T22:25:53.399 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:25:53.400 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:25:53.400 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:25:54.128 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:54.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:54.131 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:54.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:54.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:54.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:25:54.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: osdmap e269: 8 total, 8 up, 6 in 2023-09-23T22:25:54.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:54.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: osdmap e269: 8 total, 8 up, 6 in 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:54.241 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:54.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 73791 73775 78.0985 50.9375 0.0165713 0.0127989 2023-09-23T22:25:54.318 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:54.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:25:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:25:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: osdmap e269: 8 total, 8 up, 6 in 2023-09-23T22:25:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: failed to encode map e269 with expected crc 2023-09-23T22:25:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:54.653 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:54.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:54.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:54.763 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:54.790 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:54.928 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:25:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[223014]: pgmap v700: 99 pgs: 99 active+clean; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 53 MiB/s wr, 849 op/s 2023-09-23T22:25:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[223014]: failed to encode map e269 with expected crc 2023-09-23T22:25:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:55.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[215775]: pgmap v700: 99 pgs: 99 active+clean; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 53 MiB/s wr, 849 op/s 2023-09-23T22:25:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[215775]: failed to encode map e269 with expected crc 2023-09-23T22:25:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:55.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:55.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:25:55.294830+0000 min lat: 0.00118556 max lat: 0.332866 avg lat: 0.0129236 2023-09-23T22:25:55.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:25:55.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 74244 74228 77.2691 28.3125 0.0167635 0.0129236 2023-09-23T22:25:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:54 smithi070 ceph-mon[192652]: pgmap v700: 99 pgs: 99 active+clean; 4.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 53 MiB/s wr, 849 op/s 2023-09-23T22:25:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:54 smithi070 ceph-mon[192652]: failed to encode map e269 with expected crc 2023-09-23T22:25:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:25:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:25:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:25:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:25:55.583 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:55.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:55.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:55.706 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:55.741 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:55.790 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:55.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:55.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:55.863 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:55.896 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:55.985 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:56.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 75060 75044 76.8386 51 0.0104912 0.0130079 2023-09-23T22:25:56.297 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:25:56.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:56.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:25:56.357 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:25:56.398 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:25:56.409 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:56.437 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:25:56.442 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:56.484 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:25:56.497 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:56.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:56.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:56.519 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:25:56.557 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:25:56.604 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:56.605 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:25:56.637 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:56.659 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:25:56.671 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:25:56.671 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:25:56.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:25:56.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:25:56.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:25:56.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:25:56.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:25:56.672 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:25:56.693 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:56.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:56.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:56.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:56.831 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:56.888 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:56.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:56.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:56.995 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:25:57.003 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:57 smithi070 ceph-mon[192652]: pgmap v701: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 854 op/s 2023-09-23T22:25:57.003 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1941971764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:57.024 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:57.065 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:25:57.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:57.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:25:57.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:57.197 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:57.237 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:57.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:57.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:57.294 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 75824 75808 76.3696 47.75 0.0174403 0.0130885 2023-09-23T22:25:57.342 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:57.371 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:57.432 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:57.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:57.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:57.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:57 smithi026 ceph-mon[223014]: pgmap v701: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 854 op/s 2023-09-23T22:25:57.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1941971764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:57.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:56 smithi026 ceph-mon[215775]: pgmap v701: 99 pgs: 99 active+clean; 4.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 53 MiB/s wr, 854 op/s 2023-09-23T22:25:57.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1941971764' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:25:57.538 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:57.543 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:25:57.568 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:57.617 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:57.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:57.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:57.724 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:25:57.756 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:57.802 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:25:57.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:57.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:25:57.912 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:25:58.010 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:58.161 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:25:58.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:58.176 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:25:58.277 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:58.295 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 76699 76683 76.0245 54.6875 0.0207364 0.0131472 2023-09-23T22:25:58.304 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:58.359 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:58.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:58.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:58.467 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:25:58.557 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:58.699 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:25:58.727 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:25:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:58.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:25:58.807 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:25:58.807 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:25:58.835 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:58.866 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:58.922 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:58.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:59.028 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:25:59.059 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:59.113 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:25:59.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:25:59 smithi070 ceph-mon[192652]: pgmap v702: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 52 MiB/s wr, 829 op/s 2023-09-23T22:25:59.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:59.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:25:59.181 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 7, 0] out_osds: [6, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:25:59.182 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:25:59.182 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-23T22:25:59.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-23T22:25:59.221 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:59.254 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:59.296 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 77581 77565 75.698 55.125 0.00849825 0.0132047 2023-09-23T22:25:59.308 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:59.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:59.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:59.403 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:25:59 smithi026 ceph-mon[215775]: pgmap v702: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 52 MiB/s wr, 829 op/s 2023-09-23T22:25:59.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:25:59 smithi026 ceph-mon[223014]: pgmap v702: 99 pgs: 99 active+clean; 4.6 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 52 MiB/s wr, 829 op/s 2023-09-23T22:25:59.412 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:25:59.445 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:25:59.495 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:25:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:25:59.502 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:25:59.602 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:25:59.666 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:25:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:25:59] "GET /metrics HTTP/1.1" 200 36351 "" "Prometheus/2.43.0" 2023-09-23T22:25:59.668 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:00.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:26:00.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:26:00.178 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:00.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:00.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:00.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:00.296 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 78446 78430 75.3653 54.0625 0.0219767 0.0132623 2023-09-23T22:26:00.430 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:00.527 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:00.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:00.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:00.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:00.677 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:01.036 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:01.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:01.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:01.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:26:01.216 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:01.244 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:01.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 79117 79101 74.8487 41.9375 0.0285725 0.0133539 2023-09-23T22:26:01.319 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:01.370 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:01.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:01 smithi070 ceph-mon[192652]: pgmap v703: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T22:26:01.411 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:26:01.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:01.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:26:01.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:01 smithi026 ceph-mon[215775]: pgmap v703: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T22:26:01.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:01 smithi026 ceph-mon[223014]: pgmap v703: 99 pgs: 99 active+clean; 4.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s 2023-09-23T22:26:01.522 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:01.596 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:01.746 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:01.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:01.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:01.919 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:01.952 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:02.007 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:02.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:02.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:02.111 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:02.199 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:02.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 79918 79902 74.4789 50.0625 0.0139475 0.0134222 2023-09-23T22:26:03.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 80779 80763 74.1751 53.8125 0.0208626 0.0134768 2023-09-23T22:26:03.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:03 smithi026 ceph-mon[223014]: pgmap v704: 99 pgs: 99 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 804 op/s 2023-09-23T22:26:03.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2732167125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:26:03.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:03 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:26:03.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:03 smithi026 ceph-mon[215775]: pgmap v704: 99 pgs: 99 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 804 op/s 2023-09-23T22:26:03.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2732167125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:26:03.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:03 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:26:03.347 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:03.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:03.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:03.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:03 smithi070 ceph-mon[192652]: pgmap v704: 99 pgs: 99 active+clean; 4.7 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 50 MiB/s wr, 804 op/s 2023-09-23T22:26:03.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2732167125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:26:03.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:03 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:26:03.461 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:03.490 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:03.835 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:03.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:03.946 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:03.979 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:04.005 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:26:04.033 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:04.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:04.143 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:04.174 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:04.232 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:04.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:04 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:04.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:04 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:26:04.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:04 smithi026 ceph-mon[215775]: osdmap e270: 8 total, 8 up, 6 in 2023-09-23T22:26:04.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:04 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:04.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:04 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:26:04.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:04 smithi026 ceph-mon[223014]: osdmap e270: 8 total, 8 up, 6 in 2023-09-23T22:26:04.239 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:04 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:04.239 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:04 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:26:04.240 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:04 smithi070 ceph-mon[192652]: osdmap e270: 8 total, 8 up, 6 in 2023-09-23T22:26:04.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:04.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:04.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 81592 81576 73.8367 50.8125 0.00830931 0.0135364 2023-09-23T22:26:04.342 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:04.367 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:04.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:04.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:04.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:04.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:04.594 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:05.096 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:05.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:05.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:05.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:05.229 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:05.270 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:05.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:05.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:05.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 82205 82189 73.3294 38.3125 0.0202403 0.013603 2023-09-23T22:26:05.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:05 smithi070 ceph-mon[192652]: pgmap v705: 99 pgs: 99 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 47 MiB/s wr, 753 op/s 2023-09-23T22:26:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:05 smithi070 ceph-mon[192652]: failed to encode map e270 with expected crc 2023-09-23T22:26:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:05 smithi070 ceph-mon[192652]: failed to encode map e270 with expected crc 2023-09-23T22:26:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:05 smithi070 ceph-mon[192652]: failed to encode map e270 with expected crc 2023-09-23T22:26:05.374 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:05.403 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:05.454 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:05.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:05.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:05 smithi026 ceph-mon[223014]: pgmap v705: 99 pgs: 99 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 47 MiB/s wr, 753 op/s 2023-09-23T22:26:05.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:05 smithi026 ceph-mon[223014]: failed to encode map e270 with expected crc 2023-09-23T22:26:05.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:05 smithi026 ceph-mon[223014]: failed to encode map e270 with expected crc 2023-09-23T22:26:05.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:05 smithi026 ceph-mon[223014]: failed to encode map e270 with expected crc 2023-09-23T22:26:05.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:05 smithi026 ceph-mon[215775]: pgmap v705: 99 pgs: 99 active+clean; 4.8 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 47 MiB/s wr, 753 op/s 2023-09-23T22:26:05.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:05 smithi026 ceph-mon[215775]: failed to encode map e270 with expected crc 2023-09-23T22:26:05.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:05 smithi026 ceph-mon[215775]: failed to encode map e270 with expected crc 2023-09-23T22:26:05.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:05 smithi026 ceph-mon[215775]: failed to encode map e270 with expected crc 2023-09-23T22:26:05.559 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:05.597 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:05.650 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:05.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:05.755 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:05.845 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:06.057 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:06 smithi070 ceph-mon[192652]: failed to encode map e270 with expected crc 2023-09-23T22:26:06.169 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:06.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:06.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:06.276 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:06.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:06 smithi026 ceph-mon[215775]: failed to encode map e270 with expected crc 2023-09-23T22:26:06.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:06 smithi026 ceph-mon[223014]: failed to encode map e270 with expected crc 2023-09-23T22:26:06.304 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:06.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 83021 83005 73.0151 51 0.0150705 0.0136904 2023-09-23T22:26:06.352 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:06.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:06.448 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:26:06.448 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:26:06.459 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:06.488 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:06.539 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:06.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:06.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:06.646 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:26:06.674 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:06.696 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:26:06.724 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:26:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:06.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:26:06.731 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:26:06.762 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:26:06.805 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:26:06.832 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:06.843 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:26:06.863 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:06.908 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:06.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:06.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:07.015 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:26:07.044 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:07.058 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:06 smithi026 conmon[215751]: 2023-09-23T22:26:06.745+0000 7f4580f04700 -1 mon.a@0(leader).osd e270 definitely_dead 0 2023-09-23T22:26:07.067 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.2. 2023-09-23T22:26:07.092 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:26:07.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:07.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:26:07.099 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:26:07.099 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:26:07.099 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:26:07.100 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:26:07.100 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:26:07.100 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:26:07.100 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:26:07.100 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:26:07.100 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:26:07.100 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:26:07.100 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:26:07.199 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:07.224 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:07.264 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:07.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:07.305 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 83460 83444 72.3824 27.4375 0.00364337 0.0137564 2023-09-23T22:26:07.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:07 smithi070 ceph-mon[192652]: pgmap v707: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T22:26:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1785617120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1872636397' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T22:26:07.374 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:07.399 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:07.440 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:07.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:07.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:07 smithi026 ceph-mon[223014]: pgmap v707: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T22:26:07.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1785617120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:07.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1872636397' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T22:26:07.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:07 smithi026 ceph-mon[215775]: pgmap v707: 99 pgs: 99 active+clean; 4.9 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 48 MiB/s wr, 774 op/s 2023-09-23T22:26:07.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1785617120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:07.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1872636397' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T22:26:07.546 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:07.577 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:07.781 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:26:08.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:26:08.094 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:08.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:08.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:08.203 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:08.231 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:08.306 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 83460 83444 71.3915 0 - 0.0137564 2023-09-23T22:26:08.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:08 smithi070 ceph-mon[192652]: failed to encode map e271 with expected crc 2023-09-23T22:26:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:08 smithi070 ceph-mon[192652]: failed to encode map e271 with expected crc 2023-09-23T22:26:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:08 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1872636397' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T22:26:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:08 smithi070 ceph-mon[192652]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T22:26:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:08 smithi070 ceph-mon[192652]: pgmap v709: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 56 MiB/s wr, 893 op/s 2023-09-23T22:26:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:08 smithi070 ceph-mon[192652]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T22:26:08.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[223014]: failed to encode map e271 with expected crc 2023-09-23T22:26:08.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[223014]: failed to encode map e271 with expected crc 2023-09-23T22:26:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1872636397' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T22:26:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[223014]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T22:26:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[223014]: pgmap v709: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 56 MiB/s wr, 893 op/s 2023-09-23T22:26:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[223014]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T22:26:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[215775]: failed to encode map e271 with expected crc 2023-09-23T22:26:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[215775]: failed to encode map e271 with expected crc 2023-09-23T22:26:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1872636397' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T22:26:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[215775]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T22:26:08.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[215775]: pgmap v709: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 56 MiB/s wr, 893 op/s 2023-09-23T22:26:08.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:08 smithi026 ceph-mon[215775]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T22:26:08.568 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-23T22:26:08.583 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:08.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:08.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:08.609 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:08.689 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:08.717 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:08.793 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:08.932 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:08.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:08.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:09.036 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:09.065 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:09.112 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:09.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:09.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:09.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.118 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.119 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T22:26:09.120 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:09.218 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:09.246 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:09.289 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:09.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:09.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:09.306 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 83460 83444 70.4269 0 - 0.0137564 2023-09-23T22:26:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: failed to encode map e272 with expected crc 2023-09-23T22:26:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T22:26:09.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:09.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:09.433 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:09.744 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:26:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:26:09] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2023-09-23T22:26:09.818 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:09.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:09.933 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:09.960 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:10.009 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:10.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:10.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:10.114 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:10.142 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:10.190 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:10.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:10.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:10.298 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:10.306 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 83769 83753 69.7458 6.4375 0.0154481 0.0143304 2023-09-23T22:26:10.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:10 smithi070 ceph-mon[192652]: failed to encode map e273 with expected crc 2023-09-23T22:26:10.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:10 smithi070 ceph-mon[192652]: failed to encode map e273 with expected crc 2023-09-23T22:26:10.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:10 smithi070 ceph-mon[192652]: failed to encode map e273 with expected crc 2023-09-23T22:26:10.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:10 smithi070 ceph-mon[192652]: pgmap v712: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 39 MiB/s wr, 626 op/s 2023-09-23T22:26:10.324 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:10.371 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:10.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:10.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:10.479 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:10 smithi026 ceph-mon[215775]: failed to encode map e273 with expected crc 2023-09-23T22:26:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:10 smithi026 ceph-mon[215775]: failed to encode map e273 with expected crc 2023-09-23T22:26:10.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:10 smithi026 ceph-mon[215775]: failed to encode map e273 with expected crc 2023-09-23T22:26:10.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:10 smithi026 ceph-mon[215775]: pgmap v712: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 39 MiB/s wr, 626 op/s 2023-09-23T22:26:10.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:10 smithi026 ceph-mon[223014]: failed to encode map e273 with expected crc 2023-09-23T22:26:10.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:10 smithi026 ceph-mon[223014]: failed to encode map e273 with expected crc 2023-09-23T22:26:10.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:10 smithi026 ceph-mon[223014]: failed to encode map e273 with expected crc 2023-09-23T22:26:10.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:10 smithi026 ceph-mon[223014]: pgmap v712: 99 pgs: 99 active+clean; 5.0 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 39 MiB/s wr, 626 op/s 2023-09-23T22:26:10.505 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:10.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:26:10.554 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:10.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:10.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:10.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:10.763 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:11.224 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:11.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:11.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:11.307 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 84262 84246 69.2329 30.8125 0.0400642 0.014437 2023-09-23T22:26:11.341 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:11.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:11 smithi026 ceph-mon[223014]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T22:26:11.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:11 smithi026 ceph-mon[223014]: failed to encode map e274 with expected crc 2023-09-23T22:26:11.356 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:11 smithi026 ceph-mon[223014]: failed to encode map e274 with expected crc 2023-09-23T22:26:11.356 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:11 smithi026 ceph-mon[215775]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T22:26:11.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:11 smithi026 ceph-mon[215775]: failed to encode map e274 with expected crc 2023-09-23T22:26:11.357 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:11 smithi026 ceph-mon[215775]: failed to encode map e274 with expected crc 2023-09-23T22:26:11.458 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:11.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:11 smithi070 ceph-mon[192652]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T22:26:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:11 smithi070 ceph-mon[192652]: failed to encode map e274 with expected crc 2023-09-23T22:26:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:11 smithi070 ceph-mon[192652]: failed to encode map e274 with expected crc 2023-09-23T22:26:11.631 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:11.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:11.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:11.739 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:11.882 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:12.091 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:12.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:12.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:12.149 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:12 smithi026 ceph-mon[223014]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T22:26:12.149 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:12 smithi026 ceph-mon[223014]: failed to encode map e275 with expected crc 2023-09-23T22:26:12.149 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:12 smithi026 ceph-mon[223014]: pgmap v715: 99 pgs: 1 activating+remapped, 4 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 90 active+clean; 5.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 16 MiB/s wr, 251 op/s; 20901/249486 objects degraded (8.378%); 5078/249486 objects misplaced (2.035%); 1.2 KiB/s, 2 keys/s, 3 objects/s recovering 2023-09-23T22:26:12.149 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:12 smithi026 ceph-mon[215775]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T22:26:12.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:12 smithi026 ceph-mon[215775]: failed to encode map e275 with expected crc 2023-09-23T22:26:12.150 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:12 smithi026 ceph-mon[215775]: pgmap v715: 99 pgs: 1 activating+remapped, 4 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 90 active+clean; 5.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 16 MiB/s wr, 251 op/s; 20901/249486 objects degraded (8.378%); 5078/249486 objects misplaced (2.035%); 1.2 KiB/s, 2 keys/s, 3 objects/s recovering 2023-09-23T22:26:12.208 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:12.215 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:12.307 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 84693 84677 68.6839 26.9375 0.0297246 0.0145524 2023-09-23T22:26:12.324 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:12.415 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:26:12.415 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:26:12.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:26:12.415 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:26:12.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.415 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150387 5.b 5:d9a1ac1d:::benchmark_data_smithi026_286012_object56579:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.415 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.248274+0000", 2023-09-23T22:26:12.415 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.16081236500003, 2023-09-23T22:26:12.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.210601423, 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150387 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248274+0000", 2023-09-23T22:26:12.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248274+0000", 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248274+0000", 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248277+0000", 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6859999999999999e-06 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248285+0000", 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6499999999999996e-06 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248291+0000", 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5720000000000002e-06 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.257620+0000", 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093298359999999993 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.257632+0000", 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1358999999999999e-05 2023-09-23T22:26:12.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202876+0000", 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94524393299999998 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202888+0000", 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.224e-05 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.443162+0000", 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.240274028 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.443328+0000", 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000166458 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.443469+0000", 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140727 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.452748+0000", 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092784719999999994 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.452762+0000", 2023-09-23T22:26:12.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4769e-05 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.457005+0000", 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042431370000000001 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.457082+0000", 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6798000000000006e-05 2023-09-23T22:26:12.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.458875+0000", 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017932340000000001 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150389 5.b 5:d9ddf50c:::benchmark_data_smithi026_286012_object56581:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.248301+0000", 2023-09-23T22:26:12.424 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.16078578700001, 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2246119269999998, 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150389 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248301+0000", 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248301+0000", 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248300+0000", 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:12.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248303+0000", 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4959999999999999e-06 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248308+0000", 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3900000000000001e-06 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.248313+0000", 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.763e-06 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.257664+0000", 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093506289999999992 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.257672+0000", 2023-09-23T22:26:12.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3000000000000002e-06 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202927+0000", 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94525478299999999 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202936+0000", 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5699999999999999e-06 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.449098+0000", 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2461615049999999 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.449282+0000", 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018425700000000001 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.449482+0000", 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019988100000000001 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.458961+0000", 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094792399999999999 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.458975+0000", 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4269e-05 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.470682+0000", 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011706553999999999 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.470753+0000", 2023-09-23T22:26:12.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1092000000000003e-05 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.472913+0000", 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021595579999999998 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150438 5.b 5:d8e4acd1:::benchmark_data_smithi026_286012_object56630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.433 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.315867+0000", 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.09321968, 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1589761699999999, 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150438 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315867+0000", 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315867+0000", 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315865+0000", 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315872+0000", 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8040000000000004e-06 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315888+0000", 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5611999999999999e-05 2023-09-23T22:26:12.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.315894+0000", 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5860000000000001e-06 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316024+0000", 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012938899999999999 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316035+0000", 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1059e-05 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202965+0000", 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.88693064899999996 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.202973+0000", 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9109999999999993e-06 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.452813+0000", 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.249840042 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.453004+0000", 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019102799999999999 2023-09-23T22:26:12.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.453165+0000", 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000160285 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.459043+0000", 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058780220000000001 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.463292+0000", 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042492659999999998 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.473010+0000", 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097182450000000004 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.473084+0000", 2023-09-23T22:26:12.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3503000000000004e-05 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.474843+0000", 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001759172 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150443 5.b 5:d57a26b4:::benchmark_data_smithi026_286012_object56635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.442 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.316285+0000", 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.09280170599999, 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1896902159999998, 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150443 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316285+0000", 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316285+0000", 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316284+0000", 2023-09-23T22:26:12.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316289+0000", 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.147e-06 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316301+0000", 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1287999999999999e-05 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316310+0000", 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5119999999999997e-06 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316368+0000", 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8335999999999998e-05 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.316381+0000", 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.253e-05 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203015+0000", 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.88663416500000003 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203024+0000", 2023-09-23T22:26:12.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6200000000000005e-06 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.459094+0000", 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.256070204 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.459264+0000", 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000170175 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.459450+0000", 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000186274 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.470566+0000", 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011115583 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.470633+0000", 2023-09-23T22:26:12.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7287999999999997e-05 2023-09-23T22:26:12.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503835+0000", 2023-09-23T22:26:12.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033201738000000001 2023-09-23T22:26:12.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503894+0000", 2023-09-23T22:26:12.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9120999999999999e-05 2023-09-23T22:26:12.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.505975+0000", 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020808419999999998 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150448 5.b 5:da46b32b:::benchmark_data_smithi026_286012_object56640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.323598+0000", 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.08548843800003, 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1845984239999998, 2023-09-23T22:26:12.452 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150448 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323598+0000", 2023-09-23T22:26:12.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323598+0000", 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323598+0000", 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:12.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323600+0000", 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1299999999999999e-06 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323605+0000", 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.259e-06 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.323610+0000", 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4850000000000003e-06 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.330889+0000", 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072798610000000003 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.330902+0000", 2023-09-23T22:26:12.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2853e-05 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203057+0000", 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.87215510799999996 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203066+0000", 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6379999999999993e-06 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.463330+0000", 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2602637059999999 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.463498+0000", 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016803400000000001 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.463595+0000", 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7663000000000003e-05 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.474872+0000", 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011276243 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479113+0000", 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042416440000000001 2023-09-23T22:26:12.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.506089+0000", 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026975932000000001 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.506124+0000", 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5131e-05 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508196+0000", 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020720909999999999 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150486 5.b 5:d9f35565:::benchmark_data_smithi026_286012_object56678:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.381625+0000", 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.02746103200002, 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1288887700000001, 2023-09-23T22:26:12.461 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150486 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381625+0000", 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381625+0000", 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381624+0000", 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381631+0000", 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3270000000000003e-06 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381646+0000", 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4357e-05 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381651+0000", 2023-09-23T22:26:12.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.665e-06 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381699+0000", 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7746e-05 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381728+0000", 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9057999999999999e-05 2023-09-23T22:26:12.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203099+0000", 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82137101700000004 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203110+0000", 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0749e-05 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.466717+0000", 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2636070150000001 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.466867+0000", 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015026800000000001 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.466991+0000", 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000123554 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479184+0000", 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012193633000000001 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479265+0000", 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0585000000000002e-05 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508322+0000", 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029057554999999999 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508388+0000", 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5982999999999995e-05 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510514+0000", 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021257820000000001 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150488 5.b 5:df0377d0:::benchmark_data_smithi026_286012_object56680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.381878+0000", 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.02720829100002, 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.142269556, 2023-09-23T22:26:12.470 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150488 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381878+0000", 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381878+0000", 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381877+0000", 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381883+0000", 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2889999999999997e-06 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381894+0000", 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1110999999999999e-05 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381901+0000", 2023-09-23T22:26:12.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3119999999999996e-06 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381958+0000", 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6611999999999999e-05 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.381967+0000", 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1880000000000008e-06 2023-09-23T22:26:12.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203147+0000", 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82117972299999997 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203155+0000", 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9129999999999992e-06 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.474913+0000", 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.271757247 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.475101+0000", 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018805800000000001 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.475273+0000", 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000171979 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479252+0000", 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039795200000000003 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.487178+0000", 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079256190000000001 2023-09-23T22:26:12.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.522088+0000", 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034909664999999999 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.522141+0000", 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3514999999999998e-05 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524148+0000", 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020066900000000002 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150489 5.b 5:d03d5463:::benchmark_data_smithi026_286012_object56681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.382063+0000", 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.02702341899999, 2023-09-23T22:26:12.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.144385561, 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150489 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382063+0000", 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382063+0000", 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382062+0000", 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382068+0000", 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.339e-06 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382079+0000", 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1643e-05 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382086+0000", 2023-09-23T22:26:12.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0740000000000004e-06 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382133+0000", 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6912000000000001e-05 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.382142+0000", 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.217e-06 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203183+0000", 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82104097300000001 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203188+0000", 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8319999999999996e-06 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479295+0000", 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2761071349999999 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479482+0000", 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000186773 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.479658+0000", 2023-09-23T22:26:12.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017554100000000001 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491200+0000", 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011542458 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491216+0000", 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6084999999999999e-05 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524314+0000", 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033097416999999997 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524363+0000", 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9353e-05 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.526449+0000", 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020855959999999999 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150492 5.b 5:d18b0458:::benchmark_data_smithi026_286012_object56684:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.385473+0000", 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.02361367499998, 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1432154150000002, 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.488 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150492 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385473+0000", 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385473+0000", 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385471+0000", 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385478+0000", 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5880000000000003e-06 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385495+0000", 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.694e-05 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385503+0000", 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7430000000000002e-06 2023-09-23T22:26:12.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385566+0000", 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2924999999999998e-05 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.385576+0000", 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0856000000000001e-05 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203218+0000", 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81764150400000002 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203226+0000", 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8599999999999993e-06 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.483349+0000", 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2801233700000001 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.483508+0000", 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015833399999999999 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.483686+0000", 2023-09-23T22:26:12.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017833700000000001 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.495359+0000", 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011672854 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503787+0000", 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084286329999999996 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.526527+0000", 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022739796 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.526586+0000", 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8517999999999998e-05 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.528688+0000", 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021025269999999999 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150497 5.b 5:d9455892:::benchmark_data_smithi026_286012_object56689:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.391903+0000", 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 453.017183053, 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.139149631, 2023-09-23T22:26:12.497 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150497 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391903+0000", 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391903+0000", 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391903+0000", 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391906+0000", 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0060000000000001e-06 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391912+0000", 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1639999999999998e-06 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391918+0000", 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4940000000000003e-06 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391971+0000", 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3808999999999999e-05 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.391982+0000", 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0264e-05 2023-09-23T22:26:12.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203251+0000", 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81126955899999997 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203256+0000", 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3359999999999997e-06 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.487231+0000", 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2839750679999999 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.487402+0000", 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000171072 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.487509+0000", 2023-09-23T22:26:12.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106952 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.499494+0000", 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011985350000000001 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.506047+0000", 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065530850000000002 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.528764+0000", 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022716584000000001 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.528832+0000", 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8011999999999999e-05 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.531053+0000", 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002221326 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150513 5.b 5:dc4d2ce0:::benchmark_data_smithi026_286012_object56705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.412359+0000", 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 452.99672783099999, 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1316144010000002, 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.506 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150513 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412359+0000", 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412359+0000", 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412358+0000", 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412362+0000", 2023-09-23T22:26:12.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1350000000000002e-06 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412373+0000", 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0803e-05 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412380+0000", 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5100000000000001e-06 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412439+0000", 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8786999999999998e-05 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.412449+0000", 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7869999999999996e-06 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203287+0000", 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.79083858200000001 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203296+0000", 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9949999999999994e-06 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491247+0000", 2023-09-23T22:26:12.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2879502519999999 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491442+0000", 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000195491 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.491636+0000", 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019384900000000001 2023-09-23T22:26:12.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503737+0000", 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012101031999999999 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508281+0000", 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004544289 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.541884+0000", 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033602434 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.541940+0000", 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6039000000000003e-05 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.543973+0000", 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020332409999999999 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150541 5.b 5:d3c31e9d:::benchmark_data_smithi026_286012_object56733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.465404+0000", 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 452.94368238599998, 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0846849810000001, 2023-09-23T22:26:12.515 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150541 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465404+0000", 2023-09-23T22:26:12.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465404+0000", 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465403+0000", 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:26:12.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465409+0000", 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8209999999999998e-06 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465420+0000", 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1038e-05 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465427+0000", 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3130000000000001e-06 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465483+0000", 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5687999999999999e-05 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.465496+0000", 2023-09-23T22:26:12.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2700999999999999e-05 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203318+0000", 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.73782281000000005 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203323+0000", 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2910000000000001e-06 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.495388+0000", 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.292065206 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.495557+0000", 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016910800000000001 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.495740+0000", 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018281699999999999 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.503797+0000", 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080567409999999992 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510565+0000", 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067689430000000004 2023-09-23T22:26:12.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.547914+0000", 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037348627000000002 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.547980+0000", 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6103000000000001e-05 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.550089+0000", 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021088069999999999 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150552 5.b 5:d9f2880b:::benchmark_data_smithi026_286012_object56744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.490846+0000", 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 452.91824013399997, 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0703078010000002, 2023-09-23T22:26:12.524 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150552 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490846+0000", 2023-09-23T22:26:12.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490846+0000", 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490845+0000", 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490850+0000", 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8600000000000001e-06 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490858+0000", 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3850000000000002e-06 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:12.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490864+0000", 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0629999999999996e-06 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490904+0000", 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0195999999999999e-05 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.490911+0000", 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3239999999999997e-06 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203358+0000", 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.71244706400000002 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203363+0000", 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9620000000000001e-06 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.499527+0000", 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.296164345 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.499713+0000", 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018603700000000001 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.499895+0000", 2023-09-23T22:26:12.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018175900000000001 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.508222+0000", 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083265400000000003 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510605+0000", 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002382941 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.558890+0000", 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048285010000000003 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.558948+0000", 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8162000000000003e-05 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:12.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.561154+0000", 2023-09-23T22:26:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022062309999999999 2023-09-23T22:26:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:12.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.018 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150566 5.b 5:daf8d5e7:::benchmark_data_smithi026_286012_object56758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.516941+0000", 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 452.89214510699998, 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0461972589999999, 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150566 2023-09-23T22:26:13.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516941+0000", 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516941+0000", 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:13.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516940+0000", 2023-09-23T22:26:13.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:13.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:13.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516948+0000", 2023-09-23T22:26:13.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6199999999999999e-06 2023-09-23T22:26:13.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516964+0000", 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6416999999999999e-05 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.516973+0000", 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8179999999999993e-06 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.517041+0000", 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8503999999999993e-05 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.517051+0000", 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6600000000000007e-06 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203395+0000", 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68634447600000004 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203402+0000", 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6699999999999997e-06 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510628+0000", 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3072261539999999 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510814+0000", 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018611499999999999 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.510998+0000", 2023-09-23T22:26:13.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018392300000000001 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524167+0000", 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013168875999999999 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524243+0000", 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6271000000000002e-05 2023-09-23T22:26:13.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:13.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.561248+0000", 2023-09-23T22:26:13.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037004598999999999 2023-09-23T22:26:13.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:13.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.561312+0000", 2023-09-23T22:26:13.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3862999999999995e-05 2023-09-23T22:26:13.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.563139+0000", 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018266560000000001 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150579 5.b 5:d1157291:::benchmark_data_smithi026_286012_object56771:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:13.072 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.555041+0000", 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 452.85404578499998, 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0120606849999998, 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150579 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555041+0000", 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555041+0000", 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555040+0000", 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555044+0000", 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.357e-06 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:13.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555055+0000", 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0241000000000001e-05 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555059+0000", 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.763e-06 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555119+0000", 2023-09-23T22:26:13.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0109000000000003e-05 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.555133+0000", 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3774999999999999e-05 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203428+0000", 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.64829514799999999 2023-09-23T22:26:13.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203433+0000", 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2390000000000004e-06 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.514827+0000", 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3113947669999999 2023-09-23T22:26:13.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.515060+0000", 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000232835 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.515295+0000", 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000235157 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.524177+0000", 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088820050000000001 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.531120+0000", 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069427389999999999 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:13.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.564932+0000", 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033812138999999998 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.564997+0000", 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4709000000000007e-05 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.567101+0000", 2023-09-23T22:26:13.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021043540000000001 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:150581 5.b 5:d6413485:::benchmark_data_smithi026_286012_object56773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:39.564269+0000", 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 452.844817503, 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0047219690000002, 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:13.082 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 150581 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564269+0000", 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564269+0000", 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564267+0000", 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:13.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564274+0000", 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4239999999999998e-06 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564288+0000", 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3917000000000001e-05 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564297+0000", 2023-09-23T22:26:13.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9420000000000001e-06 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564349+0000", 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2439999999999999e-05 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:39.564359+0000", 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9909999999999997e-06 2023-09-23T22:26:13.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203460+0000", 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.639100432 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:40.203471+0000", 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.165e-05 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.518685+0000", 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.315213328 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.518873+0000", 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000188099 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.519036+0000", 2023-09-23T22:26:13.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016286600000000001 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.526472+0000", 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074364390000000004 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.531180+0000", 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047083710000000003 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:13.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.567156+0000", 2023-09-23T22:26:13.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.035975205000000003 2023-09-23T22:26:13.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:13.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.567214+0000", 2023-09-23T22:26:13.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7972e-05 2023-09-23T22:26:13.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:13.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:41.568991+0000", 2023-09-23T22:26:13.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001777362 2023-09-23T22:26:13.194 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.194 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:13.194 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.195 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83432 7.8 7.23f2d408 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:26:13.195 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.117800+0000", 2023-09-23T22:26:13.195 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.2912869139999996, 2023-09-23T22:26:13.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84962466199999997, 2023-09-23T22:26:13.195 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:13.196 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:26:13.196 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:13.196 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:13.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:12 smithi070 ceph-mon[192652]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T22:26:13.197 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:12 smithi070 ceph-mon[192652]: failed to encode map e275 with expected crc 2023-09-23T22:26:13.198 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:12 smithi070 ceph-mon[192652]: pgmap v715: 99 pgs: 1 activating+remapped, 4 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 90 active+clean; 5.1 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 16 MiB/s wr, 251 op/s; 20901/249486 objects degraded (8.378%); 5078/249486 objects misplaced (2.035%); 1.2 KiB/s, 2 keys/s, 3 objects/s recovering 2023-09-23T22:26:13.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:13.198 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83432 2023-09-23T22:26:13.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.198 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:13.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:13.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117800+0000", 2023-09-23T22:26:13.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:13.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117800+0000", 2023-09-23T22:26:13.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:13.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117798+0000", 2023-09-23T22:26:13.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:26:13.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:13.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117822+0000", 2023-09-23T22:26:13.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4601e-05 2023-09-23T22:26:13.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:13.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117838+0000", 2023-09-23T22:26:13.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6004000000000001e-05 2023-09-23T22:26:13.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:13.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117851+0000", 2023-09-23T22:26:13.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2478e-05 2023-09-23T22:26:13.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117942+0000", 2023-09-23T22:26:13.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1551000000000005e-05 2023-09-23T22:26:13.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117954+0000", 2023-09-23T22:26:13.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1456e-05 2023-09-23T22:26:13.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967415+0000", 2023-09-23T22:26:13.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84946135899999997 2023-09-23T22:26:13.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:13.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967424+0000", 2023-09-23T22:26:13.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0480000000000004e-06 2023-09-23T22:26:13.209 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.209 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:13.209 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.210 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83434 7.8 7.a8ea6d38 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:26:13.210 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.117865+0000", 2023-09-23T22:26:13.210 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.2912216149999995, 2023-09-23T22:26:13.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84967687400000003, 2023-09-23T22:26:13.210 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:13.210 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:26:13.211 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:13.211 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:13.211 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:13.211 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83434 2023-09-23T22:26:13.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.212 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:13.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:13.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117865+0000", 2023-09-23T22:26:13.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:13.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117865+0000", 2023-09-23T22:26:13.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:13.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117864+0000", 2023-09-23T22:26:13.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:13.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:13.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117887+0000", 2023-09-23T22:26:13.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2963e-05 2023-09-23T22:26:13.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:13.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117895+0000", 2023-09-23T22:26:13.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1529999999999997e-06 2023-09-23T22:26:13.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:13.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117901+0000", 2023-09-23T22:26:13.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1419999999999998e-06 2023-09-23T22:26:13.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117974+0000", 2023-09-23T22:26:13.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2593000000000001e-05 2023-09-23T22:26:13.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.117980+0000", 2023-09-23T22:26:13.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9460000000000003e-06 2023-09-23T22:26:13.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967533+0000", 2023-09-23T22:26:13.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.849552478 2023-09-23T22:26:13.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:13.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967542+0000", 2023-09-23T22:26:13.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1880000000000008e-06 2023-09-23T22:26:13.221 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.221 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:13.221 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.222 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83451 7.8 7.17edbb08 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:26:13.222 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.123540+0000", 2023-09-23T22:26:13.222 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.2855463350000003, 2023-09-23T22:26:13.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84403131499999995, 2023-09-23T22:26:13.222 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:13.223 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:26:13.223 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:13.223 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:13.223 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:13.223 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83451 2023-09-23T22:26:13.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.224 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:13.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:13.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123540+0000", 2023-09-23T22:26:13.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:13.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123540+0000", 2023-09-23T22:26:13.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:13.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123538+0000", 2023-09-23T22:26:13.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:26:13.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:13.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123564+0000", 2023-09-23T22:26:13.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5846999999999998e-05 2023-09-23T22:26:13.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:13.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123579+0000", 2023-09-23T22:26:13.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5095e-05 2023-09-23T22:26:13.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:13.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123588+0000", 2023-09-23T22:26:13.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7110000000000001e-06 2023-09-23T22:26:13.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123640+0000", 2023-09-23T22:26:13.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1823999999999997e-05 2023-09-23T22:26:13.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123652+0000", 2023-09-23T22:26:13.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2322e-05 2023-09-23T22:26:13.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967564+0000", 2023-09-23T22:26:13.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84391157500000002 2023-09-23T22:26:13.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:13.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967571+0000", 2023-09-23T22:26:13.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7980000000000006e-06 2023-09-23T22:26:13.232 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.233 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:13.233 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.233 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83452 7.e 7.acdf484e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:26:13.233 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.123606+0000", 2023-09-23T22:26:13.234 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.285480422, 2023-09-23T22:26:13.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84365341599999999, 2023-09-23T22:26:13.234 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:13.234 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:26:13.234 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:13.234 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:13.235 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:13.235 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83452 2023-09-23T22:26:13.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.235 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:13.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:13.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123606+0000", 2023-09-23T22:26:13.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:13.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123606+0000", 2023-09-23T22:26:13.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:13.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:13.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123605+0000", 2023-09-23T22:26:13.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:13.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:13.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123625+0000", 2023-09-23T22:26:13.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9996e-05 2023-09-23T22:26:13.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:13.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123632+0000", 2023-09-23T22:26:13.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9990000000000002e-06 2023-09-23T22:26:13.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:13.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123638+0000", 2023-09-23T22:26:13.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2469999999999999e-06 2023-09-23T22:26:13.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123820+0000", 2023-09-23T22:26:13.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018216799999999999 2023-09-23T22:26:13.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:13.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.123828+0000", 2023-09-23T22:26:13.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2320000000000001e-06 2023-09-23T22:26:13.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:13.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967248+0000", 2023-09-23T22:26:13.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84342011900000002 2023-09-23T22:26:13.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:13.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:13.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:13.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967259+0000", 2023-09-23T22:26:13.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1229e-05 2023-09-23T22:26:13.244 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.244 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:13.244 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.244 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:13.245 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:13.245 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:26:13.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:13.247 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:13.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:13.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:13 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 20901/249486 objects degraded (8.378%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:13.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3675413737' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:13.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:13 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 20901/249486 objects degraded (8.378%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:13.248 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3675413737' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:13.249 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:26:13.307 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 85139 85123 68.161 27.875 0.0199645 0.0146404 2023-09-23T22:26:13.364 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:13.392 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:13.502 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:13.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:13.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:13.568 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 7, 0, 2] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:26:13.569 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:26:13.569 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T22:26:13.569 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T22:26:13.614 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:13.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:13 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 20901/249486 objects degraded (8.378%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:13.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3675413737' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:13.642 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:13.690 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:13.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:13.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:13.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:13.894 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:13.999 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:26:14.165 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:26:14.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[215775]: failed to encode map e276 with expected crc 2023-09-23T22:26:14.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3675413737' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:26:14.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[215775]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T22:26:14.166 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[215775]: pgmap v717: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 4 activating+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 88 active+clean; 5.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 356 op/s; 26119/251955 objects degraded (10.367%); 10335/251955 objects misplaced (4.102%); 625 KiB/s, 2 keys/s, 14 objects/s recovering 2023-09-23T22:26:14.307 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 85529 85513 67.607 24.375 0.0306365 0.0147796 2023-09-23T22:26:14.476 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:26:14.476 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[223014]: failed to encode map e276 with expected crc 2023-09-23T22:26:14.476 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3675413737' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:26:14.476 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[223014]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T22:26:14.476 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:14 smithi026 ceph-mon[223014]: pgmap v717: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 4 activating+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 88 active+clean; 5.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 356 op/s; 26119/251955 objects degraded (10.367%); 10335/251955 objects misplaced (4.102%); 625 KiB/s, 2 keys/s, 14 objects/s recovering 2023-09-23T22:26:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:14 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:26:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:14 smithi070 ceph-mon[192652]: failed to encode map e276 with expected crc 2023-09-23T22:26:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:14 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3675413737' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:26:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:14 smithi070 ceph-mon[192652]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T22:26:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:14 smithi070 ceph-mon[192652]: pgmap v717: 99 pgs: 1 active+recovering+undersized+remapped, 1 activating+remapped, 4 activating+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+remapped, 88 active+clean; 5.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 356 op/s; 26119/251955 objects degraded (10.367%); 10335/251955 objects misplaced (4.102%); 625 KiB/s, 2 keys/s, 14 objects/s recovering 2023-09-23T22:26:15.269 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:15.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:15.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:15.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:26:15.308747+0000 min lat: 0.00118556 max lat: 2.89017 avg lat: 0.0148889 2023-09-23T22:26:15.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:26:15.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 86006 85990 67.1343 29.8125 0.0295272 0.0148889 2023-09-23T22:26:15.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:26:15.529 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:15.679 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:15.721 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:26:15.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:15.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:26:15.828 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:15.832 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:15.856 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:15.901 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:15.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:16.009 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:16.109 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:16.308 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 86463 86447 66.6583 28.5625 0.0329112 0.014995 2023-09-23T22:26:16.424 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:16.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:16.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:16.478 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:26:16.526 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:26:16.533 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:26:16.569 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:26:16.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:16 smithi026 ceph-mon[223014]: pgmap v718: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 29 MiB/s wr, 464 op/s; 26113/255603 objects degraded (10.216%); 10335/255603 objects misplaced (4.043%); 491 KiB/s, 1 keys/s, 10 objects/s recovering 2023-09-23T22:26:16.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/891229763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:16.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:16 smithi026 ceph-mon[215775]: pgmap v718: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 29 MiB/s wr, 464 op/s; 26113/255603 objects degraded (10.216%); 10335/255603 objects misplaced (4.043%); 491 KiB/s, 1 keys/s, 10 objects/s recovering 2023-09-23T22:26:16.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/891229763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:16.636 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:26:16.661 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:16.675 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:26:16.713 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:26:16.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:26:16.808 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:26:16.841 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:26:16.841 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:26:16.841 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:26:16.841 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:26:16.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:26:16.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:26:16.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:26:16.842 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:26:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:16 smithi070 ceph-mon[192652]: pgmap v718: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 29 MiB/s wr, 464 op/s; 26113/255603 objects degraded (10.216%); 10335/255603 objects misplaced (4.043%); 491 KiB/s, 1 keys/s, 10 objects/s recovering 2023-09-23T22:26:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/891229763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:17.310 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 86915 86899 66.189 28.25 0.0384543 0.0151028 2023-09-23T22:26:17.328 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:26:17.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:17 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26113/258585 objects degraded (10.098%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:17.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:17 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26113/258585 objects degraded (10.098%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:17.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:17 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26113/258585 objects degraded (10.098%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:18.094 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:26:18.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:18.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:26:18.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:18.237 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:18.283 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:18.287 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:18.310 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 87370 87354 65.7344 28.4375 0.0433756 0.0152066 2023-09-23T22:26:18.388 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:26:18.476 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:19.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:18 smithi026 ceph-mon[215775]: pgmap v719: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 416 op/s; 26113/258585 objects degraded (10.098%); 10243/258585 objects misplaced (3.961%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:26:19.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:18 smithi026 ceph-mon[223014]: pgmap v719: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 416 op/s; 26113/258585 objects degraded (10.098%); 10243/258585 objects misplaced (3.961%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:26:19.289 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:26:19.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:19.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:26:19.310 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 87805 87789 65.2757 27.1875 0.0374795 0.0153132 2023-09-23T22:26:19.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:18 smithi070 ceph-mon[192652]: pgmap v719: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 416 op/s; 26113/258585 objects degraded (10.098%); 10243/258585 objects misplaced (3.961%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:26:19.402 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:19.430 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:19.477 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:19.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:19.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:19.584 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:19.611 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:19.655 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:19.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:19.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:19.763 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:19.868 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:26:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:26:19] "GET /metrics HTTP/1.1" 200 36367 "" "Prometheus/2.43.0" 2023-09-23T22:26:19.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:19.996 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:20.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:20.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:20.032 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:26:20.033 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:26:20.038 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:20.108 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:20.207 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:20.315 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 88240 88224 64.825 27.1875 0.0720081 0.0154182 2023-09-23T22:26:20.993 INFO:teuthology.orchestra.run.smithi026.stdout:moved 2 / 297 (0.673401%) 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 10.4315 -> 9.75203 (expected baseline 6.03053) 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 20 -> 22 pgs (0.471381 -> 0.518519 * mean) 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.3 with 53 -> 53 pgs (1.24916 -> 1.24916 * mean) 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 0.05 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0300 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.3002 2023-09-23T22:26:20.994 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 0.5412 -> 0.5912 2023-09-23T22:26:21.211 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:26:21.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:20 smithi026 ceph-mon[215775]: pgmap v720: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 25 MiB/s wr, 400 op/s; 26113/259443 objects degraded (10.065%); 10243/259443 objects misplaced (3.948%); 1.0 MiB/s, 17 objects/s recovering 2023-09-23T22:26:21.289 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'38745'}"}]: dispatch 2023-09-23T22:26:21.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:20 smithi026 ceph-mon[223014]: pgmap v720: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 25 MiB/s wr, 400 op/s; 26113/259443 objects degraded (10.065%); 10243/259443 objects misplaced (3.948%); 1.0 MiB/s, 17 objects/s recovering 2023-09-23T22:26:21.289 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'38745'}"}]: dispatch 2023-09-23T22:26:21.314 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 88643 88627 64.364 25.1875 0.0200005 0.0155286 2023-09-23T22:26:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:20 smithi070 ceph-mon[192652]: pgmap v720: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 25 MiB/s wr, 400 op/s; 26113/259443 objects degraded (10.065%); 10243/259443 objects misplaced (3.948%); 1.0 MiB/s, 17 objects/s recovering 2023-09-23T22:26:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'38745'}"}]: dispatch 2023-09-23T22:26:21.444 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:21.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:21.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:21.550 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:21.577 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:21.585 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:26:21.620 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:21.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:21.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:21.725 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:21.783 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:22.318 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 89046 89030 63.9113 25.1875 0.0129165 0.0156408 2023-09-23T22:26:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:21 smithi070 ceph-mon[192652]: from='client.35496 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:22.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'38745'}"}]': finished 2023-09-23T22:26:22.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:21 smithi070 ceph-mon[192652]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T22:26:22.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:21 smithi070 ceph-mon[192652]: failed to encode map e277 with expected crc 2023-09-23T22:26:22.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:21 smithi070 ceph-mon[192652]: failed to encode map e277 with expected crc 2023-09-23T22:26:22.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[223014]: from='client.35496 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:22.459 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'38745'}"}]': finished 2023-09-23T22:26:22.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[223014]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T22:26:22.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[223014]: failed to encode map e277 with expected crc 2023-09-23T22:26:22.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[223014]: failed to encode map e277 with expected crc 2023-09-23T22:26:22.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[215775]: from='client.35496 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:22.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'38745'}"}]': finished 2023-09-23T22:26:22.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[215775]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T22:26:22.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[215775]: failed to encode map e277 with expected crc 2023-09-23T22:26:22.461 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:21 smithi026 ceph-mon[215775]: failed to encode map e277 with expected crc 2023-09-23T22:26:22.484 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:22.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:22.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:22.504 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T22:26:22.504 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T22:26:22.593 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:22.618 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:22.666 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:22.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:22.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:22.772 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:26:22.899 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:22.979 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:26:22.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:22.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:26:23.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:23.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[223014]: pgmap v722: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 421 op/s; 26113/262347 objects degraded (9.954%); 10243/262347 objects misplaced (3.904%); 739 KiB/s, 11 objects/s recovering 2023-09-23T22:26:23.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[223014]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T22:26:23.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[223014]: failed to encode map e278 with expected crc 2023-09-23T22:26:23.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[223014]: failed to encode map e278 with expected crc 2023-09-23T22:26:23.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[223014]: failed to encode map e278 with expected crc 2023-09-23T22:26:23.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26113/262347 objects degraded (9.954%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:23.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[215775]: pgmap v722: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 421 op/s; 26113/262347 objects degraded (9.954%); 10243/262347 objects misplaced (3.904%); 739 KiB/s, 11 objects/s recovering 2023-09-23T22:26:23.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[215775]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T22:26:23.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[215775]: failed to encode map e278 with expected crc 2023-09-23T22:26:23.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[215775]: failed to encode map e278 with expected crc 2023-09-23T22:26:23.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[215775]: failed to encode map e278 with expected crc 2023-09-23T22:26:23.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:22 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26113/262347 objects degraded (9.954%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:23.173 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:23.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 89467 89451 63.4821 26.3125 0.0196228 0.0157461 2023-09-23T22:26:23.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:22 smithi070 ceph-mon[192652]: pgmap v722: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 26 MiB/s wr, 421 op/s; 26113/262347 objects degraded (9.954%); 10243/262347 objects misplaced (3.904%); 739 KiB/s, 11 objects/s recovering 2023-09-23T22:26:23.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:22 smithi070 ceph-mon[192652]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T22:26:23.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:22 smithi070 ceph-mon[192652]: failed to encode map e278 with expected crc 2023-09-23T22:26:23.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:22 smithi070 ceph-mon[192652]: failed to encode map e278 with expected crc 2023-09-23T22:26:23.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:22 smithi070 ceph-mon[192652]: failed to encode map e278 with expected crc 2023-09-23T22:26:23.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:22 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26113/262347 objects degraded (9.954%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:23.430 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:23.438 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:23.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:23.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:23.552 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:23.578 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:23.622 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:23.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:23.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:23.666 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:23.679 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:23.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:23.838 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:24.288 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:24.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:24.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:24.321 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 89913 89897 63.0822 27.875 0.0106286 0.0158457 2023-09-23T22:26:24.401 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:24.427 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:24.471 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:24.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:24.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:24.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:24.603 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:24.649 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:24.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:24.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:24.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:24.904 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:25.321 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.0228 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 90334 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 62.7161 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 34.4291 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 174.5 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 1003 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 550.866 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 2792 2023-09-23T22:26:25.322 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T22:26:25.323 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.01594 2023-09-23T22:26:25.323 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.0401262 2023-09-23T22:26:25.323 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 2.89017 2023-09-23T22:26:25.323 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00118556 2023-09-23T22:26:25.326 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:26:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:24 smithi070 ceph-mon[192652]: pgmap v724: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 25 MiB/s wr, 395 op/s; 26113/265095 objects degraded (9.850%); 10138/265095 objects misplaced (3.824%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:26:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:26:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:26:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2114007986' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:26:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:24 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:26:25.443 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:26:25.468 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[215775]: pgmap v724: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 25 MiB/s wr, 395 op/s; 26113/265095 objects degraded (9.850%); 10138/265095 objects misplaced (3.824%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2114007986' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[223014]: pgmap v724: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 25 MiB/s wr, 395 op/s; 26113/265095 objects degraded (9.850%); 10138/265095 objects misplaced (3.824%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:26:25.469 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2114007986' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:26:25.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:24 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:26:26.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:26 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:26.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:26 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:26:26.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:26 smithi070 ceph-mon[192652]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T22:26:26.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:26 smithi070 ceph-mon[192652]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:26 smithi070 ceph-mon[192652]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:26 smithi070 ceph-mon[192652]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:26 smithi070 ceph-mon[192652]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:26:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[215775]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T22:26:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[215775]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[215775]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[215775]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[215775]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:26.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:26:26.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[223014]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T22:26:26.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[223014]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[223014]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[223014]: failed to encode map e279 with expected crc 2023-09-23T22:26:26.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:26 smithi026 ceph-mon[223014]: failed to encode map e279 with expected crc 2023-09-23T22:26:27.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:27 smithi026 ceph-mon[215775]: pgmap v725: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 26 MiB/s wr, 416 op/s; 26113/268578 objects degraded (9.723%); 10138/268578 objects misplaced (3.775%); 839 KiB/s, 13 objects/s recovering 2023-09-23T22:26:27.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:27 smithi026 ceph-mon[215775]: failed to encode map e279 with expected crc 2023-09-23T22:26:27.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:27 smithi026 ceph-mon[215775]: failed to encode map e279 with expected crc 2023-09-23T22:26:27.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:27 smithi026 ceph-mon[223014]: pgmap v725: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 26 MiB/s wr, 416 op/s; 26113/268578 objects degraded (9.723%); 10138/268578 objects misplaced (3.775%); 839 KiB/s, 13 objects/s recovering 2023-09-23T22:26:27.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:27 smithi026 ceph-mon[223014]: failed to encode map e279 with expected crc 2023-09-23T22:26:27.288 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:27 smithi026 ceph-mon[223014]: failed to encode map e279 with expected crc 2023-09-23T22:26:27.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:27 smithi070 ceph-mon[192652]: pgmap v725: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 26 MiB/s wr, 416 op/s; 26113/268578 objects degraded (9.723%); 10138/268578 objects misplaced (3.775%); 839 KiB/s, 13 objects/s recovering 2023-09-23T22:26:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:27 smithi070 ceph-mon[192652]: failed to encode map e279 with expected crc 2023-09-23T22:26:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:27 smithi070 ceph-mon[192652]: failed to encode map e279 with expected crc 2023-09-23T22:26:27.424 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:27.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:27.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:27.529 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:26:27.555 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:27.597 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:26:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:27.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:26:27.703 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:27.779 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:28.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26113/268578 objects degraded (9.723%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:28.383 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:28.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:28.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26113/268578 objects degraded (9.723%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:28.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:28.396 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26113/268578 objects degraded (9.723%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:28.497 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:28.524 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:28.570 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:28.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:28.678 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:28.798 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:29.037 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:26:29.037 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:26:29.062 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:29 smithi026 ceph-mon[223014]: pgmap v727: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 26 MiB/s wr, 477 op/s; 26113/269349 objects degraded (9.695%); 10027/269349 objects misplaced (3.723%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T22:26:29.110 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:29.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:29.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:29.216 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:29.242 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:29.286 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:29.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:29.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:29 smithi070 ceph-mon[192652]: pgmap v727: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 26 MiB/s wr, 477 op/s; 26113/269349 objects degraded (9.695%); 10027/269349 objects misplaced (3.723%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T22:26:29.393 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:26:29.420 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:29.464 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:26:29.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:29.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:26:29.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:29 smithi026 ceph-mon[215775]: pgmap v727: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 26 MiB/s wr, 477 op/s; 26113/269349 objects degraded (9.695%); 10027/269349 objects misplaced (3.723%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T22:26:29.568 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:29.688 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:29.708 INFO:teuthology.orchestra.run.smithi026.stdout:moved 9 / 297 (3.0303%) 2023-09-23T22:26:29.708 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:26:29.708 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 9.75203 -> 7.06818 (expected baseline 6.03053) 2023-09-23T22:26:29.708 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 22 -> 31 pgs (0.518519 -> 0.73064 * mean) 2023-09-23T22:26:29.708 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.3 with 53 -> 53 pgs (1.24916 -> 1.24916 * mean) 2023-09-23T22:26:29.708 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:26:29.709 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T22:26:29.709 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 3 2023-09-23T22:26:29.709 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:26:29.709 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0309 2023-09-23T22:26:29.709 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.3088 2023-09-23T22:26:29.709 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 0.5912 -> 1.0000 2023-09-23T22:26:29.716 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T22:26:29.756 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:26:29.756 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:26:29.757 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:26:29.757 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:26:29.786 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:26:29.854 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:29.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:29.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:29.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:26:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:26:29] "GET /metrics HTTP/1.1" 200 36389 "" "Prometheus/2.43.0" 2023-09-23T22:26:29.946 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:26:29 smithi026 conmon[202047]: 2023-09-23T22:26:29.689+0000 7f47954a6700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-23T22:26:29.964 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:29.993 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:30.027 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:26:30.027 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:26:30.027 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:26:30.027 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:26:30.027 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:26:30.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:26:30.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:26:30.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:26:30.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:26:30.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:26:30.028 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:26:30.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:30 smithi026 ceph-mon[223014]: pgmap v728: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 22 MiB/s wr, 418 op/s; 26113/269004 objects degraded (9.707%); 10027/269004 objects misplaced (3.727%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T22:26:30.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/840186384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:30.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:30 smithi026 ceph-mon[223014]: from='client.35508 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:30.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:30 smithi026 ceph-mon[215775]: pgmap v728: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 22 MiB/s wr, 418 op/s; 26113/269004 objects degraded (9.707%); 10027/269004 objects misplaced (3.727%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T22:26:30.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/840186384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:30.241 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:30 smithi026 ceph-mon[215775]: from='client.35508 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:30 smithi070 ceph-mon[192652]: pgmap v728: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.5 GiB data, 19 GiB used, 606 GiB / 626 GiB avail; 127 B/s rd, 22 MiB/s wr, 418 op/s; 26113/269004 objects degraded (9.707%); 10027/269004 objects misplaced (3.727%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T22:26:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/840186384' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:30 smithi070 ceph-mon[192652]: from='client.35508 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:30.598 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:26:30.844 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:30.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:30.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:30.955 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:31.014 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:31.767 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:31.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:31.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:31.829 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:26:31.830 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T22:26:31.831 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T22:26:31.874 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:32.018 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:32.230 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:32.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:32.345 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:26:32.359 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:32.450 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:32.561 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:26:32.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:32.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:26:32.666 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:32.692 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:32.736 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:32.810 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:32 smithi026 ceph-mon[215775]: pgmap v729: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 109 B/s rd, 19 MiB/s wr, 459 op/s; 26113/266880 objects degraded (9.785%); 10027/266880 objects misplaced (3.757%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:26:32.811 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:32 smithi026 ceph-mon[223014]: pgmap v729: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 109 B/s rd, 19 MiB/s wr, 459 op/s; 26113/266880 objects degraded (9.785%); 10027/266880 objects misplaced (3.757%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:26:32.811 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:32 smithi070 ceph-mon[192652]: pgmap v729: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 109 B/s rd, 19 MiB/s wr, 459 op/s; 26113/266880 objects degraded (9.785%); 10027/266880 objects misplaced (3.757%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:26:32.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:32.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:32.913 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:32.940 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:32.989 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:32.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:32.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:33.052 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:33.096 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:33.222 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:33.473 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26113/266880 objects degraded (9.785%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26113/266880 objects degraded (9.785%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:33.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26113/266880 objects degraded (9.785%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:34.061 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:34.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:34.067 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:34.147 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:26:34.153 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:34.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:34.357 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:34.726 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:34.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:34.835 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:34.862 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:34.915 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:34.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:34.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:35.021 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:26:35.048 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:35.099 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:26:35.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:35.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:26:35.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:34 smithi070 ceph-mon[192652]: pgmap v730: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 12 MiB/s wr, 435 op/s; 26113/264018 objects degraded (9.891%); 9922/264018 objects misplaced (3.758%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:26:35.205 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:35.232 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:35.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:34 smithi026 ceph-mon[223014]: pgmap v730: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 12 MiB/s wr, 435 op/s; 26113/264018 objects degraded (9.891%); 9922/264018 objects misplaced (3.758%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:26:35.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:34 smithi026 ceph-mon[215775]: pgmap v730: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.4 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 12 MiB/s wr, 435 op/s; 26113/264018 objects degraded (9.891%); 9922/264018 objects misplaced (3.758%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:26:35.276 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:35.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:35.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:35.423 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:36.038 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:26:36.122 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:36.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:36 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2143022287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:36.151 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:36.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:36.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:36.154 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:36 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2143022287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:36.154 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:36.253 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:36.344 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:36.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:36 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2143022287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:36.488 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:36.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:36.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:36.592 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:36.618 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:36.668 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:36.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:36.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:36.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:36.802 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:36.846 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:36.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:36.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:36.952 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:36.978 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:37.023 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:37.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:37.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:37.129 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:37.155 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:37.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:37.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:37.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:37.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[223014]: pgmap v731: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 5.2 MiB/s wr, 446 op/s; 26113/260208 objects degraded (10.035%); 9922/260208 objects misplaced (3.813%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:26:37.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:37.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[223014]: failed to encode map e280 with expected crc 2023-09-23T22:26:37.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:26:37.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[223014]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T22:26:37.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[223014]: failed to encode map e280 with expected crc 2023-09-23T22:26:37.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[223014]: failed to encode map e280 with expected crc 2023-09-23T22:26:37.269 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[215775]: pgmap v731: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 5.2 MiB/s wr, 446 op/s; 26113/260208 objects degraded (10.035%); 9922/260208 objects misplaced (3.813%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:26:37.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:37.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[215775]: failed to encode map e280 with expected crc 2023-09-23T22:26:37.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:26:37.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[215775]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T22:26:37.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[215775]: failed to encode map e280 with expected crc 2023-09-23T22:26:37.270 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:37 smithi026 ceph-mon[215775]: failed to encode map e280 with expected crc 2023-09-23T22:26:37.310 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:37.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:37 smithi070 ceph-mon[192652]: pgmap v731: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 5.2 MiB/s wr, 446 op/s; 26113/260208 objects degraded (10.035%); 9922/260208 objects misplaced (3.813%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:26:37.334 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:37 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:37.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:37 smithi070 ceph-mon[192652]: failed to encode map e280 with expected crc 2023-09-23T22:26:37.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:37 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:26:37.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:37 smithi070 ceph-mon[192652]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T22:26:37.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:37 smithi070 ceph-mon[192652]: failed to encode map e280 with expected crc 2023-09-23T22:26:37.335 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:37 smithi070 ceph-mon[192652]: failed to encode map e280 with expected crc 2023-09-23T22:26:37.337 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:37.384 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:37.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:37.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:37.489 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:37.516 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:37.561 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:37.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:37.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:37.666 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:37.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:37.982 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:37.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:37.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:38.091 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:38.117 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:38.118 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:38.160 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:38.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:38.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:38.268 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:38.294 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:38.338 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:38.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:38 smithi070 ceph-mon[192652]: failed to encode map e280 with expected crc 2023-09-23T22:26:38.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26113/260208 objects degraded (10.035%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:38.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:38.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:38.445 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:26:38.492 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:38 smithi026 ceph-mon[215775]: failed to encode map e280 with expected crc 2023-09-23T22:26:38.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26113/260208 objects degraded (10.035%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:38.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:38 smithi026 ceph-mon[223014]: failed to encode map e280 with expected crc 2023-09-23T22:26:38.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26113/260208 objects degraded (10.035%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:38.592 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:38.781 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:26:38.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:38.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:26:38.896 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:26:39.042 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:39.070 INFO:teuthology.orchestra.run.smithi026.stdout:moved 9 / 297 (3.0303%) 2023-09-23T22:26:39.070 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:26:39.070 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 9.75203 -> 7.06818 (expected baseline 6.03053) 2023-09-23T22:26:39.070 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 22 -> 31 pgs (0.518519 -> 0.73064 * mean) 2023-09-23T22:26:39.070 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.3 with 53 -> 53 pgs (1.24916 -> 1.24916 * mean) 2023-09-23T22:26:39.070 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:26:39.071 INFO:teuthology.orchestra.run.smithi026.stdout:oload 1000 2023-09-23T22:26:39.071 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 1 2023-09-23T22:26:39.071 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:26:39.071 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0298 2023-09-23T22:26:39.071 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.2975 2023-09-23T22:26:39.071 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2 weight 0.5912 -> 1.0000 2023-09-23T22:26:39.075 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:39 smithi026 ceph-mon[215775]: pgmap v733: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 452 op/s; 26113/257070 objects degraded (10.158%); 9821/257070 objects misplaced (3.820%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:26:39.075 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65536'}"}]: dispatch 2023-09-23T22:26:39.168 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:26:39.168 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:26:39.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93554 6.b 6:d2b835f0:::benchmark_data_smithi026_355573_object12204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.178872+0000", 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 249.98463472500001, 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0056578679999999, 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:26:39.169 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93554 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178872+0000", 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178872+0000", 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178870+0000", 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178878+0000", 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3919999999999997e-06 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178894+0000", 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5838999999999999e-05 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178903+0000", 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2730000000000008e-06 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.184954+0000", 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060513269999999996 2023-09-23T22:26:39.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.184967+0000", 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2462e-05 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.173392+0000", 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.988425043 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.173530+0000", 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138851 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.173688+0000", 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000157379 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178167+0000", 2023-09-23T22:26:39.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044787869999999997 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178182+0000", 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5404999999999999e-05 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184445+0000", 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062626799999999996 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184491+0000", 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6737999999999999e-05 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184530+0000", 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8407e-05 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93584 6.b 6:d49f848e:::benchmark_data_smithi026_355573_object12234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.233323+0000", 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 249.93018347899999, 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95130220499999996, 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93584 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233323+0000", 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233323+0000", 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233322+0000", 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233328+0000", 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5429999999999999e-06 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233341+0000", 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3261000000000001e-05 2023-09-23T22:26:39.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233351+0000", 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2459999999999992e-06 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233412+0000", 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0812e-05 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.233423+0000", 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1721e-05 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.176192+0000", 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94276894499999997 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.176311+0000", 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119238 2023-09-23T22:26:39.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.176383+0000", 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1946999999999994e-05 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178220+0000", 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001836458 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184379+0000", 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061588290000000002 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184566+0000", 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000187314 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184597+0000", 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1398e-05 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184625+0000", 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8113e-05 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93607 6.b 6:d3fcbdef:::benchmark_data_smithi026_355573_object12257:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.273341+0000", 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 249.89016525599999, 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91138181100000004, 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93607 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.185 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273341+0000", 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273341+0000", 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273340+0000", 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273347+0000", 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1049999999999997e-06 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273360+0000", 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2604e-05 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273367+0000", 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9489999999999996e-06 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273441+0000", 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3835e-05 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.273452+0000", 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0873000000000001e-05 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178246+0000", 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.90479371200000003 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178360+0000", 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114561 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178486+0000", 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012549100000000001 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184364+0000", 2023-09-23T22:26:39.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058778379999999998 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184384+0000", 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9867000000000001e-05 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184658+0000", 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027395400000000003 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184687+0000", 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9799e-05 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184723+0000", 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5917999999999997e-05 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.192 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93621 6.b 6:d0e06b28:::benchmark_data_smithi026_355573_object12271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.329808+0000", 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 249.83369907599999, 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.863240699, 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93621 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329808+0000", 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329808+0000", 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329806+0000", 2023-09-23T22:26:39.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329814+0000", 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9070000000000007e-06 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329834+0000", 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9731000000000001e-05 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329842+0000", 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8400000000000001e-06 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329932+0000", 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9689000000000001e-05 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.329945+0000", 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2563e-05 2023-09-23T22:26:39.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.181153+0000", 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.85120826599999999 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.181276+0000", 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012343 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:26:39.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.181402+0000", 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125334 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184414+0000", 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030121119999999999 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.192892+0000", 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084780399999999992 2023-09-23T22:26:39.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.192974+0000", 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1744999999999997e-05 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193006+0000", 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2427e-05 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193048+0000", 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2385000000000002e-05 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93643 6.b 6:db9397f6:::benchmark_data_smithi026_355573_object12293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.382892+0000", 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 249.780614805, 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81032079400000001, 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.200 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93643 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382892+0000", 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382892+0000", 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382891+0000", 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382895+0000", 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8350000000000003e-06 2023-09-23T22:26:39.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382907+0000", 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1114999999999999e-05 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.382914+0000", 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1760000000000004e-06 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.383035+0000", 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121657 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.383045+0000", 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0022e-05 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184771+0000", 2023-09-23T22:26:39.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80172589999999999 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184906+0000", 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013509899999999999 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.185041+0000", 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134182 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193064+0000", 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080234079999999992 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193119+0000", 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5152999999999999e-05 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193146+0000", 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.686e-05 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193175+0000", 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8615e-05 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193213+0000", 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8120000000000001e-05 2023-09-23T22:26:39.207 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93650 6.b 6:de3511b3:::benchmark_data_smithi026_355573_object12300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.385904+0000", 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 249.777603137, 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81944647999999998, 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93650 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385904+0000", 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385904+0000", 2023-09-23T22:26:39.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385903+0000", 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385906+0000", 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8569999999999999e-06 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385912+0000", 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6999999999999996e-06 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.385917+0000", 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7849999999999999e-06 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.387207+0000", 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001290081 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.387217+0000", 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0086e-05 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.187923+0000", 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80070577899999995 2023-09-23T22:26:39.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.188054+0000", 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013196200000000001 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.188190+0000", 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013539300000000001 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.193068+0000", 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048783979999999999 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.199041+0000", 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059726700000000002 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205266+0000", 2023-09-23T22:26:39.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062251429999999998 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205303+0000", 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6878999999999997e-05 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.205350+0000", 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7138999999999997e-05 2023-09-23T22:26:39.215 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83436 7.2 7:4436d6c5:::benchmark_data_smithi026_410749_object83435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.874035+0000", 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.289472013000001, 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8534463419999998, 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.216 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83436 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874035+0000", 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874035+0000", 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874034+0000", 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874054+0000", 2023-09-23T22:26:39.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0642e-05 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874063+0000", 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9560000000000003e-06 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874069+0000", 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2400000000000004e-06 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.109468+0000", 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23539858499999999 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.109634+0000", 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016552 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.109834+0000", 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000200714 2023-09-23T22:26:39.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119592+0000", 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097577359999999995 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119603+0000", 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.13e-05 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.965993+0000", 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84638983099999998 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966014+0000", 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0774000000000001e-05 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.688706+0000", 2023-09-23T22:26:39.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.722691585 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694732+0000", 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060268170000000003 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.727365+0000", 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032632845000000001 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.727481+0000", 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011591600000000001 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83439 7.2 7:41661071:::benchmark_data_smithi026_410749_object83438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875790+0000", 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.287716906999997, 2023-09-23T22:26:39.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8703235449999998, 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83439 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875790+0000", 2023-09-23T22:26:39.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875790+0000", 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875789+0000", 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:26:39.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875812+0000", 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3323999999999999e-05 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875835+0000", 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.317e-05 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875844+0000", 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4370000000000007e-06 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.113024+0000", 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.237180689 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.113148+0000", 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012339600000000001 2023-09-23T22:26:39.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.113277+0000", 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129271 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119636+0000", 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063592079999999999 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119647+0000", 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0547000000000001e-05 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966064+0000", 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84641742499999995 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966076+0000", 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1899000000000001e-05 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.699874+0000", 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.733797451 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.702993+0000", 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031193470000000002 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746002+0000", 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.04300913 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746113+0000", 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011135 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.232 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83442 7.c 7:3b003b51:::benchmark_data_smithi026_410749_object83441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.287656372000001, 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8103755399999999, 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83442 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875871+0000", 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0897999999999999e-05 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875878+0000", 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0140000000000001e-06 2023-09-23T22:26:39.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875883+0000", 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4360000000000001e-06 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875932+0000", 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8757000000000001e-05 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.876054+0000", 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122371 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.110153+0000", 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23409861300000001 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119005+0000", 2023-09-23T22:26:39.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088520830000000002 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119022+0000", 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6586999999999999e-05 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967848+0000", 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84882608400000004 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967861+0000", 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3498e-05 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.653928+0000", 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6860671979999999 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.654575+0000", 2023-09-23T22:26:39.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00064664599999999996 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686123+0000", 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031547598000000003 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686226+0000", 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000103359 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.240 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83443 7.a 7:5676589a:::benchmark_data_smithi026_410749_object83442:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.287609472, 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8791883110000001, 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.241 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83443 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875917+0000", 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0571e-05 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875925+0000", 2023-09-23T22:26:39.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4800000000000004e-06 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875930+0000", 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2549999999999997e-06 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120311+0000", 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.24438064000000001 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120321+0000", 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0532e-05 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966151+0000", 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84582989600000003 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966161+0000", 2023-09-23T22:26:39.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0161e-05 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713553+0000", 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7473917839999999 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713688+0000", 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013500599999999999 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713846+0000", 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015831800000000001 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720327+0000", 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064802289999999997 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723835+0000", 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035083179999999999 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.752272+0000", 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028437240999999999 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.752361+0000", 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8768000000000001e-05 2023-09-23T22:26:39.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.755086+0000", 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027245949999999998 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83444 7.2 7:4ab81381:::benchmark_data_smithi026_410749_object83443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:26:39.249 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.287571753999998, 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8732291189999999, 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83444 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875955+0000", 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0437000000000001e-05 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875962+0000", 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2590000000000003e-06 2023-09-23T22:26:39.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875968+0000", 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5539999999999997e-06 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120681+0000", 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.24471289900000001 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120690+0000", 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4579999999999999e-06 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966109+0000", 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84541897700000002 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966119+0000", 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4369999999999996e-06 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703041+0000", 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.73692275 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703182+0000", 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014080000000000001 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703342+0000", 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016031500000000001 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713519+0000", 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010176648 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723879+0000", 2023-09-23T22:26:39.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010359706 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746208+0000", 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022329188999999999 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746255+0000", 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6820000000000002e-05 2023-09-23T22:26:39.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.749164+0000", 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029093589999999998 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83449 7.c 7:31b18e39:::benchmark_data_smithi026_410749_object83448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.118334+0000", 2023-09-23T22:26:39.258 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.045172323000003, 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5715964209999997, 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83449 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118334+0000", 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118334+0000", 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118333+0000", 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:26:39.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118364+0000", 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1628999999999998e-05 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118376+0000", 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1888e-05 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118385+0000", 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4829999999999999e-06 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119686+0000", 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001300998 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119696+0000", 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0208e-05 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967893+0000", 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84819666800000004 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967901+0000", 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8619999999999992e-06 2023-09-23T22:26:39.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.657644+0000", 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6897430070000001 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.657855+0000", 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021073899999999999 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.658074+0000", 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021842699999999999 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664296+0000", 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062224689999999996 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.667607+0000", 2023-09-23T22:26:39.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033106160000000002 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686269+0000", 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018662099000000001 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686317+0000", 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8285999999999998e-05 2023-09-23T22:26:39.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.689931+0000", 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036137999999999999 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83454 7.a 7:5caa08ff:::benchmark_data_smithi026_410749_object83453:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:26:39.267 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.125118+0000", 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.038388325, 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.634736432, 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83454 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125118+0000", 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125118+0000", 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125116+0000", 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:26:39.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125144+0000", 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8147999999999999e-05 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125157+0000", 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2418999999999999e-05 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125167+0000", 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0088999999999999e-05 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125218+0000", 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1601000000000003e-05 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125225+0000", 2023-09-23T22:26:39.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0539999999999997e-06 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966187+0000", 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84096177699999997 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966197+0000", 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3619999999999995e-06 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.716965+0000", 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7507684270000001 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.717105+0000", 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013953199999999999 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:26:39.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.717236+0000", 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131901 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720350+0000", 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031134539999999999 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.734562+0000", 2023-09-23T22:26:39.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014212158000000001 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.756983+0000", 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022420549000000001 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.757031+0000", 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8467999999999997e-05 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.759855+0000", 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002823791 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83455 7.c 7:30f89560:::benchmark_data_smithi026_410749_object83454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.128196+0000", 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.035310674999998, 2023-09-23T22:26:39.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5655597960000001, 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83455 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128196+0000", 2023-09-23T22:26:39.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128196+0000", 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128195+0000", 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128218+0000", 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3824000000000001e-05 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128235+0000", 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6864e-05 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128245+0000", 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6709999999999994e-06 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128304+0000", 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9104000000000002e-05 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128316+0000", 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1858999999999999e-05 2023-09-23T22:26:39.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967930+0000", 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83961362900000003 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967939+0000", 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0000000000000002e-06 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664326+0000", 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.696387901 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664467+0000", 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140433 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664596+0000", 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129345 2023-09-23T22:26:39.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.667813+0000", 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032163579999999999 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.671373+0000", 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035605559999999999 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.690178+0000", 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018804521000000001 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.690239+0000", 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1784000000000003e-05 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.693756+0000", 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035164060000000001 2023-09-23T22:26:39.284 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83459 7.c 7:389583a7:::benchmark_data_smithi026_410749_object83458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.131267+0000", 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.032239251999997, 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5663957699999997, 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.285 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83459 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131267+0000", 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131267+0000", 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131266+0000", 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131287+0000", 2023-09-23T22:26:39.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0669000000000001e-05 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131296+0000", 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3179999999999996e-06 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131303+0000", 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3890000000000002e-06 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131356+0000", 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2883999999999997e-05 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131367+0000", 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1501000000000001e-05 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.968650+0000", 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83728242900000005 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.968662+0000", 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2218999999999999e-05 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.667852+0000", 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6991896950000001 2023-09-23T22:26:39.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.668041+0000", 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018923599999999999 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.668259+0000", 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000218063 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.671597+0000", 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033378510000000002 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.675784+0000", 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004187362 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.693833+0000", 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018049193000000002 2023-09-23T22:26:39.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.693899+0000", 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5381000000000003e-05 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.697663+0000", 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037646799999999998 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.293 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83460 7.2 7:4c76e5c3:::benchmark_data_smithi026_410749_object83459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.031610413000003, 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6264806219999999, 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.294 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83460 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:26:39.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131915+0000", 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9627999999999999e-05 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131927+0000", 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2150999999999999e-05 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131933+0000", 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2390000000000002e-06 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131970+0000", 2023-09-23T22:26:39.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7323000000000002e-05 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131979+0000", 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1439999999999992e-06 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966248+0000", 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83426858500000001 2023-09-23T22:26:39.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966259+0000", 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0813e-05 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.709897+0000", 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.743638695 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.710090+0000", 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000192552 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.710317+0000", 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022765700000000001 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720308+0000", 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099907369999999995 2023-09-23T22:26:39.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723922+0000", 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036138960000000001 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.755137+0000", 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031214848999999999 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.755202+0000", 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4978e-05 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.758377+0000", 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031750530000000002 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83432 7.8 7:102b4fc4:::benchmark_data_smithi026_410749_object83431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978262+0000", 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.185245127999998, 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7098934809999999, 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83432 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.303 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978262+0000", 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978262+0000", 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978260+0000", 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978316+0000", 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6307000000000001e-05 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978331+0000", 2023-09-23T22:26:39.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4329e-05 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978341+0000", 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0179e-05 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978398+0000", 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7342000000000001e-05 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978405+0000", 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2559999999999996e-06 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686224+0000", 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7078187279999999 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.688155+0000", 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019310600000000001 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83434 7.8 7:1cb65715:::benchmark_data_smithi026_410749_object83433:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978364+0000", 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.185142755000001, 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7126752629999999, 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.308 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83434 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978364+0000", 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978364+0000", 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978363+0000", 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978410+0000", 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6882999999999998e-05 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978421+0000", 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0555e-05 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978428+0000", 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.447e-06 2023-09-23T22:26:39.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978481+0000", 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2367000000000002e-05 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978491+0000", 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0348e-05 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.690940+0000", 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.712448669 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691039+0000", 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9671000000000002e-05 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.313 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83451 7.8 7:10ddb7e8:::benchmark_data_smithi026_410749_object83450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978444+0000", 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.185062922, 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7421942690000001, 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83451 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978444+0000", 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978444+0000", 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978443+0000", 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978490+0000", 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7082000000000001e-05 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978500+0000", 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9930000000000007e-06 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978507+0000", 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8789999999999997e-06 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978649+0000", 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014189399999999999 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978658+0000", 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2750000000000001e-06 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691064+0000", 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.712405483 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691807+0000", 2023-09-23T22:26:39.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00074337199999999996 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691974+0000", 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016714799999999999 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.698213+0000", 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006238848 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.704903+0000", 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066894340000000002 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705854+0000", 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00095168 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.718111+0000", 2023-09-23T22:26:39.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012256306999999999 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.718166+0000", 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5387999999999999e-05 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720638+0000", 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024721159999999999 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.321 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83458 7.8 7:14196f44:::benchmark_data_smithi026_410749_object83457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 31.184984615000001, 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.744606302, 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:39.322 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83458 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:26:39.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978566+0000", 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4564000000000002e-05 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978585+0000", 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8978999999999999e-05 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978592+0000", 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1250000000000004e-06 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978682+0000", 2023-09-23T22:26:39.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0093999999999995e-05 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978690+0000", 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3200000000000002e-06 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694418+0000", 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.715727904 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694629+0000", 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021179900000000001 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694812+0000", 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018232099999999999 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.701814+0000", 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070020819999999998 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705861+0000", 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040473699999999998 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.711791+0000", 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059297259999999997 2023-09-23T22:26:39.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720680+0000", 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088894569999999999 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720735+0000", 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5049000000000001e-05 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:26:39.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723128+0000", 2023-09-23T22:26:39.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002393109 2023-09-23T22:26:39.330 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.330 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.330 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.330 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:26:39.330 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:26:39.330 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:26:39.330 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:26:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:39 smithi070 ceph-mon[192652]: pgmap v733: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 452 op/s; 26113/257070 objects degraded (10.158%); 9821/257070 objects misplaced (3.820%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:26:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65536'}"}]: dispatch 2023-09-23T22:26:39.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:39 smithi026 ceph-mon[223014]: pgmap v733: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 1.4 MiB/s wr, 452 op/s; 26113/257070 objects degraded (10.158%); 9821/257070 objects misplaced (3.820%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:26:39.421 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'2':'65536'}"}]: dispatch 2023-09-23T22:26:39.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:26:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:26:39] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-09-23T22:26:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:40 smithi070 ceph-mon[192652]: from='client.45455 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:40 smithi070 ceph-mon[192652]: failed to encode map e281 with expected crc 2023-09-23T22:26:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536'}"}]': finished 2023-09-23T22:26:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:40 smithi070 ceph-mon[192652]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T22:26:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:40 smithi070 ceph-mon[192652]: failed to encode map e281 with expected crc 2023-09-23T22:26:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:26:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:40 smithi070 ceph-mon[192652]: pgmap v735: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 444 op/s; 26113/256194 objects degraded (10.193%); 9821/256194 objects misplaced (3.833%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:26:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:40 smithi070 ceph-mon[192652]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T22:26:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[215775]: from='client.45455 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[215775]: failed to encode map e281 with expected crc 2023-09-23T22:26:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536'}"}]': finished 2023-09-23T22:26:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[215775]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T22:26:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[215775]: failed to encode map e281 with expected crc 2023-09-23T22:26:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:26:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[215775]: pgmap v735: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 444 op/s; 26113/256194 objects degraded (10.193%); 9821/256194 objects misplaced (3.833%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:26:40.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[215775]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T22:26:40.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[223014]: from='client.45455 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:40.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[223014]: failed to encode map e281 with expected crc 2023-09-23T22:26:40.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'2':'65536'}"}]': finished 2023-09-23T22:26:40.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[223014]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T22:26:40.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[223014]: failed to encode map e281 with expected crc 2023-09-23T22:26:40.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:26:40.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:40.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[223014]: pgmap v735: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 5.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 444 op/s; 26113/256194 objects degraded (10.193%); 9821/256194 objects misplaced (3.833%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:26:40.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:40 smithi026 ceph-mon[223014]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T22:26:40.965 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:26:40.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:40.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:26:41.069 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:41.102 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:41.151 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:41.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:41.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:41.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[223014]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[223014]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[223014]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[223014]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[223014]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[223014]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T22:26:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[215775]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[215775]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[215775]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[215775]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[215775]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:41 smithi026 ceph-mon[215775]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T22:26:41.260 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:41.373 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:41 smithi070 ceph-mon[192652]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:41 smithi070 ceph-mon[192652]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:41 smithi070 ceph-mon[192652]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:41 smithi070 ceph-mon[192652]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:41 smithi070 ceph-mon[192652]: failed to encode map e282 with expected crc 2023-09-23T22:26:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:41 smithi070 ceph-mon[192652]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T22:26:41.895 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:41.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:41.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:42.005 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:42.017 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:26:42.113 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T22:26:42.113 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T22:26:42.158 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[223014]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[223014]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 280'6008 2023-09-23T22:26:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[223014]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[223014]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[223014]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[223014]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[223014]: pgmap v738: 99 pgs: 1 active+recovery_wait+remapped, 3 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 26113/253332 objects degraded (10.308%); 9821/253332 objects misplaced (3.877%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T22:26:42.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[223014]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T22:26:42.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[215775]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[215775]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 280'6008 2023-09-23T22:26:42.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[215775]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[215775]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[215775]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[215775]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[215775]: pgmap v738: 99 pgs: 1 active+recovery_wait+remapped, 3 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 26113/253332 objects degraded (10.308%); 9821/253332 objects misplaced (3.877%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T22:26:42.368 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:42 smithi026 ceph-mon[215775]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T22:26:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:42 smithi070 ceph-mon[192652]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:42 smithi070 ceph-mon[192652]: 7.b starting backfill to osd.2 from (0'0,0'0] MAX to 280'6008 2023-09-23T22:26:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:42 smithi070 ceph-mon[192652]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:42 smithi070 ceph-mon[192652]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:42 smithi070 ceph-mon[192652]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:42 smithi070 ceph-mon[192652]: failed to encode map e283 with expected crc 2023-09-23T22:26:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:42 smithi070 ceph-mon[192652]: pgmap v738: 99 pgs: 1 active+recovery_wait+remapped, 3 peering, 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 426 op/s; 26113/253332 objects degraded (10.308%); 9821/253332 objects misplaced (3.877%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T22:26:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:42 smithi070 ceph-mon[192652]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T22:26:42.384 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:26:42.393 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:42.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:42.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:42.505 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:42.600 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:42.640 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:42.807 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:26:42.848 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:42.854 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:26:42.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:42.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:42.901 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:26:42.946 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:26:42.958 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:42.986 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:26:42.992 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:43.038 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:43.039 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:26:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:43.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:43.109 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:26:43.145 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:43.156 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:26:43.170 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:26:43.170 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:26:43.170 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:26:43.171 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:26:43.171 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:26:43.171 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:26:43.171 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:26:43.171 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:26:43.175 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:43.222 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:43.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:43.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:43.315 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[215775]: failed to encode map e284 with expected crc 2023-09-23T22:26:43.316 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[215775]: failed to encode map e284 with expected crc 2023-09-23T22:26:43.316 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2417108692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:43.316 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26113/253332 objects degraded (10.308%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:43.316 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[215775]: osdmap e285: 8 total, 8 up, 7 in 2023-09-23T22:26:43.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[223014]: failed to encode map e284 with expected crc 2023-09-23T22:26:43.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[223014]: failed to encode map e284 with expected crc 2023-09-23T22:26:43.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2417108692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:43.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26113/253332 objects degraded (10.308%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:43.316 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:43 smithi026 ceph-mon[223014]: osdmap e285: 8 total, 8 up, 7 in 2023-09-23T22:26:43.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:43.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:43 smithi070 ceph-mon[192652]: failed to encode map e284 with expected crc 2023-09-23T22:26:43.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:43 smithi070 ceph-mon[192652]: failed to encode map e284 with expected crc 2023-09-23T22:26:43.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2417108692' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:43.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26113/253332 objects degraded (10.308%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:43.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:43 smithi070 ceph-mon[192652]: osdmap e285: 8 total, 8 up, 7 in 2023-09-23T22:26:43.351 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:43.397 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:43.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:43.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:43.500 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:43.527 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:43.574 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:43.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:43.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:43.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:43.711 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:44.434 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:44 smithi070 ceph-mon[192652]: failed to encode map e285 with expected crc 2023-09-23T22:26:44.434 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:44 smithi070 ceph-mon[192652]: failed to encode map e285 with expected crc 2023-09-23T22:26:44.434 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:44 smithi070 ceph-mon[192652]: failed to encode map e285 with expected crc 2023-09-23T22:26:44.435 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:44 smithi070 ceph-mon[192652]: pgmap v741: 99 pgs: 1 active+recovery_wait+remapped, 3 peering, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 30762/251301 objects degraded (12.241%); 5078/251301 objects misplaced (2.021%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:26:44.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:44 smithi026 ceph-mon[223014]: failed to encode map e285 with expected crc 2023-09-23T22:26:44.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:44 smithi026 ceph-mon[223014]: failed to encode map e285 with expected crc 2023-09-23T22:26:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:44 smithi026 ceph-mon[223014]: failed to encode map e285 with expected crc 2023-09-23T22:26:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:44 smithi026 ceph-mon[223014]: pgmap v741: 99 pgs: 1 active+recovery_wait+remapped, 3 peering, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 30762/251301 objects degraded (12.241%); 5078/251301 objects misplaced (2.021%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:26:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:44 smithi026 ceph-mon[215775]: failed to encode map e285 with expected crc 2023-09-23T22:26:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:44 smithi026 ceph-mon[215775]: failed to encode map e285 with expected crc 2023-09-23T22:26:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:44 smithi026 ceph-mon[215775]: failed to encode map e285 with expected crc 2023-09-23T22:26:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:44 smithi026 ceph-mon[215775]: pgmap v741: 99 pgs: 1 active+recovery_wait+remapped, 3 peering, 7 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 407 op/s; 30762/251301 objects degraded (12.241%); 5078/251301 objects misplaced (2.021%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:26:44.680 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:26:44.680 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:26:45.340 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:45.354 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:26:45.433 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:45.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:45.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:45.549 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:45.631 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:45.738 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:26:45.823 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:45.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:45.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:45.941 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:26:45.975 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:46.024 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:26:46.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:46.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:26:46.127 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:26:46.156 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:46.313 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:26:46.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:46.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:26:46.417 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:46.446 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:46.492 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:46.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:46.596 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:46.619 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:46 smithi070 ceph-mon[192652]: pgmap v742: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 276 op/s; 26025/248907 objects degraded (10.456%); 14849/248907 objects misplaced (5.966%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:26:46.622 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:46.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:46 smithi026 ceph-mon[215775]: pgmap v742: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 276 op/s; 26025/248907 objects degraded (10.456%); 14849/248907 objects misplaced (5.966%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:26:46.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:46 smithi026 ceph-mon[223014]: pgmap v742: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 276 op/s; 26025/248907 objects degraded (10.456%); 14849/248907 objects misplaced (5.966%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T22:26:46.880 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:46.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:46.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:46.985 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:47.129 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:47.342 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:26:47.342 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:26:47.347 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:47.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:47.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:47.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:26:47.470 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:47.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:47 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2574876588' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:26:47.576 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:26:47.624 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:47.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:47 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2574876588' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:26:47.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:47 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2574876588' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:26:48.047 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:26:48.422 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:26:48.433 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:48.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:48.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:48.550 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:48.619 INFO:teuthology.orchestra.run.smithi026.stdout:moved 44 / 297 (14.8148%) 2023-09-23T22:26:48.619 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:26:48.619 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 7.06818 -> 4.5937 (expected baseline 6.03053) 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 31 -> 42 pgs (0.73064 -> 0.989899 * mean) 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.3 with 53 -> 38 pgs (1.24916 -> 0.895623 * mean) 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 1 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0285 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0314 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 weight 1.0000 -> 0.7065 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 weight 1.0000 -> 0.6109 2023-09-23T22:26:48.620 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 weight 1.0000 -> 0.7965 2023-09-23T22:26:48.650 INFO:teuthology.orchestra.run.smithi026.stderr:no change 2023-09-23T22:26:48.672 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:48.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:26:48 smithi026 conmon[202047]: 2023-09-23T22:26:48.615+0000 7f47954a6700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T22:26:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[223014]: pgmap v743: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 26025/245892 objects degraded (10.584%); 14849/245892 objects misplaced (6.039%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:26:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[223014]: failed to encode map e286 with expected crc 2023-09-23T22:26:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2574876588' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:26:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[223014]: osdmap e286: 8 total, 8 up, 7 in 2023-09-23T22:26:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[223014]: failed to encode map e286 with expected crc 2023-09-23T22:26:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[223014]: failed to encode map e286 with expected crc 2023-09-23T22:26:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26025/245892 objects degraded (10.584%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:48.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[215775]: pgmap v743: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 26025/245892 objects degraded (10.584%); 14849/245892 objects misplaced (6.039%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:26:48.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:48.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[215775]: failed to encode map e286 with expected crc 2023-09-23T22:26:48.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2574876588' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:26:48.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[215775]: osdmap e286: 8 total, 8 up, 7 in 2023-09-23T22:26:48.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[215775]: failed to encode map e286 with expected crc 2023-09-23T22:26:48.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[215775]: failed to encode map e286 with expected crc 2023-09-23T22:26:48.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26025/245892 objects degraded (10.584%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:48.811 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:48 smithi070 ceph-mon[192652]: pgmap v743: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 26025/245892 objects degraded (10.584%); 14849/245892 objects misplaced (6.039%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T22:26:48.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:48 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:26:48.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:48 smithi070 ceph-mon[192652]: failed to encode map e286 with expected crc 2023-09-23T22:26:48.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2574876588' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:26:48.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:48 smithi070 ceph-mon[192652]: osdmap e286: 8 total, 8 up, 7 in 2023-09-23T22:26:48.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:48 smithi070 ceph-mon[192652]: failed to encode map e286 with expected crc 2023-09-23T22:26:48.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:48 smithi070 ceph-mon[192652]: failed to encode map e286 with expected crc 2023-09-23T22:26:48.812 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26025/245892 objects degraded (10.584%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:49.122 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:49.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:49.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:49.229 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:49.259 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:49.307 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:49.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:49.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:49.410 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:49.535 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:49.743 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:49.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:49.765 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T22:26:49.765 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T22:26:49.833 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:26:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:26:49] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-09-23T22:26:49.834 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:49 smithi026 ceph-mon[223014]: from='client.55288 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:49.834 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:49 smithi026 ceph-mon[215775]: from='client.55288 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:49.866 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:49 smithi070 ceph-mon[192652]: from='client.55288 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:49.913 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:49.970 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:50.140 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:26:50.140 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:26:50.288 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:50.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:50.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:50.397 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:50.428 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:50.473 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:50.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:50.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:50.523 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:50 smithi026 ceph-mon[215775]: pgmap v745: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 376 op/s; 26025/245037 objects degraded (10.621%); 14709/245037 objects misplaced (6.003%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T22:26:50.524 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:50 smithi026 ceph-mon[223014]: pgmap v745: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 376 op/s; 26025/245037 objects degraded (10.621%); 14709/245037 objects misplaced (6.003%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T22:26:50.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:50.604 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:50 smithi070 ceph-mon[192652]: pgmap v745: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 376 op/s; 26025/245037 objects degraded (10.621%); 14709/245037 objects misplaced (6.003%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T22:26:50.915 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:26:50.922 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:50.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:50.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:51.026 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:51.112 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:26:51.117 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:51.268 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:51.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:51.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:51.297 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:26:51.375 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:51.400 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:51.543 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:51.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:51.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:51.647 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:51.676 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:51.922 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:51.936 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:51.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:51.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:52.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:52.162 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:52.182 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:52.475 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:52.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:52.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:52.591 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:52.702 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:52.731 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:52 smithi070 ceph-mon[192652]: pgmap v746: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 377 op/s; 26025/241872 objects degraded (10.760%); 14587/241872 objects misplaced (6.031%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:26:52.971 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:26:52.981 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:26:52.981 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:26:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:52 smithi026 ceph-mon[223014]: pgmap v746: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 377 op/s; 26025/241872 objects degraded (10.760%); 14587/241872 objects misplaced (6.031%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:26:52.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:52 smithi026 ceph-mon[215775]: pgmap v746: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 377 op/s; 26025/241872 objects degraded (10.760%); 14587/241872 objects misplaced (6.031%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:26:53.014 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:26:53.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:26:53.111 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:26:53.148 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:26:53.190 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:26:53.246 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:26:53.290 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:26:53.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:26:53.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:26:53.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:26:53.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:26:53.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:26:53.300 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:26:53.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:26:53.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:26:53.409 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:53.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:53.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:53.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:26:53.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:53 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3895236608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:53.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26025/241872 objects degraded (10.760%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:53.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:53 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3895236608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:53.540 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26025/241872 objects degraded (10.760%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:53.547 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:53.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:26:53.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:53.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:26:53.772 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:26:53.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:53 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3895236608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:26:53.796 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26025/241872 objects degraded (10.760%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:53.818 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:53.845 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:53.899 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:53.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:53.904 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:54.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:54.034 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:54.089 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:54.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:54.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:54.174 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:26:54.197 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:26:54.225 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:54.555 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:26:54.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:54.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:26:54.668 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:54.704 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:54.814 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:54.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:54.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:54.918 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:55.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:55 smithi070 ceph-mon[192652]: pgmap v747: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 411 op/s; 26025/238953 objects degraded (10.891%); 14587/238953 objects misplaced (6.105%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:26:55.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:55.006 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:55.028 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:55.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:55 smithi026 ceph-mon[215775]: pgmap v747: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 411 op/s; 26025/238953 objects degraded (10.891%); 14587/238953 objects misplaced (6.105%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:26:55.121 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:55.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:55.122 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:55 smithi026 ceph-mon[223014]: pgmap v747: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 411 op/s; 26025/238953 objects degraded (10.891%); 14587/238953 objects misplaced (6.105%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:26:55.122 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:26:55.122 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:26:55.209 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:55.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:55.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:55.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:55.359 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:55.857 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:26:55.857 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:26:55.857 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:26:55.857 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:26:55.857 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:26:55.865 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:55.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:55.980 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:56.061 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:56.356 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:56.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:56.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:56.468 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:26:56.606 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:56.815 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:26:56.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:56.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:26:56.899 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:26:56.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:56.960 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:26:57.054 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:57.119 INFO:teuthology.orchestra.run.smithi026.stdout:moved 5 / 297 (1.6835%) 2023-09-23T22:26:57.119 INFO:teuthology.orchestra.run.smithi026.stdout:avg 42.4286 2023-09-23T22:26:57.119 INFO:teuthology.orchestra.run.smithi026.stdout:stddev 7.06818 -> 6.60859 (expected baseline 6.03053) 2023-09-23T22:26:57.119 INFO:teuthology.orchestra.run.smithi026.stdout:min osd.2 with 31 -> 31 pgs (0.73064 -> 0.73064 * mean) 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout:max osd.3 with 53 -> 53 pgs (1.24916 -> 1.24916 * mean) 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout:oload 110 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout:max_change 0.05 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout:max_change_osds 4 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout:average_utilization 0.0274 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout:overload_utilization 0.0302 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1 weight 1.0000 -> 0.9500 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T22:26:57.120 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T22:26:57.179 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:57 smithi026 ceph-mon[215775]: pgmap v748: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 26028/235098 objects degraded (11.071%); 14333/235098 objects misplaced (6.097%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:26:57.179 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:57 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260','3':'62260'}"}]: dispatch 2023-09-23T22:26:57.180 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:57 smithi026 ceph-mon[223014]: pgmap v748: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 26028/235098 objects degraded (11.071%); 14333/235098 objects misplaced (6.097%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:26:57.180 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:57 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260','3':'62260'}"}]: dispatch 2023-09-23T22:26:57.352 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:57.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:57.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:57 smithi070 ceph-mon[192652]: pgmap v748: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 26028/235098 objects degraded (11.071%); 14333/235098 objects misplaced (6.097%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T22:26:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:57 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260','3':'62260'}"}]: dispatch 2023-09-23T22:26:57.460 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:26:57.465 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:26:57.490 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:57.633 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:26:57.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:57.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:26:57.738 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:26:57.772 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:57.826 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:26:57.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:57.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:26:57.929 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:26:57.965 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:57.993 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:26:58.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: from='client.35541 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:58.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260','3':'62260'}"}]': finished 2023-09-23T22:26:58.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: osdmap e287: 8 total, 8 up, 7 in 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: failed to encode map e287 with expected crc 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/802263435' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26028/235098 objects degraded (11.071%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[215775]: osdmap e288: 8 total, 8 up, 7 in 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: from='client.35541 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260','3':'62260'}"}]': finished 2023-09-23T22:26:58.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: osdmap e287: 8 total, 8 up, 7 in 2023-09-23T22:26:58.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: failed to encode map e287 with expected crc 2023-09-23T22:26:58.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/802263435' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:58.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:58.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26028/235098 objects degraded (11.071%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:58.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:26:58.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:26:58.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:58 smithi026 ceph-mon[223014]: osdmap e288: 8 total, 8 up, 7 in 2023-09-23T22:26:58.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: from='client.35541 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T22:26:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','1':'62260','3':'62260'}"}]': finished 2023-09-23T22:26:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: osdmap e287: 8 total, 8 up, 7 in 2023-09-23T22:26:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: failed to encode map e287 with expected crc 2023-09-23T22:26:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/802263435' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:26:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26028/235098 objects degraded (11.071%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:26:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:26:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:26:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:58 smithi070 ceph-mon[192652]: osdmap e288: 8 total, 8 up, 7 in 2023-09-23T22:26:58.592 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:26:58.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:58.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:26:58.697 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:58.726 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:58.772 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:58.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:58.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:58.875 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:58.902 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:59.087 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:59 smithi070 ceph-mon[192652]: pgmap v750: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 26028/232383 objects degraded (11.200%); 14333/232383 objects misplaced (6.168%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:26:59.087 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:59 smithi070 ceph-mon[192652]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:59 smithi070 ceph-mon[192652]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:59 smithi070 ceph-mon[192652]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:59 smithi070 ceph-mon[192652]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:59 smithi070 ceph-mon[192652]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.088 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:26:59 smithi070 ceph-mon[192652]: osdmap e289: 8 total, 8 up, 7 in 2023-09-23T22:26:59.199 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:59.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:59.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:59.304 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:59.333 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:59.380 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:26:59.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:26:59.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:26:59.415 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:26:59.415 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:26:59.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:26:59.415 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:51738 7.e 7:7a19a68c:::benchmark_data_smithi026_410749_object51737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:29.524484+0000", 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 89.888238040999994, 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.222769941, 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.416 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 51738 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524484+0000", 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524484+0000", 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524483+0000", 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524552+0000", 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8769000000000005e-05 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524565+0000", 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3443e-05 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524571+0000", 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5049999999999998e-06 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524622+0000", 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.083e-05 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524754+0000", 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013249400000000001 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524892+0000", 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000138221 2023-09-23T22:26:59.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.526723+0000", 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018302489999999999 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.620491+0000", 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.093768588 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.747112+0000", 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.126621083 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.747215+0000", 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103001 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.747254+0000", 2023-09-23T22:26:59.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9026999999999998e-05 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:55106 7.e 7:71164a53:::benchmark_data_smithi026_410749_object55105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:32.988479+0000", 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 86.424243804, 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18959135999999999, 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.423 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 55106 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988479+0000", 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988479+0000", 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988478+0000", 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988536+0000", 2023-09-23T22:26:59.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7941e-05 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988543+0000", 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3370000000000002e-06 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988547+0000", 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8600000000000003e-06 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.990260+0000", 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017131119999999999 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.990359+0000", 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9418000000000003e-05 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.990481+0000", 2023-09-23T22:26:59.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121428 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.992382+0000", 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001900936 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.994279+0000", 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018973060000000001 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:33.177986+0000", 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18370663400000001 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:33.178030+0000", 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4264000000000002e-05 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:33.178070+0000", 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9923999999999999e-05 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:73944 7.e 7:79336cf0:::benchmark_data_smithi026_410749_object73943:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:54.542422+0000", 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 64.870300767000003, 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.111098656, 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.430 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 73944 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.542422+0000", 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.542422+0000", 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.542421+0000", 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.542456+0000", 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4926999999999999e-05 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.542469+0000", 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3294e-05 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.542474+0000", 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6670000000000004e-06 2023-09-23T22:26:59.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.542522+0000", 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7953e-05 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.542658+0000", 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000135742 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.542812+0000", 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000153876 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.544839+0000", 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020273880000000002 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.548382+0000", 2023-09-23T22:26:59.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003543455 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.653434+0000", 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10505120900000001 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.653484+0000", 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0430000000000003e-05 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.653520+0000", 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6219e-05 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:74105 7.e 7:788f740f:::benchmark_data_smithi026_410749_object74104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:54.917969+0000", 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 64.494753528000004, 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.111492217, 2023-09-23T22:26:59.437 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74105 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.917969+0000", 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.917969+0000", 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.917968+0000", 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.918015+0000", 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7428000000000001e-05 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.918027+0000", 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.24e-05 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.918032+0000", 2023-09-23T22:26:59.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1340000000000003e-06 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.918073+0000", 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0535999999999999e-05 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.918179+0000", 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010637 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.918283+0000", 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103349 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:54.920322+0000", 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020388350000000001 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.015131+0000", 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.094808993999999994 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.029391+0000", 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014260873 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.029433+0000", 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1897999999999998e-05 2023-09-23T22:26:59.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.029461+0000", 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7727999999999999e-05 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:74133 7.e 7:71e8988e:::benchmark_data_smithi026_410749_object74132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:55.041643+0000", 2023-09-23T22:26:59.444 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 64.371079140000006, 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098984984999999998, 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74133 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041643+0000", 2023-09-23T22:26:59.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041643+0000", 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041642+0000", 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041691+0000", 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9249999999999998e-05 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041709+0000", 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7071e-05 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041714+0000", 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2800000000000003e-06 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041764+0000", 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0212e-05 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041925+0000", 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016113 2023-09-23T22:26:59.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.042082+0000", 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000156436 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.044247+0000", 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021651840000000001 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.054053+0000", 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0098061580000000006 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.140532+0000", 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.086478792999999998 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.140578+0000", 2023-09-23T22:26:59.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6369000000000003e-05 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.140628+0000", 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0145000000000002e-05 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:74140 7.0 7:0967bb2b:::benchmark_data_smithi026_410749_object74139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:55.041724+0000", 2023-09-23T22:26:59.451 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 64.370998283000006, 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.100725195, 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74140 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041724+0000", 2023-09-23T22:26:59.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041724+0000", 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041724+0000", 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041750+0000", 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6461999999999998e-05 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041757+0000", 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6089999999999997e-06 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041761+0000", 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5399999999999997e-06 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041799+0000", 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7877999999999999e-05 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.041937+0000", 2023-09-23T22:26:59.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000137806 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.042082+0000", 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014460100000000001 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.044157+0000", 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020757670000000001 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.052542+0000", 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083848849999999999 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.142388+0000", 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.089845639000000005 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.142425+0000", 2023-09-23T22:26:59.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7004e-05 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.142449+0000", 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4462000000000001e-05 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:74224 7.e 7:776103ae:::benchmark_data_smithi026_410749_object74223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T22:26:59.458 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:55.219570+0000", 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 64.193152687999998, 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.11073899299999999, 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74224 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.219570+0000", 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.219570+0000", 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.219569+0000", 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:26:59.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.219606+0000", 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7369000000000001e-05 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.219618+0000", 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1652e-05 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.219623+0000", 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1259999999999997e-06 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.219672+0000", 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9447000000000003e-05 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.219807+0000", 2023-09-23T22:26:59.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013438600000000001 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.219899+0000", 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2644000000000003e-05 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.222025+0000", 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021255139999999998 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.226424+0000", 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043989880000000004 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.330243+0000", 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10381947499999999 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.330281+0000", 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7908999999999999e-05 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.330309+0000", 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7634999999999999e-05 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.465 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:74233 7.e 7:7d362e68:::benchmark_data_smithi026_410749_object74232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:55.223217+0000", 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 64.189505788000005, 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.107189227, 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74233 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.223217+0000", 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.223217+0000", 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.223215+0000", 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.223270+0000", 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4883000000000003e-05 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.223283+0000", 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3019e-05 2023-09-23T22:26:59.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.223289+0000", 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5670000000000002e-06 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.223329+0000", 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0373999999999997e-05 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.223426+0000", 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6917999999999996e-05 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.223527+0000", 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100325 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.226013+0000", 2023-09-23T22:26:59.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024863329999999999 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.231293+0000", 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052802930000000001 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.330346+0000", 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.099052521000000004 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.330381+0000", 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5682999999999997e-05 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.330406+0000", 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4442000000000001e-05 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:74236 7.0 7:0dea140c:::benchmark_data_smithi026_410749_object74235:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:55.230977+0000", 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 64.181745536999998, 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.110186062, 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 74236 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.473 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.230977+0000", 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.230977+0000", 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.230976+0000", 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.231014+0000", 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8214999999999999e-05 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.231026+0000", 2023-09-23T22:26:59.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2119e-05 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.231031+0000", 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.13e-06 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.231076+0000", 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4518000000000003e-05 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.231176+0000", 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010044599999999999 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.231284+0000", 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000107829 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.233157+0000", 2023-09-23T22:26:59.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018734820000000001 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.236714+0000", 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035566230000000001 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.341101+0000", 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.104386706 2023-09-23T22:26:59.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.341135+0000", 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4678999999999998e-05 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:55.341163+0000", 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7597999999999999e-05 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:85118 7.0 7:0c960c48:::benchmark_data_smithi026_410749_object85117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:13.151705+0000", 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 46.261016918000003, 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26973130099999998, 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 85118 2023-09-23T22:26:59.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151705+0000", 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151705+0000", 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151704+0000", 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151740+0000", 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5531999999999998e-05 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151753+0000", 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2944e-05 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151760+0000", 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7109999999999998e-06 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151794+0000", 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4319000000000001e-05 2023-09-23T22:26:59.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151894+0000", 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100226 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.152000+0000", 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010639799999999999 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.156285+0000", 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042846309999999997 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.164218+0000", 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0079328939999999994 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421339+0000", 2023-09-23T22:26:59.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.25712146299999999 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421396+0000", 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6419999999999999e-05 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421437+0000", 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0874999999999997e-05 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.486 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:85122 7.0 7:0bdfca06:::benchmark_data_smithi026_410749_object85121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:13.151774+0000", 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 46.260948732999999, 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26980743699999998, 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.487 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 85122 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151774+0000", 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151774+0000", 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151773+0000", 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151806+0000", 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2521999999999998e-05 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151814+0000", 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9039999999999992e-06 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151818+0000", 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9239999999999998e-06 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.154034+0000", 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022159889999999998 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.154134+0000", 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9874e-05 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.154240+0000", 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105707 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.156464+0000", 2023-09-23T22:26:59.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022235610000000002 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.164263+0000", 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007799851 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421490+0000", 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.25722648399999998 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421543+0000", 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3010000000000002e-05 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421581+0000", 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8149999999999999e-05 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.493 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:92339 7.0 7:06925a01:::benchmark_data_smithi026_410749_object2002:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:29.896629+0000", 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.516093294000001, 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12864761899999999, 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 92339 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.896629+0000", 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.896629+0000", 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.896628+0000", 2023-09-23T22:26:59.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.896633+0000", 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5469999999999998e-06 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.896643+0000", 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0706e-05 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.896648+0000", 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.578e-06 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.896688+0000", 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9808999999999998e-05 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.896783+0000", 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5433000000000006e-05 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.896880+0000", 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6984000000000005e-05 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.899137+0000", 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022571319999999998 2023-09-23T22:26:59.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.901036+0000", 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001898982 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:30.025212+0000", 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12417540000000001 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:30.025247+0000", 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5051e-05 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:30.025277+0000", 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0037000000000001e-05 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.500 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:92343 7.0 7:03306ca1:::benchmark_data_smithi026_410749_object2006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:29.908374+0000", 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.504348513, 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.117042153, 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 92343 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.908374+0000", 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.908374+0000", 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.908373+0000", 2023-09-23T22:26:59.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.908378+0000", 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4199999999999998e-06 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.908408+0000", 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9788999999999999e-05 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.908413+0000", 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6890000000000003e-06 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.908447+0000", 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4694999999999999e-05 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.908528+0000", 2023-09-23T22:26:59.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0357000000000003e-05 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.908597+0000", 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8992999999999995e-05 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.910817+0000", 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022205340000000001 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.919346+0000", 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0085283730000000006 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:30.025319+0000", 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.105973001 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:30.025373+0000", 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4126999999999999e-05 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:30.025416+0000", 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3309000000000002e-05 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.507 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:92347 7.0 7:00aa9815:::benchmark_data_smithi026_410749_object2010:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:29.912959+0000", 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.499763027, 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.112533331, 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 92347 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.912959+0000", 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.912959+0000", 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.912959+0000", 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:26:59.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.912963+0000", 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7849999999999999e-06 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.912975+0000", 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1729000000000001e-05 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.912980+0000", 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9799999999999998e-06 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.913017+0000", 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6894999999999998e-05 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.913103+0000", 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6011e-05 2023-09-23T22:26:59.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.913174+0000", 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0858000000000005e-05 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.915476+0000", 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023020720000000001 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:29.919391+0000", 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039149500000000004 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:30.025449+0000", 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.106057848 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:30.025473+0000", 2023-09-23T22:26:59.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3926000000000001e-05 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:30.025493+0000", 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0015000000000001e-05 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:93101 7.0 7:0cd916bf:::benchmark_data_smithi026_410749_object2764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-09-23T22:26:59.514 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:31.506877+0000", 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 27.905845518, 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.070145549000000001, 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 93101 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.506877+0000", 2023-09-23T22:26:59.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.506877+0000", 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.506875+0000", 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.506883+0000", 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8909999999999995e-06 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.506901+0000", 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8138000000000002e-05 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.506909+0000", 2023-09-23T22:26:59.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2289999999999997e-06 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.506961+0000", 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2596999999999998e-05 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.507065+0000", 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103538 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.507162+0000", 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7621999999999995e-05 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.509625+0000", 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002462437 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.520361+0000", 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010736626000000001 2023-09-23T22:26:59.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.576938+0000", 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.056576464999999999 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.576994+0000", 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6138999999999999e-05 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:31.577022+0000", 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8427000000000001e-05 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:94844 7.0 7:0746a5eb:::benchmark_data_smithi026_410749_object4507:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e279)", 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:35.355157+0000", 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 24.057565312000001, 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.078074616999999999, 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.521 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 94844 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.355157+0000", 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.355157+0000", 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.355156+0000", 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.355161+0000", 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6840000000000004e-06 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.355173+0000", 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2371000000000001e-05 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.355178+0000", 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7600000000000002e-06 2023-09-23T22:26:59.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.355231+0000", 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3164999999999997e-05 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.355358+0000", 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000126816 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.355443+0000", 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5301000000000002e-05 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.358025+0000", 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025817040000000002 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.377559+0000", 2023-09-23T22:26:59.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019534047999999998 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.433163+0000", 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.055603807999999998 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.433201+0000", 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8810999999999998e-05 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:35.433232+0000", 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0244999999999999e-05 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:102634 7.0 7:04110f80:::benchmark_data_smithi026_410749_object12297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:54.190216+0000", 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 5.2225062089999996, 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.14486220399999999, 2023-09-23T22:26:59.528 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 102634 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190216+0000", 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190216+0000", 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190215+0000", 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190223+0000", 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3939999999999996e-06 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190234+0000", 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0814e-05 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190240+0000", 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0619999999999997e-06 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190382+0000", 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014202099999999999 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190524+0000", 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014200199999999999 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190632+0000", 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010814099999999999 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.196327+0000", 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056952160000000003 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.319350+0000", 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.123022957 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.335017+0000", 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015667159999999999 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.335051+0000", 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3624999999999997e-05 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.335078+0000", 2023-09-23T22:26:59.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7263e-05 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:102635 7.0 7:0bf7432a:::benchmark_data_smithi026_410749_object12298:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:54.190258+0000", 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 5.222464489, 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15399560600000001, 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.535 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 102635 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190258+0000", 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190258+0000", 2023-09-23T22:26:59.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190257+0000", 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190260+0000", 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8559999999999998e-06 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190267+0000", 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3280000000000002e-06 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.190271+0000", 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9990000000000002e-06 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.193762+0000", 2023-09-23T22:26:59.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003491161 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.193879+0000", 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011691199999999999 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.193976+0000", 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7017999999999998e-05 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.196414+0000", 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024388679999999999 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.319480+0000", 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.123065591 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.344156+0000", 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.024676008999999999 2023-09-23T22:26:59.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.344206+0000", 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0078999999999999e-05 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:54.344253+0000", 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7339000000000002e-05 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103116 7.0 7:0c665476:::benchmark_data_smithi026_410749_object12779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:55.324101+0000", 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 4.088621184, 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16718866900000001, 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103116 2023-09-23T22:26:59.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.324101+0000", 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.324101+0000", 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.324100+0000", 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.324105+0000", 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7069999999999997e-06 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.324119+0000", 2023-09-23T22:26:59.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3132000000000001e-05 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.324124+0000", 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0069999999999997e-06 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.324168+0000", 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4737000000000002e-05 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.324270+0000", 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010151 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.324343+0000", 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3169000000000004e-05 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.327004+0000", 2023-09-23T22:26:59.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026606469999999999 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332102+0000", 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050987330000000003 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.491219+0000", 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.159117061 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.491257+0000", 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7849000000000003e-05 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.491290+0000", 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2583000000000003e-05 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103118 7.0 7:0666e1d6:::benchmark_data_smithi026_410749_object12781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:55.332165+0000", 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 4.080557378, 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.159217672, 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103118 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.549 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332165+0000", 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332165+0000", 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:26:59.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332164+0000", 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332169+0000", 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7589999999999997e-06 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:26:59.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332178+0000", 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7890000000000006e-06 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332183+0000", 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4050000000000002e-06 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332238+0000", 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5362000000000003e-05 2023-09-23T22:26:59.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332401+0000", 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000162592 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.332550+0000", 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014884300000000001 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.336311+0000", 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0037614430000000002 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.336388+0000", 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7162000000000004e-05 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.491324+0000", 2023-09-23T22:26:59.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.15493552699999999 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.491357+0000", 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2937e-05 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:55.491383+0000", 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6083000000000001e-05 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.555 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.556 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.556 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:26:59.556 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:26:59.556 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:26:59.556 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:26:59.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[223014]: pgmap v750: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 26028/232383 objects degraded (11.200%); 14333/232383 objects misplaced (6.168%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:26:59.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[223014]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[223014]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[223014]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[223014]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[223014]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[223014]: osdmap e289: 8 total, 8 up, 7 in 2023-09-23T22:26:59.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[215775]: pgmap v750: 99 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 26028/232383 objects degraded (11.200%); 14333/232383 objects misplaced (6.168%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:26:59.557 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[215775]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[215775]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[215775]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[215775]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[215775]: failed to encode map e288 with expected crc 2023-09-23T22:26:59.558 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:26:59 smithi026 ceph-mon[215775]: osdmap e289: 8 total, 8 up, 7 in 2023-09-23T22:26:59.590 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:26:59.948 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:26:59.989 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:26:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:26:59] "GET /metrics HTTP/1.1" 200 36376 "" "Prometheus/2.43.0" 2023-09-23T22:27:00.426 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:00 smithi070 ceph-mon[192652]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.426 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:00 smithi070 ceph-mon[192652]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.426 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:00 smithi070 ceph-mon[192652]: 7.1 starting backfill to osd.0 from (0'0,0'0] MAX to 286'6378 2023-09-23T22:27:00.426 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:00 smithi070 ceph-mon[192652]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.426 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:00 smithi070 ceph-mon[192652]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.427 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:00 smithi070 ceph-mon[192652]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.427 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:00 smithi070 ceph-mon[192652]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.444 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[223014]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.444 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[223014]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.444 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[223014]: 7.1 starting backfill to osd.0 from (0'0,0'0] MAX to 286'6378 2023-09-23T22:27:00.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[223014]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[223014]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[223014]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[223014]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.446 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:27:00.448 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:00.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[215775]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[215775]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[215775]: 7.1 starting backfill to osd.0 from (0'0,0'0] MAX to 286'6378 2023-09-23T22:27:00.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[215775]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[215775]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[215775]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:00 smithi026 ceph-mon[215775]: failed to encode map e289 with expected crc 2023-09-23T22:27:00.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:00.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:00.552 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:00.586 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:00.693 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:00.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:00.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:00.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:00.833 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:00.890 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:27:01.043 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:01.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:01.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:01.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:01 smithi026 ceph-mon[223014]: failed to encode map e289 with expected crc 2023-09-23T22:27:01.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:01 smithi026 ceph-mon[223014]: pgmap v753: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 26028/231711 objects degraded (11.233%); 14219/231711 objects misplaced (6.137%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:27:01.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:01 smithi026 ceph-mon[215775]: failed to encode map e289 with expected crc 2023-09-23T22:27:01.142 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:01 smithi026 ceph-mon[215775]: pgmap v753: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 26028/231711 objects degraded (11.233%); 14219/231711 objects misplaced (6.137%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:27:01.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:01.176 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:01.276 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:01.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:01.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:01 smithi070 ceph-mon[192652]: failed to encode map e289 with expected crc 2023-09-23T22:27:01.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:01 smithi070 ceph-mon[192652]: pgmap v753: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 402 op/s; 26028/231711 objects degraded (11.233%); 14219/231711 objects misplaced (6.137%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:27:01.381 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:01.416 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:01.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:01.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:01.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:01.675 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:01.707 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:01.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:27:02.242 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:02.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:02.249 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:02.349 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:02.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:02 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:02.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:02 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:02.451 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:02 smithi026 ceph-mon[223014]: pgmap v754: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 26053/229959 objects degraded (11.329%); 18695/229959 objects misplaced (8.130%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T22:27:02.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:02 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:02.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:02 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:02.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:02 smithi026 ceph-mon[215775]: pgmap v754: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 26053/229959 objects degraded (11.329%); 18695/229959 objects misplaced (8.130%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T22:27:02.471 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:02.472 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:02 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:02.472 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:02 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:02.472 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:02 smithi070 ceph-mon[192652]: pgmap v754: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 285 op/s; 26053/229959 objects degraded (11.329%); 18695/229959 objects misplaced (8.130%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T22:27:02.641 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:02.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:02.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:02.704 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:27:02.704 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:27:02.705 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:27:02.705 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:27:02.777 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:02.810 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:02.960 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:02.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:02.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:03.063 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:03.090 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:03.138 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:03.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:03.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:03.145 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:27:03.195 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:27:03.242 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:03.244 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:27:03.269 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:03.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:27:03.322 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:03.340 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:27:03.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26053/229959 objects degraded (11.329%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:27:03.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/629650549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:03.381 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:27:03.425 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:27:03.463 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:27:03.487 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:03.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:03.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:03.503 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:27:03.503 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:27:03.503 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:27:03.503 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:27:03.503 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:27:03.503 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:27:03.503 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:27:03.504 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:27:03.545 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:27:03.591 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:03.705 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26053/229959 objects degraded (11.329%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:27:03.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/629650549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:03.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26053/229959 objects degraded (11.329%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:27:03.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/629650549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:03.716 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:03.930 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:27:04.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:04 smithi070 ceph-mon[192652]: pgmap v755: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 26053/227997 objects degraded (11.427%); 18695/227997 objects misplaced (8.200%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T22:27:04.624 INFO:tasks.thrashosds.thrasher:in_osds: [1, 3, 4, 5, 7, 0, 2] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:27:04.625 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:27:04.625 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [1, 3, 4, 5, 7, 0, 2] 2023-09-23T22:27:04.625 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 1 2023-09-23T22:27:04.682 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:04 smithi026 ceph-mon[215775]: pgmap v755: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 26053/227997 objects degraded (11.427%); 18695/227997 objects misplaced (8.200%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T22:27:04.682 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:04 smithi026 ceph-mon[223014]: pgmap v755: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 374 op/s; 26053/227997 objects degraded (11.427%); 18695/227997 objects misplaced (8.200%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T22:27:05.005 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:05.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:05.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:05.121 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:05.242 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:05.557 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:27:05.557 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:27:06.354 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:27:06.426 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:06.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:06.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:06.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:06 smithi026 ceph-mon[223014]: pgmap v756: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 26053/224337 objects degraded (11.613%); 17755/224337 objects misplaced (7.914%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:27:06.535 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:06.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:06 smithi070 ceph-mon[192652]: pgmap v756: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 26053/224337 objects degraded (11.613%); 17755/224337 objects misplaced (7.914%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:27:06.669 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:06.745 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:27:06.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:06 smithi026 ceph-mon[215775]: pgmap v756: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 334 op/s; 26053/224337 objects degraded (11.613%); 17755/224337 objects misplaced (7.914%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:27:06.813 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:06.880 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:06.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:06.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:06.993 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:07.111 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:07.398 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:07.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:07.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:07.502 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:07.531 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:07.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:07.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:07.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:07.679 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:07.705 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:07.899 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:07.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:07.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:08.003 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:08.150 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:08.231 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:27:08.421 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:27:08.422 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:27:08.683 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:27:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:08 smithi070 ceph-mon[192652]: pgmap v757: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 26053/221511 objects degraded (11.761%); 17755/221511 objects misplaced (8.015%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:27:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:08 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:08 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:08 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1916294558' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:27:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26053/221511 objects degraded (11.761%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:27:08.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[223014]: pgmap v757: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 26053/221511 objects degraded (11.761%); 17755/221511 objects misplaced (8.015%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:27:08.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:08.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:08.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1916294558' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:27:08.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26053/221511 objects degraded (11.761%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:27:08.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[215775]: pgmap v757: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 383 op/s; 26053/221511 objects degraded (11.761%); 17755/221511 objects misplaced (8.015%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:27:08.992 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:08.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:08.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1916294558' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:27:08.993 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26053/221511 objects degraded (11.761%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:27:09.215 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:27:09.615 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:27:09.787 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:09.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:09.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:09.896 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:09.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:09.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1916294558' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:27:09.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: osdmap e290: 8 total, 8 up, 7 in 2023-09-23T22:27:09.923 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.924 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.924 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.924 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:09.924 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:09.924 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:09.924 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:09.926 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:09.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:27:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:27:09] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-09-23T22:27:09.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:09.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1916294558' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:27:09.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: osdmap e290: 8 total, 8 up, 7 in 2023-09-23T22:27:09.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:09.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:09.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:09.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:09.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1916294558' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: osdmap e290: 8 total, 8 up, 7 in 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: failed to encode map e290 with expected crc 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:27:09.992 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:10.006 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:10.175 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:10.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:10.279 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:10.307 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:10.352 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:10.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:10.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:10.457 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:10.555 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:10.710 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:10.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:10.826 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:10.857 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:10.968 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:10.970 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:10 smithi070 ceph-mon[192652]: pgmap v759: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 26053/220728 objects degraded (11.803%); 17621/220728 objects misplaced (7.983%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T22:27:10.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:10.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:10.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:10 smithi026 ceph-mon[215775]: pgmap v759: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 26053/220728 objects degraded (11.803%); 17621/220728 objects misplaced (7.983%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T22:27:10.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:10 smithi026 ceph-mon[223014]: pgmap v759: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 26053/220728 objects degraded (11.803%); 17621/220728 objects misplaced (7.983%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T22:27:11.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:11.188 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:11.275 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:11.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:11.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:11.290 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:27:11.291 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:27:11.391 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:11.424 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:11.492 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:11 smithi026 conmon[215751]: 2023-09-23T22:27:11.054+0000 7f4580f04700 -1 mon.a@0(leader).osd e290 definitely_dead 0 2023-09-23T22:27:11.523 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:11.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:11.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:11.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:11.653 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:11.697 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.1. 2023-09-23T22:27:11.712 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:11.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:11.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:11.816 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:11.824 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:27:11.845 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:11.902 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:11.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:11.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:11.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/969494897' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T22:27:11.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T22:27:11.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:11 smithi026 ceph-mon[215775]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T22:27:11.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/969494897' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T22:27:11.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T22:27:11.956 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:11 smithi026 ceph-mon[223014]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T22:27:12.009 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:12.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/969494897' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T22:27:12.034 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T22:27:12.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:11 smithi070 ceph-mon[192652]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T22:27:12.037 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:12.230 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:12.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:12.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:12.281 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:27:12.333 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:12.364 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:12.620 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:27:12.658 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:12.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:12.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:12.762 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:12.815 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:12.902 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:12.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:12.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:12.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[215775]: pgmap v760: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 26053/218187 objects degraded (11.941%); 16909/218187 objects misplaced (7.750%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T22:27:12.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[215775]: failed to encode map e291 with expected crc 2023-09-23T22:27:12.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T22:27:12.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[215775]: osdmap e291: 8 total, 8 up, 6 in 2023-09-23T22:27:12.912 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[215775]: failed to encode map e291 with expected crc 2023-09-23T22:27:12.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[223014]: pgmap v760: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 26053/218187 objects degraded (11.941%); 16909/218187 objects misplaced (7.750%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T22:27:12.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[223014]: failed to encode map e291 with expected crc 2023-09-23T22:27:12.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T22:27:12.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[223014]: osdmap e291: 8 total, 8 up, 6 in 2023-09-23T22:27:12.914 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:12 smithi026 ceph-mon[223014]: failed to encode map e291 with expected crc 2023-09-23T22:27:13.008 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:13.019 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:12 smithi070 ceph-mon[192652]: pgmap v760: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 391 op/s; 26053/218187 objects degraded (11.941%); 16909/218187 objects misplaced (7.750%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T22:27:13.019 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:12 smithi070 ceph-mon[192652]: failed to encode map e291 with expected crc 2023-09-23T22:27:13.019 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:12 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T22:27:13.019 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:12 smithi070 ceph-mon[192652]: osdmap e291: 8 total, 8 up, 6 in 2023-09-23T22:27:13.019 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:12 smithi070 ceph-mon[192652]: failed to encode map e291 with expected crc 2023-09-23T22:27:13.050 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:13.102 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:13.238 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:13.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:13.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:13.332 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:27:13.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:13.373 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:27:13.414 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:27:13.414 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:13.467 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:27:13.515 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:27:13.544 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:27:13.604 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:27:13.605 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:13.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:13.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:13.645 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:27:13.675 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:27:13.675 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:27:13.675 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:27:13.675 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:27:13.675 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:27:13.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:27:13.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:27:13.676 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:27:13.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: osdmap e292: 8 total, 8 up, 6 in 2023-09-23T22:27:13.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: 7.1 continuing backfill to osd.0 from (264'400,290'6737] 7:8423e91c:::benchmark_data_smithi026_410749_object38221:head to 290'6737 2023-09-23T22:27:13.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: 7.1 starting backfill to osd.7 from (0'0,0'0] MAX to 290'6737 2023-09-23T22:27:13.698 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26053/218187 objects degraded (11.941%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: osdmap e293: 8 total, 8 up, 6 in 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: 7.b continuing backfill to osd.2 from (264'500,290'6814] MIN to 290'6814 2023-09-23T22:27:13.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2526357604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:13.724 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:13.756 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:13.940 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:13.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:13.943 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: osdmap e292: 8 total, 8 up, 6 in 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: 7.1 continuing backfill to osd.0 from (264'400,290'6737] 7:8423e91c:::benchmark_data_smithi026_410749_object38221:head to 290'6737 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: 7.1 starting backfill to osd.7 from (0'0,0'0] MAX to 290'6737 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26053/218187 objects degraded (11.941%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: osdmap e293: 8 total, 8 up, 6 in 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.944 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.945 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.945 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.945 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: 7.b continuing backfill to osd.2 from (264'500,290'6814] MIN to 290'6814 2023-09-23T22:27:13.945 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2526357604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: osdmap e292: 8 total, 8 up, 6 in 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: 7.1 continuing backfill to osd.0 from (264'400,290'6737] 7:8423e91c:::benchmark_data_smithi026_410749_object38221:head to 290'6737 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: 7.1 starting backfill to osd.7 from (0'0,0'0] MAX to 290'6737 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e292 with expected crc 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26053/218187 objects degraded (11.941%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: osdmap e293: 8 total, 8 up, 6 in 2023-09-23T22:27:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: failed to encode map e293 with expected crc 2023-09-23T22:27:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: 7.b continuing backfill to osd.2 from (264'500,290'6814] MIN to 290'6814 2023-09-23T22:27:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2526357604' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:14.044 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:14.155 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:14.294 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:14.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:14.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:14.404 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:14.509 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:14.570 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:27:14.570 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:27:14.570 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:27:14.570 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:27:14.570 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:27:14.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: 7.9 starting backfill to osd.7 from (0'0,0'0] MAX to 290'6937 2023-09-23T22:27:14.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: 7.d starting backfill to osd.7 from (0'0,0'0] MAX to 290'6885 2023-09-23T22:27:14.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: 7.4 starting backfill to osd.5 from (0'0,0'0] MAX to 290'6889 2023-09-23T22:27:14.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: 7.7 starting backfill to osd.7 from (0'0,0'0] MAX to 290'6791 2023-09-23T22:27:14.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: pgmap v764: 99 pgs: 2 unknown, 12 peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 417 op/s; 26053/215544 objects degraded (12.087%); 16909/215544 objects misplaced (7.845%); 7.8 MiB/s, 120 objects/s recovering 2023-09-23T22:27:14.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:27:14.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: osdmap e294: 8 total, 8 up, 6 in 2023-09-23T22:27:14.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[215775]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: 7.9 starting backfill to osd.7 from (0'0,0'0] MAX to 290'6937 2023-09-23T22:27:14.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: 7.d starting backfill to osd.7 from (0'0,0'0] MAX to 290'6885 2023-09-23T22:27:14.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: 7.4 starting backfill to osd.5 from (0'0,0'0] MAX to 290'6889 2023-09-23T22:27:14.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: 7.7 starting backfill to osd.7 from (0'0,0'0] MAX to 290'6791 2023-09-23T22:27:14.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: pgmap v764: 99 pgs: 2 unknown, 12 peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 417 op/s; 26053/215544 objects degraded (12.087%); 16909/215544 objects misplaced (7.845%); 7.8 MiB/s, 120 objects/s recovering 2023-09-23T22:27:14.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:27:14.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: osdmap e294: 8 total, 8 up, 6 in 2023-09-23T22:27:14.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:14 smithi070 ceph-mon[192652]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: 7.9 starting backfill to osd.7 from (0'0,0'0] MAX to 290'6937 2023-09-23T22:27:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: 7.d starting backfill to osd.7 from (0'0,0'0] MAX to 290'6885 2023-09-23T22:27:14.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: 7.4 starting backfill to osd.5 from (0'0,0'0] MAX to 290'6889 2023-09-23T22:27:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: 7.7 starting backfill to osd.7 from (0'0,0'0] MAX to 290'6791 2023-09-23T22:27:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: pgmap v764: 99 pgs: 2 unknown, 12 peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 75 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 417 op/s; 26053/215544 objects degraded (12.087%); 16909/215544 objects misplaced (7.845%); 7.8 MiB/s, 120 objects/s recovering 2023-09-23T22:27:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:27:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: osdmap e294: 8 total, 8 up, 6 in 2023-09-23T22:27:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: failed to encode map e294 with expected crc 2023-09-23T22:27:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:14 smithi026 ceph-mon[223014]: failed to encode map e294 with expected crc 2023-09-23T22:27:15.874 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:27:16.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:16 smithi070 ceph-mon[192652]: osdmap e295: 8 total, 8 up, 6 in 2023-09-23T22:27:16.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:16 smithi070 ceph-mon[192652]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:16 smithi070 ceph-mon[192652]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:16 smithi070 ceph-mon[192652]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:16 smithi070 ceph-mon[192652]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:16 smithi070 ceph-mon[192652]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.131 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:16.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:16.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:16.246 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:16.277 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:16.281 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:27:16.335 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:16.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:16.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:16.441 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:16.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[223014]: osdmap e295: 8 total, 8 up, 6 in 2023-09-23T22:27:16.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[223014]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[223014]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[223014]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[223014]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[223014]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[215775]: osdmap e295: 8 total, 8 up, 6 in 2023-09-23T22:27:16.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[215775]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[215775]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[215775]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[215775]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:16 smithi026 ceph-mon[215775]: failed to encode map e295 with expected crc 2023-09-23T22:27:16.574 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:16.749 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:16.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:16.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:16.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:16.967 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:17.129 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:17.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:17.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:17.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:17 smithi026 ceph-mon[215775]: pgmap v767: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 2 unknown, 15 peering, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 22 KiB/s rd, 59 KiB/s wr, 478 op/s; 37058/213000 objects degraded (17.398%); 16012/213000 objects misplaced (7.517%); 24 MiB/s, 365 objects/s recovering 2023-09-23T22:27:17.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:17 smithi026 ceph-mon[223014]: pgmap v767: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 2 unknown, 15 peering, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 22 KiB/s rd, 59 KiB/s wr, 478 op/s; 37058/213000 objects degraded (17.398%); 16012/213000 objects misplaced (7.517%); 24 MiB/s, 365 objects/s recovering 2023-09-23T22:27:17.239 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:17.327 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:17.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:17 smithi070 ceph-mon[192652]: pgmap v767: 99 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 2 unknown, 15 peering, 1 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 72 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 22 KiB/s rd, 59 KiB/s wr, 478 op/s; 37058/213000 objects degraded (17.398%); 16012/213000 objects misplaced (7.517%); 24 MiB/s, 365 objects/s recovering 2023-09-23T22:27:17.488 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:17.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:17.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:17.597 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:17.682 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:17.809 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:17.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:17.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:17.844 INFO:tasks.thrashosds.thrasher:in_osds: [3, 4, 5, 7, 0, 2] out_osds: [6, 1] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:27:17.845 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:27:17.845 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [3, 4, 5, 7, 0, 2] 2023-09-23T22:27:17.845 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T22:27:17.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:17.964 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:18.096 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:27:18.111 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:27:18.121 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:18.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:18.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:18.182 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:27:18.182 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101023 5.5 e161/152 5:a887f58c:::benchmark_data_smithi026_286012_object7215:head v 161'6460, mlcod=0'0)", 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.654324+0000", 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 597.523921728, 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.053302833000000001, 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.183 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654324+0000", 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654324+0000", 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654323+0000", 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654333+0000", 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8530000000000004e-06 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.185 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654346+0000", 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3842000000000001e-05 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.654353+0000", 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.489e-06 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.186 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.664025+0000", 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096725869999999999 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.664052+0000", 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6594999999999999e-05 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707599+0000", 2023-09-23T22:27:18.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.043547359000000001 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707627+0000", 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7282000000000001e-05 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101021 5.d e161/111 5:ba53bdac:::benchmark_data_smithi026_286012_object7213:head v 161'6371, mlcod=161'6368)", 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.656838+0000", 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 597.52140751299999, 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050757414000000001, 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656838+0000", 2023-09-23T22:27:18.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656838+0000", 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656836+0000", 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:27:18.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656849+0000", 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2418e-05 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656861+0000", 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2558000000000001e-05 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.656869+0000", 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3440000000000003e-06 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659763+0000", 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002893825 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659783+0000", 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0205999999999999e-05 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707552+0000", 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047768928000000002 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707595+0000", 2023-09-23T22:27:18.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3696000000000003e-05 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101026 5.d e161/111 5:bd4efaeb:::benchmark_data_smithi026_286012_object7218:head v 161'6372, mlcod=161'6368)", 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.657139+0000", 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 597.52110673499999, 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050510059000000003, 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.194 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657139+0000", 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657139+0000", 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657138+0000", 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657144+0000", 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9499999999999998e-06 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657156+0000", 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2381999999999999e-05 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.657162+0000", 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5629999999999999e-06 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.662814+0000", 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005652341 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.662834+0000", 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9296999999999998e-05 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707629+0000", 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.044795436000000001 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707649+0000", 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9789999999999999e-05 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.199 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101024 5.5 e161/152 5:adf8d131:::benchmark_data_smithi026_286012_object7216:head v 161'6461, mlcod=0'0)", 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.659691+0000", 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 597.51855476799994, 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048001597999999999, 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659691+0000", 2023-09-23T22:27:18.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659691+0000", 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659689+0000", 2023-09-23T22:27:18.201 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659701+0000", 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2172e-05 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659726+0000", 2023-09-23T22:27:18.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5106e-05 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.659733+0000", 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1729999999999998e-06 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.673568+0000", 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013834377 2023-09-23T22:27:18.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.673593+0000", 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4805999999999999e-05 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707672+0000", 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034079600000000002 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.204 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707692+0000", 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0287000000000001e-05 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101025 5.5 e161/152 5:ac250153:::benchmark_data_smithi026_286012_object7217:head v 161'6462, mlcod=0'0)", 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:20.660139+0000", 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 597.51810693899995, 2023-09-23T22:27:18.205 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.047578966, 2023-09-23T22:27:18.206 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.206 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.206 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.206 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.206 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.206 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660139+0000", 2023-09-23T22:27:18.206 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.206 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.206 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660139+0000", 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660138+0000", 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.207 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660144+0000", 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8930000000000002e-06 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660153+0000", 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8540000000000003e-06 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.208 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.660158+0000", 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8180000000000003e-06 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.680575+0000", 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020417219 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.680606+0000", 2023-09-23T22:27:18.209 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0701000000000003e-05 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707696+0000", 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.02709056 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:20.707718+0000", 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1460000000000001e-05 2023-09-23T22:27:18.210 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101956 5.d e161/111 5:b91a2013:::benchmark_data_smithi026_286012_object8148:head v 161'6423, mlcod=161'6418)", 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 596.20462737000003, 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.075111800000000006, 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.211 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.212 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973618+0000", 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973635+0000", 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6799e-05 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.213 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973642+0000", 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9199999999999998e-06 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.973646+0000", 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1540000000000004e-06 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.979441+0000", 2023-09-23T22:27:18.214 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057947739999999999 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.979459+0000", 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7938999999999999e-05 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048696+0000", 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.069237189000000005 2023-09-23T22:27:18.215 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048730+0000", 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4269e-05 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.34674.0:101969 5.4 e161/111 5:2c36eb9f:::benchmark_data_smithi026_286012_object8161:head v 161'6383, mlcod=161'6381)", 2023-09-23T22:27:18.216 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:17:21.982992+0000", 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 596.19525404900003, 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.071878993000000002, 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982992+0000", 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.217 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.218 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.218 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982992+0000", 2023-09-23T22:27:18.218 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.218 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.218 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.218 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.218 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982991+0000", 2023-09-23T22:27:18.219 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:27:18.219 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.219 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.219 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.219 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:21.982997+0000", 2023-09-23T22:27:18.219 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1310000000000003e-06 2023-09-23T22:27:18.219 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.219 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.219 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048677+0000", 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.065680429999999998 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048692+0000", 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4908e-05 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048737+0000", 2023-09-23T22:27:18.220 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4496000000000002e-05 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.048761+0000", 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4675999999999999e-05 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.054828+0000", 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006067037 2023-09-23T22:27:18.221 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:17:22.054871+0000", 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2276e-05 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45002.0:104782 6.8 e243/240 6:131c13f0:::benchmark_data_smithi026_355573_object23432:head v 243'6710, mlcod=208'1215)", 2023-09-23T22:27:18.222 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:22:52.789793+0000", 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 265.388452917, 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.055001656000000003, 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.789793+0000", 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.223 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.789793+0000", 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.789792+0000", 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.789806+0000", 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3899e-05 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.823581+0000", 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033775740999999998 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.823594+0000", 2023-09-23T22:27:18.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2738999999999999e-05 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.823741+0000", 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000147243 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.823770+0000", 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8470999999999999e-05 2023-09-23T22:27:18.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.227 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.227 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.227 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.844747+0000", 2023-09-23T22:27:18.227 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020977182 2023-09-23T22:27:18.227 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.227 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.228 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.228 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:22:52.844794+0000", 2023-09-23T22:27:18.228 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7302999999999998e-05 2023-09-23T22:27:18.228 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.228 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.229 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.229 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.229 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.229 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16555 7.5 e264/260 7:a4423849:::benchmark_data_smithi026_410749_object16554:head v 264'1083, mlcod=264'1081)", 2023-09-23T22:27:18.229 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.326981+0000", 2023-09-23T22:27:18.229 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 135.85126501600001, 2023-09-23T22:27:18.229 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17186283799999999, 2023-09-23T22:27:18.229 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.229 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.230 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.230 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.230 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.230 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.326981+0000", 2023-09-23T22:27:18.230 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.230 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.230 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.230 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.326981+0000", 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.326980+0000", 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327020+0000", 2023-09-23T22:27:18.231 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0092999999999997e-05 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327027+0000", 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0530000000000001e-06 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327031+0000", 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3840000000000001e-06 2023-09-23T22:27:18.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327068+0000", 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7005000000000002e-05 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327085+0000", 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7181e-05 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498806+0000", 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17172121400000001 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498843+0000", 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7351000000000002e-05 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.234 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16562 7.5 e264/260 7:ab68d64a:::benchmark_data_smithi026_410749_object16561:head v 264'1084, mlcod=264'1081)", 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.327040+0000", 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 135.85120601200001, 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.171824014, 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.235 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327040+0000", 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327040+0000", 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.236 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327039+0000", 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967296 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327186+0000", 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014651900000000001 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327190+0000", 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1849999999999997e-06 2023-09-23T22:27:18.237 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327193+0000", 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.497e-06 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327646+0000", 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045361700000000002 2023-09-23T22:27:18.238 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327663+0000", 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6540000000000001e-05 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498849+0000", 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17118603199999999 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.239 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498864+0000", 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4829e-05 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16567 7.3 e264/260 7:cd2ae393:::benchmark_data_smithi026_410749_object16566:head v 264'1045, mlcod=264'1042)", 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.327226+0000", 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 135.85101916400001, 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17163550499999999, 2023-09-23T22:27:18.240 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327226+0000", 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327226+0000", 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.241 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327226+0000", 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327292+0000", 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5415999999999998e-05 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.242 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327296+0000", 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.977e-06 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327299+0000", 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5739999999999998e-06 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.243 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327323+0000", 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4108999999999999e-05 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.327335+0000", 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2143e-05 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.244 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498824+0000", 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.171488848 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498862+0000", 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7737999999999997e-05 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.245 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16570 7.5 e264/260 7:ab27bf89:::benchmark_data_smithi026_410749_object16569:head v 264'1085, mlcod=264'1081)", 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.330592+0000", 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 135.84765379500001, 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17050346699999999, 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330592+0000", 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.246 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330592+0000", 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330591+0000", 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.247 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330624+0000", 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2951999999999999e-05 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498796+0000", 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.168172236 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498807+0000", 2023-09-23T22:27:18.248 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0747999999999999e-05 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498888+0000", 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1130000000000004e-05 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498909+0000", 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0795e-05 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.249 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.501072+0000", 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002162841 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.501095+0000", 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3391999999999999e-05 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.250 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:16572 7.2 e264/260 7:4b8eea19:::benchmark_data_smithi026_410749_object16571:head v 264'1051, mlcod=264'1049)", 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:02.330913+0000", 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 135.84733269099999, 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17039949600000001, 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.251 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330913+0000", 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330913+0000", 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330912+0000", 2023-09-23T22:27:18.252 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.330974+0000", 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1835999999999995e-05 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498801+0000", 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16782661900000001 2023-09-23T22:27:18.253 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.498809+0000", 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6200000000000005e-06 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.499112+0000", 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030310500000000002 2023-09-23T22:27:18.254 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.499130+0000", 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8016000000000002e-05 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.501278+0000", 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021478320000000001 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.255 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:02.501312+0000", 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4060999999999998e-05 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:72266 7.2 e268/260 7:4693bc36:::benchmark_data_smithi026_410749_object72265:head v 268'4506, mlcod=268'4503)", 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:52.217476+0000", 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 85.960769497000001, 2023-09-23T22:27:18.256 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.109211474, 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.217476+0000", 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.217476+0000", 2023-09-23T22:27:18.257 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.217475+0000", 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.217564+0000", 2023-09-23T22:27:18.258 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9355000000000001e-05 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.217585+0000", 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0965999999999999e-05 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.217595+0000", 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0246e-05 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.259 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322023+0000", 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10442800000000001 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322046+0000", 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.249e-05 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.326645+0000", 2023-09-23T22:27:18.260 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004599054 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.326688+0000", 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2830000000000002e-05 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:72274 7.c e268/260 7:39737cff:::benchmark_data_smithi026_410749_object72273:head v 268'4439, mlcod=268'4437)", 2023-09-23T22:27:18.261 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:52.226067+0000", 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 85.952178098000005, 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.102720094, 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.226067+0000", 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.262 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.226067+0000", 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.226066+0000", 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.263 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.226121+0000", 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5148000000000001e-05 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.321997+0000", 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.095876014999999995 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.264 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322007+0000", 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6919999999999997e-06 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322044+0000", 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6613999999999999e-05 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322061+0000", 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7269e-05 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.265 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.328748+0000", 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066866879999999997 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.328788+0000", 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9938000000000003e-05 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.266 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:72289 7.5 e268/260 7:af8212b6:::benchmark_data_smithi026_410749_object72288:head v 268'4604, mlcod=268'4602)", 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:52.249858+0000", 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 85.928387107999995, 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.076824745, 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.249858+0000", 2023-09-23T22:27:18.267 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.249858+0000", 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.249857+0000", 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:27:18.268 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.249908+0000", 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1845000000000003e-05 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322002+0000", 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.072093982000000001 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.269 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322012+0000", 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8060000000000006e-06 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322124+0000", 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011156 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322145+0000", 2023-09-23T22:27:18.270 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0990000000000001e-05 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.326649+0000", 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004503974 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.326683+0000", 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4490999999999999e-05 2023-09-23T22:27:18.271 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:72293 7.8 e268/260 7:1ad54cbb:::benchmark_data_smithi026_410749_object72292:head v 268'4401, mlcod=268'4399)", 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:52.251304+0000", 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 85.926941647999996, 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.073182581999999996, 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.272 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.251304+0000", 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.251304+0000", 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.273 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.251303+0000", 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.251326+0000", 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3260000000000001e-05 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.274 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.321994+0000", 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.070667468999999997 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322007+0000", 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3589e-05 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322091+0000", 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3561999999999997e-05 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.275 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.322112+0000", 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0557999999999999e-05 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.324437+0000", 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023254119999999998 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.276 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:52.324486+0000", 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9469999999999999e-05 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:110942 7.f e295/260 7:f7637fba:::benchmark_data_smithi026_410749_object20605:head v 295'6814, mlcod=295'6812)", 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:27:17.143301+0000", 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.0349446870000001, 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10253398, 2023-09-23T22:27:18.277 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143301+0000", 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143301+0000", 2023-09-23T22:27:18.278 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143299+0000", 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143312+0000", 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2422999999999999e-05 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.279 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143333+0000", 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1243e-05 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143343+0000", 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0393e-05 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.280 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143436+0000", 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2167e-05 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143468+0000", 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2613000000000001e-05 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.245798+0000", 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.102330226 2023-09-23T22:27:18.281 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.245835+0000", 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6402000000000001e-05 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:110940 7.c e295/272 7:33cbdb82:::benchmark_data_smithi026_410749_object20603:head v 295'6850, mlcod=264'1024)", 2023-09-23T22:27:18.282 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:27:17.143715+0000", 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.0345309620000001, 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.105135408, 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143715+0000", 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.283 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143715+0000", 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143714+0000", 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.284 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143719+0000", 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.118e-06 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143727+0000", 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5549999999999997e-06 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.143731+0000", 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8020000000000002e-06 2023-09-23T22:27:18.285 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.245833+0000", 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10210296000000001 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.245858+0000", 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4167000000000001e-05 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.248804+0000", 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002946621 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.248850+0000", 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5725e-05 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:110943 7.5 e295/272 7:a117030d:::benchmark_data_smithi026_410749_object20606:head v 295'7090, mlcod=264'651)", 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:27:17.145481+0000", 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.032765057, 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.105506327, 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.145481+0000", 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.145481+0000", 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.145480+0000", 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.289 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.145486+0000", 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6389999999999997e-06 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.245806+0000", 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.100319994 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.290 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.245814+0000", 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7669999999999992e-06 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.245866+0000", 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2071999999999999e-05 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.245887+0000", 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0716999999999998e-05 2023-09-23T22:27:18.291 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.250934+0000", 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050464860000000002 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:17.250987+0000", 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3251999999999998e-05 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.292 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.293 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:27:18.293 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:27:18.293 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:27:18.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:18.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3407698215' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:27:18.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:18 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:27:18.294 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 37058/213000 objects degraded (17.398%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:18.295 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3407698215' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:27:18.295 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:18 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:27:18.295 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 37058/213000 objects degraded (17.398%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:18.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3407698215' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:27:18.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:18 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:27:18.295 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 37058/213000 objects degraded (17.398%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:18.325 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:18.426 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:18.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:18.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:18.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:18.651 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:19.121 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:27:19.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:19 smithi070 ceph-mon[192652]: pgmap v768: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 3 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 17 KiB/s rd, 46 KiB/s wr, 364 op/s; 33025/210555 objects degraded (15.685%); 37390/210555 objects misplaced (17.758%); 21 MiB/s, 320 objects/s recovering 2023-09-23T22:27:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:19 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:27:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:19 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:27:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:19 smithi070 ceph-mon[192652]: osdmap e296: 8 total, 8 up, 6 in 2023-09-23T22:27:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:19 smithi070 ceph-mon[192652]: failed to encode map e296 with expected crc 2023-09-23T22:27:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:19 smithi070 ceph-mon[192652]: failed to encode map e296 with expected crc 2023-09-23T22:27:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[223014]: pgmap v768: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 3 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 17 KiB/s rd, 46 KiB/s wr, 364 op/s; 33025/210555 objects degraded (15.685%); 37390/210555 objects misplaced (17.758%); 21 MiB/s, 320 objects/s recovering 2023-09-23T22:27:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:27:19.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:19.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:27:19.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[223014]: osdmap e296: 8 total, 8 up, 6 in 2023-09-23T22:27:19.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[223014]: failed to encode map e296 with expected crc 2023-09-23T22:27:19.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[223014]: failed to encode map e296 with expected crc 2023-09-23T22:27:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[215775]: pgmap v768: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 3 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 17 KiB/s rd, 46 KiB/s wr, 364 op/s; 33025/210555 objects degraded (15.685%); 37390/210555 objects misplaced (17.758%); 21 MiB/s, 320 objects/s recovering 2023-09-23T22:27:19.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:27:19.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:19.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:27:19.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[215775]: osdmap e296: 8 total, 8 up, 6 in 2023-09-23T22:27:19.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[215775]: failed to encode map e296 with expected crc 2023-09-23T22:27:19.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:19 smithi026 ceph-mon[215775]: failed to encode map e296 with expected crc 2023-09-23T22:27:19.800 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:19.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:19.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:19.922 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:19.987 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:27:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:27:19] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-09-23T22:27:20.074 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:20.246 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:20.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:20.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:20.267 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:20.316 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:20.361 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:20.389 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:20.491 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:20.695 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:20.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:20.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:20.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:20.830 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:20.880 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:20.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:20.988 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:21.112 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:21.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:21 smithi070 ceph-mon[192652]: pgmap v770: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 3 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 288 op/s; 33025/210555 objects degraded (15.685%); 37390/210555 objects misplaced (17.758%); 16 MiB/s, 253 objects/s recovering 2023-09-23T22:27:21.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:21 smithi026 ceph-mon[223014]: pgmap v770: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 3 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 288 op/s; 33025/210555 objects degraded (15.685%); 37390/210555 objects misplaced (17.758%); 16 MiB/s, 253 objects/s recovering 2023-09-23T22:27:21.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:21 smithi026 ceph-mon[215775]: pgmap v770: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 3 peering, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 80 active+clean; 4.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 288 op/s; 33025/210555 objects degraded (15.685%); 37390/210555 objects misplaced (17.758%); 16 MiB/s, 253 objects/s recovering 2023-09-23T22:27:22.066 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:27:22.122 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:22.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:22.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:22.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:22.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:22.766 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:22.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:22.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:22.880 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:22.907 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:23 smithi026 ceph-mon[215775]: pgmap v771: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 369 op/s; 32913/207540 objects degraded (15.859%); 35599/207540 objects misplaced (17.153%); 21 MiB/s, 329 objects/s recovering 2023-09-23T22:27:23.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 32913/207540 objects degraded (15.859%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:23.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:23 smithi026 ceph-mon[223014]: pgmap v771: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 369 op/s; 32913/207540 objects degraded (15.859%); 35599/207540 objects misplaced (17.153%); 21 MiB/s, 329 objects/s recovering 2023-09-23T22:27:23.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 32913/207540 objects degraded (15.859%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:23.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:23 smithi070 ceph-mon[192652]: pgmap v771: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 11 KiB/s rd, 30 KiB/s wr, 369 op/s; 32913/207540 objects degraded (15.859%); 35599/207540 objects misplaced (17.153%); 21 MiB/s, 329 objects/s recovering 2023-09-23T22:27:23.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 32913/207540 objects degraded (15.859%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:23.727 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:23.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:23.839 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:23.868 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:23.924 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:23.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:24.028 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:24.104 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:24.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:24.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:24.494 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:24.920 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:24.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:25.043 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:25.075 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:25.126 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:25.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:25.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:25.194 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:25 smithi026 ceph-mon[223014]: pgmap v772: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 32913/204351 objects degraded (16.106%); 34956/204351 objects misplaced (17.106%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:27:25.195 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:25 smithi026 ceph-mon[215775]: pgmap v772: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 32913/204351 objects degraded (16.106%); 34956/204351 objects misplaced (17.106%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:27:25.230 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:25.325 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:25.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:25 smithi070 ceph-mon[192652]: pgmap v772: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.2 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 343 op/s; 32913/204351 objects degraded (16.106%); 34956/204351 objects misplaced (17.106%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:27:25.462 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35406.0:114387 7.e e296/272)", 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:25.457685+0000", 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0037611509999999999, 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037894700000000001, 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.457685+0000", 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.457685+0000", 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:25.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.457684+0000", 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.457692+0000", 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2330000000000006e-06 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.457706+0000", 2023-09-23T22:27:25.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.402e-05 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.457724+0000", 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7516999999999998e-05 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114392 7.7 7:e12ba598:::benchmark_data_smithi026_410749_object24055:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:25.466 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:25.458908+0000", 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0025375150000000002, 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026292690000000001, 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114392 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:25.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.458908+0000", 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.458908+0000", 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:25.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.458907+0000", 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.458913+0000", 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.263e-06 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.458926+0000", 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2860000000000001e-05 2023-09-23T22:27:25.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.458933+0000", 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4919999999999996e-06 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.459215+0000", 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000281651 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.459367+0000", 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015164700000000001 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.459525+0000", 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000158214 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114383 7.5 7:a5ce1b7f:::benchmark_data_smithi026_410749_object24046:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:25.419199+0000", 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.042247093999999999, 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.042426799000000001, 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:25.472 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114383 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.419199+0000", 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:25.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.419199+0000", 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.419198+0000", 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.419201+0000", 2023-09-23T22:27:25.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1739999999999999e-06 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.419206+0000", 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0610000000000003e-06 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.419210+0000", 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3970000000000004e-06 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.419262+0000", 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1779e-05 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.419420+0000", 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015847499999999999 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.419576+0000", 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015542299999999999 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.450873+0000", 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031297208 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.455109+0000", 2023-09-23T22:27:25.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004236025 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114387 7.e 7:7a88c5e2:::benchmark_data_smithi026_410749_object24050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:25.428649+0000", 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.032796944000000001, 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033081065999999999, 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:25.478 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114387 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.428649+0000", 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.428649+0000", 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.428647+0000", 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:25.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.428655+0000", 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.306e-06 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.428669+0000", 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4283e-05 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.428678+0000", 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8589999999999994e-06 2023-09-23T22:27:25.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.441922+0000", 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013244685000000001 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.446189+0000", 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042669149999999996 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.446379+0000", 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018976 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114390 7.5 7:acf93de9:::benchmark_data_smithi026_410749_object24053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:25.441855+0000", 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.019590533, 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019957415999999999, 2023-09-23T22:27:25.483 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114390 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.441855+0000", 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.441855+0000", 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.441853+0000", 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.441862+0000", 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9819999999999997e-06 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.441882+0000", 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0091000000000001e-05 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:25.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.441900+0000", 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7617000000000001e-05 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.441978+0000", 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8443999999999996e-05 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.442212+0000", 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023405499999999999 2023-09-23T22:27:25.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.442420+0000", 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020780500000000001 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.455157+0000", 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012737155 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:25.488 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPushReply(7.e 296/272 [PushReplyOp(7:7b1cb86e:::benchmark_data_smithi026_410749_object24061:head)])", 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:25.450737+0000", 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.010708789, 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011160408, 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.450737+0000", 2023-09-23T22:27:25.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.450737+0000", 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.450737+0000", 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:25.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.450742+0000", 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9649999999999999e-06 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.450750+0000", 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9100000000000005e-06 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.450756+0000", 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2199999999999997e-06 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:25.450850+0000", 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4283000000000005e-05 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:25.492 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:27:25.493 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 6 2023-09-23T22:27:25.493 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:27:25.493 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:25.520 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:25.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:25.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:25.640 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:25.733 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:26.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:25 smithi026 conmon[215751]: 2023-09-23T22:27:25.941+0000 7f4580f04700 -1 mon.a@0(leader).osd e296 definitely_dead 0 2023-09-23T22:27:26.131 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:26 smithi026 ceph-mon[215775]: pgmap v773: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 394 op/s; 32797/201162 objects degraded (16.304%); 33693/201162 objects misplaced (16.749%); 11 MiB/s, 177 objects/s recovering 2023-09-23T22:27:26.156 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.3. 2023-09-23T22:27:26.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:26 smithi026 ceph-mon[223014]: pgmap v773: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 394 op/s; 32797/201162 objects degraded (16.304%); 33693/201162 objects misplaced (16.749%); 11 MiB/s, 177 objects/s recovering 2023-09-23T22:27:26.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/966848559' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:27:26.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:26 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:27:26.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:26 smithi026 ceph-mon[223014]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T22:27:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/966848559' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:27:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:26 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:27:26.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:26 smithi026 ceph-mon[215775]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T22:27:26.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:26 smithi070 ceph-mon[192652]: pgmap v773: 99 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 83 active+clean; 4.1 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 394 op/s; 32797/201162 objects degraded (16.304%); 33693/201162 objects misplaced (16.749%); 11 MiB/s, 177 objects/s recovering 2023-09-23T22:27:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/966848559' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:27:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:26 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:27:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:26 smithi070 ceph-mon[192652]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T22:27:26.930 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:26.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:27.034 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:27.063 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:27.111 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:27.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:27.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:27.217 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:27.245 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:27.309 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:27.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:27 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T22:27:27.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:27 smithi070 ceph-mon[192652]: osdmap e297: 8 total, 8 up, 5 in 2023-09-23T22:27:27.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:27 smithi070 ceph-mon[192652]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:27 smithi070 ceph-mon[192652]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:27 smithi070 ceph-mon[192652]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:27 smithi070 ceph-mon[192652]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:27.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:27.423 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:27.448 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T22:27:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[215775]: osdmap e297: 8 total, 8 up, 5 in 2023-09-23T22:27:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[215775]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[215775]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[215775]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[215775]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T22:27:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[223014]: osdmap e297: 8 total, 8 up, 5 in 2023-09-23T22:27:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[223014]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[223014]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[223014]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:27 smithi026 ceph-mon[223014]: failed to encode map e297 with expected crc 2023-09-23T22:27:27.489 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:27.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:27.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:27.595 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:27.621 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:27.663 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:27.664 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:27:27.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:27.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:27.767 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:27.796 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:28.043 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:27:28.070 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:27:28.090 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:27:28.103 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:28.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:28.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:28.114 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:27:28.162 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:27:28.208 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:27:28.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:28.248 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:28.281 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:27:28.301 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:28.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:28.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:28.320 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:27:28.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:27:28.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:27:28.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:27:28.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:27:28.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:27:28.352 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:27:28.352 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:27:28.352 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:27:28.406 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:28.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: osdmap e298: 8 total, 8 up, 5 in 2023-09-23T22:27:28.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: 7.d starting backfill to osd.5 from (0'0,0'0] MAX to 296'7216 2023-09-23T22:27:28.411 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: 7.7 starting backfill to osd.4 from (0'0,0'0] MAX to 296'7122 2023-09-23T22:27:28.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: 7.d continuing backfill to osd.7 from (265'2101,296'7216] MIN to 296'7216 2023-09-23T22:27:28.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: 7.7 starting backfill to osd.7 from (265'2007,296'7121] MIN to 296'7122 2023-09-23T22:27:28.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: pgmap v776: 99 pgs: 1 activating+degraded+remapped, 3 remapped+peering, 8 peering, 1 unknown, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 488 op/s; 36449/198831 objects degraded (18.332%); 37663/198831 objects misplaced (18.942%); 14 MiB/s, 223 objects/s recovering 2023-09-23T22:27:28.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 32797/201162 objects degraded (16.304%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:28.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3612333886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:28.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[215775]: osdmap e299: 8 total, 8 up, 5 in 2023-09-23T22:27:28.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: osdmap e298: 8 total, 8 up, 5 in 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: 7.d starting backfill to osd.5 from (0'0,0'0] MAX to 296'7216 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: 7.7 starting backfill to osd.4 from (0'0,0'0] MAX to 296'7122 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: 7.d continuing backfill to osd.7 from (265'2101,296'7216] MIN to 296'7216 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: 7.7 starting backfill to osd.7 from (265'2007,296'7121] MIN to 296'7122 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: pgmap v776: 99 pgs: 1 activating+degraded+remapped, 3 remapped+peering, 8 peering, 1 unknown, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 488 op/s; 36449/198831 objects degraded (18.332%); 37663/198831 objects misplaced (18.942%); 14 MiB/s, 223 objects/s recovering 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 32797/201162 objects degraded (16.304%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:28.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3612333886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:28.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:28 smithi026 ceph-mon[223014]: osdmap e299: 8 total, 8 up, 5 in 2023-09-23T22:27:28.504 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:28.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: osdmap e298: 8 total, 8 up, 5 in 2023-09-23T22:27:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: failed to encode map e298 with expected crc 2023-09-23T22:27:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: 7.d starting backfill to osd.5 from (0'0,0'0] MAX to 296'7216 2023-09-23T22:27:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: 7.7 starting backfill to osd.4 from (0'0,0'0] MAX to 296'7122 2023-09-23T22:27:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: 7.d continuing backfill to osd.7 from (265'2101,296'7216] MIN to 296'7216 2023-09-23T22:27:28.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: 7.7 starting backfill to osd.7 from (265'2007,296'7121] MIN to 296'7122 2023-09-23T22:27:28.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: pgmap v776: 99 pgs: 1 activating+degraded+remapped, 3 remapped+peering, 8 peering, 1 unknown, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 488 op/s; 36449/198831 objects degraded (18.332%); 37663/198831 objects misplaced (18.942%); 14 MiB/s, 223 objects/s recovering 2023-09-23T22:27:28.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 32797/201162 objects degraded (16.304%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:28.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3612333886' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:28.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:28 smithi070 ceph-mon[192652]: osdmap e299: 8 total, 8 up, 5 in 2023-09-23T22:27:29.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: 7.a starting backfill to osd.4 from (0'0,0'0] MAX to 296'7305 2023-09-23T22:27:29.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: 7.2 starting backfill to osd.0 from (0'0,0'0] MAX to 296'7148 2023-09-23T22:27:29.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:27:29.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: 7.f starting backfill to osd.4 from (0'0,0'0] MAX to 296'7042 2023-09-23T22:27:29.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: 7.6 starting backfill to osd.5 from (0'0,0'0] MAX to 296'7187 2023-09-23T22:27:29.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[215775]: osdmap e300: 8 total, 8 up, 5 in 2023-09-23T22:27:29.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: 7.a starting backfill to osd.4 from (0'0,0'0] MAX to 296'7305 2023-09-23T22:27:29.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: 7.2 starting backfill to osd.0 from (0'0,0'0] MAX to 296'7148 2023-09-23T22:27:29.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:27:29.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: 7.f starting backfill to osd.4 from (0'0,0'0] MAX to 296'7042 2023-09-23T22:27:29.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: 7.6 starting backfill to osd.5 from (0'0,0'0] MAX to 296'7187 2023-09-23T22:27:29.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:29 smithi026 ceph-mon[223014]: osdmap e300: 8 total, 8 up, 5 in 2023-09-23T22:27:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: 7.a starting backfill to osd.4 from (0'0,0'0] MAX to 296'7305 2023-09-23T22:27:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: failed to encode map e299 with expected crc 2023-09-23T22:27:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: 7.2 starting backfill to osd.0 from (0'0,0'0] MAX to 296'7148 2023-09-23T22:27:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:27:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: 7.f starting backfill to osd.4 from (0'0,0'0] MAX to 296'7042 2023-09-23T22:27:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: 7.6 starting backfill to osd.5 from (0'0,0'0] MAX to 296'7187 2023-09-23T22:27:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:29 smithi070 ceph-mon[192652]: osdmap e300: 8 total, 8 up, 5 in 2023-09-23T22:27:29.627 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:29.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:29.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:29.731 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:29.770 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:29.833 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:29.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:29.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:29.869 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:27:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:27:29] "GET /metrics HTTP/1.1" 200 36440 "" "Prometheus/2.43.0" 2023-09-23T22:27:29.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:29.970 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:30.021 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:30.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:30.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:30.127 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:30.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[223014]: failed to encode map e299 with expected crc 2023-09-23T22:27:30.170 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[223014]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[223014]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[223014]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[223014]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[223014]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[223014]: pgmap v779: 99 pgs: 1 activating+degraded+remapped, 3 remapped+peering, 8 peering, 1 unknown, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 194 op/s; 36449/198831 objects degraded (18.332%); 37663/198831 objects misplaced (18.942%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:27:30.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[223014]: osdmap e301: 8 total, 8 up, 5 in 2023-09-23T22:27:30.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[215775]: failed to encode map e299 with expected crc 2023-09-23T22:27:30.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[215775]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[215775]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[215775]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[215775]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[215775]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[215775]: pgmap v779: 99 pgs: 1 activating+degraded+remapped, 3 remapped+peering, 8 peering, 1 unknown, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 194 op/s; 36449/198831 objects degraded (18.332%); 37663/198831 objects misplaced (18.942%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:27:30.172 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:30 smithi026 ceph-mon[215775]: osdmap e301: 8 total, 8 up, 5 in 2023-09-23T22:27:30.183 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:30 smithi070 ceph-mon[192652]: failed to encode map e299 with expected crc 2023-09-23T22:27:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:30 smithi070 ceph-mon[192652]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:30 smithi070 ceph-mon[192652]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:30 smithi070 ceph-mon[192652]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:30 smithi070 ceph-mon[192652]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:30 smithi070 ceph-mon[192652]: failed to encode map e300 with expected crc 2023-09-23T22:27:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:30 smithi070 ceph-mon[192652]: pgmap v779: 99 pgs: 1 activating+degraded+remapped, 3 remapped+peering, 8 peering, 1 unknown, 1 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 4 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 71 active+clean; 4.0 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 194 op/s; 36449/198831 objects degraded (18.332%); 37663/198831 objects misplaced (18.942%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T22:27:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:30 smithi070 ceph-mon[192652]: osdmap e301: 8 total, 8 up, 5 in 2023-09-23T22:27:30.747 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:30.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:30.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:30.851 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:30.862 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:30.883 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:30.931 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:30.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:30.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:31.040 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:31.068 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:31.113 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:31.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:31.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:31.218 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:31.245 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:31.286 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:31.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:31.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:31.391 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:31.417 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:31.458 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:31.461 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:31 smithi070 ceph-mon[192652]: failed to encode map e301 with expected crc 2023-09-23T22:27:31.461 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:31 smithi070 ceph-mon[192652]: failed to encode map e301 with expected crc 2023-09-23T22:27:31.461 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:31 smithi070 ceph-mon[192652]: failed to encode map e301 with expected crc 2023-09-23T22:27:31.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:31.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:31.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:31 smithi026 ceph-mon[223014]: failed to encode map e301 with expected crc 2023-09-23T22:27:31.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:31 smithi026 ceph-mon[223014]: failed to encode map e301 with expected crc 2023-09-23T22:27:31.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:31 smithi026 ceph-mon[223014]: failed to encode map e301 with expected crc 2023-09-23T22:27:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:31 smithi026 ceph-mon[215775]: failed to encode map e301 with expected crc 2023-09-23T22:27:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:31 smithi026 ceph-mon[215775]: failed to encode map e301 with expected crc 2023-09-23T22:27:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:31 smithi026 ceph-mon[215775]: failed to encode map e301 with expected crc 2023-09-23T22:27:31.563 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:31.618 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:31.703 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:31.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:31.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:31.809 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:31.849 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:31.921 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:31.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:31.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:32.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:32.060 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:32.129 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:32.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:32.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:32.176 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:32 smithi026 ceph-mon[223014]: failed to encode map e301 with expected crc 2023-09-23T22:27:32.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:32 smithi026 ceph-mon[223014]: pgmap v781: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 3 remapped+peering, 9 peering, 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 66 active+clean; 4.0 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 39 KiB/s rd, 102 KiB/s wr, 250 op/s; 41371/198321 objects degraded (20.861%); 40935/198321 objects misplaced (20.641%); 16 MiB/s, 247 objects/s recovering 2023-09-23T22:27:32.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:32 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2664127721' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:27:32.177 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:32 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:27:32.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:32 smithi026 ceph-mon[215775]: failed to encode map e301 with expected crc 2023-09-23T22:27:32.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:32 smithi026 ceph-mon[215775]: pgmap v781: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 3 remapped+peering, 9 peering, 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 66 active+clean; 4.0 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 39 KiB/s rd, 102 KiB/s wr, 250 op/s; 41371/198321 objects degraded (20.861%); 40935/198321 objects misplaced (20.641%); 16 MiB/s, 247 objects/s recovering 2023-09-23T22:27:32.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:32 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2664127721' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:27:32.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:32 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:27:32.186 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:27:32.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:32.270 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:32.331 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:32.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:32.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:32.436 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:32.463 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:32 smithi070 ceph-mon[192652]: failed to encode map e301 with expected crc 2023-09-23T22:27:32.464 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:32 smithi070 ceph-mon[192652]: pgmap v781: 99 pgs: 1 activating+undersized+degraded+remapped, 1 activating+degraded+remapped, 3 remapped+peering, 9 peering, 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 4 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 66 active+clean; 4.0 GiB data, 8.8 GiB used, 438 GiB / 447 GiB avail; 39 KiB/s rd, 102 KiB/s wr, 250 op/s; 41371/198321 objects degraded (20.861%); 40935/198321 objects misplaced (20.641%); 16 MiB/s, 247 objects/s recovering 2023-09-23T22:27:32.464 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:32 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2664127721' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:27:32.464 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:32 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:27:32.466 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:32.514 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:32.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:32.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:32.622 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:32.649 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:32.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:32.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:32.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:32.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:32.825 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:32.866 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:32.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:32.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:32.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:27:32.972 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:33.017 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:33.122 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 2] out_osds: [6, 1, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:27:33.122 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:27:33.122 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T22:27:33.122 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:27:33.123 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:27:33.123 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T22:27:33.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:33.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[215775]: failed to encode map e302 with expected crc 2023-09-23T22:27:33.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:27:33.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[215775]: osdmap e302: 8 total, 8 up, 5 in 2023-09-23T22:27:33.330 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:33.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:33.438 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:33.439 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[215775]: failed to encode map e302 with expected crc 2023-09-23T22:27:33.439 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[215775]: failed to encode map e302 with expected crc 2023-09-23T22:27:33.439 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 41371/198321 objects degraded (20.861%), 14 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:33.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:33.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[223014]: failed to encode map e302 with expected crc 2023-09-23T22:27:33.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:27:33.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[223014]: osdmap e302: 8 total, 8 up, 5 in 2023-09-23T22:27:33.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[223014]: failed to encode map e302 with expected crc 2023-09-23T22:27:33.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[223014]: failed to encode map e302 with expected crc 2023-09-23T22:27:33.440 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 41371/198321 objects degraded (20.861%), 14 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:33.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:33 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:33.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:33 smithi070 ceph-mon[192652]: failed to encode map e302 with expected crc 2023-09-23T22:27:33.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:33 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:27:33.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:33 smithi070 ceph-mon[192652]: osdmap e302: 8 total, 8 up, 5 in 2023-09-23T22:27:33.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:33 smithi070 ceph-mon[192652]: failed to encode map e302 with expected crc 2023-09-23T22:27:33.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:33 smithi070 ceph-mon[192652]: failed to encode map e302 with expected crc 2023-09-23T22:27:33.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 41371/198321 objects degraded (20.861%), 14 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:33.468 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:33.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:33.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:33.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:33.616 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:33.685 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:33.830 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:33.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:33.918 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:33.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:33.973 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:27:33.973 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:27:33.992 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:34.007 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:34.246 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:34.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:34.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:34.360 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:34.392 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:34.439 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:34.442 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:34 smithi070 ceph-mon[192652]: failed to encode map e302 with expected crc 2023-09-23T22:27:34.443 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:34 smithi070 ceph-mon[192652]: pgmap v783: 99 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 1 unknown, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 67 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 31 KiB/s rd, 83 KiB/s wr, 59 op/s; 41369/198285 objects degraded (20.863%); 49165/198285 objects misplaced (24.795%); 11 MiB/s, 173 objects/s recovering 2023-09-23T22:27:34.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:34.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:34.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:34 smithi026 ceph-mon[215775]: failed to encode map e302 with expected crc 2023-09-23T22:27:34.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:34 smithi026 ceph-mon[215775]: pgmap v783: 99 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 1 unknown, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 67 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 31 KiB/s rd, 83 KiB/s wr, 59 op/s; 41369/198285 objects degraded (20.863%); 49165/198285 objects misplaced (24.795%); 11 MiB/s, 173 objects/s recovering 2023-09-23T22:27:34.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:34 smithi026 ceph-mon[223014]: failed to encode map e302 with expected crc 2023-09-23T22:27:34.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:34 smithi026 ceph-mon[223014]: pgmap v783: 99 pgs: 1 activating+undersized+degraded+remapped, 9 peering, 1 unknown, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 67 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 31 KiB/s rd, 83 KiB/s wr, 59 op/s; 41369/198285 objects degraded (20.863%); 49165/198285 objects misplaced (24.795%); 11 MiB/s, 173 objects/s recovering 2023-09-23T22:27:34.544 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:34.572 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:34.615 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:34.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:34.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:34.719 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:34.746 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:34.786 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:34.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:34.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:34.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:34.930 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:35.121 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:35.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:35.127 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:35.227 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:35.255 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:35.298 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:35.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:35.306 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:35.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:27:35.406 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:35.432 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:35.467 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:27:35.472 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:35.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:35.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:35.577 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:35.667 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:35.962 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:35.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:36.067 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:36.096 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:36.141 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:36.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:36.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:36.246 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:36.273 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:36.316 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:36.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:36.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:36.422 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:36.448 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:36.490 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:36.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:36.594 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:36.620 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:36.661 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:36.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:36.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:36.767 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:36.800 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:36 smithi026 ceph-mon[223014]: pgmap v784: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 26 KiB/s rd, 70 KiB/s wr, 49 op/s; 41369/198285 objects degraded (20.863%); 48852/198285 objects misplaced (24.637%); 12 MiB/s, 195 objects/s recovering 2023-09-23T22:27:36.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:36 smithi026 ceph-mon[215775]: pgmap v784: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 26 KiB/s rd, 70 KiB/s wr, 49 op/s; 41369/198285 objects degraded (20.863%); 48852/198285 objects misplaced (24.637%); 12 MiB/s, 195 objects/s recovering 2023-09-23T22:27:36.805 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:36 smithi070 ceph-mon[192652]: pgmap v784: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 26 KiB/s rd, 70 KiB/s wr, 49 op/s; 41369/198285 objects degraded (20.863%); 48852/198285 objects misplaced (24.637%); 12 MiB/s, 195 objects/s recovering 2023-09-23T22:27:37.180 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:37.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:37.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:37.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:37.335 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:37.363 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:37.377 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:37.489 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:37.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:37.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:37.596 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:37.625 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:37.670 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:37.677 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:37.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:37 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:27:37.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:37 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:27:37.776 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:27:37.778 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:37.808 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:27:37.843 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:37.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:37 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:27:38.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:38.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:38.114 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:27:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:27:38.115 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:27:38.189 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:38.218 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:38.262 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:38.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:38.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:38.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:38.407 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:38 smithi026 ceph-mon[223014]: pgmap v785: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 39 op/s; 41369/198285 objects degraded (20.863%); 48852/198285 objects misplaced (24.637%); 9.7 MiB/s, 153 objects/s recovering 2023-09-23T22:27:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:38 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3253397476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:38.698 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 41369/198285 objects degraded (20.863%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:38.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:38 smithi026 ceph-mon[215775]: pgmap v785: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 39 op/s; 41369/198285 objects degraded (20.863%); 48852/198285 objects misplaced (24.637%); 9.7 MiB/s, 153 objects/s recovering 2023-09-23T22:27:38.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:38 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3253397476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:38.699 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 41369/198285 objects degraded (20.863%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:38.730 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:38.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:38.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:38.834 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:38.859 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:38 smithi070 ceph-mon[192652]: pgmap v785: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 39 op/s; 41369/198285 objects degraded (20.863%); 48852/198285 objects misplaced (24.637%); 9.7 MiB/s, 153 objects/s recovering 2023-09-23T22:27:38.859 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:38 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3253397476' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:27:38.859 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 41369/198285 objects degraded (20.863%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:38.862 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:38.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:38.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:38.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:39.010 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:39.037 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:39.077 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:39.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:39.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:39.184 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:39.210 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:39.250 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:39.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:39.355 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:39.381 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:39.422 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:39.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:39.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:39.527 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:39.553 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:39.594 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:39.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:39.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:39.702 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:39.749 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:39.846 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:39.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:39.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:39.872 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:27:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:27:39] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.43.0" 2023-09-23T22:27:39.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:39.981 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:40.022 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:40.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:40.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:40.127 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:40.152 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:40.194 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:40.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:40.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:40.299 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:40.336 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:40.834 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:40.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:40.940 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:40.967 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:41.009 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:41.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:41.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:41.114 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:41.122 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:41 smithi026 ceph-mon[223014]: pgmap v786: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 41369/198285 objects degraded (20.863%); 48852/198285 objects misplaced (24.637%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T22:27:41.122 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:41 smithi026 ceph-mon[215775]: pgmap v786: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 41369/198285 objects degraded (20.863%); 48852/198285 objects misplaced (24.637%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T22:27:41.167 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:41.233 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:41.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:41.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:41.338 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:41.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:41 smithi070 ceph-mon[192652]: pgmap v786: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 41369/198285 objects degraded (20.863%); 48852/198285 objects misplaced (24.637%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T22:27:41.393 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:41.593 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:41.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:41.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:41.706 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:41.724 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:27:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:27:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:27:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:27:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160066 5.3 5:cf864644:::benchmark_data_smithi026_286012_object66258:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:27:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724400+0000", 2023-09-23T22:27:41.724 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 523.99511476600003, 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5659008060000001, 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160066 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724400+0000", 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724400+0000", 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.726 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724400+0000", 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724402+0000", 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1710000000000001e-06 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724409+0000", 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3950000000000003e-06 2023-09-23T22:27:41.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724415+0000", 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3349999999999997e-06 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725871+0000", 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014559379999999999 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725875+0000", 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5360000000000003e-06 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.035756+0000", 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3098814270000001 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.036147+0000", 2023-09-23T22:27:41.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000390276 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.053539+0000", 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017392409000000001 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.053674+0000", 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135306 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.054299+0000", 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00062442400000000005 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.070040+0000", 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015741027000000001 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.070085+0000", 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4923999999999999e-05 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.083065+0000", 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012980047999999999 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.288967+0000", 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20590188000000001 2023-09-23T22:27:41.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.289024+0000", 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6818999999999999e-05 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.290301+0000", 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001277296 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.733 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160067 5.3 5:c73fe88c:::benchmark_data_smithi026_286012_object66259:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724420+0000", 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 523.99509513999999, 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5671979359999999, 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:27:41.734 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160067 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724420+0000", 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724420+0000", 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724419+0000", 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724421+0000", 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.984e-06 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724426+0000", 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0690000000000001e-06 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724431+0000", 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9110000000000001e-06 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725889+0000", 2023-09-23T22:27:41.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014581290000000001 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725894+0000", 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0189999999999997e-06 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.036192+0000", 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3102978089999999 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.036767+0000", 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057512800000000005 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.073186+0000", 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.036419066 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.073316+0000", 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129686 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.073488+0000", 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017221599999999999 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.089235+0000", 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015746687999999998 2023-09-23T22:27:41.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.119472+0000", 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030236613999999998 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.119541+0000", 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9079999999999996e-05 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.290378+0000", 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.170836919 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.290448+0000", 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0296000000000006e-05 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.291618+0000", 2023-09-23T22:27:41.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011696880000000001 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160081 5.3 5:cfd739d8:::benchmark_data_smithi026_286012_object66273:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724435+0000", 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 523.995079369, 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.568329584, 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.743 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160081 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724435+0000", 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724435+0000", 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724435+0000", 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724437+0000", 2023-09-23T22:27:41.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9420000000000002e-06 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724442+0000", 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9760000000000003e-06 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724447+0000", 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9049999999999996e-06 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725908+0000", 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014606 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725912+0000", 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3100000000000002e-06 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.036814+0000", 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3109024380000001 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037101+0000", 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028662199999999999 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.079431+0000", 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.042330206000000002 2023-09-23T22:27:41.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.079562+0000", 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130839 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.079752+0000", 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000189469 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.095018+0000", 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01526634 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.119583+0000", 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024565441 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125033+0000", 2023-09-23T22:27:41.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054497169999999998 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.291683+0000", 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.166650353 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.291749+0000", 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5491e-05 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.292765+0000", 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010162089999999999 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160090 5.3 5:c5c01081:::benchmark_data_smithi026_286012_object66282:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724451+0000", 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 523.99506401400004, 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5694129160000001, 2023-09-23T22:27:41.752 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160090 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724451+0000", 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724451+0000", 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724451+0000", 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724456+0000", 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6899999999999997e-06 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724461+0000", 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0520000000000001e-06 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724466+0000", 2023-09-23T22:27:41.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8890000000000001e-06 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725924+0000", 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014574060000000001 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725928+0000", 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.33e-06 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037149+0000", 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3112209639999999 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037542+0000", 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039318900000000001 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.085738+0000", 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048196211000000003 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.085840+0000", 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000101345 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.085972+0000", 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132791 2023-09-23T22:27:41.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.097699+0000", 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011726221 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125076+0000", 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027376948000000002 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125123+0000", 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7067999999999997e-05 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.292841+0000", 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16771852700000001 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.292904+0000", 2023-09-23T22:27:41.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2728e-05 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.293864+0000", 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000959826 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160098 5.3 5:cfe0d09f:::benchmark_data_smithi026_286012_object66290:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:27:41.761 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724470+0000", 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 523.99504475100002, 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.570344789, 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160098 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724470+0000", 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724470+0000", 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724470+0000", 2023-09-23T22:27:41.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724472+0000", 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.323e-06 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724477+0000", 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0780000000000003e-06 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724482+0000", 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6009999999999997e-06 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725940+0000", 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001457898 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725944+0000", 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2869999999999998e-06 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037585+0000", 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3116414009999999 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.037953+0000", 2023-09-23T22:27:41.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036729699999999999 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.091574+0000", 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.053620959000000003 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.091698+0000", 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012485 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.091877+0000", 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017887099999999999 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.101104+0000", 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092271279999999994 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125165+0000", 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024060350000000001 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125208+0000", 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3374000000000002e-05 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.293938+0000", 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.168729515 2023-09-23T22:27:41.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.293999+0000", 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1302000000000004e-05 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.294815+0000", 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081585700000000002 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.770 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160108 5.3 5:c2b2bc92:::benchmark_data_smithi026_286012_object66300:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724485+0000", 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 523.995029413, 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5712676249999999, 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:27:41.771 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160108 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724485+0000", 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724485+0000", 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724485+0000", 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724487+0000", 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8530000000000001e-06 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724492+0000", 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1819999999999997e-06 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724497+0000", 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6179999999999997e-06 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725956+0000", 2023-09-23T22:27:41.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014589029999999999 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725960+0000", 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2069999999999997e-06 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038012+0000", 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3120517139999999 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038386+0000", 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037387599999999998 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.097736+0000", 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059349986 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.097802+0000", 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6389000000000004e-05 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.097927+0000", 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012475100000000001 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.109875+0000", 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011948215999999999 2023-09-23T22:27:41.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.130691+0000", 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020815904 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.130756+0000", 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5492000000000002e-05 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.294890+0000", 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16413367700000001 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.294945+0000", 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4957999999999998e-05 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.295753+0000", 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00080815300000000004 2023-09-23T22:27:41.779 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160125 5.3 5:c0b4c552:::benchmark_data_smithi026_286012_object66317:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.724501+0000", 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 523.99501414400004, 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.572192045, 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.780 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160125 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724501+0000", 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724501+0000", 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724500+0000", 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724502+0000", 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.891e-06 2023-09-23T22:27:41.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724507+0000", 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.062e-06 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.724512+0000", 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2100000000000003e-06 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725972+0000", 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001460097 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.725976+0000", 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3669999999999999e-06 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038446+0000", 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.312469847 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038851+0000", 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040529699999999999 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.104126+0000", 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065275193999999995 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.104252+0000", 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012552799999999999 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.104352+0000", 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010051399999999999 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.116000+0000", 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011647210999999999 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135693+0000", 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019693161000000001 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135744+0000", 2023-09-23T22:27:41.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1597000000000001e-05 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.295827+0000", 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16008249399999999 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.295880+0000", 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3418999999999998e-05 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.296693+0000", 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081249700000000002 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160141 5.3 5:c5e24ba2:::benchmark_data_smithi026_286012_object66333:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.948336+0000", 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 523.77117869899996, 2023-09-23T22:27:41.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3492636490000001, 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160141 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948336+0000", 2023-09-23T22:27:41.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948336+0000", 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948335+0000", 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948341+0000", 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6309999999999997e-06 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948355+0000", 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3841e-05 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948362+0000", 2023-09-23T22:27:41.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0090000000000001e-06 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948422+0000", 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9821000000000002e-05 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.948431+0000", 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5830000000000007e-06 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.038909+0000", 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0904784590000001 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.039292+0000", 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038332500000000001 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.109910+0000", 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.070617271999999995 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.109988+0000", 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8589000000000003e-05 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.110092+0000", 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010372 2023-09-23T22:27:41.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.119489+0000", 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093965960000000001 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135777+0000", 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016287978000000002 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135823+0000", 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6122000000000003e-05 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.296763+0000", 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16094037 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.296819+0000", 2023-09-23T22:27:41.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6005999999999999e-05 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.297600+0000", 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00078078999999999996 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.34674.0:160147 5.3 5:c3a1d842:::benchmark_data_smithi026_286012_object66339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T22:27:41.798 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:18:57.966597+0000", 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 523.75291772100002, 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3318658239999999, 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.34674", 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 160147 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966597+0000", 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966597+0000", 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966595+0000", 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966603+0000", 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8990000000000001e-06 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966623+0000", 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9882e-05 2023-09-23T22:27:41.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966632+0000", 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0129999999999999e-06 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966699+0000", 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6631999999999995e-05 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:57.966720+0000", 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1373e-05 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.039353+0000", 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0726333219999999 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:27:41.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.039739+0000", 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038586399999999998 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.116044+0000", 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.076304935000000004 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.116162+0000", 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118059 2023-09-23T22:27:41.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,5", 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.116276+0000", 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011349399999999999 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.125088+0000", 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088123579999999993 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135904+0000", 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010815791 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.135983+0000", 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9462999999999995e-05 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.297682+0000", 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16169909599999999 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.297747+0000", 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4618999999999999e-05 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:18:59.298463+0000", 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00071606700000000003 2023-09-23T22:27:41.807 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83452 7.e 7:7212fb35:::benchmark_data_smithi026_410749_object83451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.976183+0000", 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 93.743331967000003, 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.748405781, 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.808 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83452 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976183+0000", 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976183+0000", 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976181+0000", 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976213+0000", 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2251e-05 2023-09-23T22:27:41.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976230+0000", 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6982000000000001e-05 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976242+0000", 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.219e-05 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976670+0000", 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042763099999999999 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976680+0000", 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9739999999999997e-06 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703132+0000", 2023-09-23T22:27:41.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7264517640000001 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705750+0000", 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026186209999999998 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705939+0000", 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018901399999999999 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.712672+0000", 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067324810000000002 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.716114+0000", 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034423779999999998 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720906+0000", 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047918270000000002 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720954+0000", 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8099000000000001e-05 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.724589+0000", 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036344390000000002 2023-09-23T22:27:41.815 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83457 7.5 7:ae7c1643:::benchmark_data_smithi026_410749_object83456:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.976262+0000", 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 93.743253018000004, 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.709729018, 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.816 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83457 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976262+0000", 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976262+0000", 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976261+0000", 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976329+0000", 2023-09-23T22:27:41.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7922999999999993e-05 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976339+0000", 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0276e-05 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976347+0000", 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5920000000000003e-06 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976407+0000", 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0486000000000003e-05 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976419+0000", 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1871e-05 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.662197+0000", 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6857776609999999 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.662351+0000", 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153938 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:27:41.821 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.662471+0000", 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012044099999999999 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.669804+0000", 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073327050000000001 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.670014+0000", 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000209817 2023-09-23T22:27:41.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.681794+0000", 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011780163 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.681867+0000", 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3206000000000003e-05 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.685991+0000", 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041235569999999999 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:111802 7.6 7:6a21129c:::benchmark_data_smithi026_410749_object21465:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:19.116576+0000", 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.602939152000001, 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.231539205, 2023-09-23T22:27:41.824 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 111802 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116576+0000", 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116576+0000", 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116574+0000", 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116583+0000", 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8810000000000008e-06 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116603+0000", 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9077999999999999e-05 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116613+0000", 2023-09-23T22:27:41.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0168e-05 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116674+0000", 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1011000000000003e-05 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116829+0000", 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015495799999999999 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116950+0000", 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121752 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.124229+0000", 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072788690000000003 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.133493+0000", 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092635210000000003 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.348033+0000", 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.21453991 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.348076+0000", 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3470000000000002e-05 2023-09-23T22:27:41.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.348115+0000", 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8682e-05 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:111803 7.d 7:bb8c2292:::benchmark_data_smithi026_410749_object21466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:19.116630+0000", 2023-09-23T22:27:41.831 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.602884973999998, 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22956916999999999, 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 111803 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116630+0000", 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116630+0000", 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116629+0000", 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116632+0000", 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.92e-06 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116639+0000", 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4790000000000001e-06 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116644+0000", 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2270000000000001e-06 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116702+0000", 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8304999999999998e-05 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116865+0000", 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000162359 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.117004+0000", 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139221 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.119906+0000", 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002902338 2023-09-23T22:27:41.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.124239+0000", 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043322730000000002 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.125508+0000", 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012695880000000001 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.346005+0000", 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22049674499999999 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.346071+0000", 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6291000000000006e-05 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.346199+0000", 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012788599999999999 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:111804 7.6 7:629b927d:::benchmark_data_smithi026_410749_object21467:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:19.116658+0000", 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.602856727999999, 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.24124799, 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.839 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 111804 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116658+0000", 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116658+0000", 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116658+0000", 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116660+0000", 2023-09-23T22:27:41.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.503e-06 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116676+0000", 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5665e-05 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.116681+0000", 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2800000000000003e-06 2023-09-23T22:27:41.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.119839+0000", 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031582239999999998 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.119963+0000", 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000123504 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:27:41.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.120078+0000", 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115419 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.124349+0000", 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042706460000000003 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.133500+0000", 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0091505530000000005 2023-09-23T22:27:41.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.357803+0000", 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22430365299999999 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.357859+0000", 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6137999999999997e-05 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.357906+0000", 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6745000000000001e-05 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:111806 7.6 7:66c97b66:::benchmark_data_smithi026_410749_object21469:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:19.127831+0000", 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.591683644, 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.230269106, 2023-09-23T22:27:41.846 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 111806 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127831+0000", 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127831+0000", 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127830+0000", 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127839+0000", 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0319999999999992e-06 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127851+0000", 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2349e-05 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127858+0000", 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6490000000000002e-06 2023-09-23T22:27:41.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127904+0000", 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6381000000000002e-05 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.128042+0000", 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013826299999999999 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.128178+0000", 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013629500000000001 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.133509+0000", 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053306339999999999 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.134370+0000", 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00086081199999999997 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.358015+0000", 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.223645446 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.358064+0000", 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8736000000000002e-05 2023-09-23T22:27:41.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.358100+0000", 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6189000000000002e-05 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:111810 7.6 7:635cf152:::benchmark_data_smithi026_410749_object21473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:41.853 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:19.127882+0000", 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.591632751999999, 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23032530200000001, 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 111810 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127882+0000", 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127882+0000", 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127882+0000", 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127885+0000", 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8519999999999997e-06 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127893+0000", 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6199999999999999e-06 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.127899+0000", 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8680000000000004e-06 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.130681+0000", 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027816659999999999 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.130857+0000", 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017637299999999999 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.131015+0000", 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015768100000000001 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.133675+0000", 2023-09-23T22:27:41.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026608410000000002 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.134436+0000", 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00076030900000000003 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.358136+0000", 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22370051299999999 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.358173+0000", 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6736000000000002e-05 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:19.358207+0000", 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4415000000000002e-05 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114677 7.7 7:eb7f7b8f:::benchmark_data_smithi026_410749_object24340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.131224+0000", 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 15.588291337999999, 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2516092140000001, 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114677 2023-09-23T22:27:41.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.131224+0000", 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.131224+0000", 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.131223+0000", 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.131226+0000", 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0589999999999998e-06 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.131233+0000", 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7240000000000003e-06 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.131239+0000", 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9290000000000003e-06 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.131290+0000", 2023-09-23T22:27:41.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1377e-05 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.131427+0000", 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013670300000000001 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.131565+0000", 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013782700000000001 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.137603+0000", 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060386579999999997 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141094+0000", 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034903569999999999 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141125+0000", 2023-09-23T22:27:41.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1565e-05 2023-09-23T22:27:41.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.264882+0000", 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12375641 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.264893+0000", 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.13e-05 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.382722+0000", 2023-09-23T22:27:41.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.117828606 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.382833+0000", 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111123 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114678 7.d 7:bd720119:::benchmark_data_smithi026_410749_object24341:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:41.869 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.141141+0000", 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 15.578373779, 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2232911950000001, 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114678 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141141+0000", 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141141+0000", 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141140+0000", 2023-09-23T22:27:41.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141145+0000", 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6189999999999999e-06 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141156+0000", 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0851e-05 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141163+0000", 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7340000000000002e-06 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141227+0000", 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4270000000000006e-05 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141378+0000", 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015066299999999999 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141564+0000", 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000186563 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.144348+0000", 2023-09-23T22:27:41.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027832809999999999 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.256622+0000", 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.11227493500000001 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.258704+0000", 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020810949999999998 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.258727+0000", 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3034999999999999e-05 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364366+0000", 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.105638941 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364432+0000", 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6752000000000001e-05 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114680 7.7 7:ef6f5091:::benchmark_data_smithi026_410749_object24343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.141180+0000", 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 15.578335129999999, 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2545092410000001, 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.877 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114680 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141180+0000", 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141180+0000", 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141179+0000", 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141182+0000", 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9979999999999999e-06 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141188+0000", 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6520000000000003e-06 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141193+0000", 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9899999999999997e-06 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.148288+0000", 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070954750000000004 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.148458+0000", 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016946499999999999 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.148610+0000", 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015177100000000001 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.264928+0000", 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.116318199 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.264934+0000", 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9349999999999999e-06 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.383233+0000", 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1182987209999999 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.383419+0000", 2023-09-23T22:27:41.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018637600000000001 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.395636+0000", 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012216912 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.395689+0000", 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3134999999999999e-05 2023-09-23T22:27:41.884 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114683 7.7 7:e04d9b9e:::benchmark_data_smithi026_410749_object24346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.144374+0000", 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 15.575141169, 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.250866671, 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:27:41.885 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114683 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.144374+0000", 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:27:41.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.144374+0000", 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.144372+0000", 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.144378+0000", 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.772e-06 2023-09-23T22:27:41.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.144389+0000", 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1208e-05 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.144397+0000", 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1880000000000002e-06 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.264956+0000", 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12055904200000001 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.264964+0000", 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5490000000000001e-06 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.383487+0000", 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1185233400000001 2023-09-23T22:27:41.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.383618+0000", 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130424 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,3,4,7", 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.384120+0000", 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000502423 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.390703+0000", 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065829679999999998 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.390747+0000", 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3943999999999998e-05 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:27:41.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.391072+0000", 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032494000000000002 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.393744+0000", 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026720250000000002 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.395032+0000", 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012878830000000001 2023-09-23T22:27:41.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.395098+0000", 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5906000000000002e-05 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.395240+0000", 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014244299999999999 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.893 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.894 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:27:41.894 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:27:41.894 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:27:41.894 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:41.996 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:41.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:41.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:42.065 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2097449716' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:27:42.083 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:27:42.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:42.147 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:42.211 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:42.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:42.314 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:42.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2097449716' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:27:42.346 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:42.396 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:42.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:42.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:42.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2097449716' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:27:42.503 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:42.531 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:42.575 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:42.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:42.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:42.680 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:42.706 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:42.747 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:42.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:42.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:42.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:42.894 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:43.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[223014]: pgmap v787: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 41369/198285 objects degraded (20.863%); 48528/198285 objects misplaced (24.474%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T22:27:43.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:27:43.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[223014]: failed to encode map e303 with expected crc 2023-09-23T22:27:43.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2097449716' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:27:43.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[223014]: osdmap e303: 8 total, 8 up, 5 in 2023-09-23T22:27:43.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[223014]: failed to encode map e303 with expected crc 2023-09-23T22:27:43.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 3 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:27:43.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[215775]: pgmap v787: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 41369/198285 objects degraded (20.863%); 48528/198285 objects misplaced (24.474%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T22:27:43.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:27:43.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[215775]: failed to encode map e303 with expected crc 2023-09-23T22:27:43.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2097449716' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:27:43.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[215775]: osdmap e303: 8 total, 8 up, 5 in 2023-09-23T22:27:43.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[215775]: failed to encode map e303 with expected crc 2023-09-23T22:27:43.240 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:43 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 3 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:27:43.299 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:43.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:43.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:43 smithi070 ceph-mon[192652]: pgmap v787: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 41369/198285 objects degraded (20.863%); 48528/198285 objects misplaced (24.474%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T22:27:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:43 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:27:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:43 smithi070 ceph-mon[192652]: failed to encode map e303 with expected crc 2023-09-23T22:27:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2097449716' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:27:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:43 smithi070 ceph-mon[192652]: osdmap e303: 8 total, 8 up, 5 in 2023-09-23T22:27:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:43 smithi070 ceph-mon[192652]: failed to encode map e303 with expected crc 2023-09-23T22:27:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:43 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 3 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:27:43.403 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:43.440 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:43.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:27:44.075 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:44.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:44.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:44.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:44.217 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:44.290 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:44.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:44.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:44.396 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:44.437 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:44.516 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:44.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:44.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:44.585 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:44.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:44.662 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:44.739 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:44.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:44.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:44.827 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 2] out_osds: [6, 1, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:27:44.827 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:27:44.827 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-09-23T22:27:44.828 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 1 2023-09-23T22:27:44.843 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:44.876 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:44.927 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:44.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:44.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:45.033 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:45.061 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:45.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:45.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:45.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:45 smithi070 ceph-mon[192652]: pgmap v789: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 41369/198285 objects degraded (20.863%); 48528/198285 objects misplaced (24.474%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T22:27:45.124 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:27:44 smithi070 conmon[172396]: logger=cleanup t=2023-09-23T22:27:44.954666613Z level=info msg="Completed cleanup jobs" duration=12.619689ms 2023-09-23T22:27:45.216 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:45.226 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:45 smithi026 ceph-mon[223014]: pgmap v789: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 41369/198285 objects degraded (20.863%); 48528/198285 objects misplaced (24.474%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T22:27:45.226 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:45 smithi026 ceph-mon[215775]: pgmap v789: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 41369/198285 objects degraded (20.863%); 48528/198285 objects misplaced (24.474%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T22:27:45.242 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:45.283 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:45.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:45.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:45.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:45.430 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:45.491 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:45.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:45.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:45.599 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:45.626 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:45.668 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:45.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:45.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:45.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:45.810 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:45.870 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:45.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:45.974 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:46.002 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:46.045 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:46.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:46.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:46.152 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:46.177 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:27:46.177 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:46.218 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:46.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:46.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:46.324 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:46.348 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:46.426 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:46.564 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:46.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:46.571 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:46.672 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:46.700 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:46.743 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:46.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:46.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:46.851 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:46.878 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:46.919 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:46.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:46.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:47.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:47.104 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:47.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:47 smithi026 ceph-mon[223014]: pgmap v790: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 41369/198285 objects degraded (20.863%); 48171/198285 objects misplaced (24.294%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:27:47.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:47 smithi026 ceph-mon[215775]: pgmap v790: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 41369/198285 objects degraded (20.863%); 48171/198285 objects misplaced (24.294%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:27:47.355 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:27:47.355 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:27:47.355 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:27:47.355 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:27:47.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:47 smithi070 ceph-mon[192652]: pgmap v790: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 41369/198285 objects degraded (20.863%); 48171/198285 objects misplaced (24.294%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:27:47.387 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:47.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:47.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:47.492 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:47.519 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:47.560 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:47.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:47.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:47.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:47.691 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:47.735 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:47.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:47.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:47.840 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:47.866 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:47.910 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:47.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:47.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:48.017 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:48.058 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:48.529 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:48.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:48.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:48.636 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:48.663 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:48.707 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:48.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:48.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:48.815 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:48.860 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:49.197 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:49.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:49 smithi026 ceph-mon[223014]: pgmap v791: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 41369/198267 objects degraded (20.865%); 48164/198267 objects misplaced (24.292%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:27:49.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:49 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 41369/198267 objects degraded (20.865%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:49.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:49.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:49.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:49 smithi026 ceph-mon[215775]: pgmap v791: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 41369/198267 objects degraded (20.865%); 48164/198267 objects misplaced (24.292%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:27:49.204 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:49 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 41369/198267 objects degraded (20.865%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:49.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:49.342 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:49.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:49 smithi070 ceph-mon[192652]: pgmap v791: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 41369/198267 objects degraded (20.865%); 48164/198267 objects misplaced (24.292%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:27:49.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:49 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 41369/198267 objects degraded (20.865%), 13 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:49.377 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:27:49.416 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:49.430 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:49.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:49.540 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:49.568 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:49.611 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:49.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:49.617 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:49.717 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:49.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:27:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:27:49] "GET /metrics HTTP/1.1" 200 36433 "" "Prometheus/2.43.0" 2023-09-23T22:27:49.743 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:49.785 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:49.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:49.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:49.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:27:50.006 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:50.129 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:27:50.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:50.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:27:50.235 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:50.262 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:50.304 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:50.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:50.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:50.411 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:27:50.436 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:50.477 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:27:50.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:50 smithi026 conmon[215751]: 2023-09-23T22:27:50.204+0000 7f4580f04700 -1 mon.a@0(leader).osd e303 definitely_dead 0 2023-09-23T22:27:50.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:50.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:27:50.582 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:50.675 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:50.746 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:50.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:50.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:50.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:50.886 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:50.946 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:50.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:50.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:51.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:51 smithi026 ceph-mon[223014]: pgmap v792: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 41369/198267 objects degraded (20.865%); 48164/198267 objects misplaced (24.292%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:27:51.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:51 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3531572518' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T22:27:51.020 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:51 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T22:27:51.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:51 smithi026 ceph-mon[215775]: pgmap v792: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 41369/198267 objects degraded (20.865%); 48164/198267 objects misplaced (24.292%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:27:51.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:51 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3531572518' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T22:27:51.021 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:51 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T22:27:51.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:51.065 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.1. 2023-09-23T22:27:51.080 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:51.130 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:51.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:51.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:51.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:51.268 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:51.369 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:27:51.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:51 smithi070 ceph-mon[192652]: pgmap v792: 99 pgs: 1 activating+undersized+degraded+remapped, 8 peering, 1 unknown, 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 68 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 0 op/s; 41369/198267 objects degraded (20.865%); 48164/198267 objects misplaced (24.292%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:27:51.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:51 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3531572518' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T22:27:51.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:51 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T22:27:51.396 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:27:51.419 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:27:51.443 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:27:51.443 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:51.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:51.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:51.464 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:27:51.485 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:27:51.487 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:27:51.487 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:27:51.487 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:27:51.487 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:27:51.487 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:27:51.487 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:27:51.487 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:27:51.487 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:27:51.549 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:51.575 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:51.616 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:51.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:51.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:51.725 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:51.756 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:52.028 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:52 smithi026 ceph-mon[223014]: failed to encode map e304 with expected crc 2023-09-23T22:27:52.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:52 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T22:27:52.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:52 smithi026 ceph-mon[223014]: osdmap e304: 8 total, 8 up, 6 in 2023-09-23T22:27:52.029 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:52 smithi026 ceph-mon[223014]: failed to encode map e304 with expected crc 2023-09-23T22:27:52.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:52 smithi026 ceph-mon[215775]: failed to encode map e304 with expected crc 2023-09-23T22:27:52.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:52 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T22:27:52.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:52 smithi026 ceph-mon[215775]: osdmap e304: 8 total, 8 up, 6 in 2023-09-23T22:27:52.029 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:52 smithi026 ceph-mon[215775]: failed to encode map e304 with expected crc 2023-09-23T22:27:52.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:52 smithi070 ceph-mon[192652]: failed to encode map e304 with expected crc 2023-09-23T22:27:52.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:52 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T22:27:52.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:52 smithi070 ceph-mon[192652]: osdmap e304: 8 total, 8 up, 6 in 2023-09-23T22:27:52.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:52 smithi070 ceph-mon[192652]: failed to encode map e304 with expected crc 2023-09-23T22:27:52.983 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:52.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:52.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:53.090 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:53.157 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:53.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: pgmap v794: 99 pgs: 1 activating+undersized+degraded+remapped, 1 remapped, 6 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 71 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 18 op/s; 39906/197778 objects degraded (20.177%); 47783/197778 objects misplaced (24.160%); 5.1 MiB/s, 83 objects/s recovering 2023-09-23T22:27:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: failed to encode map e304 with expected crc 2023-09-23T22:27:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2023-09-23T22:27:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: osdmap e305: 8 total, 8 up, 6 in 2023-09-23T22:27:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: 7.1 continuing backfill to osd.0 from (265'1795,303'7068] 7:8b632fac:::benchmark_data_smithi026_410749_object81429:head to 303'7068 2023-09-23T22:27:53.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:53 smithi070 ceph-mon[192652]: osdmap e306: 8 total, 8 up, 6 in 2023-09-23T22:27:53.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: pgmap v794: 99 pgs: 1 activating+undersized+degraded+remapped, 1 remapped, 6 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 71 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 18 op/s; 39906/197778 objects degraded (20.177%); 47783/197778 objects misplaced (24.160%); 5.1 MiB/s, 83 objects/s recovering 2023-09-23T22:27:53.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: failed to encode map e304 with expected crc 2023-09-23T22:27:53.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2023-09-23T22:27:53.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: osdmap e305: 8 total, 8 up, 6 in 2023-09-23T22:27:53.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.399 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.399 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: 7.1 continuing backfill to osd.0 from (265'1795,303'7068] 7:8b632fac:::benchmark_data_smithi026_410749_object81429:head to 303'7068 2023-09-23T22:27:53.399 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[223014]: osdmap e306: 8 total, 8 up, 6 in 2023-09-23T22:27:53.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: pgmap v794: 99 pgs: 1 activating+undersized+degraded+remapped, 1 remapped, 6 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 71 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 18 op/s; 39906/197778 objects degraded (20.177%); 47783/197778 objects misplaced (24.160%); 5.1 MiB/s, 83 objects/s recovering 2023-09-23T22:27:53.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: failed to encode map e304 with expected crc 2023-09-23T22:27:53.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 2 pgs peering) 2023-09-23T22:27:53.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: osdmap e305: 8 total, 8 up, 6 in 2023-09-23T22:27:53.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: failed to encode map e305 with expected crc 2023-09-23T22:27:53.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: 7.1 continuing backfill to osd.0 from (265'1795,303'7068] 7:8b632fac:::benchmark_data_smithi026_410749_object81429:head to 303'7068 2023-09-23T22:27:53.401 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:53 smithi026 ceph-mon[215775]: osdmap e306: 8 total, 8 up, 6 in 2023-09-23T22:27:53.435 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-09-23T22:27:53.506 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:53.600 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:53.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:53.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:53.709 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:53.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:53.780 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:53.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:53.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:53.887 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:53.913 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:53.959 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:53.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:53.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:54.064 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:54.090 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:54.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:54.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:54.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:54.238 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:54.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[223014]: failed to encode map e306 with expected crc 2023-09-23T22:27:54.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[223014]: failed to encode map e306 with expected crc 2023-09-23T22:27:54.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[223014]: failed to encode map e306 with expected crc 2023-09-23T22:27:54.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[223014]: 7.b continuing backfill to osd.2 from (265'1827,303'7190] MIN to 303'7190 2023-09-23T22:27:54.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[223014]: 7.d continuing backfill to osd.7 from (266'2741,303'7228] MIN to 303'7228 2023-09-23T22:27:54.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[223014]: 7.7 continuing backfill to osd.7 from (266'2647,303'7142] MIN to 303'7142 2023-09-23T22:27:54.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[223014]: osdmap e307: 8 total, 8 up, 6 in 2023-09-23T22:27:54.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[215775]: failed to encode map e306 with expected crc 2023-09-23T22:27:54.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[215775]: failed to encode map e306 with expected crc 2023-09-23T22:27:54.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[215775]: failed to encode map e306 with expected crc 2023-09-23T22:27:54.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[215775]: 7.b continuing backfill to osd.2 from (265'1827,303'7190] MIN to 303'7190 2023-09-23T22:27:54.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[215775]: 7.d continuing backfill to osd.7 from (266'2741,303'7228] MIN to 303'7228 2023-09-23T22:27:54.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[215775]: 7.7 continuing backfill to osd.7 from (266'2647,303'7142] MIN to 303'7142 2023-09-23T22:27:54.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:54 smithi026 ceph-mon[215775]: osdmap e307: 8 total, 8 up, 6 in 2023-09-23T22:27:54.322 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:54.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:54 smithi070 ceph-mon[192652]: failed to encode map e306 with expected crc 2023-09-23T22:27:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:54 smithi070 ceph-mon[192652]: failed to encode map e306 with expected crc 2023-09-23T22:27:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:54 smithi070 ceph-mon[192652]: failed to encode map e306 with expected crc 2023-09-23T22:27:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:54 smithi070 ceph-mon[192652]: 7.b continuing backfill to osd.2 from (265'1827,303'7190] MIN to 303'7190 2023-09-23T22:27:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:54 smithi070 ceph-mon[192652]: 7.d continuing backfill to osd.7 from (266'2741,303'7228] MIN to 303'7228 2023-09-23T22:27:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:54 smithi070 ceph-mon[192652]: 7.7 continuing backfill to osd.7 from (266'2647,303'7142] MIN to 303'7142 2023-09-23T22:27:54.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:54 smithi070 ceph-mon[192652]: osdmap e307: 8 total, 8 up, 6 in 2023-09-23T22:27:54.400 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:54.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:54.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:54.505 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:54.564 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:54.658 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:54.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:54.812 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:54.839 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:54.881 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:54.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:54.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:54.986 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:55.011 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:55.037 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:27:55.056 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:55.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:55.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: pgmap v797: 99 pgs: 3 remapped+peering, 1 activating+undersized+degraded+remapped, 1 remapped, 7 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 70 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 39 op/s; 36170/197553 objects degraded (18.309%); 34986/197553 objects misplaced (17.710%); 4.4 MiB/s, 73 objects/s recovering 2023-09-23T22:27:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: failed to encode map e306 with expected crc 2023-09-23T22:27:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: failed to encode map e307 with expected crc 2023-09-23T22:27:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: failed to encode map e307 with expected crc 2023-09-23T22:27:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: failed to encode map e307 with expected crc 2023-09-23T22:27:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:27:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 36170/197553 objects degraded (18.309%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/875981844' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:27:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:27:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:27:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:55 smithi070 ceph-mon[192652]: osdmap e308: 8 total, 8 up, 6 in 2023-09-23T22:27:55.163 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:55.189 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:55.230 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:55.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:55.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:55.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:55.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: pgmap v797: 99 pgs: 3 remapped+peering, 1 activating+undersized+degraded+remapped, 1 remapped, 7 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 70 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 39 op/s; 36170/197553 objects degraded (18.309%); 34986/197553 objects misplaced (17.710%); 4.4 MiB/s, 73 objects/s recovering 2023-09-23T22:27:55.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: failed to encode map e306 with expected crc 2023-09-23T22:27:55.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: failed to encode map e307 with expected crc 2023-09-23T22:27:55.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: failed to encode map e307 with expected crc 2023-09-23T22:27:55.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: failed to encode map e307 with expected crc 2023-09-23T22:27:55.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:27:55.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 36170/197553 objects degraded (18.309%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:55.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:55.370 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:55.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/875981844' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:27:55.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:27:55.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:55.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:27:55.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[223014]: osdmap e308: 8 total, 8 up, 6 in 2023-09-23T22:27:55.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: pgmap v797: 99 pgs: 3 remapped+peering, 1 activating+undersized+degraded+remapped, 1 remapped, 7 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 70 active+clean; 4.0 GiB data, 8.9 GiB used, 438 GiB / 447 GiB avail; 0 B/s wr, 39 op/s; 36170/197553 objects degraded (18.309%); 34986/197553 objects misplaced (17.710%); 4.4 MiB/s, 73 objects/s recovering 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: failed to encode map e306 with expected crc 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: failed to encode map e307 with expected crc 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: failed to encode map e307 with expected crc 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: failed to encode map e307 with expected crc 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 36170/197553 objects degraded (18.309%), 12 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/875981844' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:27:55.372 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:27:55.373 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:27:55.373 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:55 smithi026 ceph-mon[215775]: osdmap e308: 8 total, 8 up, 6 in 2023-09-23T22:27:55.373 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:55.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:27:55.682 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:55.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:55.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:55.786 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:55.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:55.855 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:55.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:55.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:55.961 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:55.986 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:56.028 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:56.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:56.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:56.134 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:27:56.162 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:56.206 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:27:56.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:56.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:27:56.312 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:56.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:56 smithi070 ceph-mon[192652]: failed to encode map e308 with expected crc 2023-09-23T22:27:56.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:56 smithi070 ceph-mon[192652]: failed to encode map e308 with expected crc 2023-09-23T22:27:56.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:56 smithi070 ceph-mon[192652]: failed to encode map e308 with expected crc 2023-09-23T22:27:56.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:56 smithi070 ceph-mon[192652]: osdmap e309: 8 total, 8 up, 6 in 2023-09-23T22:27:56.338 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:56.383 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:56.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:56.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:56 smithi026 ceph-mon[215775]: failed to encode map e308 with expected crc 2023-09-23T22:27:56.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:56 smithi026 ceph-mon[215775]: failed to encode map e308 with expected crc 2023-09-23T22:27:56.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:56 smithi026 ceph-mon[215775]: failed to encode map e308 with expected crc 2023-09-23T22:27:56.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:56 smithi026 ceph-mon[215775]: osdmap e309: 8 total, 8 up, 6 in 2023-09-23T22:27:56.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:56 smithi026 ceph-mon[223014]: failed to encode map e308 with expected crc 2023-09-23T22:27:56.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:56 smithi026 ceph-mon[223014]: failed to encode map e308 with expected crc 2023-09-23T22:27:56.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:56 smithi026 ceph-mon[223014]: failed to encode map e308 with expected crc 2023-09-23T22:27:56.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:56 smithi026 ceph-mon[223014]: osdmap e309: 8 total, 8 up, 6 in 2023-09-23T22:27:56.490 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:56.517 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:56.559 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:56.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:56.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:56.666 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:56.692 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:56.735 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:56.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:56.841 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:56.867 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:56.869 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:56.910 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:56.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:56.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:57.008 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:27:57.008 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:27:57.009 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:27:57.009 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:27:57.009 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:27:57.018 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:27:57.058 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:57.139 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:27:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:57.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:27:57.246 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:57.274 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:57.319 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:57.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:57 smithi070 ceph-mon[192652]: pgmap v800: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating+undersized+degraded+remapped, 6 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 40 op/s; 28990/197289 objects degraded (14.694%); 33627/197289 objects misplaced (17.045%); 22 MiB/s, 349 objects/s recovering 2023-09-23T22:27:57.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:57 smithi070 ceph-mon[192652]: failed to encode map e309 with expected crc 2023-09-23T22:27:57.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:57 smithi070 ceph-mon[192652]: failed to encode map e309 with expected crc 2023-09-23T22:27:57.322 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:57 smithi070 ceph-mon[192652]: failed to encode map e309 with expected crc 2023-09-23T22:27:57.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:57.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:57.424 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:57.449 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:57.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:57 smithi026 ceph-mon[215775]: pgmap v800: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating+undersized+degraded+remapped, 6 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 40 op/s; 28990/197289 objects degraded (14.694%); 33627/197289 objects misplaced (17.045%); 22 MiB/s, 349 objects/s recovering 2023-09-23T22:27:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:57 smithi026 ceph-mon[215775]: failed to encode map e309 with expected crc 2023-09-23T22:27:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:57 smithi026 ceph-mon[215775]: failed to encode map e309 with expected crc 2023-09-23T22:27:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:57 smithi026 ceph-mon[215775]: failed to encode map e309 with expected crc 2023-09-23T22:27:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:57 smithi026 ceph-mon[223014]: pgmap v800: 99 pgs: 1 active+clean+remapped, 3 remapped+peering, 1 activating+undersized+degraded+remapped, 6 peering, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 75 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 40 op/s; 28990/197289 objects degraded (14.694%); 33627/197289 objects misplaced (17.045%); 22 MiB/s, 349 objects/s recovering 2023-09-23T22:27:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:57 smithi026 ceph-mon[223014]: failed to encode map e309 with expected crc 2023-09-23T22:27:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:57 smithi026 ceph-mon[223014]: failed to encode map e309 with expected crc 2023-09-23T22:27:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:57 smithi026 ceph-mon[223014]: failed to encode map e309 with expected crc 2023-09-23T22:27:57.490 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:57.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:57.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:57.598 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:57.632 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:58.345 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:58.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:58.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:58.436 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 2, 1] out_osds: [6, 3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:27:58.437 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:27:58.437 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [4, 5, 7, 0, 2, 1] 2023-09-23T22:27:58.437 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-23T22:27:58.453 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:27:58.516 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:58.611 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:27:58.909 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:27:59.168 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:27:59 smithi026 ceph-mon[215775]: pgmap v802: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 peering, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 28965/196617 objects degraded (14.732%); 37325/196617 objects misplaced (18.984%); 24 MiB/s, 382 objects/s recovering 2023-09-23T22:27:59.168 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:27:59 smithi026 ceph-mon[223014]: pgmap v802: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 peering, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 28965/196617 objects degraded (14.732%); 37325/196617 objects misplaced (18.984%); 24 MiB/s, 382 objects/s recovering 2023-09-23T22:27:59.169 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:27:59.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:59.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:27:59.276 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:59.302 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:59.343 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:27:59.349 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:27:59 smithi070 ceph-mon[192652]: pgmap v802: 99 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 peering, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 79 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 70 op/s; 28965/196617 objects degraded (14.732%); 37325/196617 objects misplaced (18.984%); 24 MiB/s, 382 objects/s recovering 2023-09-23T22:27:59.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:59.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:27:59.395 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:59.451 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:27:59.477 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:59.502 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:27:59.518 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:27:59.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:59.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:27:59.624 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:27:59.697 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:59.825 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:27:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:27:59.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:27:59.873 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:27:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:27:59] "GET /metrics HTTP/1.1" 200 36392 "" "Prometheus/2.43.0" 2023-09-23T22:27:59.930 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:27:59.956 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:27:59.999 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:00.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:00.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:00.107 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:00.132 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:00.176 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:00.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:00.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:00 smithi026 ceph-mon[223014]: pgmap v803: 99 pgs: 1 active+clean+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 60 op/s; 28965/196458 objects degraded (14.744%); 37272/196458 objects misplaced (18.972%); 18 MiB/s, 282 objects/s recovering 2023-09-23T22:28:00.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:00 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 28965/196458 objects degraded (14.744%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:28:00.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:00 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T22:28:00.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:00 smithi026 ceph-mon[215775]: pgmap v803: 99 pgs: 1 active+clean+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 60 op/s; 28965/196458 objects degraded (14.744%); 37272/196458 objects misplaced (18.972%); 18 MiB/s, 282 objects/s recovering 2023-09-23T22:28:00.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:00 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 28965/196458 objects degraded (14.744%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:28:00.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:00 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T22:28:00.284 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:00.308 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:00 smithi070 ceph-mon[192652]: pgmap v803: 99 pgs: 1 active+clean+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 81 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 60 op/s; 28965/196458 objects degraded (14.744%); 37272/196458 objects misplaced (18.972%); 18 MiB/s, 282 objects/s recovering 2023-09-23T22:28:00.308 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:00 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 28965/196458 objects degraded (14.744%), 11 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:28:00.308 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:00 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T22:28:00.310 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:00.351 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:00.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:00.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:00.461 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:00.501 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:01.017 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:28:01.017 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:28:01.017 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:28:01.027 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:28:01.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:28:01.211 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:28:01.212 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:28:01.212 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:28:01.212 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:28:01.212 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:28:01.212 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:28:01.212 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:28:01.212 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:28:01.212 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:28:01.213 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:28:01.213 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:28:01.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1042648107' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:01.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1042648107' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1042648107' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:02.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:02 smithi026 ceph-mon[215775]: pgmap v804: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 28931/196050 objects degraded (14.757%); 37243/196050 objects misplaced (18.997%); 15 MiB/s, 237 objects/s recovering 2023-09-23T22:28:02.407 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:02 smithi026 ceph-mon[223014]: pgmap v804: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 28931/196050 objects degraded (14.757%); 37243/196050 objects misplaced (18.997%); 15 MiB/s, 237 objects/s recovering 2023-09-23T22:28:02.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:02 smithi070 ceph-mon[192652]: pgmap v804: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 64 op/s; 28931/196050 objects degraded (14.757%); 37243/196050 objects misplaced (18.997%); 15 MiB/s, 237 objects/s recovering 2023-09-23T22:28:02.913 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:02.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:02.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:03.019 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:03.045 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:03.089 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:03.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:03.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:03.157 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:03.195 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:03.221 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:03.263 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:03.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:03.371 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:03.455 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:04.025 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:03 smithi026 conmon[215751]: 2023-09-23T22:28:03.973+0000 7f4580f04700 -1 mon.a@0(leader).osd e309 definitely_dead 0 2023-09-23T22:28:04.025 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1240516200' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:28:04.065 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.2. 2023-09-23T22:28:04.104 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:04.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:04.227 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:04.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:04 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:28:04.291 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:04 smithi026 ceph-mon[215775]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T22:28:04.292 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1240516200' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:28:04.292 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:04 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:28:04.292 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:04 smithi026 ceph-mon[223014]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T22:28:04.295 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:04.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1240516200' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:28:04.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:04 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T22:28:04.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:04 smithi070 ceph-mon[192652]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T22:28:04.410 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:04.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:04.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:04.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:04.543 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:04.591 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:04.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:04.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:04.696 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:04.724 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:04.771 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:04.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:04.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:04.875 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:04.908 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:04.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:28:04.965 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:04.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:04.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:05.058 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:28:05.070 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:05.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: pgmap v805: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 28875/195195 objects degraded (14.793%); 36848/195195 objects misplaced (18.878%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T22:28:05.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T22:28:05.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: osdmap e310: 8 total, 8 up, 5 in 2023-09-23T22:28:05.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.198 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/176210810' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:05.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:05.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: pgmap v805: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 28875/195195 objects degraded (14.793%); 36848/195195 objects misplaced (18.878%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T22:28:05.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T22:28:05.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: osdmap e310: 8 total, 8 up, 5 in 2023-09-23T22:28:05.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.199 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/176210810' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:05.200 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:05 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:05.224 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:05.348 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:05.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:05.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:05.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: pgmap v805: 99 pgs: 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 82 op/s; 28875/195195 objects degraded (14.793%); 36848/195195 objects misplaced (18.878%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T22:28:05.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T22:28:05.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: osdmap e310: 8 total, 8 up, 5 in 2023-09-23T22:28:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: failed to encode map e310 with expected crc 2023-09-23T22:28:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/176210810' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:05 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:05.459 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:05.612 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:06.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: failed to encode map e310 with expected crc 2023-09-23T22:28:06.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:06.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: osdmap e311: 8 total, 8 up, 5 in 2023-09-23T22:28:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: 7.3 starting backfill to osd.1 from (0'0,0'0] MAX to 309'7236 2023-09-23T22:28:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:06 smithi070 ceph-mon[192652]: osdmap e312: 8 total, 8 up, 5 in 2023-09-23T22:28:06.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: failed to encode map e310 with expected crc 2023-09-23T22:28:06.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:06.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: osdmap e311: 8 total, 8 up, 5 in 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: 7.3 starting backfill to osd.1 from (0'0,0'0] MAX to 309'7236 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[215775]: osdmap e312: 8 total, 8 up, 5 in 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: failed to encode map e310 with expected crc 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:06.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: osdmap e311: 8 total, 8 up, 5 in 2023-09-23T22:28:06.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: failed to encode map e311 with expected crc 2023-09-23T22:28:06.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: 7.3 starting backfill to osd.1 from (0'0,0'0] MAX to 309'7236 2023-09-23T22:28:06.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:06 smithi026 ceph-mon[223014]: osdmap e312: 8 total, 8 up, 5 in 2023-09-23T22:28:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: pgmap v808: 99 pgs: 5 remapped+peering, 16 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 86 op/s; 18899/194550 objects degraded (9.714%); 23273/194550 objects misplaced (11.962%); 4.1 MiB/s, 59 objects/s recovering 2023-09-23T22:28:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: 7.8 starting backfill to osd.5 from (0'0,0'0] MAX to 309'7058 2023-09-23T22:28:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: 7.c starting backfill to osd.1 from (0'0,0'0] MAX to 309'7142 2023-09-23T22:28:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: 7.2 starting backfill to osd.0 from (265'2301,303'7164] MIN to 309'7237 2023-09-23T22:28:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: 7.a starting backfill to osd.4 from (265'2423,303'7317] MIN to 309'7382 2023-09-23T22:28:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:28:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 18899/194550 objects degraded (9.714%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:28:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:07 smithi070 ceph-mon[192652]: 7.e starting backfill to osd.1 from (0'0,0'0] MAX to 309'7245 2023-09-23T22:28:07.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: pgmap v808: 99 pgs: 5 remapped+peering, 16 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 86 op/s; 18899/194550 objects degraded (9.714%); 23273/194550 objects misplaced (11.962%); 4.1 MiB/s, 59 objects/s recovering 2023-09-23T22:28:07.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: 7.8 starting backfill to osd.5 from (0'0,0'0] MAX to 309'7058 2023-09-23T22:28:07.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: 7.c starting backfill to osd.1 from (0'0,0'0] MAX to 309'7142 2023-09-23T22:28:07.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: 7.2 starting backfill to osd.0 from (265'2301,303'7164] MIN to 309'7237 2023-09-23T22:28:07.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: 7.a starting backfill to osd.4 from (265'2423,303'7317] MIN to 309'7382 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 18899/194550 objects degraded (9.714%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[223014]: 7.e starting backfill to osd.1 from (0'0,0'0] MAX to 309'7245 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: pgmap v808: 99 pgs: 5 remapped+peering, 16 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 86 op/s; 18899/194550 objects degraded (9.714%); 23273/194550 objects misplaced (11.962%); 4.1 MiB/s, 59 objects/s recovering 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: failed to encode map e312 with expected crc 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: 7.8 starting backfill to osd.5 from (0'0,0'0] MAX to 309'7058 2023-09-23T22:28:07.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: 7.c starting backfill to osd.1 from (0'0,0'0] MAX to 309'7142 2023-09-23T22:28:07.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: 7.2 starting backfill to osd.0 from (265'2301,303'7164] MIN to 309'7237 2023-09-23T22:28:07.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: 7.a starting backfill to osd.4 from (265'2423,303'7317] MIN to 309'7382 2023-09-23T22:28:07.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:28:07.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 18899/194550 objects degraded (9.714%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:28:07.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:07 smithi026 ceph-mon[215775]: 7.e starting backfill to osd.1 from (0'0,0'0] MAX to 309'7245 2023-09-23T22:28:07.617 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:07.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:07.723 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:07.760 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:07.904 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:07.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:08.009 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:08.011 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:08.037 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:08.089 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:08.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:08.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:08.156 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:28:08.156 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:28:08.197 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:08.224 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:08.271 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:08.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:08.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:08.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:08 smithi026 ceph-mon[223014]: failed to encode map e313 with expected crc 2023-09-23T22:28:08.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:08 smithi026 ceph-mon[223014]: osdmap e313: 8 total, 8 up, 5 in 2023-09-23T22:28:08.318 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:08 smithi026 ceph-mon[223014]: osdmap e314: 8 total, 8 up, 5 in 2023-09-23T22:28:08.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:08 smithi026 ceph-mon[215775]: failed to encode map e313 with expected crc 2023-09-23T22:28:08.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:08 smithi026 ceph-mon[215775]: osdmap e313: 8 total, 8 up, 5 in 2023-09-23T22:28:08.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:08 smithi026 ceph-mon[215775]: osdmap e314: 8 total, 8 up, 5 in 2023-09-23T22:28:08.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:08 smithi070 ceph-mon[192652]: failed to encode map e313 with expected crc 2023-09-23T22:28:08.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:08 smithi070 ceph-mon[192652]: osdmap e313: 8 total, 8 up, 5 in 2023-09-23T22:28:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:08 smithi070 ceph-mon[192652]: osdmap e314: 8 total, 8 up, 5 in 2023-09-23T22:28:08.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:08.422 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:09.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[215775]: pgmap v811: 99 pgs: 5 remapped+peering, 16 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 53 op/s; 18899/194550 objects degraded (9.714%); 23273/194550 objects misplaced (11.962%); 1.9 MiB/s, 17 objects/s recovering 2023-09-23T22:28:09.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[215775]: failed to encode map e314 with expected crc 2023-09-23T22:28:09.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[215775]: failed to encode map e314 with expected crc 2023-09-23T22:28:09.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[215775]: failed to encode map e314 with expected crc 2023-09-23T22:28:09.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:09.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:09.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[223014]: pgmap v811: 99 pgs: 5 remapped+peering, 16 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 53 op/s; 18899/194550 objects degraded (9.714%); 23273/194550 objects misplaced (11.962%); 1.9 MiB/s, 17 objects/s recovering 2023-09-23T22:28:09.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[223014]: failed to encode map e314 with expected crc 2023-09-23T22:28:09.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[223014]: failed to encode map e314 with expected crc 2023-09-23T22:28:09.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[223014]: failed to encode map e314 with expected crc 2023-09-23T22:28:09.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:09.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:09.362 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:09.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:09.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:09.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:09 smithi070 ceph-mon[192652]: pgmap v811: 99 pgs: 5 remapped+peering, 16 peering, 2 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 66 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 53 op/s; 18899/194550 objects degraded (9.714%); 23273/194550 objects misplaced (11.962%); 1.9 MiB/s, 17 objects/s recovering 2023-09-23T22:28:09.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:09 smithi070 ceph-mon[192652]: failed to encode map e314 with expected crc 2023-09-23T22:28:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:09 smithi070 ceph-mon[192652]: failed to encode map e314 with expected crc 2023-09-23T22:28:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:09 smithi070 ceph-mon[192652]: failed to encode map e314 with expected crc 2023-09-23T22:28:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:09.469 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:09.513 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:09.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:28:09.710 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:28:09.921 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:28:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:28:09] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-09-23T22:28:09.939 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:09.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:10.051 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:10.113 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:10.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:10.252 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:10.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:10.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:10.358 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:28:10.392 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:10.448 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:10.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:10.511 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:10.554 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:10.585 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:10.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:10.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:10.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:10.737 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:10.798 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:10.849 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:11.086 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:11 smithi070 ceph-mon[192652]: pgmap v813: 99 pgs: 4 remapped+peering, 16 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 18895/194550 objects degraded (9.712%); 19504/194550 objects misplaced (10.025%); 8.5 MiB/s, 172 objects/s recovering 2023-09-23T22:28:11.087 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4009185667' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:11.087 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:11.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:11.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:11.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:11 smithi026 ceph-mon[223014]: pgmap v813: 99 pgs: 4 remapped+peering, 16 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 18895/194550 objects degraded (9.712%); 19504/194550 objects misplaced (10.025%); 8.5 MiB/s, 172 objects/s recovering 2023-09-23T22:28:11.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4009185667' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:11.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:11 smithi026 ceph-mon[215775]: pgmap v813: 99 pgs: 4 remapped+peering, 16 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 68 active+clean; 4.0 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 18895/194550 objects degraded (9.712%); 19504/194550 objects misplaced (10.025%); 8.5 MiB/s, 172 objects/s recovering 2023-09-23T22:28:11.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4009185667' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:11.192 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:11.233 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:11.357 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:11.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:11.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:11.463 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:11.492 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:11.603 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:11.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:11.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:11.708 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:11.740 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 1] out_osds: [6, 3, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:28:11.740 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:28:11.740 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T22:28:11.740 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:28:11.740 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:28:11.740 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T22:28:11.744 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:12.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:12 smithi026 ceph-mon[215775]: pgmap v814: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 29387/194523 objects misplaced (15.107%); 30 MiB/s, 1 keys/s, 531 objects/s recovering 2023-09-23T22:28:12.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:12 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 20130/194523 objects degraded (10.348%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:28:12.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:12 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:28:12.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:12 smithi026 ceph-mon[223014]: pgmap v814: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 29387/194523 objects misplaced (15.107%); 30 MiB/s, 1 keys/s, 531 objects/s recovering 2023-09-23T22:28:12.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:12 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 20130/194523 objects degraded (10.348%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:28:12.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:12 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:28:12.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:12 smithi070 ceph-mon[192652]: pgmap v814: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 29387/194523 objects misplaced (15.107%); 30 MiB/s, 1 keys/s, 531 objects/s recovering 2023-09-23T22:28:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:12 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 20130/194523 objects degraded (10.348%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:28:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:12 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:28:13.268 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:13.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:13.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:13.373 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:13.427 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:13.779 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:13.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:13.783 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:13.802 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:13.884 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:13.912 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:13.957 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:13.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:13.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:14.061 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:14.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:14 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:14.101 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:14 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:14.101 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:28:14.101 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:28:14.119 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:14.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:14 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:14.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:14 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:14.443 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:14.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:14 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:14.445 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:14 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:14.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:14.547 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:14.574 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:14.620 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:14.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:14.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:14.726 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:14.763 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:15.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:15.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:15.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:15.125 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:28:15.135 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:15.162 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:15.210 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:15.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:15.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:15.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[215775]: pgmap v815: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 29387/194523 objects misplaced (15.107%); 24 MiB/s, 1 keys/s, 430 objects/s recovering 2023-09-23T22:28:15.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[215775]: osdmap e315: 8 total, 8 up, 5 in 2023-09-23T22:28:15.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[215775]: failed to encode map e315 with expected crc 2023-09-23T22:28:15.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[215775]: failed to encode map e315 with expected crc 2023-09-23T22:28:15.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[215775]: failed to encode map e315 with expected crc 2023-09-23T22:28:15.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1252367495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:15.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:15.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[223014]: pgmap v815: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 29387/194523 objects misplaced (15.107%); 24 MiB/s, 1 keys/s, 430 objects/s recovering 2023-09-23T22:28:15.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[223014]: osdmap e315: 8 total, 8 up, 5 in 2023-09-23T22:28:15.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[223014]: failed to encode map e315 with expected crc 2023-09-23T22:28:15.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[223014]: failed to encode map e315 with expected crc 2023-09-23T22:28:15.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[223014]: failed to encode map e315 with expected crc 2023-09-23T22:28:15.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1252367495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:15.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:15 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:15.350 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:15.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:15 smithi070 ceph-mon[192652]: pgmap v815: 99 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 1 active+recovering+degraded, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 29387/194523 objects misplaced (15.107%); 24 MiB/s, 1 keys/s, 430 objects/s recovering 2023-09-23T22:28:15.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:15 smithi070 ceph-mon[192652]: osdmap e315: 8 total, 8 up, 5 in 2023-09-23T22:28:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:15 smithi070 ceph-mon[192652]: failed to encode map e315 with expected crc 2023-09-23T22:28:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:15 smithi070 ceph-mon[192652]: failed to encode map e315 with expected crc 2023-09-23T22:28:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:15 smithi070 ceph-mon[192652]: failed to encode map e315 with expected crc 2023-09-23T22:28:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:15 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1252367495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:15 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:16.092 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:28:16.112 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:16.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:16.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:16.216 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:16.241 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:16.286 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:16.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:16.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:16.390 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:16.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:16.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:28:16.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[223014]: osdmap e316: 8 total, 8 up, 5 in 2023-09-23T22:28:16.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[223014]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[223014]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[223014]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[223014]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[223014]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:16.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:28:16.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[215775]: osdmap e316: 8 total, 8 up, 5 in 2023-09-23T22:28:16.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[215775]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[215775]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[215775]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[215775]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.421 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:16 smithi026 ceph-mon[215775]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.422 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:16.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:16 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:16 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:28:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:16 smithi070 ceph-mon[192652]: osdmap e316: 8 total, 8 up, 5 in 2023-09-23T22:28:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:16 smithi070 ceph-mon[192652]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:16 smithi070 ceph-mon[192652]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:16 smithi070 ceph-mon[192652]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:16 smithi070 ceph-mon[192652]: failed to encode map e316 with expected crc 2023-09-23T22:28:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:16 smithi070 ceph-mon[192652]: failed to encode map e316 with expected crc 2023-09-23T22:28:17.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:17 smithi026 ceph-mon[223014]: pgmap v818: 99 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 26853/194523 objects misplaced (13.805%); 42 MiB/s, 1 keys/s, 784 objects/s recovering 2023-09-23T22:28:17.471 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:17 smithi026 ceph-mon[223014]: failed to encode map e316 with expected crc 2023-09-23T22:28:17.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:17 smithi026 ceph-mon[215775]: pgmap v818: 99 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 26853/194523 objects misplaced (13.805%); 42 MiB/s, 1 keys/s, 784 objects/s recovering 2023-09-23T22:28:17.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:17 smithi026 ceph-mon[215775]: failed to encode map e316 with expected crc 2023-09-23T22:28:17.506 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:17.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:17.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:17.620 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:17.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:17 smithi070 ceph-mon[192652]: pgmap v818: 99 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 26853/194523 objects misplaced (13.805%); 42 MiB/s, 1 keys/s, 784 objects/s recovering 2023-09-23T22:28:17.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:17 smithi070 ceph-mon[192652]: failed to encode map e316 with expected crc 2023-09-23T22:28:17.646 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:17.689 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:17.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:17.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:17.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:17.825 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:17.870 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:17.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:17.977 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:18.034 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:18.230 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:18.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:18.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:18.236 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:18.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:18.383 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:18 smithi026 ceph-mon[223014]: pgmap v819: 99 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 26853/194523 objects misplaced (13.805%); 34 MiB/s, 1 keys/s, 631 objects/s recovering 2023-09-23T22:28:18.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 20130/194523 objects degraded (10.348%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:28:18.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:18 smithi026 ceph-mon[215775]: pgmap v819: 99 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 26853/194523 objects misplaced (13.805%); 34 MiB/s, 1 keys/s, 631 objects/s recovering 2023-09-23T22:28:18.384 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 20130/194523 objects degraded (10.348%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:28:18.392 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:18.395 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:28:18.395 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:28:18.395 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:28:18.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:18 smithi070 ceph-mon[192652]: pgmap v819: 99 pgs: 1 peering, 1 remapped+peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 20130/194523 objects degraded (10.348%); 26853/194523 objects misplaced (13.805%); 34 MiB/s, 1 keys/s, 631 objects/s recovering 2023-09-23T22:28:18.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 20130/194523 objects degraded (10.348%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:28:18.632 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:18.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:18.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:18.741 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:18.785 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:19.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:28:19.859 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:19.861 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:28:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:28:19] "GET /metrics HTTP/1.1" 200 36450 "" "Prometheus/2.43.0" 2023-09-23T22:28:19.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:19.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:19.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:19.991 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:20.037 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:20.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:20.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:20.142 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:20.194 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:20.477 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:20.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:20.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:20.581 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:20.643 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:20.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:20 smithi026 ceph-mon[223014]: pgmap v820: 99 pgs: 2 active+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 20130/194523 objects degraded (10.348%); 29404/194523 objects misplaced (15.116%); 28 MiB/s, 650 objects/s recovering 2023-09-23T22:28:20.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:20.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:20 smithi026 ceph-mon[215775]: pgmap v820: 99 pgs: 2 active+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 20130/194523 objects degraded (10.348%); 29404/194523 objects misplaced (15.116%); 28 MiB/s, 650 objects/s recovering 2023-09-23T22:28:20.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:20.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:20 smithi070 ceph-mon[192652]: pgmap v820: 99 pgs: 2 active+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 20130/194523 objects degraded (10.348%); 29404/194523 objects misplaced (15.116%); 28 MiB/s, 650 objects/s recovering 2023-09-23T22:28:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:20.927 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:20.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:20.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:21.036 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:21.081 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:21.159 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:21.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:21.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:21.264 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:21.305 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:21.374 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:21.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:21.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:21.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:21.506 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:21.554 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:21.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:21.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:21.661 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:21.688 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:21.733 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:21.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:21.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:21.837 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:21.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:21.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:21.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:21.865 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:21.911 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:21.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:21.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:21.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:21.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:21.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:28:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:28:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:22.017 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:22.062 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:22.132 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:22.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:22.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:22.237 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:22.276 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:22.463 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:22.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:22.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:22.475 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:22.568 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:22.610 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:22.860 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:22.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:22.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:22.965 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:23.017 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:23.179 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:28:23.180 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:28:23.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:23 smithi070 ceph-mon[192652]: pgmap v821: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 19042/194475 objects degraded (9.791%); 29028/194475 objects misplaced (14.926%); 31 MiB/s, 699 objects/s recovering 2023-09-23T22:28:23.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:23 smithi026 ceph-mon[223014]: pgmap v821: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 19042/194475 objects degraded (9.791%); 29028/194475 objects misplaced (14.926%); 31 MiB/s, 699 objects/s recovering 2023-09-23T22:28:23.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:23 smithi026 ceph-mon[215775]: pgmap v821: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 19042/194475 objects degraded (9.791%); 29028/194475 objects misplaced (14.926%); 31 MiB/s, 699 objects/s recovering 2023-09-23T22:28:24.169 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:24.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:24.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:24.275 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:24.298 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:24 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 19042/194475 objects degraded (9.791%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:28:24.300 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:24.344 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:24.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:24.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:24.448 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:24.475 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:24.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:24 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 19042/194475 objects degraded (9.791%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:28:24.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:24 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 19042/194475 objects degraded (9.791%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:28:24.519 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:24.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:24.624 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:24.649 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:24.695 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:24.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:24.802 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:24.828 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:24.873 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:24.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:24.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:24.981 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:25.021 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:25.033 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:28:25.074 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:28:25.248 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:25.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:25.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:25.353 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:25.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:25 smithi070 ceph-mon[192652]: pgmap v822: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 19042/194475 objects degraded (9.791%); 29028/194475 objects misplaced (14.926%); 24 MiB/s, 524 objects/s recovering 2023-09-23T22:28:25.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:25 smithi070 ceph-mon[192652]: failed to encode map e317 with expected crc 2023-09-23T22:28:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:25 smithi070 ceph-mon[192652]: osdmap e317: 8 total, 8 up, 5 in 2023-09-23T22:28:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:25 smithi070 ceph-mon[192652]: failed to encode map e317 with expected crc 2023-09-23T22:28:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:25 smithi070 ceph-mon[192652]: failed to encode map e317 with expected crc 2023-09-23T22:28:25.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[223014]: pgmap v822: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 19042/194475 objects degraded (9.791%); 29028/194475 objects misplaced (14.926%); 24 MiB/s, 524 objects/s recovering 2023-09-23T22:28:25.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[223014]: failed to encode map e317 with expected crc 2023-09-23T22:28:25.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[223014]: osdmap e317: 8 total, 8 up, 5 in 2023-09-23T22:28:25.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[223014]: failed to encode map e317 with expected crc 2023-09-23T22:28:25.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:25.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:25.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[223014]: failed to encode map e317 with expected crc 2023-09-23T22:28:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[215775]: pgmap v822: 99 pgs: 1 activating+undersized+degraded+remapped, 2 active+remapped+backfilling, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 85 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1 op/s; 19042/194475 objects degraded (9.791%); 29028/194475 objects misplaced (14.926%); 24 MiB/s, 524 objects/s recovering 2023-09-23T22:28:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[215775]: failed to encode map e317 with expected crc 2023-09-23T22:28:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[215775]: osdmap e317: 8 total, 8 up, 5 in 2023-09-23T22:28:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[215775]: failed to encode map e317 with expected crc 2023-09-23T22:28:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:28:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:25.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:25 smithi026 ceph-mon[215775]: failed to encode map e317 with expected crc 2023-09-23T22:28:25.384 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:25.525 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 1] out_osds: [6, 3, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:28:25.525 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:28:25.525 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:28:25.525 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 1.000000 2023-09-23T22:28:25.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T22:28:25.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:25.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:25.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:25.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:28:25.861 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:25.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:26.023 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:26.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:26.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:26.128 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:26.143 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:26.154 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:26.201 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:26.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:26.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:26.311 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:26.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:26 smithi070 ceph-mon[192652]: failed to encode map e317 with expected crc 2023-09-23T22:28:26.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:26 smithi070 ceph-mon[192652]: osdmap e318: 8 total, 8 up, 5 in 2023-09-23T22:28:26.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:26 smithi070 ceph-mon[192652]: failed to encode map e318 with expected crc 2023-09-23T22:28:26.336 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:26 smithi070 ceph-mon[192652]: failed to encode map e318 with expected crc 2023-09-23T22:28:26.339 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:26.384 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:26.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:26.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:26 smithi026 ceph-mon[223014]: failed to encode map e317 with expected crc 2023-09-23T22:28:26.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:26 smithi026 ceph-mon[223014]: osdmap e318: 8 total, 8 up, 5 in 2023-09-23T22:28:26.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:26 smithi026 ceph-mon[223014]: failed to encode map e318 with expected crc 2023-09-23T22:28:26.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:26 smithi026 ceph-mon[223014]: failed to encode map e318 with expected crc 2023-09-23T22:28:26.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:26 smithi026 ceph-mon[215775]: failed to encode map e317 with expected crc 2023-09-23T22:28:26.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:26 smithi026 ceph-mon[215775]: osdmap e318: 8 total, 8 up, 5 in 2023-09-23T22:28:26.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:26 smithi026 ceph-mon[215775]: failed to encode map e318 with expected crc 2023-09-23T22:28:26.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:26 smithi026 ceph-mon[215775]: failed to encode map e318 with expected crc 2023-09-23T22:28:26.490 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:26.526 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:26.621 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:26.633 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:26.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:26.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:26.738 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:26.765 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:26.809 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:26.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:26.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:26.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:26.943 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:26.990 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:26.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:26.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:27.068 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:28:27.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:27.131 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:27.188 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:27.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:27.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[223014]: pgmap v825: 99 pgs: 1 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 86 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 9 op/s; 19797/194295 objects degraded (10.189%); 19642/194295 objects misplaced (10.109%); 66 MiB/s, 1.19k objects/s recovering 2023-09-23T22:28:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[223014]: osdmap e319: 8 total, 8 up, 5 in 2023-09-23T22:28:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[223014]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[223014]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[223014]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[223014]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[223014]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/641556173' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[215775]: pgmap v825: 99 pgs: 1 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 86 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 9 op/s; 19797/194295 objects degraded (10.189%); 19642/194295 objects misplaced (10.109%); 66 MiB/s, 1.19k objects/s recovering 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[215775]: osdmap e319: 8 total, 8 up, 5 in 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[215775]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[215775]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[215775]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[215775]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[215775]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/641556173' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:27.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:27 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:27.292 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:27.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:27 smithi070 ceph-mon[192652]: pgmap v825: 99 pgs: 1 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 86 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 9 op/s; 19797/194295 objects degraded (10.189%); 19642/194295 objects misplaced (10.109%); 66 MiB/s, 1.19k objects/s recovering 2023-09-23T22:28:27.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:27 smithi070 ceph-mon[192652]: osdmap e319: 8 total, 8 up, 5 in 2023-09-23T22:28:27.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:27 smithi070 ceph-mon[192652]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:27 smithi070 ceph-mon[192652]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:27 smithi070 ceph-mon[192652]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:27 smithi070 ceph-mon[192652]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:27 smithi070 ceph-mon[192652]: failed to encode map e319 with expected crc 2023-09-23T22:28:27.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/641556173' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:27.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:27 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:27.319 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:27.361 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:27.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:27.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:27.469 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:27.489 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:28:27.505 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:27.514 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:28:27.544 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:28:27.565 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:27.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:27.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:27.578 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:28:27.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:27.680 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:28:27.699 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:27.740 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:27.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:27.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:27.761 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:28:27.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:28:27.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:28:27.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:28:27.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:28:27.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:28:27.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:28:27.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:28:27.762 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:28:27.846 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:27.871 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:27.914 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:27.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:27.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:28.022 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:28.049 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:28.090 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:28.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:28.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:28.194 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:28.220 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:28.261 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:28.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:28.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:28.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:28.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:28 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:28:28.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:28 smithi070 ceph-mon[192652]: failed to encode map e320 with expected crc 2023-09-23T22:28:28.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:28 smithi070 ceph-mon[192652]: failed to encode map e320 with expected crc 2023-09-23T22:28:28.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:28 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:28 smithi070 ceph-mon[192652]: osdmap e320: 8 total, 8 up, 5 in 2023-09-23T22:28:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:28 smithi070 ceph-mon[192652]: failed to encode map e320 with expected crc 2023-09-23T22:28:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 19797/194295 objects degraded (10.189%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:28.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:28:28.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[223014]: failed to encode map e320 with expected crc 2023-09-23T22:28:28.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[223014]: failed to encode map e320 with expected crc 2023-09-23T22:28:28.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:28.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[223014]: osdmap e320: 8 total, 8 up, 5 in 2023-09-23T22:28:28.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[223014]: failed to encode map e320 with expected crc 2023-09-23T22:28:28.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 19797/194295 objects degraded (10.189%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:28.396 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:28:28.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[215775]: failed to encode map e320 with expected crc 2023-09-23T22:28:28.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[215775]: failed to encode map e320 with expected crc 2023-09-23T22:28:28.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:28.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[215775]: osdmap e320: 8 total, 8 up, 5 in 2023-09-23T22:28:28.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[215775]: failed to encode map e320 with expected crc 2023-09-23T22:28:28.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 19797/194295 objects degraded (10.189%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:28.400 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:28.750 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:28.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:28.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:28.855 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:28.883 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:28.926 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:28.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:28.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:29.031 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:29.072 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:29.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:29 smithi026 ceph-mon[223014]: pgmap v828: 99 pgs: 1 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 86 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 14 op/s; 19797/194295 objects degraded (10.189%); 19642/194295 objects misplaced (10.109%); 103 MiB/s, 1.64k objects/s recovering 2023-09-23T22:28:29.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:29 smithi026 ceph-mon[215775]: pgmap v828: 99 pgs: 1 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 86 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 14 op/s; 19797/194295 objects degraded (10.189%); 19642/194295 objects misplaced (10.109%); 103 MiB/s, 1.64k objects/s recovering 2023-09-23T22:28:29.331 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:29.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:29.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:29.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:29 smithi070 ceph-mon[192652]: pgmap v828: 99 pgs: 1 active+clean+remapped, 2 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 1 active+recovering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 86 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 14 op/s; 19797/194295 objects degraded (10.189%); 19642/194295 objects misplaced (10.109%); 103 MiB/s, 1.64k objects/s recovering 2023-09-23T22:28:29.437 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:29.463 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:29.506 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:29.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:29.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:29.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:28:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:28:29] "GET /metrics HTTP/1.1" 200 36445 "" "Prometheus/2.43.0" 2023-09-23T22:28:29.654 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:29.779 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:29.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:29.883 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:29.908 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:29.949 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:29.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:30.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:30.088 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T22:28:30.113 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:30.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:30 smithi026 ceph-mon[223014]: failed to encode map e320 with expected crc 2023-09-23T22:28:30.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1579994637' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:28:30.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:30 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:28:30.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:30 smithi026 ceph-mon[215775]: failed to encode map e320 with expected crc 2023-09-23T22:28:30.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1579994637' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:28:30.328 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:30 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:28:30.346 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:30.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:30 smithi070 ceph-mon[192652]: failed to encode map e320 with expected crc 2023-09-23T22:28:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1579994637' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:28:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:30 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:28:30.454 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114688 7.c 7:35dbdb8d:::benchmark_data_smithi026_410749_object24351:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e298)", 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:27.145310+0000", 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.305578764000003, 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 25.772119308000001, 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.455 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114688 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.145310+0000", 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.145310+0000", 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.145310+0000", 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.145313+0000", 2023-09-23T22:28:30.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5229999999999998e-06 2023-09-23T22:28:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.145319+0000", 2023-09-23T22:28:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9989999999999997e-06 2023-09-23T22:28:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.145324+0000", 2023-09-23T22:28:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.057e-06 2023-09-23T22:28:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.145361+0000", 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7001e-05 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.145371+0000", 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.047e-05 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.062200+0000", 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 24.916828567 2023-09-23T22:28:30.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for readable", 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.087716+0000", 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025516377999999999 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.174574+0000", 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.086858311999999993 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.190637+0000", 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016062941000000001 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.232032+0000", 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041395213 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.238397+0000", 2023-09-23T22:28:30.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063645680000000001 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.868218+0000", 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.62982054899999995 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.874118+0000", 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059002029999999997 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.885438+0000", 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011320258999999999 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.898771+0000", 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013332696999999999 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.899537+0000", 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00076623200000000005 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.901836+0000", 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022988520000000001 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.905341+0000", 2023-09-23T22:28:30.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035047339999999998 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.917430+0000", 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012089156 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.482 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114695 7.c 7:3095d1d2:::benchmark_data_smithi026_410749_object24358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e298)", 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:27.364644+0000", 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.086244882999999, 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 25.692039958999999, 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114695 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.483 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364644+0000", 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364644+0000", 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364643+0000", 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364651+0000", 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0979999999999994e-06 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364668+0000", 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7428999999999999e-05 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364680+0000", 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1705999999999999e-05 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364744+0000", 2023-09-23T22:28:30.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4319000000000005e-05 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.364755+0000", 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0874e-05 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.239588+0000", 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 24.874833253999999 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.246582+0000", 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069937929999999999 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.331498+0000", 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.084915918000000007 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.338979+0000", 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074813019999999996 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.933771+0000", 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.59479144900000003 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.939264+0000", 2023-09-23T22:28:30.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005493291 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.949394+0000", 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010129994999999999 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.012332+0000", 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.062938146 2023-09-23T22:28:30.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.013471+0000", 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001138888 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.017658+0000", 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041866960000000002 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.026285+0000", 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086268690000000006 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.056684+0000", 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030399625999999999 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114702 7.c 7:3fb410f3:::benchmark_data_smithi026_410749_object24365:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e298)", 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:27.411726+0000", 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.039163494999997, 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 25.746545143999999, 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.493 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114702 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.411726+0000", 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.411726+0000", 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.411724+0000", 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.411733+0000", 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6819999999999992e-06 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.411747+0000", 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.411e-05 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.411756+0000", 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3389999999999999e-06 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.411806+0000", 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9719000000000003e-05 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.411820+0000", 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4466e-05 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.249810+0000", 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 24.837989328999999 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.300811+0000", 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.051001896999999997 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.388303+0000", 2023-09-23T22:28:30.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.087491283000000003 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.395561+0000", 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007258685 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.975504+0000", 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.57994304500000005 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.987952+0000", 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012447387000000001 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.000725+0000", 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01277359 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.061289+0000", 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060563628000000001 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.113104+0000", 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.051814589000000001 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.117961+0000", 2023-09-23T22:28:30.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004857531 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.126470+0000", 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085090719999999995 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.158271+0000", 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031800541000000002 2023-09-23T22:28:30.502 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114911 7.c 7:3e77fea2:::benchmark_data_smithi026_410749_object24574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:53.421670+0000", 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 37.029218956999998, 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3147603380000001, 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.503 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114911 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421670+0000", 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421670+0000", 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421668+0000", 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421677+0000", 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4350000000000003e-06 2023-09-23T22:28:30.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421696+0000", 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8627999999999999e-05 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421706+0000", 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.039e-05 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.637038+0000", 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.21533159299999999 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.660388+0000", 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023350196 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.752370+0000", 2023-09-23T22:28:30.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.091982265999999993 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.769070+0000", 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016699711999999999 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.637198+0000", 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8681277650000001 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.644126+0000", 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006927791 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.653252+0000", 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0091262840000000001 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.705760+0000", 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.052507695 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.707669+0000", 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001909736 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.709644+0000", 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019750499999999999 2023-09-23T22:28:30.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.712822+0000", 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031774709999999999 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.736431+0000", 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023608589999999999 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114966 7.c 7:3a942ff9:::benchmark_data_smithi026_410749_object24629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:53.806941+0000", 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 36.643948584, 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3599143910000002, 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114966 2023-09-23T22:28:30.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.806941+0000", 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.806941+0000", 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.806939+0000", 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.806978+0000", 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9634e-05 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.807023+0000", 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4372999999999997e-05 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.807037+0000", 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4270999999999999e-05 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.932073+0000", 2023-09-23T22:28:30.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.125036286 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.948418+0000", 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01634472 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.186340+0000", 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.237922314 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.196343+0000", 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010002305 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.070957+0000", 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8746143819999999 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.075952+0000", 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049950079999999996 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.084821+0000", 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088687560000000002 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.122964+0000", 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038143073 2023-09-23T22:28:30.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.126606+0000", 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003641881 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.127956+0000", 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013507899999999999 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.135068+0000", 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071115500000000003 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.166855+0000", 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031786994999999998 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114971 7.c 7:3079c21b:::benchmark_data_smithi026_410749_object24634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e306)", 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:54.016788+0000", 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 36.434101290000001, 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4142198389999998, 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114971 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.016788+0000", 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.016788+0000", 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.016786+0000", 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.016795+0000", 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8789999999999992e-06 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.016810+0000", 2023-09-23T22:28:30.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4823e-05 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.016818+0000", 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4260000000000003e-06 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.238570+0000", 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22175110000000001 2023-09-23T22:28:30.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.255812+0000", 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017242475 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.417934+0000", 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16212169900000001 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.433369+0000", 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015435620000000001 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.205604+0000", 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7722350009999999 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.221401+0000", 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015796748999999999 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.250094+0000", 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028692973 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.373859+0000", 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.123764745 2023-09-23T22:28:30.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.379397+0000", 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055379849999999996 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.384160+0000", 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047628740000000003 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.392169+0000", 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080088550000000005 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.431008+0000", 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.038839208 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115036 7.c 7:35ee2cf4:::benchmark_data_smithi026_410749_object24699:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:54.613102+0000", 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 35.837787397, 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5037636939999999, 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115036 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.530 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.613102+0000", 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.613102+0000", 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.613100+0000", 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.613108+0000", 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2810000000000008e-06 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.613119+0000", 2023-09-23T22:28:30.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1018e-05 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.613126+0000", 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9940000000000003e-06 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.695811+0000", 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082684975999999993 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.707228+0000", 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011416327 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.753351+0000", 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.046123190000000001 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.761081+0000", 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077301619999999996 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.979937+0000", 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.218855434 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.991518+0000", 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011581912999999999 2023-09-23T22:28:30.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.014029+0000", 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022510186000000001 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.035765+0000", 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021736385 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.042448+0000", 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066832230000000003 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.076596+0000", 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034148120999999997 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.082335+0000", 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057390569999999997 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.116865+0000", 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034530061000000001 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115041 7.c 7:393c7926:::benchmark_data_smithi026_410749_object24704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:54.661055+0000", 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 35.789834034000002, 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6461448949999999, 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115041 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.539 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.661055+0000", 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.661055+0000", 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.661053+0000", 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.661063+0000", 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7079999999999992e-06 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.661079+0000", 2023-09-23T22:28:30.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5860000000000001e-05 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.661087+0000", 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7490000000000004e-06 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.710297+0000", 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.049209548999999998 2023-09-23T22:28:30.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.718094+0000", 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077966340000000002 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.790825+0000", 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.072731334999999994 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.801893+0000", 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011068318000000001 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.118341+0000", 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.316447884 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.133529+0000", 2023-09-23T22:28:30.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015188198999999999 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.158160+0000", 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024630282999999999 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.261500+0000", 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.103340194 2023-09-23T22:28:30.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.266330+0000", 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048297430000000001 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.271607+0000", 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052775180000000001 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.279449+0000", 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078419319999999994 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.307200+0000", 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027750988000000001 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115050 7.c 7:3f7e75c4:::benchmark_data_smithi026_410749_object24713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:54.898563+0000", 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 35.552325717000002, 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4472782620000002, 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115050 2023-09-23T22:28:30.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.898563+0000", 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.898563+0000", 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.898562+0000", 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.898568+0000", 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8279999999999999e-06 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.898578+0000", 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0325e-05 2023-09-23T22:28:30.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.898584+0000", 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.173e-06 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.084758+0000", 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.186173437 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.097152+0000", 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012394347 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.177658+0000", 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080505808999999998 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.192704+0000", 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015046222999999999 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.203484+0000", 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0107801670000001 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.218308+0000", 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014823311 2023-09-23T22:28:30.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.243744+0000", 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025436251999999999 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.308300+0000", 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.064556283000000006 2023-09-23T22:28:30.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.311938+0000", 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036381669999999999 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.315226+0000", 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032873199999999998 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.322244+0000", 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070180570000000003 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.345842+0000", 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023597969 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115052 7.c 7:387ca568:::benchmark_data_smithi026_410749_object24715:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:54.916961+0000", 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 35.533927753, 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5737012340000001, 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115052 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.916961+0000", 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.916961+0000", 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.916960+0000", 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.916968+0000", 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9489999999999996e-06 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.916980+0000", 2023-09-23T22:28:30.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2870999999999999e-05 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:54.916988+0000", 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7719999999999994e-06 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.136187+0000", 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.21919844899999999 2023-09-23T22:28:30.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.147510+0000", 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011323125999999999 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.230988+0000", 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.083477993 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.238185+0000", 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007196933 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.377518+0000", 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1393332790000001 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.387365+0000", 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0098469750000000009 2023-09-23T22:28:30.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.401010+0000", 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013644743000000001 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.456829+0000", 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.055819204999999997 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.457524+0000", 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00069553699999999998 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.460808+0000", 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032838500000000001 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.464658+0000", 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038500919999999998 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.490663+0000", 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026004200000000002 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.565 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115067 7.c 7:304e9f58:::benchmark_data_smithi026_410749_object24730:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:55.092276+0000", 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 35.358613181000003, 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5026813250000002, 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115067 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.092276+0000", 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.092276+0000", 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.092274+0000", 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.092283+0000", 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2979999999999992e-06 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.092299+0000", 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6254000000000001e-05 2023-09-23T22:28:30.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.092308+0000", 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9770000000000006e-06 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.240430+0000", 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.148122266 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.247087+0000", 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066564930000000003 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.319886+0000", 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.072799718999999999 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.326243+0000", 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00635623 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.510669+0000", 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.18442632 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.516897+0000", 2023-09-23T22:28:30.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062277959999999999 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.530705+0000", 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013808097 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.557765+0000", 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027060055 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.560851+0000", 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030864719999999998 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.561722+0000", 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00087086200000000002 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.566778+0000", 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005055609 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.594957+0000", 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028179525 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.574 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115079 7.c 7:3dfc7cf5:::benchmark_data_smithi026_410749_object24742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:55.480266+0000", 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 34.970623199000002, 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3668626690000001, 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115079 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.480266+0000", 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.480266+0000", 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.480264+0000", 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.480272+0000", 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8120000000000008e-06 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.480288+0000", 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5699e-05 2023-09-23T22:28:30.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.480296+0000", 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7609999999999996e-06 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.540793+0000", 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.060496226 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.550321+0000", 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095283079999999992 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.612032+0000", 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.061710963000000001 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.618368+0000", 2023-09-23T22:28:30.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063363999999999998 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.689588+0000", 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0712196490000001 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.703920+0000", 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014332157999999999 2023-09-23T22:28:30.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.720493+0000", 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016572519000000001 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.814125+0000", 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.093631930000000002 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.817047+0000", 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029222509999999998 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.821131+0000", 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040846839999999999 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.825544+0000", 2023-09-23T22:28:30.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044121350000000002 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.847129+0000", 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021585033 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115115 7.c 7:3a7fcb1f:::benchmark_data_smithi026_410749_object24778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T22:28:30.583 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:55.895610+0000", 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 34.555279398000003, 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2789327859999999, 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115115 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.895610+0000", 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.895610+0000", 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.895608+0000", 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.895616+0000", 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0970000000000006e-06 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.895630+0000", 2023-09-23T22:28:30.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3641e-05 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.895641+0000", 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1371e-05 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.908616+0000", 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012974757 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:55.915233+0000", 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066167209999999999 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.005516+0000", 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.090283215999999999 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.012443+0000", 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069273700000000004 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.074782+0000", 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0623387520000001 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.082587+0000", 2023-09-23T22:28:30.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078052149999999999 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.102885+0000", 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020298252999999999 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.130889+0000", 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.02800323 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.132617+0000", 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017286809999999999 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.136831+0000", 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042137010000000003 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.143462+0000", 2023-09-23T22:28:30.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066307220000000004 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.174543+0000", 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031080869000000001 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115187 7.c 7:38eb6e68:::benchmark_data_smithi026_410749_object24850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T22:28:30.592 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:56.350520+0000", 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 34.100369164999996, 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4969085999999998, 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115187 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.350520+0000", 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.350520+0000", 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.350518+0000", 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.350525+0000", 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.041e-06 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.350538+0000", 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2299e-05 2023-09-23T22:28:30.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.350547+0000", 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1439999999999992e-06 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.505685+0000", 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.155138206 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.521643+0000", 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015958063000000001 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.739076+0000", 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.217432449 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.756015+0000", 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016939124 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.705309+0000", 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.949294045 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.718827+0000", 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013518125000000001 2023-09-23T22:28:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.735236+0000", 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016408821000000001 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.817594+0000", 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082358114999999996 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.821497+0000", 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039035699999999999 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.824448+0000", 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029502740000000001 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.827812+0000", 2023-09-23T22:28:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003364574 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.847429+0000", 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019616359 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115217 7.c 7:3f4acd75:::benchmark_data_smithi026_410749_object24880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T22:28:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:56.490414+0000", 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 33.960474751, 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5950759699999999, 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115217 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.490414+0000", 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.490414+0000", 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.490413+0000", 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.490422+0000", 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9360000000000005e-06 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.490439+0000", 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7356000000000001e-05 2023-09-23T22:28:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.490448+0000", 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2760000000000006e-06 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.761396+0000", 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.27094757600000002 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.778299+0000", 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016903133000000001 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.850860+0000", 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.072560863000000003 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.869101+0000", 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018241468 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.913232+0000", 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.044130923 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.929173+0000", 2023-09-23T22:28:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015940580999999999 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.955996+0000", 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026823329999999999 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.031747+0000", 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.075750451999999996 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.036539+0000", 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047920259999999996 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.047048+0000", 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010509772000000001 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.055172+0000", 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0081237710000000001 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.085490+0000", 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030318271000000001 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115236 7.c 7:30b06529:::benchmark_data_smithi026_410749_object24899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:56.668210+0000", 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 33.782679295999998, 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.712533788, 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115236 2023-09-23T22:28:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.668210+0000", 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.668210+0000", 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.668208+0000", 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.668217+0000", 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5939999999999994e-06 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.668235+0000", 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8850000000000001e-05 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.668245+0000", 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4110000000000002e-06 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.783695+0000", 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.115450416 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.799179+0000", 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015484256 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.930353+0000", 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13117303499999999 2023-09-23T22:28:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:56.949424+0000", 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019071182999999998 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.164955+0000", 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2155314170000002 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.178489+0000", 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013533895000000001 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.205076+0000", 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026587422999999999 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.289554+0000", 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.084477318999999995 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.319405+0000", 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029851283999999999 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.326222+0000", 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068167929999999998 2023-09-23T22:28:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.334839+0000", 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086169780000000008 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.380744+0000", 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045904832 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115287 7.c 7:3a61fc07:::benchmark_data_smithi026_410749_object24950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:57.241979+0000", 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 33.208910398, 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6220771410000001, 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115287 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.241979+0000", 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.241979+0000", 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.241977+0000", 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.241984+0000", 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1e-06 2023-09-23T22:28:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.241997+0000", 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3487e-05 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.242007+0000", 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4299999999999995e-06 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.346961+0000", 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10495462799999999 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.355066+0000", 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0081050470000000006 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.445174+0000", 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.090107904000000003 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.454643+0000", 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094693840000000008 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.709986+0000", 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2553426549999998 2023-09-23T22:28:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.721520+0000", 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011533844999999999 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.742372+0000", 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020852057 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.821844+0000", 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.079471604000000001 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.827045+0000", 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052011109999999996 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.828840+0000", 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017956510000000001 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.836752+0000", 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079114860000000006 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:59.864056+0000", 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027304030999999999 2023-09-23T22:28:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115302 7.c 7:3eef061a:::benchmark_data_smithi026_410749_object24965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:57.480095+0000", 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.970794300000001, 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7558104810000001, 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115302 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.480095+0000", 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.480095+0000", 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.480093+0000", 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.480102+0000", 2023-09-23T22:28:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5070000000000006e-06 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.480120+0000", 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7825999999999998e-05 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.480130+0000", 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0047000000000001e-05 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.546056+0000", 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065925531999999995 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.555237+0000", 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0091811789999999994 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.643645+0000", 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.088407547000000003 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.658616+0000", 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014971099 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.111811+0000", 2023-09-23T22:28:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4531957769999999 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.123275+0000", 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011463930000000001 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.138153+0000", 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014877796 2023-09-23T22:28:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.208423+0000", 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.070269465000000003 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.210144+0000", 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00172156 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.212077+0000", 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019330230000000001 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.215608+0000", 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035310960000000001 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.235905+0000", 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020297061000000002 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115329 7.c 7:3cd8cb13:::benchmark_data_smithi026_410749_object24992:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:57.770205+0000", 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.680683932999997, 2023-09-23T22:28:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5179037360000001, 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115329 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.770205+0000", 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.770205+0000", 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.770204+0000", 2023-09-23T22:28:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.770211+0000", 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4529999999999996e-06 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.770222+0000", 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1076e-05 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.770229+0000", 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0929999999999997e-06 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.853091+0000", 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.082861798 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:57.860788+0000", 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076966680000000003 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.003437+0000", 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.14264934200000001 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.014349+0000", 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01091204 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.181827+0000", 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1674775030000002 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.187490+0000", 2023-09-23T22:28:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056628069999999997 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.199821+0000", 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012331526000000001 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.237695+0000", 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037873872000000003 2023-09-23T22:28:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.243190+0000", 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054948949999999996 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.248648+0000", 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005458638 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.259831+0000", 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011182084 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.288109+0000", 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028278431 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115385 7.c 7:3771d310:::benchmark_data_smithi026_410749_object25048:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:58.310125+0000", 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 32.140764543000003, 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5303352380000002, 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115385 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.310125+0000", 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.310125+0000", 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.310122+0000", 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.310131+0000", 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0070000000000003e-06 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.310151+0000", 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0142000000000001e-05 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.310163+0000", 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1600000000000001e-05 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.407426+0000", 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.097262888000000006 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.422543+0000", 2023-09-23T22:28:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015116635999999999 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.623582+0000", 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20103960900000001 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:58.641909+0000", 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01832686 2023-09-23T22:28:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.672154+0000", 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0302447830000001 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.677585+0000", 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054307089999999997 2023-09-23T22:28:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.688098+0000", 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010513743000000001 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.747158+0000", 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.059059884 2023-09-23T22:28:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.748634+0000", 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014757100000000001 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.816935+0000", 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.068301034999999996 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.820155+0000", 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003219871 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:00.840460+0000", 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020305058000000001 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:30.658 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:28:30.658 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:30.685 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:30.690 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:28:30.726 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:30.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:30.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:28:30.832 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:30.866 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:31.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:31 smithi070 ceph-mon[192652]: pgmap v829: 99 pgs: 1 active+undersized+degraded+remapped+laggy, 1 activating+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 11 op/s; 19797/194295 objects degraded (10.189%); 18877/194295 objects misplaced (9.716%); 25 MiB/s, 393 objects/s recovering 2023-09-23T22:28:31.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:31 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:28:31.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:31 smithi070 ceph-mon[192652]: osdmap e321: 8 total, 8 up, 5 in 2023-09-23T22:28:31.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:31 smithi070 ceph-mon[192652]: failed to encode map e321 with expected crc 2023-09-23T22:28:31.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:31 smithi026 ceph-mon[215775]: pgmap v829: 99 pgs: 1 active+undersized+degraded+remapped+laggy, 1 activating+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 11 op/s; 19797/194295 objects degraded (10.189%); 18877/194295 objects misplaced (9.716%); 25 MiB/s, 393 objects/s recovering 2023-09-23T22:28:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:31 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:28:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:31 smithi026 ceph-mon[215775]: osdmap e321: 8 total, 8 up, 5 in 2023-09-23T22:28:31.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:31 smithi026 ceph-mon[215775]: failed to encode map e321 with expected crc 2023-09-23T22:28:31.486 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:31.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:31.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:31 smithi026 ceph-mon[223014]: pgmap v829: 99 pgs: 1 active+undersized+degraded+remapped+laggy, 1 activating+undersized+degraded+remapped, 1 active+recovering, 3 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 11 op/s; 19797/194295 objects degraded (10.189%); 18877/194295 objects misplaced (9.716%); 25 MiB/s, 393 objects/s recovering 2023-09-23T22:28:31.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:31 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:28:31.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:31 smithi026 ceph-mon[223014]: osdmap e321: 8 total, 8 up, 5 in 2023-09-23T22:28:31.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:31 smithi026 ceph-mon[223014]: failed to encode map e321 with expected crc 2023-09-23T22:28:31.590 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:31.600 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:28:31.633 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:31.676 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:31.738 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:31.768 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:31.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:31.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:31.882 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:31.907 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:31.948 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:31.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:31.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:32.053 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:32.094 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:32.415 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:32.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:32.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:32.520 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:32.555 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:33.066 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:33.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:33.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:33.174 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:33.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:33 smithi026 ceph-mon[215775]: pgmap v831: 99 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+laggy, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 7 op/s; 24054/194163 objects degraded (12.389%); 15584/194163 objects misplaced (8.026%); 11 MiB/s, 168 objects/s recovering 2023-09-23T22:28:33.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 24054/194163 objects degraded (12.389%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:33.218 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:33 smithi026 ceph-mon[223014]: pgmap v831: 99 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+laggy, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 7 op/s; 24054/194163 objects degraded (12.389%); 15584/194163 objects misplaced (8.026%); 11 MiB/s, 168 objects/s recovering 2023-09-23T22:28:33.219 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 24054/194163 objects degraded (12.389%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:33.224 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:33.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:33 smithi070 ceph-mon[192652]: pgmap v831: 99 pgs: 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+laggy, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 7 op/s; 24054/194163 objects degraded (12.389%); 15584/194163 objects misplaced (8.026%); 11 MiB/s, 168 objects/s recovering 2023-09-23T22:28:33.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 24054/194163 objects degraded (12.389%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:33.636 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:33.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:33.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:33.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:33.765 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:33.807 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:33.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:33.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:33.820 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:33.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:33.952 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:34.114 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:34.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:34.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:34.218 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:34.256 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:34.484 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:34.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:34.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:34.588 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:34.614 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:34.655 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:34.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:34.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:34.760 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:34.787 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:34.831 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:28:34.835 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:34.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:34.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:34.847 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:28:34.869 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:28:34.893 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:28:34.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:28:34.939 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:28:34.943 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:34.969 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:34.970 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:28:34.990 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:28:34.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:28:34.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:28:34.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:28:34.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:28:34.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:28:34.991 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:28:34.992 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:28:34.992 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:28:35.011 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:35.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:35.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:35.104 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:28:35.118 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:35.160 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:35.234 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:35 smithi026 ceph-mon[215775]: pgmap v832: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 7 op/s; 18035/194127 objects degraded (9.290%); 20041/194127 objects misplaced (10.324%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T22:28:35.234 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/292478035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:28:35.234 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:35 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:28:35.234 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3984685906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:35.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:35 smithi026 ceph-mon[223014]: pgmap v832: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 7 op/s; 18035/194127 objects degraded (9.290%); 20041/194127 objects misplaced (10.324%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T22:28:35.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/292478035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:28:35.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:35 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:28:35.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3984685906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:35.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:35 smithi070 ceph-mon[192652]: pgmap v832: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 4.0 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 7 op/s; 18035/194127 objects degraded (9.290%); 20041/194127 objects misplaced (10.324%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T22:28:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/292478035' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:28:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:35 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:28:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3984685906' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:36.092 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:36.092 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[223014]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.092 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[223014]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.092 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:28:36.092 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[223014]: osdmap e322: 8 total, 8 up, 5 in 2023-09-23T22:28:36.092 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[223014]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.092 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[223014]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:36.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[215775]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[215775]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:28:36.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[215775]: osdmap e322: 8 total, 8 up, 5 in 2023-09-23T22:28:36.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[215775]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.093 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:36 smithi026 ceph-mon[215775]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.229 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 1] out_osds: [6, 3, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:28:36.229 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:28:36.229 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:28:36.229 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.707903 2023-09-23T22:28:36.230 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 0.7079031548314793 2023-09-23T22:28:36.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:36.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:36.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:36.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:36 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:36 smithi070 ceph-mon[192652]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:36 smithi070 ceph-mon[192652]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:28:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:36 smithi070 ceph-mon[192652]: osdmap e322: 8 total, 8 up, 5 in 2023-09-23T22:28:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:36 smithi070 ceph-mon[192652]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:36 smithi070 ceph-mon[192652]: failed to encode map e322 with expected crc 2023-09-23T22:28:36.466 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:36.491 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:36.534 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:36.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:36.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:36.642 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:36.681 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:36.796 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:36.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:36.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:36.853 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:36.856 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:28:36.856 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:28:36.901 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:36.942 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:37.194 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:37.197 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:37 smithi026 ceph-mon[223014]: pgmap v834: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 62 op/s; 17962/192801 objects degraded (9.316%); 19921/192801 objects misplaced (10.332%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T22:28:37.197 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:37 smithi026 ceph-mon[215775]: pgmap v834: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 62 op/s; 17962/192801 objects degraded (9.316%); 19921/192801 objects misplaced (10.332%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T22:28:37.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:37.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:37.299 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:37.328 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:37.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:37 smithi070 ceph-mon[192652]: pgmap v834: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 62 op/s; 17962/192801 objects degraded (9.316%); 19921/192801 objects misplaced (10.332%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T22:28:37.397 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:37.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:37.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:37.507 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:37.532 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:37.575 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:37.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:37.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:37.681 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:37.709 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:37.753 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:37.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:37.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:37.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:28:37.863 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:37.929 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:38.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17962/192801 objects degraded (9.316%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:38.208 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:28:38.371 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:28:38.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17962/192801 objects degraded (9.316%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:38.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17962/192801 objects degraded (9.316%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:38.603 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:38.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:38.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:38.708 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:38.733 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:38.776 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:38.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:38.834 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:38.881 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:38.906 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:38.945 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:38.947 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:38.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:38.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:39.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:39.097 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:39.198 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:39.212 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:39.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:39.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:39.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:39 smithi026 ceph-mon[223014]: pgmap v835: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 62 op/s; 17962/192801 objects degraded (9.316%); 19921/192801 objects misplaced (10.332%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:28:39.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:39.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:39 smithi026 ceph-mon[215775]: pgmap v835: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 62 op/s; 17962/192801 objects degraded (9.316%); 19921/192801 objects misplaced (10.332%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:28:39.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:39.317 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:39.342 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:39 smithi070 ceph-mon[192652]: pgmap v835: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 62 op/s; 17962/192801 objects degraded (9.316%); 19921/192801 objects misplaced (10.332%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:28:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:39.384 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:39.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:39.490 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:39.516 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:39.560 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:39.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:39.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:39.669 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:39.696 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:39.740 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:39.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:39.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:39.805 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:28:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:28:39] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.43.0" 2023-09-23T22:28:39.848 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:39.873 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:39.916 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:39.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:39.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:40.023 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:40.050 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:40.091 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:40.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:40.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:40.115 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.1 primary-affinity to 0.707903 (8463932) 2023-09-23T22:28:40.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:40.223 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:40.264 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:40.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:40.371 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:40.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3985954304' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7079031548314793}]: dispatch 2023-09-23T22:28:40.409 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3985954304' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7079031548314793}]: dispatch 2023-09-23T22:28:40.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3985954304' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7079031548314793}]: dispatch 2023-09-23T22:28:40.412 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:40.519 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:40.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:40.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:40.623 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:40.649 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:40.695 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:40.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:40.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:40.801 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:40.833 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:41.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[215775]: pgmap v836: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 69 op/s; 17962/192318 objects degraded (9.340%); 19895/192318 objects misplaced (10.345%); 1008 KiB/s, 15 objects/s recovering 2023-09-23T22:28:41.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[215775]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3985954304' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7079031548314793}]': finished 2023-09-23T22:28:41.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[215775]: osdmap e323: 8 total, 8 up, 5 in 2023-09-23T22:28:41.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[215775]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[215775]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[215775]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.336 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[223014]: pgmap v836: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 69 op/s; 17962/192318 objects degraded (9.340%); 19895/192318 objects misplaced (10.345%); 1008 KiB/s, 15 objects/s recovering 2023-09-23T22:28:41.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[223014]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3985954304' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7079031548314793}]': finished 2023-09-23T22:28:41.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[223014]: osdmap e323: 8 total, 8 up, 5 in 2023-09-23T22:28:41.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[223014]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[223014]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:41 smithi026 ceph-mon[223014]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:41 smithi070 ceph-mon[192652]: pgmap v836: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 69 op/s; 17962/192318 objects degraded (9.340%); 19895/192318 objects misplaced (10.345%); 1008 KiB/s, 15 objects/s recovering 2023-09-23T22:28:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:41 smithi070 ceph-mon[192652]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3985954304' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.7079031548314793}]': finished 2023-09-23T22:28:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:41 smithi070 ceph-mon[192652]: osdmap e323: 8 total, 8 up, 5 in 2023-09-23T22:28:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:41 smithi070 ceph-mon[192652]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:41 smithi070 ceph-mon[192652]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:41 smithi070 ceph-mon[192652]: failed to encode map e323 with expected crc 2023-09-23T22:28:41.709 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:41.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:41.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:41.815 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:41.840 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:41.886 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:41.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:41.893 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:41.993 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:42.021 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:42.065 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:42.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:42.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:42.169 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:42.203 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:42.311 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:42.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:42.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:42 smithi070 ceph-mon[192652]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:42 smithi070 ceph-mon[192652]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:42 smithi070 ceph-mon[192652]: osdmap e324: 8 total, 8 up, 5 in 2023-09-23T22:28:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:42 smithi070 ceph-mon[192652]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:42 smithi070 ceph-mon[192652]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.417 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:42.443 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:42.483 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:42.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[223014]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[223014]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[223014]: osdmap e324: 8 total, 8 up, 5 in 2023-09-23T22:28:42.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[223014]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[223014]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[215775]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[215775]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[215775]: osdmap e324: 8 total, 8 up, 5 in 2023-09-23T22:28:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[215775]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:42 smithi026 ceph-mon[215775]: failed to encode map e324 with expected crc 2023-09-23T22:28:42.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:42.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:42.526 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:28:42.548 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:28:42.570 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:28:42.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:42.602 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:28:42.635 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:28:42.644 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:42.658 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:28:42.683 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:28:42.709 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:28:42.711 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:28:42.711 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:28:42.711 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:28:42.711 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:28:42.711 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:28:42.711 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:28:42.711 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:28:42.711 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:28:42.962 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:42.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:42.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:43.051 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:28:43.051 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:28:43.051 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:28:43.051 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:28:43.051 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:28:43.067 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:43.102 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:43.163 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:43.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:43.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:43.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:43 smithi026 ceph-mon[223014]: pgmap v839: 99 pgs: 4 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 128 op/s; 17898/191007 objects degraded (9.370%); 19770/191007 objects misplaced (10.350%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:28:43.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2583077700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:43.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17898/191007 objects degraded (9.370%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:43.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:43 smithi026 ceph-mon[215775]: pgmap v839: 99 pgs: 4 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 128 op/s; 17898/191007 objects degraded (9.370%); 19770/191007 objects misplaced (10.350%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:28:43.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2583077700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:43.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17898/191007 objects degraded (9.370%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:43.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:43.294 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:43.337 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:43.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:43.442 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:43.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:43 smithi070 ceph-mon[192652]: pgmap v839: 99 pgs: 4 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 128 op/s; 17898/191007 objects degraded (9.370%); 19770/191007 objects misplaced (10.350%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T22:28:43.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2583077700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:43.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17898/191007 objects degraded (9.370%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:43.469 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:43.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:43.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:43.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:43.618 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:43.648 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:43.706 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:43.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:43.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:43.810 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:43.835 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:43.878 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:43.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:43.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:43.982 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:44.013 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:44.147 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:28:44.160 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:44.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:44.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:44.165 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:28:44.266 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:44.293 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:44.335 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:44.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:44.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:44.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:44 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/11440849' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:44.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:44 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/11440849' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:44.442 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:44.478 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:44.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:44 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/11440849' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:44.785 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:44.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:44.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:44.891 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:44.916 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:44.952 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:44.958 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:44.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:44.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:45.066 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:45.092 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:45.127 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[215775]: pgmap v840: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 79 op/s; 17898/190899 objects degraded (9.376%); 19742/190899 objects misplaced (10.342%); 944 KiB/s, 14 objects/s recovering 2023-09-23T22:28:45.127 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:45.127 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/11440849' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:45.127 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[215775]: osdmap e325: 8 total, 8 up, 5 in 2023-09-23T22:28:45.127 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[215775]: failed to encode map e325 with expected crc 2023-09-23T22:28:45.127 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[215775]: failed to encode map e325 with expected crc 2023-09-23T22:28:45.133 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:45.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:45.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:45.242 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:45.277 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:45.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:45 smithi070 ceph-mon[192652]: pgmap v840: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 79 op/s; 17898/190899 objects degraded (9.376%); 19742/190899 objects misplaced (10.342%); 944 KiB/s, 14 objects/s recovering 2023-09-23T22:28:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:45 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/11440849' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:45 smithi070 ceph-mon[192652]: osdmap e325: 8 total, 8 up, 5 in 2023-09-23T22:28:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:45 smithi070 ceph-mon[192652]: failed to encode map e325 with expected crc 2023-09-23T22:28:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:45 smithi070 ceph-mon[192652]: failed to encode map e325 with expected crc 2023-09-23T22:28:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:45 smithi070 ceph-mon[192652]: failed to encode map e325 with expected crc 2023-09-23T22:28:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[223014]: pgmap v840: 99 pgs: 1 active+recovering+undersized+remapped, 4 peering, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 79 op/s; 17898/190899 objects degraded (9.376%); 19742/190899 objects misplaced (10.342%); 944 KiB/s, 14 objects/s recovering 2023-09-23T22:28:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/11440849' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[223014]: osdmap e325: 8 total, 8 up, 5 in 2023-09-23T22:28:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[223014]: failed to encode map e325 with expected crc 2023-09-23T22:28:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[223014]: failed to encode map e325 with expected crc 2023-09-23T22:28:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[223014]: failed to encode map e325 with expected crc 2023-09-23T22:28:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:45 smithi026 ceph-mon[215775]: failed to encode map e325 with expected crc 2023-09-23T22:28:45.546 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:45.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:45.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:45.627 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:28:45.650 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:45.676 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:45.718 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:45.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:45.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:45.823 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:45.892 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:45.935 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:45.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:45.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:46.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:46.081 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:46.157 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:46.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:46.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:46.200 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:46.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:46.305 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:46 smithi026 ceph-mon[223014]: failed to encode map e325 with expected crc 2023-09-23T22:28:46.305 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:46 smithi026 ceph-mon[215775]: failed to encode map e325 with expected crc 2023-09-23T22:28:46.309 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:46.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:46 smithi070 ceph-mon[192652]: failed to encode map e325 with expected crc 2023-09-23T22:28:46.419 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:46.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:46.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:46.525 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:46.551 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:46.594 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:46.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:46.651 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45002.0:93554 6.b 6:d2b835f0:::benchmark_data_smithi026_355573_object12204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e236)", 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:22:29.178872+0000", 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 377.46843414799997, 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0056578679999999, 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.652 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45002", 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/718616586", 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 93554 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178872+0000", 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178872+0000", 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178870+0000", 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178878+0000", 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3919999999999997e-06 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178894+0000", 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5838999999999999e-05 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.178903+0000", 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2730000000000008e-06 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.184954+0000", 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060513269999999996 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:29.184967+0000", 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2462e-05 2023-09-23T22:28:46.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.173392+0000", 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.988425043 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.173530+0000", 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000138851 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.173688+0000", 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000157379 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178167+0000", 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044787869999999997 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.178182+0000", 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5404999999999999e-05 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184445+0000", 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062626799999999996 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184491+0000", 2023-09-23T22:28:46.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6737999999999999e-05 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:22:30.184530+0000", 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8407e-05 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.660 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83436 7.2 7:4436d6c5:::benchmark_data_smithi026_410749_object83435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.874035+0000", 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.77327143599999, 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8534463419999998, 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83436 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.661 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874035+0000", 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874035+0000", 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874034+0000", 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874054+0000", 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0642e-05 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874063+0000", 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9560000000000003e-06 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874069+0000", 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2400000000000004e-06 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.109468+0000", 2023-09-23T22:28:46.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23539858499999999 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.109634+0000", 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016552 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.109834+0000", 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000200714 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119592+0000", 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097577359999999995 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119603+0000", 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.13e-05 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.965993+0000", 2023-09-23T22:28:46.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84638983099999998 2023-09-23T22:28:46.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966014+0000", 2023-09-23T22:28:46.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0774000000000001e-05 2023-09-23T22:28:46.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.688706+0000", 2023-09-23T22:28:46.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.722691585 2023-09-23T22:28:46.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694732+0000", 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060268170000000003 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.727365+0000", 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032632845000000001 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.727481+0000", 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011591600000000001 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83439 7.2 7:41661071:::benchmark_data_smithi026_410749_object83438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875790+0000", 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.77151633, 2023-09-23T22:28:46.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8703235449999998, 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83439 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875790+0000", 2023-09-23T22:28:46.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875790+0000", 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875789+0000", 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875812+0000", 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3323999999999999e-05 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875835+0000", 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.317e-05 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875844+0000", 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4370000000000007e-06 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.113024+0000", 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.237180689 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.113148+0000", 2023-09-23T22:28:46.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012339600000000001 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.113277+0000", 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129271 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119636+0000", 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063592079999999999 2023-09-23T22:28:46.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119647+0000", 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0547000000000001e-05 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966064+0000", 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84641742499999995 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966076+0000", 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1899000000000001e-05 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.699874+0000", 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.733797451 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.702993+0000", 2023-09-23T22:28:46.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031193470000000002 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746002+0000", 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.04300913 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746113+0000", 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011135 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.679 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83442 7.c 7:3b003b51:::benchmark_data_smithi026_410749_object83441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.77145579500001, 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8103755399999999, 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.680 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83442 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875871+0000", 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0897999999999999e-05 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875878+0000", 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0140000000000001e-06 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875883+0000", 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4360000000000001e-06 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875932+0000", 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8757000000000001e-05 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.876054+0000", 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122371 2023-09-23T22:28:46.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.110153+0000", 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23409861300000001 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119005+0000", 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088520830000000002 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119022+0000", 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6586999999999999e-05 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967848+0000", 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84882608400000004 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967861+0000", 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3498e-05 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.653928+0000", 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6860671979999999 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.654575+0000", 2023-09-23T22:28:46.687 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00064664599999999996 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686123+0000", 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031547598000000003 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686226+0000", 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000103359 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.688 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83443 7.a 7:5676589a:::benchmark_data_smithi026_410749_object83442:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.77140889500001, 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8791883110000001, 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.689 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83443 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.690 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875917+0000", 2023-09-23T22:28:46.691 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0571e-05 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875925+0000", 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4800000000000004e-06 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875930+0000", 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2549999999999997e-06 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.692 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120311+0000", 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.24438064000000001 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120321+0000", 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0532e-05 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.693 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966151+0000", 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84582989600000003 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966161+0000", 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0161e-05 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713553+0000", 2023-09-23T22:28:46.694 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7473917839999999 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713688+0000", 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013500599999999999 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713846+0000", 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015831800000000001 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.695 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720327+0000", 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064802289999999997 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723835+0000", 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035083179999999999 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.696 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.752272+0000", 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028437240999999999 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.752361+0000", 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8768000000000001e-05 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.755086+0000", 2023-09-23T22:28:46.697 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027245949999999998 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83444 7.2 7:4ab81381:::benchmark_data_smithi026_410749_object83443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.77137117699999, 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8732291189999999, 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.698 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83444 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.699 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875955+0000", 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0437000000000001e-05 2023-09-23T22:28:46.700 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875962+0000", 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2590000000000003e-06 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875968+0000", 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5539999999999997e-06 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.701 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120681+0000", 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.24471289900000001 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120690+0000", 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4579999999999999e-06 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.702 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966109+0000", 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84541897700000002 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966119+0000", 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4369999999999996e-06 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703041+0000", 2023-09-23T22:28:46.703 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.73692275 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703182+0000", 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014080000000000001 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703342+0000", 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016031500000000001 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.704 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713519+0000", 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010176648 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723879+0000", 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010359706 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.705 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746208+0000", 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022329188999999999 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746255+0000", 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6820000000000002e-05 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.749164+0000", 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029093589999999998 2023-09-23T22:28:46.706 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83449 7.c 7:31b18e39:::benchmark_data_smithi026_410749_object83448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.118334+0000", 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.528971746, 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5715964209999997, 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.707 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83449 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118334+0000", 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.708 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118334+0000", 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118333+0000", 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118364+0000", 2023-09-23T22:28:46.709 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1628999999999998e-05 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118376+0000", 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1888e-05 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118385+0000", 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4829999999999999e-06 2023-09-23T22:28:46.710 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119686+0000", 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001300998 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119696+0000", 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0208e-05 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.711 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967893+0000", 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84819666800000004 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967901+0000", 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8619999999999992e-06 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.712 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.657644+0000", 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6897430070000001 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.657855+0000", 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021073899999999999 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.658074+0000", 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021842699999999999 2023-09-23T22:28:46.713 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664296+0000", 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062224689999999996 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.667607+0000", 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033106160000000002 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.714 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686269+0000", 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018662099000000001 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686317+0000", 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8285999999999998e-05 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.715 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.689931+0000", 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036137999999999999 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83454 7.a 7:5caa08ff:::benchmark_data_smithi026_410749_object83453:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.125118+0000", 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.52218774799999, 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.634736432, 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.716 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83454 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125118+0000", 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.717 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125118+0000", 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125116+0000", 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.718 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125144+0000", 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8147999999999999e-05 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125157+0000", 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2418999999999999e-05 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.719 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125167+0000", 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0088999999999999e-05 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125218+0000", 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1601000000000003e-05 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.720 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125225+0000", 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0539999999999997e-06 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966187+0000", 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84096177699999997 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966197+0000", 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3619999999999995e-06 2023-09-23T22:28:46.721 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.716965+0000", 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7507684270000001 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.717105+0000", 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013953199999999999 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.722 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.717236+0000", 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131901 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720350+0000", 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031134539999999999 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.723 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.734562+0000", 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014212158000000001 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.756983+0000", 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022420549000000001 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.757031+0000", 2023-09-23T22:28:46.724 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8467999999999997e-05 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.759855+0000", 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002823791 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83455 7.c 7:30f89560:::benchmark_data_smithi026_410749_object83454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:28:46.725 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.128196+0000", 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.519110098, 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5655597960000001, 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83455 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.726 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128196+0000", 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128196+0000", 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.727 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128195+0000", 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128218+0000", 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3824000000000001e-05 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128235+0000", 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6864e-05 2023-09-23T22:28:46.728 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128245+0000", 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6709999999999994e-06 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128304+0000", 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9104000000000002e-05 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.729 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128316+0000", 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1858999999999999e-05 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967930+0000", 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83961362900000003 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967939+0000", 2023-09-23T22:28:46.730 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0000000000000002e-06 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664326+0000", 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.696387901 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664467+0000", 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140433 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.731 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664596+0000", 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129345 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.667813+0000", 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032163579999999999 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.732 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.671373+0000", 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035605559999999999 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.690178+0000", 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018804521000000001 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.690239+0000", 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1784000000000003e-05 2023-09-23T22:28:46.733 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.693756+0000", 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035164060000000001 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83459 7.c 7:389583a7:::benchmark_data_smithi026_410749_object83458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.131267+0000", 2023-09-23T22:28:46.734 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.516038675, 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5663957699999997, 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83459 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.735 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131267+0000", 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131267+0000", 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131266+0000", 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:28:46.736 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131287+0000", 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0669000000000001e-05 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131296+0000", 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3179999999999996e-06 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.737 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131303+0000", 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3890000000000002e-06 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131356+0000", 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2883999999999997e-05 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.738 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131367+0000", 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1501000000000001e-05 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.968650+0000", 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83728242900000005 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.968662+0000", 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2218999999999999e-05 2023-09-23T22:28:46.739 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.667852+0000", 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6991896950000001 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.668041+0000", 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018923599999999999 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.740 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.668259+0000", 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000218063 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.671597+0000", 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033378510000000002 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.675784+0000", 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004187362 2023-09-23T22:28:46.741 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.693833+0000", 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018049193000000002 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.693899+0000", 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5381000000000003e-05 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.742 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.697663+0000", 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037646799999999998 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83460 7.2 7:4c76e5c3:::benchmark_data_smithi026_410749_object83459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 159.515409836, 2023-09-23T22:28:46.743 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6264806219999999, 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83460 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:28:46.744 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.745 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131915+0000", 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9627999999999999e-05 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131927+0000", 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2150999999999999e-05 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131933+0000", 2023-09-23T22:28:46.746 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2390000000000002e-06 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131970+0000", 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7323000000000002e-05 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131979+0000", 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1439999999999992e-06 2023-09-23T22:28:46.747 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966248+0000", 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83426858500000001 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966259+0000", 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0813e-05 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.748 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.709897+0000", 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.743638695 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.710090+0000", 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000192552 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.710317+0000", 2023-09-23T22:28:46.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022765700000000001 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720308+0000", 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099907369999999995 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723922+0000", 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036138960000000001 2023-09-23T22:28:46.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.755137+0000", 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031214848999999999 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.755202+0000", 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4978e-05 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.758377+0000", 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031750530000000002 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83432 7.8 7:102b4fc4:::benchmark_data_smithi026_410749_object83431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978262+0000", 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.66904455100001, 2023-09-23T22:28:46.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7098934809999999, 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83432 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978262+0000", 2023-09-23T22:28:46.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978262+0000", 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978260+0000", 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:46.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978316+0000", 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6307000000000001e-05 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978331+0000", 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4329e-05 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978341+0000", 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0179e-05 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978398+0000", 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7342000000000001e-05 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978405+0000", 2023-09-23T22:28:46.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2559999999999996e-06 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686224+0000", 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7078187279999999 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.688155+0000", 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019310600000000001 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.757 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83434 7.8 7:1cb65715:::benchmark_data_smithi026_410749_object83433:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978364+0000", 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.66894217800001, 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7126752629999999, 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.758 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83434 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978364+0000", 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978364+0000", 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978363+0000", 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978410+0000", 2023-09-23T22:28:46.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6882999999999998e-05 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978421+0000", 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0555e-05 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978428+0000", 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.447e-06 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978481+0000", 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2367000000000002e-05 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978491+0000", 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0348e-05 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.690940+0000", 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.712448669 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691039+0000", 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9671000000000002e-05 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.763 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83451 7.8 7:10ddb7e8:::benchmark_data_smithi026_410749_object83450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978444+0000", 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.66886234500001, 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7421942690000001, 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83451 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.764 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978444+0000", 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978444+0000", 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978443+0000", 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978490+0000", 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7082000000000001e-05 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978500+0000", 2023-09-23T22:28:46.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9930000000000007e-06 2023-09-23T22:28:46.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.767 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:46.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978507+0000", 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8789999999999997e-06 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978649+0000", 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014189399999999999 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978658+0000", 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2750000000000001e-06 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691064+0000", 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.712405483 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691807+0000", 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00074337199999999996 2023-09-23T22:28:46.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691974+0000", 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016714799999999999 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.698213+0000", 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006238848 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.704903+0000", 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066894340000000002 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705854+0000", 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00095168 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.718111+0000", 2023-09-23T22:28:46.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012256306999999999 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.718166+0000", 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5387999999999999e-05 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720638+0000", 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024721159999999999 2023-09-23T22:28:46.773 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83458 7.8 7:14196f44:::benchmark_data_smithi026_410749_object83457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.66878403800001, 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.744606302, 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.774 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83458 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978566+0000", 2023-09-23T22:28:46.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4564000000000002e-05 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978585+0000", 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8978999999999999e-05 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978592+0000", 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1250000000000004e-06 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978682+0000", 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0093999999999995e-05 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978690+0000", 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3200000000000002e-06 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694418+0000", 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.715727904 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694629+0000", 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021179900000000001 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694812+0000", 2023-09-23T22:28:46.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018232099999999999 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.701814+0000", 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070020819999999998 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705861+0000", 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040473699999999998 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.711791+0000", 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059297259999999997 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720680+0000", 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088894569999999999 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720735+0000", 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5049000000000001e-05 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723128+0000", 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002393109 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.782 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114666 7.a 7:5b4d1c8f:::benchmark_data_smithi026_410749_object24329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.103730+0000", 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 80.543576111999997, 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0325042339999999, 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114666 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103730+0000", 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103730+0000", 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103729+0000", 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103735+0000", 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.331e-06 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103747+0000", 2023-09-23T22:28:46.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1623999999999999e-05 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103755+0000", 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2430000000000005e-06 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.130376+0000", 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026621280000000001 2023-09-23T22:28:46.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.130574+0000", 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019795899999999999 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.135541+0000", 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049666739999999999 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141802+0000", 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062614180000000004 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.144696+0000", 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028941489999999999 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.283811+0000", 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.139114184 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.283833+0000", 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1937e-05 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.136221+0000", 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.85238819899999996 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.136234+0000", 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3553000000000001e-05 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114679 7.3 7:c9482514:::benchmark_data_smithi026_410749_object24342:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.138279+0000", 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 80.509027024999995, 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99640990200000001, 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.791 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114679 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138279+0000", 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138279+0000", 2023-09-23T22:28:46.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138278+0000", 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138285+0000", 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4590000000000001e-06 2023-09-23T22:28:46.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138298+0000", 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3152999999999999e-05 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138307+0000", 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7600000000000008e-06 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138382+0000", 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4432999999999997e-05 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138592+0000", 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020998499999999999 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:28:46.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138787+0000", 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019538800000000001 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.286686+0000", 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.147899482 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.286694+0000", 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7000000000000008e-06 2023-09-23T22:28:46.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.134671+0000", 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84797734300000005 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.134689+0000", 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7621e-05 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.797 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114682 7.3 7:c74ef5a2:::benchmark_data_smithi026_410749_object24345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.138323+0000", 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 80.508983203, 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99642287399999996, 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.798 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114682 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138323+0000", 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138323+0000", 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138322+0000", 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138326+0000", 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1779999999999999e-06 2023-09-23T22:28:46.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138333+0000", 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5440000000000001e-06 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138339+0000", 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8529999999999997e-06 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141825+0000", 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034864309999999999 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.142018+0000", 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019217 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.142174+0000", 2023-09-23T22:28:46.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015678799999999999 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.292592+0000", 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.15041737199999999 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.292603+0000", 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.128e-05 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.134733+0000", 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84213020299999997 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.134746+0000", 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2549e-05 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114854 7.a 7:514305bc:::benchmark_data_smithi026_410749_object24517:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e305)", 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:52.042477+0000", 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 54.604829664999997, 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.374932499, 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114854 2023-09-23T22:28:46.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042477+0000", 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042477+0000", 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042475+0000", 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042482+0000", 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6210000000000004e-06 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042496+0000", 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3213e-05 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042508+0000", 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2310000000000001e-05 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043832+0000", 2023-09-23T22:28:46.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013236859999999999 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043845+0000", 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3722e-05 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.392953+0000", 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3491075079999999 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393122+0000", 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016948199999999999 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.417296+0000", 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024174047000000001 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.417409+0000", 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112722 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114867 7.a 7:5bbaaced:::benchmark_data_smithi026_410749_object24530:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e305)", 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:52.042515+0000", 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 54.604791155000001, 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3830621110000001, 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:28:46.811 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114867 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042515+0000", 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042515+0000", 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042514+0000", 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:28:46.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042518+0000", 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0249999999999998e-06 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042525+0000", 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1919999999999999e-06 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042531+0000", 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8989999999999998e-06 2023-09-23T22:28:46.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043871+0000", 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00134023 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043879+0000", 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7200000000000006e-06 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393153+0000", 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3492744459999999 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393299+0000", 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145838 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T22:28:46.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393456+0000", 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000156928 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.400332+0000", 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068758229999999997 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.400343+0000", 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0963e-05 2023-09-23T22:28:46.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.417452+0000", 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017109017000000001 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421935+0000", 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044831059999999997 2023-09-23T22:28:46.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421995+0000", 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9889000000000002e-05 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.425577+0000", 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035825710000000001 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.819 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.820 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.820 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:28:46.820 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:28:46.820 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:28:46.821 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:46.836 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:46.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:46.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:46.846 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:28:46.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:46.976 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:47.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:47 smithi026 ceph-mon[223014]: pgmap v842: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 170 op/s; 17829/189246 objects degraded (9.421%); 19595/189246 objects misplaced (10.354%); 1.7 MiB/s, 25 objects/s recovering 2023-09-23T22:28:47.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:47 smithi026 ceph-mon[215775]: pgmap v842: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 170 op/s; 17829/189246 objects degraded (9.421%); 19595/189246 objects misplaced (10.354%); 1.7 MiB/s, 25 objects/s recovering 2023-09-23T22:28:47.316 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 1] out_osds: [6, 3, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:28:47.316 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:28:47.317 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T22:28:47.317 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:28:47.317 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:28:47.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T22:28:47.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:47 smithi070 ceph-mon[192652]: pgmap v842: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 170 op/s; 17829/189246 objects degraded (9.421%); 19595/189246 objects misplaced (10.354%); 1.7 MiB/s, 25 objects/s recovering 2023-09-23T22:28:47.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:28:47.897 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:47.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:47.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:48.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:48.068 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:48.114 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:48.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:48.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:48.136 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:28:48.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:48.261 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:48.262 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:48.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:48.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:48.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:48.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:48 smithi070 ceph-mon[192652]: pgmap v843: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 17829/189246 objects degraded (9.421%); 19595/189246 objects misplaced (10.354%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:28:48.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17829/189246 objects degraded (9.421%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:48.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:48 smithi026 ceph-mon[215775]: pgmap v843: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 17829/189246 objects degraded (9.421%); 19595/189246 objects misplaced (10.354%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:28:48.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17829/189246 objects degraded (9.421%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:48.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:48 smithi026 ceph-mon[223014]: pgmap v843: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.9 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 17829/189246 objects degraded (9.421%); 19595/189246 objects misplaced (10.354%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T22:28:48.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17829/189246 objects degraded (9.421%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:48.444 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:48.446 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:48.492 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:48.644 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:28:48.672 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:28:48.698 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:28:48.709 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:48.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:48.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:48.725 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:28:48.749 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:28:48.781 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:28:48.812 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:28:48.814 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:48.842 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:28:48.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:28:48.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:28:48.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:28:48.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:28:48.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:28:48.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:28:48.844 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:28:48.845 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:28:48.866 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:49.096 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:49.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:49.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:49.204 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:49.236 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:49 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2861926275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:49.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:49 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2861926275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:49.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:49 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2861926275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:28:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:28:49] "GET /metrics HTTP/1.1" 200 36415 "" "Prometheus/2.43.0" 2023-09-23T22:28:50.094 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:50.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:50.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:50.199 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:50.236 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:50.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:50 smithi026 ceph-mon[223014]: pgmap v844: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 79 op/s; 17829/189027 objects degraded (9.432%); 19573/189027 objects misplaced (10.355%); 919 KiB/s, 13 objects/s recovering 2023-09-23T22:28:50.423 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:50 smithi026 ceph-mon[215775]: pgmap v844: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 79 op/s; 17829/189027 objects degraded (9.432%); 19573/189027 objects misplaced (10.355%); 919 KiB/s, 13 objects/s recovering 2023-09-23T22:28:50.580 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:50 smithi070 ceph-mon[192652]: pgmap v844: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 79 op/s; 17829/189027 objects degraded (9.432%); 19573/189027 objects misplaced (10.355%); 919 KiB/s, 13 objects/s recovering 2023-09-23T22:28:50.706 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:50.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:50.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:50.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:28:50.854 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:51.063 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:28:51.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:51.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:28:51.084 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:28:51.084 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:28:51.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:51.221 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:51.302 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:28:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "" 2023-09-23T22:28:51.302 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:28:51.304 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:51.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:51.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:51.408 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:51.446 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:28:51.447 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:51.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:51 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3925266288' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:51 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:51.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:28:51.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:51 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3925266288' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:51.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:51 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:51.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:51 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3925266288' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:51.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:51 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:28:52.153 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:28:52.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:52.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:28:52.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[223014]: osdmap e326: 8 total, 8 up, 5 in 2023-09-23T22:28:52.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[223014]: failed to encode map e326 with expected crc 2023-09-23T22:28:52.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[223014]: failed to encode map e326 with expected crc 2023-09-23T22:28:52.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[223014]: failed to encode map e326 with expected crc 2023-09-23T22:28:52.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[223014]: pgmap v846: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 148 op/s; 17758/187326 objects degraded (9.480%); 19440/187326 objects misplaced (10.378%); 1.3 MiB/s, 19 objects/s recovering 2023-09-23T22:28:52.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:52.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:28:52.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[215775]: osdmap e326: 8 total, 8 up, 5 in 2023-09-23T22:28:52.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[215775]: failed to encode map e326 with expected crc 2023-09-23T22:28:52.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[215775]: failed to encode map e326 with expected crc 2023-09-23T22:28:52.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[215775]: failed to encode map e326 with expected crc 2023-09-23T22:28:52.451 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:52 smithi026 ceph-mon[215775]: pgmap v846: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 148 op/s; 17758/187326 objects degraded (9.480%); 19440/187326 objects misplaced (10.378%); 1.3 MiB/s, 19 objects/s recovering 2023-09-23T22:28:52.569 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:52.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:52.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:52.653 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:52 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:28:52.653 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:52 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:28:52.653 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:52 smithi070 ceph-mon[192652]: osdmap e326: 8 total, 8 up, 5 in 2023-09-23T22:28:52.653 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:52 smithi070 ceph-mon[192652]: failed to encode map e326 with expected crc 2023-09-23T22:28:52.653 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:52 smithi070 ceph-mon[192652]: failed to encode map e326 with expected crc 2023-09-23T22:28:52.654 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:52 smithi070 ceph-mon[192652]: failed to encode map e326 with expected crc 2023-09-23T22:28:52.654 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:52 smithi070 ceph-mon[192652]: pgmap v846: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 148 op/s; 17758/187326 objects degraded (9.480%); 19440/187326 objects misplaced (10.378%); 1.3 MiB/s, 19 objects/s recovering 2023-09-23T22:28:52.677 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:52.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:28:53.258 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:53.267 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:53.352 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:53.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:53.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:53.456 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:53.490 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:53.551 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:53.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:53.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17758/187326 objects degraded (9.480%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:53.655 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:53.689 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:53.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17758/187326 objects degraded (9.480%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:53.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17758/187326 objects degraded (9.480%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:53.903 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:28:53.903 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:28:53.950 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:53.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:53.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:54.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:28:54.083 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:54.402 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:28:54.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:54.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:28:54.508 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:54.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:54.554 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:28:54.571 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:28:54.585 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:54.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:54.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:54.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:54 smithi026 ceph-mon[215775]: pgmap v847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 126 op/s; 17758/187245 objects degraded (9.484%); 19414/187245 objects misplaced (10.368%); 1.3 MiB/s, 19 objects/s recovering 2023-09-23T22:28:54.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:54.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:54 smithi026 ceph-mon[223014]: pgmap v847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 126 op/s; 17758/187245 objects degraded (9.484%); 19414/187245 objects misplaced (10.368%); 1.3 MiB/s, 19 objects/s recovering 2023-09-23T22:28:54.666 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:54.690 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:54.725 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:54.834 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:54.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:54.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:54 smithi070 ceph-mon[192652]: pgmap v847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 126 op/s; 17758/187245 objects degraded (9.484%); 19414/187245 objects misplaced (10.368%); 1.3 MiB/s, 19 objects/s recovering 2023-09-23T22:28:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:28:54.931 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:28:54.938 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:54.960 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:28:54.973 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:28:54.990 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:54.999 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:28:55.032 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:28:55.041 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:55.056 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:28:55.059 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:55.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:55.083 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:28:55.109 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:28:55.138 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:28:55.139 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:28:55.140 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:28:55.140 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:28:55.140 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:28:55.140 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:28:55.140 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:28:55.140 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:28:55.140 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:28:55.164 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:55.193 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:55.240 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:28:55.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:55.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:28:55.344 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:55.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:55.518 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:55.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:55.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:55.622 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:55.649 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:55.694 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:55.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:55.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:55.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/449913382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:55.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/449913382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:55.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:55.820 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/449913382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:28:55.822 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:56.036 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:56.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:56.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:56.139 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:56.176 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:56.341 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:56.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:56.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:56.445 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:56.477 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:56.523 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:56.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:56.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:56.599 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:28:56.599 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:28:56.599 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:28:56.599 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:28:56.599 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.599 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161176 5.9 5:945d6bac:::benchmark_data_smithi026_286012_object67368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:28:56.599 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.166084+0000", 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 595.42958553000005, 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99363560299999998, 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161176 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.600 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166084+0000", 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166084+0000", 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.601 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166082+0000", 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166091+0000", 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7220000000000005e-06 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166109+0000", 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8105e-05 2023-09-23T22:28:56.602 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166116+0000", 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4440000000000002e-06 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166180+0000", 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3662000000000001e-05 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.603 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.166194+0000", 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4032000000000001e-05 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.156511+0000", 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99031726200000003 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.604 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.156629+0000", 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011803600000000001 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.156707+0000", 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7717000000000001e-05 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159591+0000", 2023-09-23T22:28:56.605 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002884148 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159602+0000", 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1090999999999999e-05 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159659+0000", 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7086000000000002e-05 2023-09-23T22:28:56.606 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159698+0000", 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8359000000000003e-05 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159720+0000", 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1844e-05 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.607 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161183 5.9 5:9d2505ba:::benchmark_data_smithi026_286012_object67375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.174941+0000", 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 595.42072873699999, 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99311035800000003, 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:28:56.608 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161183 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174941+0000", 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174941+0000", 2023-09-23T22:28:56.609 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174940+0000", 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174956+0000", 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6464e-05 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.610 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174973+0000", 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6818000000000001e-05 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.174978+0000", 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5899999999999998e-06 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.611 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.175022+0000", 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4005999999999998e-05 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.175030+0000", 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1430000000000001e-06 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159758+0000", 2023-09-23T22:28:56.612 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98472881700000003 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159842+0000", 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3813999999999994e-05 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.159906+0000", 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3689000000000006e-05 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.167923+0000", 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0080168219999999998 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.167986+0000", 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2785000000000003e-05 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.167992+0000", 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7510000000000002e-06 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168025+0000", 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2543999999999999e-05 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168051+0000", 2023-09-23T22:28:56.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6344000000000001e-05 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161184 5.9 5:9ee6e8a5:::benchmark_data_smithi026_286012_object67376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.184105+0000", 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 595.41156461100002, 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98725831399999997, 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.616 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.617 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:28:56.617 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:28:56.617 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161184 2023-09-23T22:28:56.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.617 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184105+0000", 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184105+0000", 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184104+0000", 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:28:56.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184109+0000", 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2270000000000001e-06 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184123+0000", 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3202e-05 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.619 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184127+0000", 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7500000000000003e-06 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184175+0000", 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7769000000000003e-05 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.184184+0000", 2023-09-23T22:28:56.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9900000000000003e-06 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.162842+0000", 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97865803299999998 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.162967+0000", 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012465299999999999 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.163088+0000", 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012078699999999999 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168063+0000", 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049753080000000003 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.168098+0000", 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4718000000000002e-05 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171274+0000", 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031765180000000001 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171324+0000", 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9665e-05 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171363+0000", 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9535e-05 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.34674.0:161224 5.9 5:9cc03ca7:::benchmark_data_smithi026_286012_object67416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T22:28:56.624 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:19:01.255475+0000", 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 595.340194494, 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91600351700000004, 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.34674", 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2373899709", 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 161224 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255475+0000", 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255475+0000", 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255474+0000", 2023-09-23T22:28:56.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255482+0000", 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8900000000000007e-06 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255495+0000", 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3318e-05 2023-09-23T22:28:56.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255501+0000", 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9490000000000001e-06 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255544+0000", 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3299e-05 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:01.255551+0000", 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7850000000000003e-06 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.165528+0000", 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90997731400000004 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.165626+0000", 2023-09-23T22:28:56.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7516000000000006e-05 2023-09-23T22:28:56.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:28:56.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.165692+0000", 2023-09-23T22:28:56.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5684000000000003e-05 2023-09-23T22:28:56.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171210+0000", 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055180790000000004 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171245+0000", 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5228999999999998e-05 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171403+0000", 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015777799999999999 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171445+0000", 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2277000000000002e-05 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:19:02.171479+0000", 2023-09-23T22:28:56.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3555e-05 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103831 7.1 7:8db69374:::benchmark_data_smithi026_410749_object13494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.108957+0000", 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.486712122, 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3785559219999999, 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.633 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103831 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.108957+0000", 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.108957+0000", 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.108956+0000", 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.108964+0000", 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8890000000000002e-06 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.108978+0000", 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4202000000000001e-05 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.108985+0000", 2023-09-23T22:28:56.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0520000000000004e-06 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109103+0000", 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000118044 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109120+0000", 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6636000000000001e-05 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984819+0000", 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87569942700000003 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984832+0000", 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2116000000000001e-05 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.469088+0000", 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.484256794 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.469204+0000", 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011589199999999999 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.469297+0000", 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2349999999999995e-05 2023-09-23T22:28:56.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.471894+0000", 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025973039999999999 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.471903+0000", 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8349999999999993e-06 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.482954+0000", 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011051442999999999 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.487394+0000", 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044394329999999996 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.487433+0000", 2023-09-23T22:28:56.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9190999999999998e-05 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.487513+0000", 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0607999999999998e-05 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103832 7.1 7:886ceb7b:::benchmark_data_smithi026_410749_object13495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T22:28:56.642 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.108998+0000", 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.486671728, 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3913696440000001, 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103832 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.108998+0000", 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.108998+0000", 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.108998+0000", 2023-09-23T22:28:56.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109000+0000", 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4980000000000001e-06 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109005+0000", 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2510000000000002e-06 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109009+0000", 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.569e-06 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109162+0000", 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000153296 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109176+0000", 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3393e-05 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984857+0000", 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87568148199999996 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984864+0000", 2023-09-23T22:28:56.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4529999999999999e-06 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.471920+0000", 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4870565339999999 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.472004+0000", 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3671999999999994e-05 2023-09-23T22:28:56.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.472082+0000", 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8305000000000003e-05 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.476609+0000", 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045272029999999996 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.476759+0000", 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015019799999999999 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.487321+0000", 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010561434 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.500233+0000", 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012911911 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.500272+0000", 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.947e-05 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.500367+0000", 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5253000000000004e-05 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103834 7.1 7:812b6642:::benchmark_data_smithi026_410749_object13497:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.109034+0000", 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.486635268, 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3914885219999999, 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.652 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103834 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109034+0000", 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109034+0000", 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109034+0000", 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.654 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109036+0000", 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.137e-06 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109040+0000", 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.377e-06 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109042+0000", 2023-09-23T22:28:56.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5799999999999999e-06 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109213+0000", 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000170395 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.109223+0000", 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0319999999999999e-05 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984877+0000", 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87565365799999995 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984880+0000", 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5389999999999999e-06 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.473565+0000", 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.488685134 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.473639+0000", 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3986000000000006e-05 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.473716+0000", 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7318999999999996e-05 2023-09-23T22:28:56.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.476767+0000", 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030509040000000001 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.484410+0000", 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0076426189999999998 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.487361+0000", 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029508160000000002 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.500412+0000", 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013051599000000001 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.500447+0000", 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4557000000000001e-05 2023-09-23T22:28:56.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.500523+0000", 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5847999999999996e-05 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103856 7.1 7:837d67bd:::benchmark_data_smithi026_410749_object13519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.661 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.212998+0000", 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.38267115799999, 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2884717549999998, 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103856 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.212998+0000", 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.212998+0000", 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.212997+0000", 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:56.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213004+0000", 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9689999999999997e-06 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213018+0000", 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4005000000000001e-05 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213025+0000", 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.635e-06 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213073+0000", 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7942000000000003e-05 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213080+0000", 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6059999999999997e-06 2023-09-23T22:28:56.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984900+0000", 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.77181938299999997 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984907+0000", 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1760000000000004e-06 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.475237+0000", 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.49033013 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.475307+0000", 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0347999999999998e-05 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.475382+0000", 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5098000000000005e-05 2023-09-23T22:28:56.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.479929+0000", 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045469009999999999 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494049+0000", 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014119526 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.500570+0000", 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006520858 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.501377+0000", 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00080741899999999998 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.501405+0000", 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7569999999999999e-05 2023-09-23T22:28:56.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.501470+0000", 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5594999999999998e-05 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103860 7.1 7:86638d28:::benchmark_data_smithi026_410749_object13523:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.670 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.213034+0000", 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.382635118, 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2969892989999998, 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103860 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213034+0000", 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213034+0000", 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213034+0000", 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:56.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213036+0000", 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2400000000000002e-06 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213040+0000", 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3409999999999999e-06 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213043+0000", 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0599999999999999e-06 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213111+0000", 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7806999999999996e-05 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.213120+0000", 2023-09-23T22:28:56.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6979999999999993e-06 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984930+0000", 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.77180985599999996 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984934+0000", 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1989999999999999e-06 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.476796+0000", 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.491861232 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.476886+0000", 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0684000000000001e-05 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.476977+0000", 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1261000000000006e-05 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.482961+0000", 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059839230000000004 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494091+0000", 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011129238 2023-09-23T22:28:56.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.500596+0000", 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0065057709999999996 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.509764+0000", 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0091673669999999992 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.509849+0000", 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5421999999999996e-05 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.510024+0000", 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017455899999999999 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.679 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103868 7.1 7:8eaeaa3d:::benchmark_data_smithi026_410749_object13531:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.247080+0000", 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.34858932900001, 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2768763519999999, 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103868 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.247080+0000", 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.247080+0000", 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.247078+0000", 2023-09-23T22:28:56.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.247090+0000", 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2408e-05 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.247122+0000", 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2205000000000001e-05 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.247133+0000", 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0882e-05 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.247195+0000", 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1629000000000002e-05 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.247207+0000", 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1796e-05 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984954+0000", 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73774721899999995 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984958+0000", 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7390000000000001e-06 2023-09-23T22:28:56.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.478303+0000", 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4933457409999999 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.478382+0000", 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8288999999999995e-05 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.478458+0000", 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6339000000000002e-05 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.484418+0000", 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059602170000000003 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494117+0000", 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096985030000000007 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.510084+0000", 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015967315999999999 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.523762+0000", 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013677814 2023-09-23T22:28:56.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.523823+0000", 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0924000000000001e-05 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.523957+0000", 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013376600000000001 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.688 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103889 7.1 7:8cffea8f:::benchmark_data_smithi026_410749_object13552:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.312120+0000", 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.283549753, 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2120587880000002, 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103889 2023-09-23T22:28:56.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312120+0000", 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312120+0000", 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312118+0000", 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312133+0000", 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5139e-05 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312161+0000", 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8639000000000001e-05 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312172+0000", 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0577000000000001e-05 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312235+0000", 2023-09-23T22:28:56.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2589999999999995e-05 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312244+0000", 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6930000000000002e-06 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984969+0000", 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67272521900000004 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984973+0000", 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4240000000000002e-06 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.479965+0000", 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.494992331 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.480042+0000", 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6616e-05 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.480119+0000", 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7257999999999998e-05 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.484424+0000", 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043051679999999998 2023-09-23T22:28:56.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494151+0000", 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.009726396 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.510131+0000", 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015980557999999999 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524011+0000", 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013879636000000001 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524055+0000", 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4467e-05 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524179+0000", 2023-09-23T22:28:56.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000123275 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103890 7.1 7:87e8d097:::benchmark_data_smithi026_410749_object13553:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.312189+0000", 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.283480967, 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2122174339999998, 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.698 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103890 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312189+0000", 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312189+0000", 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312188+0000", 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312191+0000", 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2830000000000002e-06 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312197+0000", 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0070000000000003e-06 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312202+0000", 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3429999999999998e-06 2023-09-23T22:28:56.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312287+0000", 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5131999999999997e-05 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.312299+0000", 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2046e-05 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984984+0000", 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.67268516899999997 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984988+0000", 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.658e-06 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.481396+0000", 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.496407925 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.481466+0000", 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0235999999999996e-05 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.481539+0000", 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3102999999999994e-05 2023-09-23T22:28:56.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.484430+0000", 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028906859999999999 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494182+0000", 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0097518410000000007 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.510199+0000", 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016017422 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524237+0000", 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014038418 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524289+0000", 2023-09-23T22:28:56.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1966999999999999e-05 2023-09-23T22:28:56.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524406+0000", 2023-09-23T22:28:56.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000116636 2023-09-23T22:28:56.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.709 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103902 7.1 7:849116c8:::benchmark_data_smithi026_410749_object13565:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.359793+0000", 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.235876262, 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1648110350000001, 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.710 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103902 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.359793+0000", 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.359793+0000", 2023-09-23T22:28:56.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.359792+0000", 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.359798+0000", 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2199999999999997e-06 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.359812+0000", 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3858e-05 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.359817+0000", 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1209999999999998e-06 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.359867+0000", 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9486e-05 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.359881+0000", 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3832000000000001e-05 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.984999+0000", 2023-09-23T22:28:56.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.62511845600000004 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985002+0000", 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4120000000000001e-06 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.482989+0000", 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.497986933 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.483062+0000", 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2187000000000005e-05 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.483151+0000", 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9875000000000003e-05 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.489196+0000", 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060449689999999999 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.510240+0000", 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021043593999999999 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.510831+0000", 2023-09-23T22:28:56.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059077299999999995 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524463+0000", 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013632296 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524499+0000", 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5499e-05 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524604+0000", 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105744 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103910 7.1 7:8224f2d4:::benchmark_data_smithi026_410749_object13573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.388020+0000", 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.20764946, 2023-09-23T22:28:56.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1368085790000002, 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103910 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.388020+0000", 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.388020+0000", 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.388019+0000", 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.388025+0000", 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3690000000000003e-06 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.388041+0000", 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5719e-05 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.388047+0000", 2023-09-23T22:28:56.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.789e-06 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.388101+0000", 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4657999999999997e-05 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.388109+0000", 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4390000000000003e-06 2023-09-23T22:28:56.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985014+0000", 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.59690502400000001 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985017+0000", 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4180000000000001e-06 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.484454+0000", 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.499436416 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.484529+0000", 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5187999999999999e-05 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.484602+0000", 2023-09-23T22:28:56.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3459000000000003e-05 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.489202+0000", 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045993609999999997 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.510857+0000", 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021655029999999999 2023-09-23T22:28:56.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.520338+0000", 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094807410000000009 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524660+0000", 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043225140000000004 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524711+0000", 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0912999999999999e-05 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524829+0000", 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011772799999999999 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103961 7.1 7:807ee380:::benchmark_data_smithi026_410749_object13624:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.516057+0000", 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.079612243, 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0090139159999998, 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103961 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.729 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.516057+0000", 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.516057+0000", 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.516055+0000", 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.516066+0000", 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0540000000000001e-05 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.516090+0000", 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4428000000000001e-05 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.516100+0000", 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7799999999999995e-06 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.516170+0000", 2023-09-23T22:28:56.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0135000000000005e-05 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.516185+0000", 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4905e-05 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985029+0000", 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.46884341800000001 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985032+0000", 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3960000000000002e-06 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.485886+0000", 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5008537390000001 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.485963+0000", 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7051000000000005e-05 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.486039+0000", 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5951999999999994e-05 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.492459+0000", 2023-09-23T22:28:56.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064198270000000003 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.510877+0000", 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018417914000000001 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.520375+0000", 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094982950000000003 2023-09-23T22:28:56.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524891+0000", 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004515748 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.524958+0000", 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7659999999999999e-05 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525071+0000", 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113037 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103965 7.1 7:86d0216d:::benchmark_data_smithi026_410749_object13628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.527958+0000", 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.067711793, 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.997434637, 2023-09-23T22:28:56.738 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103965 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.527958+0000", 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.527958+0000", 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.527955+0000", 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.527969+0000", 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3267e-05 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.527993+0000", 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4481000000000002e-05 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.528004+0000", 2023-09-23T22:28:56.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.094e-05 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.528062+0000", 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7558e-05 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.528071+0000", 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0089999999999996e-06 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985043+0000", 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.45697256200000003 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985047+0000", 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5420000000000001e-06 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.487545+0000", 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.502498654 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.487623+0000", 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7941999999999993e-05 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.487702+0000", 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8558000000000002e-05 2023-09-23T22:28:56.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494055+0000", 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063536310000000002 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.510956+0000", 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016900577999999999 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.520413+0000", 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094566100000000007 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525196+0000", 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047831050000000002 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525257+0000", 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1093999999999995e-05 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525392+0000", 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000135563 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:103978 7.1 7:8430557b:::benchmark_data_smithi026_410749_object13641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.562309+0000", 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 119.03336033799999, 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9633077839999999, 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103978 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.748 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.562309+0000", 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.562309+0000", 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.562308+0000", 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.562314+0000", 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6300000000000003e-06 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.562327+0000", 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3205e-05 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.562333+0000", 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4749999999999998e-06 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.562381+0000", 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7519999999999999e-05 2023-09-23T22:28:56.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.562393+0000", 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2252000000000001e-05 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985059+0000", 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.42266537900000001 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985062+0000", 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4740000000000001e-06 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.489240+0000", 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5041776279999999 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.489367+0000", 2023-09-23T22:28:56.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000127136 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.489511+0000", 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014463099999999999 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494062+0000", 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045509160000000003 2023-09-23T22:28:56.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.520451+0000", 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026389105999999999 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525130+0000", 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046783199999999997 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525512+0000", 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038217600000000002 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525544+0000", 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2257e-05 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525617+0000", 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2812999999999996e-05 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:104037 7.1 7:8605c9ed:::benchmark_data_smithi026_410749_object13700:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e287)", 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:57.716135+0000", 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 118.87953478199999, 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.809648194, 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.757 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104037 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.716135+0000", 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.716135+0000", 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.716134+0000", 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.716140+0000", 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.353e-06 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.716172+0000", 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1940000000000003e-05 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.716177+0000", 2023-09-23T22:28:56.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2920000000000003e-06 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.716229+0000", 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1721999999999997e-05 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.716241+0000", 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1695000000000001e-05 2023-09-23T22:28:56.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985139+0000", 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26889845000000001 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:57.985146+0000", 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3839999999999999e-06 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.490955+0000", 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.505808815 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.491042+0000", 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6802e-05 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.491122+0000", 2023-09-23T22:28:56.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0302000000000005e-05 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494122+0000", 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029998450000000001 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.522100+0000", 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027977637 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525462+0000", 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033621789999999999 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525688+0000", 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022637300000000001 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525715+0000", 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6838999999999998e-05 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525783+0000", 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7821e-05 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.766 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:104110 7.1 7:84597c34:::benchmark_data_smithi026_410749_object13773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:58.101171+0000", 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 118.494498595, 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4247870919999999, 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104110 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.101171+0000", 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.101171+0000", 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.101168+0000", 2023-09-23T22:28:56.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999971 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.101193+0000", 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4423999999999999e-05 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.101203+0000", 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0643e-05 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.101209+0000", 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6690000000000003e-06 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.101248+0000", 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9028e-05 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.101255+0000", 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3579999999999997e-06 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.492495+0000", 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.39123957 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.492621+0000", 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012624700000000001 2023-09-23T22:28:56.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.492760+0000", 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013875899999999999 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494188+0000", 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001427509 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.522143+0000", 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027955649999999999 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525663+0000", 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035195809999999999 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525837+0000", 2023-09-23T22:28:56.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017396600000000001 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525862+0000", 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5412e-05 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525958+0000", 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5906e-05 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.774 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:104111 7.1 7:83d0d88f:::benchmark_data_smithi026_410749_object13774:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:58.106033+0000", 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 118.48963686099999, 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4200714400000001, 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:56.775 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104111 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.106033+0000", 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.106033+0000", 2023-09-23T22:28:56.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.106031+0000", 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.106042+0000", 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1403000000000001e-05 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.106067+0000", 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5252000000000001e-05 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.106080+0000", 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.293e-05 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.106159+0000", 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8898000000000003e-05 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:58.106165+0000", 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.755e-06 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494211+0000", 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3880460110000001 2023-09-23T22:28:56.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494286+0000", 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5209000000000004e-05 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.494362+0000", 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5757e-05 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:56.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.500533+0000", 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061713560000000002 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.522195+0000", 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021662163000000002 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525812+0000", 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036168419999999999 2023-09-23T22:28:56.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.525997+0000", 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000184515 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.526024+0000", 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7407e-05 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:28:56.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:59.526104+0000", 2023-09-23T22:28:56.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9968000000000004e-05 2023-09-23T22:28:56.783 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.783 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.783 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.783 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:56.783 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:56.783 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:28:56.783 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:28:56.784 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:56 smithi026 ceph-mon[215775]: pgmap v848: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 84 op/s; 17732/186699 objects degraded (9.498%); 19384/186699 objects misplaced (10.382%); 920 KiB/s, 14 objects/s recovering 2023-09-23T22:28:56.785 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:56 smithi070 ceph-mon[192652]: pgmap v848: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 84 op/s; 17732/186699 objects degraded (9.498%); 19384/186699 objects misplaced (10.382%); 920 KiB/s, 14 objects/s recovering 2023-09-23T22:28:56.785 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:56 smithi026 ceph-mon[223014]: pgmap v848: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 84 op/s; 17732/186699 objects degraded (9.498%); 19384/186699 objects misplaced (10.382%); 920 KiB/s, 14 objects/s recovering 2023-09-23T22:28:56.814 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:56.856 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:28:56.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:56.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:28:56.959 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:56.986 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:57.228 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:57.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:57.297 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:28:57.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:57.367 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:57.411 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:57.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:57.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:57.516 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:57.521 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 1] out_osds: [6, 3, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:28:57.521 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:28:57.522 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T22:28:57.522 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T22:28:57.547 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:57.593 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:57.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:57.670 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:28:57.702 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:57.737 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:57.797 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:57.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:57.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:57.902 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:57.942 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:57.947 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:28:57.970 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:58.005 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:58.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:58.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:58.109 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:28:58.143 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:58.257 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:28:58.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:58.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:28:58.364 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:28:58.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:58.434 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:28:58.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:58.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:28:58.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:28:58.542 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:28:58.564 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:58.663 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:28:58.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:58.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:28:58.769 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:28:58.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:58 smithi026 ceph-mon[223014]: pgmap v849: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 113 op/s; 17696/185841 objects degraded (9.522%); 19301/185841 objects misplaced (10.386%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T22:28:58.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1710573937' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:58.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:58.799 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17696/185841 objects degraded (9.522%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:58 smithi026 ceph-mon[215775]: pgmap v849: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 113 op/s; 17696/185841 objects degraded (9.522%); 19301/185841 objects misplaced (10.386%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T22:28:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1710573937' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:58.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17696/185841 objects degraded (9.522%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:58.803 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:58.810 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:58 smithi070 ceph-mon[192652]: pgmap v849: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 113 op/s; 17696/185841 objects degraded (9.522%); 19301/185841 objects misplaced (10.386%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T22:28:58.811 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1710573937' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:58.811 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:28:58.811 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17696/185841 objects degraded (9.522%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T22:28:59.305 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:28:59.305 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:28:59.305 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.305 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:119168 7.a 7:556e1a0e:::benchmark_data_smithi026_410749_object28831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T22:28:59.305 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:28:59.299984+0000", 2023-09-23T22:28:59.305 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0041917159999999998, 2023-09-23T22:28:59.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042135369999999998, 2023-09-23T22:28:59.305 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119168 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.299984+0000", 2023-09-23T22:28:59.306 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.299984+0000", 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.299983+0000", 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.299989+0000", 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.144e-06 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.300002+0000", 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3166000000000001e-05 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.300007+0000", 2023-09-23T22:28:59.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3179999999999997e-06 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.300052+0000", 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4969999999999998e-05 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.300160+0000", 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010778200000000001 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3,4", 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.300242+0000", 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2565000000000007e-05 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.302334+0000", 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020918880000000001 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.302424+0000", 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0137999999999997e-05 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:28:59.303812+0000", 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013870550000000001 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T22:28:59.311 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 1 2023-09-23T22:28:59.312 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:28:59.621 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:28:59.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:28:59.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:28:59.726 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:28:59.760 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:28:59.835 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:59 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:28:59.835 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:59 smithi070 ceph-mon[192652]: failed to encode map e327 with expected crc 2023-09-23T22:28:59.835 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:59 smithi070 ceph-mon[192652]: failed to encode map e327 with expected crc 2023-09-23T22:28:59.835 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:59 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:59.835 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:28:59 smithi070 ceph-mon[192652]: osdmap e327: 8 total, 8 up, 5 in 2023-09-23T22:28:59.838 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:28:59.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:28:59.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[223014]: failed to encode map e327 with expected crc 2023-09-23T22:28:59.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[223014]: failed to encode map e327 with expected crc 2023-09-23T22:28:59.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:59.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[223014]: osdmap e327: 8 total, 8 up, 5 in 2023-09-23T22:28:59.926 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:28:59.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[215775]: failed to encode map e327 with expected crc 2023-09-23T22:28:59.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[215775]: failed to encode map e327 with expected crc 2023-09-23T22:28:59.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:28:59.927 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:28:59 smithi026 ceph-mon[215775]: osdmap e327: 8 total, 8 up, 5 in 2023-09-23T22:28:59.927 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:28:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:28:59] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T22:29:00.067 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:29:00.085 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:00.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:00.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:00.191 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:00.218 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:00.264 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:00.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:00.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:00.370 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:29:00.426 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:00.480 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:29:00.568 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.6. 2023-09-23T22:29:00.572 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:29:00.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:00.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:29:00.687 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:00.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:00.761 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:00.764 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:00.865 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:00 smithi070 ceph-mon[192652]: pgmap v851: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 69 op/s; 17696/185643 objects degraded (9.532%); 19267/185643 objects misplaced (10.379%); 979 KiB/s, 15 objects/s recovering 2023-09-23T22:29:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/17744434' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T22:29:00.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:00 smithi026 conmon[215751]: 2023-09-23T22:29:00.489+0000 7f4580f04700 -1 mon.a@0(leader).osd e327 definitely_dead 0 2023-09-23T22:29:00.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:00 smithi026 ceph-mon[215775]: pgmap v851: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 69 op/s; 17696/185643 objects degraded (9.532%); 19267/185643 objects misplaced (10.379%); 979 KiB/s, 15 objects/s recovering 2023-09-23T22:29:00.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/17744434' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T22:29:00.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:00 smithi026 ceph-mon[223014]: pgmap v851: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 88 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 69 op/s; 17696/185643 objects degraded (9.532%); 19267/185643 objects misplaced (10.379%); 979 KiB/s, 15 objects/s recovering 2023-09-23T22:29:00.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/17744434' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T22:29:00.896 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:01.444 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:01.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:01.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:01.549 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:01.603 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:01.699 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:29:01.724 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:01 smithi026 ceph-mon[223014]: failed to encode map e328 with expected crc 2023-09-23T22:29:01.724 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/17744434' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T22:29:01.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:01 smithi026 ceph-mon[223014]: osdmap e328: 8 total, 8 up, 6 in 2023-09-23T22:29:01.725 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:01 smithi026 ceph-mon[223014]: failed to encode map e328 with expected crc 2023-09-23T22:29:01.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:01 smithi026 ceph-mon[215775]: failed to encode map e328 with expected crc 2023-09-23T22:29:01.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/17744434' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T22:29:01.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:01 smithi026 ceph-mon[215775]: osdmap e328: 8 total, 8 up, 6 in 2023-09-23T22:29:01.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:01 smithi026 ceph-mon[215775]: failed to encode map e328 with expected crc 2023-09-23T22:29:01.763 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:01.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:01.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:01.868 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:01.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:01 smithi070 ceph-mon[192652]: failed to encode map e328 with expected crc 2023-09-23T22:29:01.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/17744434' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T22:29:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:01 smithi070 ceph-mon[192652]: osdmap e328: 8 total, 8 up, 6 in 2023-09-23T22:29:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:01 smithi070 ceph-mon[192652]: failed to encode map e328 with expected crc 2023-09-23T22:29:01.900 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:02.108 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:02.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:02.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:02.181 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T22:29:02.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:02.246 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:02.305 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:29:02.305 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:29:02.306 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:29:02.306 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:29:02.306 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:29:02.350 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:02.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:02.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:02.450 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:02.454 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:02.487 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:02.536 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:02.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:02.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:02.569 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:29:02.600 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:29:02.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:29:02.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:02.644 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:29:02.671 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:02.701 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:29:02.733 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:29:02.742 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: pgmap v853: 99 pgs: 2 remapped+peering, 5 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 10136/184440 objects degraded (5.496%); 15565/184440 objects misplaced (8.439%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:29:02.742 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: osdmap e329: 8 total, 8 up, 6 in 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: 7.c starting backfill to osd.6 from (0'0,0'0] MAX to 327'7368 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: 7.7 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7427 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.743 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.744 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.744 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.744 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.744 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.744 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.744 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1952372704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:02.744 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:02.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:02.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:02.770 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:29:02.791 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:29:02.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:29:02.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:29:02.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:29:02.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:29:02.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:29:02.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:29:02.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:29:02.794 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:29:02.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: pgmap v853: 99 pgs: 2 remapped+peering, 5 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 10136/184440 objects degraded (5.496%); 15565/184440 objects misplaced (8.439%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:29:02.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: osdmap e329: 8 total, 8 up, 6 in 2023-09-23T22:29:02.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: 7.c starting backfill to osd.6 from (0'0,0'0] MAX to 327'7368 2023-09-23T22:29:02.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: 7.7 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7427 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.818 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1952372704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: pgmap v853: 99 pgs: 2 remapped+peering, 5 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 116 op/s; 10136/184440 objects degraded (5.496%); 15565/184440 objects misplaced (8.439%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: osdmap e329: 8 total, 8 up, 6 in 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: 7.c starting backfill to osd.6 from (0'0,0'0] MAX to 327'7368 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: 7.7 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7427 2023-09-23T22:29:02.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: failed to encode map e329 with expected crc 2023-09-23T22:29:02.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1952372704' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:02.853 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:03.546 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: osdmap e330: 8 total, 8 up, 6 in 2023-09-23T22:29:03.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.663 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:03.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:03.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:03.767 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:03.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: osdmap e330: 8 total, 8 up, 6 in 2023-09-23T22:29:03.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: 7.b starting backfill to osd.6 from (0'0,0'0] MAX to 327'7482 2023-09-23T22:29:03.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: 7.2 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7457 2023-09-23T22:29:03.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: 7.d starting backfill to osd.6 from (0'0,0'0] MAX to 327'7509 2023-09-23T22:29:03.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: 7.1 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7302 2023-09-23T22:29:03.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7440 2023-09-23T22:29:03.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10136/184440 objects degraded (5.496%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:29:03.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: failed to encode map e331 with expected crc 2023-09-23T22:29:03.795 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:03 smithi070 ceph-mon[192652]: osdmap e331: 8 total, 8 up, 6 in 2023-09-23T22:29:03.795 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:03.843 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:03.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:03.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: 7.b starting backfill to osd.6 from (0'0,0'0] MAX to 327'7482 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: 7.2 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7457 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: 7.d starting backfill to osd.6 from (0'0,0'0] MAX to 327'7509 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: 7.1 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7302 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7440 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10136/184440 objects degraded (5.496%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: failed to encode map e331 with expected crc 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[215775]: osdmap e331: 8 total, 8 up, 6 in 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: osdmap e330: 8 total, 8 up, 6 in 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: 7.b starting backfill to osd.6 from (0'0,0'0] MAX to 327'7482 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: 7.2 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7457 2023-09-23T22:29:03.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: 7.d starting backfill to osd.6 from (0'0,0'0] MAX to 327'7509 2023-09-23T22:29:03.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: 7.1 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7302 2023-09-23T22:29:03.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 327'7440 2023-09-23T22:29:03.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: failed to encode map e330 with expected crc 2023-09-23T22:29:03.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10136/184440 objects degraded (5.496%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:29:03.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: failed to encode map e331 with expected crc 2023-09-23T22:29:03.941 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:03 smithi026 ceph-mon[223014]: osdmap e331: 8 total, 8 up, 6 in 2023-09-23T22:29:03.946 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:03.978 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:04.537 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:29:04.553 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:04.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:04.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:04.658 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:04.686 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:04.736 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:04.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:04.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:04.842 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:04.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[223014]: failed to encode map e331 with expected crc 2023-09-23T22:29:04.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[223014]: failed to encode map e331 with expected crc 2023-09-23T22:29:04.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[223014]: pgmap v857: 99 pgs: 2 remapped+peering, 5 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 100 op/s; 10136/184440 objects degraded (5.496%); 15565/184440 objects misplaced (8.439%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T22:29:04.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[223014]: failed to encode map e332 with expected crc 2023-09-23T22:29:04.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[223014]: osdmap e332: 8 total, 8 up, 6 in 2023-09-23T22:29:04.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[223014]: failed to encode map e332 with expected crc 2023-09-23T22:29:04.864 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[223014]: failed to encode map e332 with expected crc 2023-09-23T22:29:04.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[215775]: failed to encode map e331 with expected crc 2023-09-23T22:29:04.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[215775]: failed to encode map e331 with expected crc 2023-09-23T22:29:04.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[215775]: pgmap v857: 99 pgs: 2 remapped+peering, 5 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 100 op/s; 10136/184440 objects degraded (5.496%); 15565/184440 objects misplaced (8.439%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T22:29:04.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[215775]: failed to encode map e332 with expected crc 2023-09-23T22:29:04.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[215775]: osdmap e332: 8 total, 8 up, 6 in 2023-09-23T22:29:04.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[215775]: failed to encode map e332 with expected crc 2023-09-23T22:29:04.865 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:04 smithi026 ceph-mon[215775]: failed to encode map e332 with expected crc 2023-09-23T22:29:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:04 smithi070 ceph-mon[192652]: failed to encode map e331 with expected crc 2023-09-23T22:29:04.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:04 smithi070 ceph-mon[192652]: failed to encode map e331 with expected crc 2023-09-23T22:29:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:04 smithi070 ceph-mon[192652]: pgmap v857: 99 pgs: 2 remapped+peering, 5 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.8 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 100 op/s; 10136/184440 objects degraded (5.496%); 15565/184440 objects misplaced (8.439%); 1.1 MiB/s, 18 objects/s recovering 2023-09-23T22:29:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:04 smithi070 ceph-mon[192652]: failed to encode map e332 with expected crc 2023-09-23T22:29:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:04 smithi070 ceph-mon[192652]: osdmap e332: 8 total, 8 up, 6 in 2023-09-23T22:29:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:04 smithi070 ceph-mon[192652]: failed to encode map e332 with expected crc 2023-09-23T22:29:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:04 smithi070 ceph-mon[192652]: failed to encode map e332 with expected crc 2023-09-23T22:29:04.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:05.045 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:05.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:05.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:05.149 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:05.176 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:05.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:05.393 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:05.394 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:05.495 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:05.529 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:05.587 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:29:05.591 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:05.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:05.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:05.630 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:29:05.699 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:05.733 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:05.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[223014]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[223014]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[223014]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[223014]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4285085607' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:29:05.820 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:29:05.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[215775]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[215775]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[215775]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[215775]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4285085607' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:29:05.821 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:05 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:29:05.840 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:05.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:05.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:05 smithi070 ceph-mon[192652]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:05 smithi070 ceph-mon[192652]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:05 smithi070 ceph-mon[192652]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:05 smithi070 ceph-mon[192652]: failed to encode map e332 with expected crc 2023-09-23T22:29:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:05 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4285085607' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:29:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:05 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:29:05.946 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:05.988 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:06.223 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:06.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:06.327 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:06.359 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:06.415 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:06.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:06.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:06.519 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:29:06.551 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:06.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:06 smithi070 ceph-mon[192652]: pgmap v859: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 6 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 13540/184095 objects degraded (7.355%); 21261/184095 objects misplaced (11.549%); 16 MiB/s, 2 keys/s, 266 objects/s recovering 2023-09-23T22:29:06.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:06 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:06.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:06 smithi070 ceph-mon[192652]: failed to encode map e333 with expected crc 2023-09-23T22:29:06.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:06 smithi070 ceph-mon[192652]: failed to encode map e333 with expected crc 2023-09-23T22:29:06.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:06 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:29:06.572 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:06 smithi070 ceph-mon[192652]: osdmap e333: 8 total, 8 up, 6 in 2023-09-23T22:29:06.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:29:06.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[223014]: pgmap v859: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 6 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 13540/184095 objects degraded (7.355%); 21261/184095 objects misplaced (11.549%); 16 MiB/s, 2 keys/s, 266 objects/s recovering 2023-09-23T22:29:06.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:06.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[223014]: failed to encode map e333 with expected crc 2023-09-23T22:29:06.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[223014]: failed to encode map e333 with expected crc 2023-09-23T22:29:06.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:29:06.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[223014]: osdmap e333: 8 total, 8 up, 6 in 2023-09-23T22:29:06.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[215775]: pgmap v859: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 6 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 28 op/s; 13540/184095 objects degraded (7.355%); 21261/184095 objects misplaced (11.549%); 16 MiB/s, 2 keys/s, 266 objects/s recovering 2023-09-23T22:29:06.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:06.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[215775]: failed to encode map e333 with expected crc 2023-09-23T22:29:06.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[215775]: failed to encode map e333 with expected crc 2023-09-23T22:29:06.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:29:06.608 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:06 smithi026 ceph-mon[215775]: osdmap e333: 8 total, 8 up, 6 in 2023-09-23T22:29:06.612 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:29:06.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:06.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:29:06.717 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:06.748 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:06.973 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:06.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:07.017 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:07.076 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:07.110 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:07.182 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 1, 6] out_osds: [3, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:29:07.182 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:29:07.182 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T22:29:07.182 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T22:29:07.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T22:29:07.215 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:07.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:07.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:07.321 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:07.349 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:07.396 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:07.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:07.486 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:07 smithi026 systemd[1]: Stopping Ceph osd.2 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:29:07.499 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:07.527 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:07.575 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:07.578 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:07.679 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:07.706 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:07.925 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:07.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:07.931 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:08.032 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:08.064 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:08.120 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:08.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:08.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:08.225 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:08.263 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:08.467 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:08.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:08.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:08.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:08.589 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:29:08.589 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:29:08.614 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:08 smithi070 ceph-mon[192652]: pgmap v861: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 6 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 23 op/s; 13540/184095 objects degraded (7.355%); 21261/184095 objects misplaced (11.549%); 13 MiB/s, 2 keys/s, 216 objects/s recovering 2023-09-23T22:29:08.615 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 13540/184095 objects degraded (7.355%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:08.615 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:08 smithi070 ceph-mon[192652]: osd.2 marked itself down and dead 2023-09-23T22:29:08.626 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:08.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:08 smithi026 ceph-mon[215775]: pgmap v861: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 6 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 23 op/s; 13540/184095 objects degraded (7.355%); 21261/184095 objects misplaced (11.549%); 13 MiB/s, 2 keys/s, 216 objects/s recovering 2023-09-23T22:29:08.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 13540/184095 objects degraded (7.355%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:08.776 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:08 smithi026 ceph-mon[215775]: osd.2 marked itself down and dead 2023-09-23T22:29:08.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:08 smithi026 ceph-mon[223014]: pgmap v861: 99 pgs: 1 active+remapped+backfilling, 2 remapped+peering, 6 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 77 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 23 op/s; 13540/184095 objects degraded (7.355%); 21261/184095 objects misplaced (11.549%); 13 MiB/s, 2 keys/s, 216 objects/s recovering 2023-09-23T22:29:08.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 13540/184095 objects degraded (7.355%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:08.777 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:08 smithi026 ceph-mon[223014]: osd.2 marked itself down and dead 2023-09-23T22:29:08.777 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:08 smithi026 conmon[253575]: 2023-09-23T22:29:08.560+0000 7f8552879700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:29:08.777 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:08 smithi026 conmon[253575]: 2023-09-23T22:29:08.560+0000 7f8552879700 -1 osd.2 333 *** Got signal Terminated *** 2023-09-23T22:29:08.778 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:08 smithi026 conmon[253575]: 2023-09-23T22:29:08.560+0000 7f8552879700 -1 osd.2 333 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:29:08.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:29:09.331 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:09.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:09.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:09.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:09.677 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:09.709 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:09.761 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:09.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:09.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:09.840 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:29:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:29:09] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-09-23T22:29:09.841 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:09 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2131162575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:09.841 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:09 smithi026 ceph-mon[223014]: osdmap e334: 8 total, 7 up, 6 in 2023-09-23T22:29:09.841 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:09 smithi026 ceph-mon[223014]: failed to encode map e334 with expected crc 2023-09-23T22:29:09.841 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:09.842 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:09 smithi026 bash[466112]: 82c8c1d0efd8845863a26477a1c21350419a3ace307213c38c9a86a282c23d73 2023-09-23T22:29:09.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:09 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2131162575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:09.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:09 smithi026 ceph-mon[215775]: osdmap e334: 8 total, 7 up, 6 in 2023-09-23T22:29:09.842 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:09 smithi026 ceph-mon[215775]: failed to encode map e334 with expected crc 2023-09-23T22:29:09.843 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:09.864 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:09 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2131162575' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:09 smithi070 ceph-mon[192652]: osdmap e334: 8 total, 7 up, 6 in 2023-09-23T22:29:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:09 smithi070 ceph-mon[192652]: failed to encode map e334 with expected crc 2023-09-23T22:29:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:09.903 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:10 smithi070 ceph-mon[192652]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:10 smithi070 ceph-mon[192652]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:10 smithi070 ceph-mon[192652]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:10 smithi070 ceph-mon[192652]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:10 smithi070 ceph-mon[192652]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:10 smithi070 ceph-mon[192652]: pgmap v863: 99 pgs: 1 active+clean+remapped, 2 remapped+peering, 5 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 19 op/s; 13540/184095 objects degraded (7.355%); 18147/184095 objects misplaced (9.857%); 43 MiB/s, 1 keys/s, 696 objects/s recovering 2023-09-23T22:29:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:10 smithi070 ceph-mon[192652]: osdmap e335: 8 total, 7 up, 6 in 2023-09-23T22:29:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:10 smithi070 ceph-mon[192652]: failed to encode map e335 with expected crc 2023-09-23T22:29:10.961 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:10.963 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[215775]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[215775]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[215775]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[215775]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[215775]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[215775]: pgmap v863: 99 pgs: 1 active+clean+remapped, 2 remapped+peering, 5 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 19 op/s; 13540/184095 objects degraded (7.355%); 18147/184095 objects misplaced (9.857%); 43 MiB/s, 1 keys/s, 696 objects/s recovering 2023-09-23T22:29:10.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[215775]: osdmap e335: 8 total, 7 up, 6 in 2023-09-23T22:29:10.964 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[215775]: failed to encode map e335 with expected crc 2023-09-23T22:29:10.964 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[223014]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[223014]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[223014]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[223014]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[223014]: failed to encode map e334 with expected crc 2023-09-23T22:29:10.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[223014]: pgmap v863: 99 pgs: 1 active+clean+remapped, 2 remapped+peering, 5 peering, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 78 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 19 op/s; 13540/184095 objects degraded (7.355%); 18147/184095 objects misplaced (9.857%); 43 MiB/s, 1 keys/s, 696 objects/s recovering 2023-09-23T22:29:10.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[223014]: osdmap e335: 8 total, 7 up, 6 in 2023-09-23T22:29:10.965 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:10 smithi026 ceph-mon[223014]: failed to encode map e335 with expected crc 2023-09-23T22:29:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:10.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:11.066 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:11.128 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:11.179 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:29:11.187 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:11.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:11.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:11.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:11.318 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:11.363 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:11.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:11.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:11.469 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:11.494 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:11.538 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:11.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:11.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:11.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:11 smithi070 ceph-mon[192652]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:11 smithi070 ceph-mon[192652]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:11 smithi070 ceph-mon[192652]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:11 smithi070 ceph-mon[192652]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:11 smithi070 ceph-mon[192652]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:11 smithi070 ceph-mon[192652]: failed to encode map e336 with expected crc 2023-09-23T22:29:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:11 smithi070 ceph-mon[192652]: osdmap e336: 8 total, 7 up, 6 in 2023-09-23T22:29:11.645 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:11.670 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:11.713 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:11.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:11.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[215775]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[215775]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[215775]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[215775]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[215775]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[215775]: failed to encode map e336 with expected crc 2023-09-23T22:29:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[215775]: osdmap e336: 8 total, 7 up, 6 in 2023-09-23T22:29:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[223014]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[223014]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[223014]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[223014]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[223014]: failed to encode map e335 with expected crc 2023-09-23T22:29:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[223014]: failed to encode map e336 with expected crc 2023-09-23T22:29:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:11 smithi026 ceph-mon[223014]: osdmap e336: 8 total, 7 up, 6 in 2023-09-23T22:29:11.821 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:11.846 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:11.891 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:11.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:11.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:11.996 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:12.033 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:12.092 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:12.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:12.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:12.196 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:12.224 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:12.270 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:12.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:12.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:12.376 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:12.405 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:12.444 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:29:12.451 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:12.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:12.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:12.556 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:12.584 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:12.592 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:29:12.685 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:12.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:12.687 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:12.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[215775]: failed to encode map e336 with expected crc 2023-09-23T22:29:12.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[215775]: failed to encode map e336 with expected crc 2023-09-23T22:29:12.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[215775]: failed to encode map e336 with expected crc 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[215775]: 7.e continuing backfill to osd.1 from (266'2684,331'7450] MIN to 331'7450 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[215775]: 7.3 continuing backfill to osd.6 from (265'2322,331'7463] MIN to 331'7463 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[215775]: pgmap v866: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 4 remapped+peering, 5 peering, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 79 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 6062/184095 objects degraded (3.293%); 10749/184095 objects misplaced (5.839%); 46 MiB/s, 734 objects/s recovering 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3320685515' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[223014]: failed to encode map e336 with expected crc 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[223014]: failed to encode map e336 with expected crc 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[223014]: failed to encode map e336 with expected crc 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[223014]: 7.e continuing backfill to osd.1 from (266'2684,331'7450] MIN to 331'7450 2023-09-23T22:29:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[223014]: 7.3 continuing backfill to osd.6 from (265'2322,331'7463] MIN to 331'7463 2023-09-23T22:29:12.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[223014]: pgmap v866: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 4 remapped+peering, 5 peering, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 79 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 6062/184095 objects degraded (3.293%); 10749/184095 objects misplaced (5.839%); 46 MiB/s, 734 objects/s recovering 2023-09-23T22:29:12.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3320685515' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:12.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:12 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:12.789 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:12.823 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:12 smithi070 ceph-mon[192652]: failed to encode map e336 with expected crc 2023-09-23T22:29:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:12 smithi070 ceph-mon[192652]: failed to encode map e336 with expected crc 2023-09-23T22:29:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:12 smithi070 ceph-mon[192652]: failed to encode map e336 with expected crc 2023-09-23T22:29:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:12 smithi070 ceph-mon[192652]: 7.e continuing backfill to osd.1 from (266'2684,331'7450] MIN to 331'7450 2023-09-23T22:29:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:12 smithi070 ceph-mon[192652]: 7.3 continuing backfill to osd.6 from (265'2322,331'7463] MIN to 331'7463 2023-09-23T22:29:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:12 smithi070 ceph-mon[192652]: pgmap v866: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 4 remapped+peering, 5 peering, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 79 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 6062/184095 objects degraded (3.293%); 10749/184095 objects misplaced (5.839%); 46 MiB/s, 734 objects/s recovering 2023-09-23T22:29:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3320685515' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:12 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:12.961 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:12.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:12.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:13.064 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:13.098 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:13.314 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:29:13.342 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:29:13.366 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:13.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:13.370 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:13.471 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:13.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:13.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:13.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[223014]: failed to encode map e337 with expected crc 2023-09-23T22:29:13.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[223014]: failed to encode map e337 with expected crc 2023-09-23T22:29:13.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:29:13.630 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[223014]: osdmap e337: 8 total, 7 up, 6 in 2023-09-23T22:29:13.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6062/184095 objects degraded (3.293%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:13.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:13.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[215775]: failed to encode map e337 with expected crc 2023-09-23T22:29:13.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[215775]: failed to encode map e337 with expected crc 2023-09-23T22:29:13.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:29:13.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[215775]: osdmap e337: 8 total, 7 up, 6 in 2023-09-23T22:29:13.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6062/184095 objects degraded (3.293%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:13.710 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:13.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:13.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:13.764 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T22:29:13.814 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:13.835 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:29:13.836 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T22:29:13.845 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:13 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:13 smithi070 ceph-mon[192652]: failed to encode map e337 with expected crc 2023-09-23T22:29:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:13 smithi070 ceph-mon[192652]: failed to encode map e337 with expected crc 2023-09-23T22:29:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:13 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:29:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:13 smithi070 ceph-mon[192652]: osdmap e337: 8 total, 7 up, 6 in 2023-09-23T22:29:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6062/184095 objects degraded (3.293%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:13.947 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:13.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:13.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:13.995 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:29:14.015 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:29:14.033 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:29:14.052 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:14.056 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:29:14.080 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:29:14.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:29:14.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:29:14.081 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:29:14.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:29:14.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:29:14.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:29:14.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:29:14.081 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:29:14.085 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:14.282 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:14.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:14.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:14.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:14.414 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:14.461 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:14.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:14.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:14.565 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:14.593 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:14.641 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:14.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:14.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:14.744 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:14.774 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:14.792 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:29:14.793 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:29:14.818 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:14.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:14.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:14.869 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:29:14.869 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:29:14.870 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:29:14.870 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:29:14.923 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:14.953 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:15.100 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:15.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:15.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:15.228 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:15 smithi070 ceph-mon[192652]: pgmap v868: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 3 remapped+peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 82 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 23 op/s; 7670/183786 objects degraded (4.173%); 21927/183786 objects misplaced (11.931%); 20 MiB/s, 327 objects/s recovering 2023-09-23T22:29:15.228 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:15 smithi070 ceph-mon[192652]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.228 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:15 smithi070 ceph-mon[192652]: osdmap e338: 8 total, 7 up, 6 in 2023-09-23T22:29:15.229 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:15 smithi070 ceph-mon[192652]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.229 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:15 smithi070 ceph-mon[192652]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.229 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:15 smithi070 ceph-mon[192652]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.231 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:15.278 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:15.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:15.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:15.381 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:15.408 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:15.469 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:15.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:15.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[215775]: pgmap v868: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 3 remapped+peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 82 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 23 op/s; 7670/183786 objects degraded (4.173%); 21927/183786 objects misplaced (11.931%); 20 MiB/s, 327 objects/s recovering 2023-09-23T22:29:15.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[215775]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[215775]: osdmap e338: 8 total, 7 up, 6 in 2023-09-23T22:29:15.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[215775]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[215775]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[215775]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[223014]: pgmap v868: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 3 remapped+peering, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 82 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 23 op/s; 7670/183786 objects degraded (4.173%); 21927/183786 objects misplaced (11.931%); 20 MiB/s, 327 objects/s recovering 2023-09-23T22:29:15.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[223014]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[223014]: osdmap e338: 8 total, 7 up, 6 in 2023-09-23T22:29:15.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[223014]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[223014]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:15 smithi026 ceph-mon[223014]: failed to encode map e338 with expected crc 2023-09-23T22:29:15.573 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:15.599 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:15.642 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:15.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:15.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:15.745 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:15.771 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:15.815 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:15.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:15.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:15.919 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:15.933 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:29:15.951 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:16.006 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:16.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:16.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:16.112 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:16.140 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:16.191 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:16.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:16.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[223014]: failed to encode map e339 with expected crc 2023-09-23T22:29:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[223014]: osdmap e339: 8 total, 7 up, 6 in 2023-09-23T22:29:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[223014]: failed to encode map e339 with expected crc 2023-09-23T22:29:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[223014]: 7.c continuing backfill to osd.6 from (266'2797,327'7368] MIN to 327'7368 2023-09-23T22:29:16.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[223014]: failed to encode map e339 with expected crc 2023-09-23T22:29:16.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[215775]: failed to encode map e339 with expected crc 2023-09-23T22:29:16.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[215775]: osdmap e339: 8 total, 7 up, 6 in 2023-09-23T22:29:16.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[215775]: failed to encode map e339 with expected crc 2023-09-23T22:29:16.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[215775]: 7.c continuing backfill to osd.6 from (266'2797,327'7368] MIN to 327'7368 2023-09-23T22:29:16.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:16 smithi026 ceph-mon[215775]: failed to encode map e339 with expected crc 2023-09-23T22:29:16.278 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:29:16.295 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:16.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:16 smithi070 ceph-mon[192652]: failed to encode map e339 with expected crc 2023-09-23T22:29:16.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:16 smithi070 ceph-mon[192652]: osdmap e339: 8 total, 7 up, 6 in 2023-09-23T22:29:16.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:16 smithi070 ceph-mon[192652]: failed to encode map e339 with expected crc 2023-09-23T22:29:16.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:16 smithi070 ceph-mon[192652]: 7.c continuing backfill to osd.6 from (266'2797,327'7368] MIN to 327'7368 2023-09-23T22:29:16.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:16 smithi070 ceph-mon[192652]: failed to encode map e339 with expected crc 2023-09-23T22:29:16.361 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:16.476 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:16.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:16.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:16.521 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:16.585 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:16.642 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:16.725 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:16.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:16.733 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:16.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:29:16.834 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:16.877 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:16.948 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:16.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:16.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:16.994 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:29:17.052 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:17.099 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:17.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:17 smithi026 ceph-mon[215775]: failed to encode map e339 with expected crc 2023-09-23T22:29:17.123 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:17 smithi026 ceph-mon[215775]: pgmap v871: 99 pgs: 1 active+recovering+remapped, 1 remapped+peering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 35 op/s; 11797/183624 objects degraded (6.425%); 26003/183624 objects misplaced (14.161%); 18 MiB/s, 488 objects/s recovering 2023-09-23T22:29:17.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:17 smithi026 ceph-mon[223014]: failed to encode map e339 with expected crc 2023-09-23T22:29:17.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:17 smithi026 ceph-mon[223014]: pgmap v871: 99 pgs: 1 active+recovering+remapped, 1 remapped+peering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 35 op/s; 11797/183624 objects degraded (6.425%); 26003/183624 objects misplaced (14.161%); 18 MiB/s, 488 objects/s recovering 2023-09-23T22:29:17.161 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:17.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:17.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:17.275 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:17 smithi070 ceph-mon[192652]: failed to encode map e339 with expected crc 2023-09-23T22:29:17.275 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:17 smithi070 ceph-mon[192652]: pgmap v871: 99 pgs: 1 active+recovering+remapped, 1 remapped+peering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 35 op/s; 11797/183624 objects degraded (6.425%); 26003/183624 objects misplaced (14.161%); 18 MiB/s, 488 objects/s recovering 2023-09-23T22:29:17.358 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:17.482 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:17.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:17.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:17.599 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:17.665 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:17.716 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:17.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:17.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:17.820 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:17.896 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:18.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 11797/183624 objects degraded (6.425%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:18.239 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:18 smithi026 ceph-mon[223014]: osdmap e340: 8 total, 7 up, 6 in 2023-09-23T22:29:18.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 11797/183624 objects degraded (6.425%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:18.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:18 smithi026 ceph-mon[215775]: osdmap e340: 8 total, 7 up, 6 in 2023-09-23T22:29:18.301 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:18.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:18.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:18.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 11797/183624 objects degraded (6.425%), 8 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:18.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:18 smithi070 ceph-mon[192652]: osdmap e340: 8 total, 7 up, 6 in 2023-09-23T22:29:18.419 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:18.437 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:18.459 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:18.539 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:18.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:18.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:18.652 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:18.746 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:18.837 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 1, 6] out_osds: [3, 2] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-09-23T22:29:18.837 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:29:18.837 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:29:18.837 INFO:tasks.thrashosds.thrasher:Setting osd 6 primary_affinity to 1.000000 2023-09-23T22:29:18.837 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T22:29:19.003 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:19.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:19.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:19.069 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:29:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:29:19.069 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:29:19.070 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:29:19.070 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:29:19.099 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:19 smithi070 ceph-mon[192652]: pgmap v872: 99 pgs: 1 active+recovering+remapped, 1 remapped+peering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 26 op/s; 11797/183624 objects degraded (6.425%); 26003/183624 objects misplaced (14.161%); 13 MiB/s, 360 objects/s recovering 2023-09-23T22:29:19.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:19 smithi070 ceph-mon[192652]: failed to encode map e340 with expected crc 2023-09-23T22:29:19.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:19 smithi070 ceph-mon[192652]: failed to encode map e340 with expected crc 2023-09-23T22:29:19.100 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:19 smithi070 ceph-mon[192652]: failed to encode map e340 with expected crc 2023-09-23T22:29:19.121 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:19.240 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:19.354 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:19.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:19.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:19.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:19 smithi026 ceph-mon[215775]: pgmap v872: 99 pgs: 1 active+recovering+remapped, 1 remapped+peering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 26 op/s; 11797/183624 objects degraded (6.425%); 26003/183624 objects misplaced (14.161%); 13 MiB/s, 360 objects/s recovering 2023-09-23T22:29:19.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:19 smithi026 ceph-mon[215775]: failed to encode map e340 with expected crc 2023-09-23T22:29:19.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:19 smithi026 ceph-mon[215775]: failed to encode map e340 with expected crc 2023-09-23T22:29:19.358 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:19 smithi026 ceph-mon[215775]: failed to encode map e340 with expected crc 2023-09-23T22:29:19.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:19 smithi026 ceph-mon[223014]: pgmap v872: 99 pgs: 1 active+recovering+remapped, 1 remapped+peering, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 26 op/s; 11797/183624 objects degraded (6.425%); 26003/183624 objects misplaced (14.161%); 13 MiB/s, 360 objects/s recovering 2023-09-23T22:29:19.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:19 smithi026 ceph-mon[223014]: failed to encode map e340 with expected crc 2023-09-23T22:29:19.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:19 smithi026 ceph-mon[223014]: failed to encode map e340 with expected crc 2023-09-23T22:29:19.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:19 smithi026 ceph-mon[223014]: failed to encode map e340 with expected crc 2023-09-23T22:29:19.459 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:19.568 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:19.613 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:29:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:29:19] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-09-23T22:29:19.919 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:29:20.155 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:29:20.331 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:20.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:20.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:20 smithi070 ceph-mon[192652]: osdmap e341: 8 total, 7 up, 6 in 2023-09-23T22:29:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:20 smithi070 ceph-mon[192652]: failed to encode map e341 with expected crc 2023-09-23T22:29:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2954756503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:29:20.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:20 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:29:20.433 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:29:20.443 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:20.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:20 smithi026 ceph-mon[223014]: osdmap e341: 8 total, 7 up, 6 in 2023-09-23T22:29:20.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:20 smithi026 ceph-mon[223014]: failed to encode map e341 with expected crc 2023-09-23T22:29:20.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2954756503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:29:20.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:20 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:29:20.488 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:20.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:20 smithi026 ceph-mon[215775]: osdmap e341: 8 total, 7 up, 6 in 2023-09-23T22:29:20.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:20 smithi026 ceph-mon[215775]: failed to encode map e341 with expected crc 2023-09-23T22:29:20.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2954756503' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:29:20.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:20 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:29:20.624 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:20.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:20.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:20.727 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:20.796 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:20.842 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:20.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:20.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:20.945 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:20.990 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:21.215 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:21.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:21.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:21.322 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:21.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[223014]: pgmap v875: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 12433/180543 objects degraded (6.886%); 28838/180543 objects misplaced (15.973%); 12 MiB/s, 378 objects/s recovering 2023-09-23T22:29:21.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:21.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[223014]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.347 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[223014]: osdmap e342: 8 total, 7 up, 6 in 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[223014]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[223014]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[223014]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[215775]: pgmap v875: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 12433/180543 objects degraded (6.886%); 28838/180543 objects misplaced (15.973%); 12 MiB/s, 378 objects/s recovering 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[215775]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[215775]: osdmap e342: 8 total, 7 up, 6 in 2023-09-23T22:29:21.348 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[215775]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[215775]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:21 smithi026 ceph-mon[215775]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:21 smithi070 ceph-mon[192652]: pgmap v875: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 6 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 83 active+clean; 3.7 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 197 op/s; 12433/180543 objects degraded (6.886%); 28838/180543 objects misplaced (15.973%); 12 MiB/s, 378 objects/s recovering 2023-09-23T22:29:21.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:21 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:21.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:21 smithi070 ceph-mon[192652]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:21 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:29:21.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:21 smithi070 ceph-mon[192652]: osdmap e342: 8 total, 7 up, 6 in 2023-09-23T22:29:21.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:21 smithi070 ceph-mon[192652]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:21 smithi070 ceph-mon[192652]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:21 smithi070 ceph-mon[192652]: failed to encode map e342 with expected crc 2023-09-23T22:29:21.440 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:21.523 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:21.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:21.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:21.628 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:21.657 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:21.658 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:21.877 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:29:21.912 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:29:21.934 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:21.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:21.958 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:29:21.997 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:29:22.032 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:29:22.039 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:22.072 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:29:22.080 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:22.113 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:29:22.113 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:29:22.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:29:22.114 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:29:22.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:29:22.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:29:22.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:29:22.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:29:22.114 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:29:22.254 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:22.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:22.264 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:22 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:22.264 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2984177760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:22.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:22.366 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:22.423 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:22.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:22 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:22.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2984177760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:22.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:22 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:29:22.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2984177760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:22.517 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:22.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:22.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:22.623 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:22.674 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:22.762 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:22.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:22.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:22.807 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:29:22.807 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:29:22.865 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:22.902 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:23.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:29:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:23 smithi026 ceph-mon[223014]: pgmap v877: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 12431/177825 objects degraded (6.991%); 32171/177825 objects misplaced (18.091%); 43 KiB/s, 135 objects/s recovering 2023-09-23T22:29:23.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12431/177825 objects degraded (6.991%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:23 smithi026 ceph-mon[215775]: pgmap v877: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 12431/177825 objects degraded (6.991%); 32171/177825 objects misplaced (18.091%); 43 KiB/s, 135 objects/s recovering 2023-09-23T22:29:23.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12431/177825 objects degraded (6.991%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:23 smithi070 ceph-mon[192652]: pgmap v877: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.6 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 322 op/s; 12431/177825 objects degraded (6.991%); 32171/177825 objects misplaced (18.091%); 43 KiB/s, 135 objects/s recovering 2023-09-23T22:29:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12431/177825 objects degraded (6.991%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:23.842 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:29:23.842 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:29:23.919 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:23.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:23.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:24.011 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:29:24.025 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:24.070 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:24.137 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:24.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:24.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:24.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:24.330 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:24.344 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:24 smithi026 ceph-mon[215775]: pgmap v878: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 635 op/s; 12431/172179 objects degraded (7.220%); 30979/172179 objects misplaced (17.992%); 43 KiB/s, 176 objects/s recovering 2023-09-23T22:29:24.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:24.345 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:24 smithi026 ceph-mon[223014]: pgmap v878: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 635 op/s; 12431/172179 objects degraded (7.220%); 30979/172179 objects misplaced (17.992%); 43 KiB/s, 176 objects/s recovering 2023-09-23T22:29:24.345 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:24.354 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:24.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:24 smithi070 ceph-mon[192652]: pgmap v878: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.5 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 635 op/s; 12431/172179 objects degraded (7.220%); 30979/172179 objects misplaced (17.992%); 43 KiB/s, 176 objects/s recovering 2023-09-23T22:29:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:24.972 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:24.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:24.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:25.080 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:25.117 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:25.240 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:25.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:25.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:25.344 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:25.384 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:25.555 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:25.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:25.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:25.663 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:25.720 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:25.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:29:25.936 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:25.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:25.941 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:26.041 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:26.134 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:26.501 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:26.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:26.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:26.580 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T22:29:26.606 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:26.660 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:26.698 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:26.828 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:26.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:26.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:26 smithi026 ceph-mon[223014]: pgmap v879: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 12431/167940 objects degraded (7.402%); 30219/167940 objects misplaced (17.994%); 34 KiB/s, 142 objects/s recovering 2023-09-23T22:29:26.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4074012140' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:29:26.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:26 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:29:26.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:26.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:26 smithi026 ceph-mon[215775]: pgmap v879: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 12431/167940 objects degraded (7.402%); 30219/167940 objects misplaced (17.994%); 34 KiB/s, 142 objects/s recovering 2023-09-23T22:29:26.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4074012140' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:29:26.833 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:26 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:29:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:26 smithi070 ceph-mon[192652]: pgmap v879: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 12431/167940 objects degraded (7.402%); 30219/167940 objects misplaced (17.994%); 34 KiB/s, 142 objects/s recovering 2023-09-23T22:29:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4074012140' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:29:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:26 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:29:26.933 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:26.987 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:27.112 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:27.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:27.119 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:27.220 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:27.274 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:27.373 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:27.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:27.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:27.483 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:27.508 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:27.550 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:27.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:27.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:27.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:27.711 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:27 smithi070 ceph-mon[192652]: failed to encode map e343 with expected crc 2023-09-23T22:29:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:27 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:29:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:27 smithi070 ceph-mon[192652]: osdmap e343: 8 total, 7 up, 6 in 2023-09-23T22:29:27.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:27 smithi070 ceph-mon[192652]: failed to encode map e343 with expected crc 2023-09-23T22:29:27.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:27 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:27.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:27 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:27.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[215775]: failed to encode map e343 with expected crc 2023-09-23T22:29:27.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:29:27.930 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[215775]: osdmap e343: 8 total, 7 up, 6 in 2023-09-23T22:29:27.931 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[215775]: failed to encode map e343 with expected crc 2023-09-23T22:29:27.931 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:27.931 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:27.931 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[223014]: failed to encode map e343 with expected crc 2023-09-23T22:29:27.931 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:29:27.931 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[223014]: osdmap e343: 8 total, 7 up, 6 in 2023-09-23T22:29:27.931 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[223014]: failed to encode map e343 with expected crc 2023-09-23T22:29:27.931 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:27.931 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:27 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:28.286 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:28.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:28.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:28.394 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:28.506 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:28.615 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:28.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:28.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:28.723 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:28 smithi026 ceph-mon[223014]: pgmap v881: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 12431/167940 objects degraded (7.402%); 30219/167940 objects misplaced (17.994%); 32 KiB/s, 117 objects/s recovering 2023-09-23T22:29:28.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12431/167940 objects degraded (7.402%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:28.746 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:28 smithi026 ceph-mon[215775]: pgmap v881: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 12431/167940 objects degraded (7.402%); 30219/167940 objects misplaced (17.994%); 32 KiB/s, 117 objects/s recovering 2023-09-23T22:29:28.747 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12431/167940 objects degraded (7.402%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:28.806 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:28 smithi070 ceph-mon[192652]: pgmap v881: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.4 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 524 op/s; 12431/167940 objects degraded (7.402%); 30219/167940 objects misplaced (17.994%); 32 KiB/s, 117 objects/s recovering 2023-09-23T22:29:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12431/167940 objects degraded (7.402%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:28.889 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:28.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:28.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:28.998 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:29.036 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:29.096 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:29.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:29.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:29.182 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:29.202 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:29.233 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:29.291 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:29.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:29.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:29.397 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:29.434 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:29.497 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:29.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:29.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:29.605 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:29.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:29 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/653330985' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:29.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:29 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:29.647 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:29.677 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:29:29.721 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:29.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:29.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:29.832 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:29.924 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:29:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:29:29] "GET /metrics HTTP/1.1" 200 36439 "" "Prometheus/2.43.0" 2023-09-23T22:29:29.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:29 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/653330985' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:29.924 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:29 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:29.925 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:29 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/653330985' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:29.926 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:29 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:29.935 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:30.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[215775]: pgmap v882: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 12431/161565 objects degraded (7.694%); 28971/161565 objects misplaced (17.931%); 0 B/s, 60 objects/s recovering 2023-09-23T22:29:30.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:29:30.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[215775]: failed to encode map e344 with expected crc 2023-09-23T22:29:30.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:29:30.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[215775]: osdmap e344: 8 total, 7 up, 6 in 2023-09-23T22:29:30.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[215775]: failed to encode map e344 with expected crc 2023-09-23T22:29:30.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[223014]: pgmap v882: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 12431/161565 objects degraded (7.694%); 28971/161565 objects misplaced (17.931%); 0 B/s, 60 objects/s recovering 2023-09-23T22:29:30.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:29:30.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[223014]: failed to encode map e344 with expected crc 2023-09-23T22:29:30.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:29:30.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[223014]: osdmap e344: 8 total, 7 up, 6 in 2023-09-23T22:29:30.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:30 smithi026 ceph-mon[223014]: failed to encode map e344 with expected crc 2023-09-23T22:29:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:30 smithi070 ceph-mon[192652]: pgmap v882: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.3 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 578 op/s; 12431/161565 objects degraded (7.694%); 28971/161565 objects misplaced (17.931%); 0 B/s, 60 objects/s recovering 2023-09-23T22:29:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:30 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:29:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:30 smithi070 ceph-mon[192652]: failed to encode map e344 with expected crc 2023-09-23T22:29:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:30 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:29:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:30 smithi070 ceph-mon[192652]: osdmap e344: 8 total, 7 up, 6 in 2023-09-23T22:29:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:30 smithi070 ceph-mon[192652]: failed to encode map e344 with expected crc 2023-09-23T22:29:31.189 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:29:31.189 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:29:32.145 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:32.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:32.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:32.257 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:32.291 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:32.292 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:29:32.349 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:32.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:32.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:32.454 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:32.495 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:32.550 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:32.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:32.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:32.656 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:32.686 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:32.754 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:32.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:32.762 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:32.863 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:32 smithi070 ceph-mon[192652]: pgmap v884: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 12431/157695 objects degraded (7.883%); 28203/157695 objects misplaced (17.885%); 0 B/s, 39 objects/s recovering 2023-09-23T22:29:32.901 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:32.972 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:32.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:32.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:32.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:32 smithi026 ceph-mon[223014]: pgmap v884: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 12431/157695 objects degraded (7.883%); 28203/157695 objects misplaced (17.885%); 0 B/s, 39 objects/s recovering 2023-09-23T22:29:32.991 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:32 smithi026 ceph-mon[215775]: pgmap v884: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.2 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 12431/157695 objects degraded (7.883%); 28203/157695 objects misplaced (17.885%); 0 B/s, 39 objects/s recovering 2023-09-23T22:29:33.002 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:29:33.054 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:29:33.079 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:33.115 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:29:33.116 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:33.153 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:29:33.187 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:29:33.195 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:33.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:33.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:33.210 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 7, 0, 1, 6] out_osds: [3, 2] dead_osds: [2] live_osds: [0, 1, 3, 4, 5, 6, 7] 2023-09-23T22:29:33.210 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-09-23T22:29:33.211 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-09-23T22:29:33.211 DEBUG:teuthology.orchestra.run.smithi026:osd.2> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T22:29:33.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl start ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T22:29:33.228 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:29:33.274 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:29:33.275 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:29:33.275 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:29:33.275 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:29:33.275 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:29:33.275 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:29:33.275 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:29:33.275 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:29:33.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:29:33.297 INFO:journalctl@ceph.osd.2.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T22:29:33.303 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:33.404 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:33.641 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:33.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:33.645 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:33 smithi026 systemd[1]: Starting Ceph osd.2 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:29:33.746 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:33.830 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:33 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3204778235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:33.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12431/157695 objects degraded (7.883%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:33 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3204778235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12431/157695 objects degraded (7.883%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:34.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:29:34.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:33 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3204778235' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:34.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12431/157695 objects degraded (7.883%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:34.413 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:34.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:34.521 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:34.536 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:34.548 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:34.587 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:29:34.588 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:29:34.605 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:34.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:34.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:34.714 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:34.751 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:34.824 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:34.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:34.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:34.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:35.016 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:35.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:35 smithi070 ceph-mon[192652]: pgmap v885: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 12431/151347 objects degraded (8.214%); 26805/151347 objects misplaced (17.711%); 0 B/s, 75 objects/s recovering 2023-09-23T22:29:35.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:35 smithi026 ceph-mon[215775]: pgmap v885: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 12431/151347 objects degraded (8.214%); 26805/151347 objects misplaced (17.711%); 0 B/s, 75 objects/s recovering 2023-09-23T22:29:35.126 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:35 smithi026 ceph-mon[223014]: pgmap v885: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 12431/151347 objects degraded (8.214%); 26805/151347 objects misplaced (17.711%); 0 B/s, 75 objects/s recovering 2023-09-23T22:29:35.148 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:36.078 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:36.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:36.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:36.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:36.295 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:36.460 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:36.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:36.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:36.568 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:36.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:29:36.603 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:36.663 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:36.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:36.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:36.773 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:36.809 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:36.871 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:36.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:36.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:36.979 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:37.076 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:37 smithi026 ceph-mon[223014]: pgmap v886: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 12431/146778 objects degraded (8.469%); 26027/146778 objects misplaced (17.732%); 0 B/s, 67 objects/s recovering 2023-09-23T22:29:37.076 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:37.076 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:37.077 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:37 smithi026 ceph-mon[215775]: pgmap v886: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 12431/146778 objects degraded (8.469%); 26027/146778 objects misplaced (17.732%); 0 B/s, 67 objects/s recovering 2023-09-23T22:29:37.077 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:37.077 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:37.123 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:37.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:37 smithi070 ceph-mon[192652]: pgmap v886: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 12431/146778 objects degraded (8.469%); 26027/146778 objects misplaced (17.732%); 0 B/s, 67 objects/s recovering 2023-09-23T22:29:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:37.488 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:37.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:37.501 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:29:37.501 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:29:37.502 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:29:37.502 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:29:37.502 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:29:37.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:37.603 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:37.650 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:37.725 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:37.736 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:37 smithi026 bash[472128]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:29:37.736 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:37 smithi026 bash[472128]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:29:37.737 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:37 smithi026 bash[472128]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T22:29:37.737 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:37 smithi026 bash[472128]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:29:37.737 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:37 smithi026 bash[472128]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T22:29:37.737 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:37 smithi026 bash[472128]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T22:29:37.737 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:37 smithi026 bash[472128]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T22:29:37.737 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:37 smithi026 bash[472128]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T22:29:37.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:37.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:37.839 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:37.887 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:37.977 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:37.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:37.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:38.083 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:38.120 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:38.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12431/146778 objects degraded (8.469%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:38.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:38.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:38.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:38.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12431/146778 objects degraded (8.469%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:29:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:29:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12431/146778 objects degraded (8.469%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:39.309 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:39.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:39.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:39 smithi070 ceph-mon[192652]: pgmap v887: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 12431/146778 objects degraded (8.469%); 26027/146778 objects misplaced (17.732%); 0 B/s, 60 objects/s recovering 2023-09-23T22:29:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:39.428 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:39.465 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:39.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:39 smithi026 ceph-mon[223014]: pgmap v887: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 12431/146778 objects degraded (8.469%); 26027/146778 objects misplaced (17.732%); 0 B/s, 60 objects/s recovering 2023-09-23T22:29:39.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:39.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:39 smithi026 ceph-mon[215775]: pgmap v887: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 3.0 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 12431/146778 objects degraded (8.469%); 26027/146778 objects misplaced (17.732%); 0 B/s, 60 objects/s recovering 2023-09-23T22:29:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:39.534 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:39.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:39.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:39.645 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:39.692 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:39.743 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:39.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:39.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:39.848 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:39.877 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:29:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:29:39] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-09-23T22:29:39.880 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:40.227 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:40.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:40.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:40.337 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:40.369 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:40.434 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:40.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:40.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:40.448 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:40.469 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:29:40.539 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:40.624 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:41.032 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:41.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:41.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:41.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:29:41.182 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:41.187 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:41 smithi026 ceph-mon[223014]: pgmap v888: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 12431/140412 objects degraded (8.853%); 24798/140412 objects misplaced (17.661%); 0 B/s, 55 objects/s recovering 2023-09-23T22:29:41.188 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:41 smithi026 ceph-mon[215775]: pgmap v888: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 12431/140412 objects degraded (8.853%); 24798/140412 objects misplaced (17.661%); 0 B/s, 55 objects/s recovering 2023-09-23T22:29:41.257 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:29:41.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:41.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:29:41.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:29:41.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:41 smithi070 ceph-mon[192652]: pgmap v888: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 2.9 GiB data, 9.7 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 704 op/s; 12431/140412 objects degraded (8.853%); 24798/140412 objects misplaced (17.661%); 0 B/s, 55 objects/s recovering 2023-09-23T22:29:41.450 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:41 smithi026 bash[473525]: 11475c0ad819687fa136cd691f2eb1319f98924d4f7f7c39233e1a595327b27a 2023-09-23T22:29:41.450 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:41 smithi026 systemd[1]: Started Ceph osd.2 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:29:41.470 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:41.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:41.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:41.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:41.740 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:41 smithi026 conmon[473736]: 2023-09-23T22:29:41.597+0000 7fd615389640 -1 Falling back to public interface 2023-09-23T22:29:41.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:41.847 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:41.929 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:41.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:42.035 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:42.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:42.142 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:29:42.143 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:42.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:42.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:42.254 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:42.331 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:42.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2787365487' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:29:42.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2787365487' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:29:42.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2787365487' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:29:43.095 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:43.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:43.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:43.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:43.235 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:43.239 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:43.300 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:43.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:43.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: pgmap v889: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 707 op/s; 12431/136365 objects degraded (9.116%); 23969/136365 objects misplaced (17.577%); 0 B/s, 46 objects/s recovering 2023-09-23T22:29:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2787365487' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:29:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: osdmap e345: 8 total, 7 up, 6 in 2023-09-23T22:29:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12431/136365 objects degraded (9.116%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:43.407 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:43.451 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:43.469 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:43.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: pgmap v889: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 707 op/s; 12431/136365 objects degraded (9.116%); 23969/136365 objects misplaced (17.577%); 0 B/s, 46 objects/s recovering 2023-09-23T22:29:43.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:43.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2787365487' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:29:43.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: osdmap e345: 8 total, 7 up, 6 in 2023-09-23T22:29:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12431/136365 objects degraded (9.116%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:43.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: pgmap v889: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 2.8 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 707 op/s; 12431/136365 objects degraded (9.116%); 23969/136365 objects misplaced (17.577%); 0 B/s, 46 objects/s recovering 2023-09-23T22:29:43.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:43.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2787365487' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:29:43.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: osdmap e345: 8 total, 7 up, 6 in 2023-09-23T22:29:43.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: failed to encode map e345 with expected crc 2023-09-23T22:29:43.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12431/136365 objects degraded (9.116%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:43.519 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:43.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:43.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:43.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:43.660 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:43.732 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:43.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:43.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:43.838 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:43.902 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:44.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:44.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:44.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:44.341 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:44.416 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:29:44.436 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:44.451 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:29:44.493 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:29:44.534 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:29:44.571 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:29:44.615 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:29:44.645 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:29:44.645 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:29:44.645 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:29:44.645 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T22:29:44.645 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:29:44.645 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:29:44.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:29:44.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:29:44.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:29:44.841 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:44.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:44.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:44.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:45.015 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:45.129 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:45 smithi026 ceph-mon[215775]: pgmap v891: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 12431/130356 objects degraded (9.536%); 22658/130356 objects misplaced (17.382%); 0 B/s, 58 objects/s recovering 2023-09-23T22:29:45.130 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2468525599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:45.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:29:45.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:45 smithi026 ceph-mon[223014]: pgmap v891: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 12431/130356 objects degraded (9.536%); 22658/130356 objects misplaced (17.382%); 0 B/s, 58 objects/s recovering 2023-09-23T22:29:45.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2468525599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:45 smithi070 ceph-mon[192652]: pgmap v891: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 84 active+clean; 2.7 GiB data, 9.3 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 699 op/s; 12431/130356 objects degraded (9.536%); 22658/130356 objects misplaced (17.382%); 0 B/s, 58 objects/s recovering 2023-09-23T22:29:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2468525599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:46.361 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:46.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:46.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:46.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:46 smithi026 ceph-mon[223014]: osdmap e346: 8 total, 7 up, 6 in 2023-09-23T22:29:46.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:46 smithi026 ceph-mon[223014]: failed to encode map e346 with expected crc 2023-09-23T22:29:46.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:46 smithi026 ceph-mon[215775]: osdmap e346: 8 total, 7 up, 6 in 2023-09-23T22:29:46.455 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:46 smithi026 ceph-mon[215775]: failed to encode map e346 with expected crc 2023-09-23T22:29:46.473 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:46.502 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:46 smithi070 ceph-mon[192652]: osdmap e346: 8 total, 7 up, 6 in 2023-09-23T22:29:46.502 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:46 smithi070 ceph-mon[192652]: failed to encode map e346 with expected crc 2023-09-23T22:29:46.505 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:46.569 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:46.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:46.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:46.658 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:29:46.659 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:29:46.675 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:46.761 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:47.039 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:47.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:47.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:47.152 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:47.222 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:47.252 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:29:47.252 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:29:47.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[215775]: pgmap v893: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 872 op/s; 10632/125820 objects degraded (8.450%); 19998/125820 objects misplaced (15.894%); 0 B/s, 76 objects/s recovering 2023-09-23T22:29:47.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[215775]: failed to encode map e347 with expected crc 2023-09-23T22:29:47.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[215775]: osdmap e347: 8 total, 7 up, 6 in 2023-09-23T22:29:47.253 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[215775]: failed to encode map e347 with expected crc 2023-09-23T22:29:47.254 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[215775]: failed to encode map e347 with expected crc 2023-09-23T22:29:47.254 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[215775]: from='osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:29:47.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[223014]: pgmap v893: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 872 op/s; 10632/125820 objects degraded (8.450%); 19998/125820 objects misplaced (15.894%); 0 B/s, 76 objects/s recovering 2023-09-23T22:29:47.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[223014]: failed to encode map e347 with expected crc 2023-09-23T22:29:47.254 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[223014]: osdmap e347: 8 total, 7 up, 6 in 2023-09-23T22:29:47.255 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[223014]: failed to encode map e347 with expected crc 2023-09-23T22:29:47.255 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[223014]: failed to encode map e347 with expected crc 2023-09-23T22:29:47.255 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:47 smithi026 ceph-mon[223014]: from='osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:29:47.255 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:47 smithi026 conmon[473736]: 2023-09-23T22:29:47.003+0000 7fd615389640 -1 osd.2 333 log_to_monitors true 2023-09-23T22:29:47.255 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:29:47 smithi026 conmon[473736]: 2023-09-23T22:29:47.203+0000 7fd6037f0700 -1 osd.2 333 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:29:47.256 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:47.396 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:29:47.396 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:29:47.396 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:29:47.396 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:29:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:47 smithi070 ceph-mon[192652]: pgmap v893: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 872 op/s; 10632/125820 objects degraded (8.450%); 19998/125820 objects misplaced (15.894%); 0 B/s, 76 objects/s recovering 2023-09-23T22:29:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:47 smithi070 ceph-mon[192652]: failed to encode map e347 with expected crc 2023-09-23T22:29:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:47 smithi070 ceph-mon[192652]: osdmap e347: 8 total, 7 up, 6 in 2023-09-23T22:29:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:47 smithi070 ceph-mon[192652]: failed to encode map e347 with expected crc 2023-09-23T22:29:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:47 smithi070 ceph-mon[192652]: failed to encode map e347 with expected crc 2023-09-23T22:29:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:47 smithi070 ceph-mon[192652]: from='osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T22:29:47.891 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:47.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:47.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:47.999 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:48.086 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:48.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:48.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:48.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:48.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:48.302 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:48.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:48 smithi070 ceph-mon[192652]: from='osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:29:48.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:48 smithi070 ceph-mon[192652]: osdmap e348: 8 total, 7 up, 6 in 2023-09-23T22:29:48.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:48 smithi070 ceph-mon[192652]: failed to encode map e348 with expected crc 2023-09-23T22:29:48.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:48 smithi070 ceph-mon[192652]: from='osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:29:48.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:48 smithi070 ceph-mon[192652]: failed to encode map e348 with expected crc 2023-09-23T22:29:48.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:48 smithi070 ceph-mon[192652]: pgmap v896: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 654 op/s; 10632/125820 objects degraded (8.450%); 19998/125820 objects misplaced (15.894%); 0 B/s, 63 objects/s recovering 2023-09-23T22:29:48.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10632/125820 objects degraded (8.450%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[215775]: from='osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:29:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[215775]: osdmap e348: 8 total, 7 up, 6 in 2023-09-23T22:29:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[215775]: failed to encode map e348 with expected crc 2023-09-23T22:29:48.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[215775]: from='osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:29:48.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[215775]: failed to encode map e348 with expected crc 2023-09-23T22:29:48.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[215775]: pgmap v896: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 654 op/s; 10632/125820 objects degraded (8.450%); 19998/125820 objects misplaced (15.894%); 0 B/s, 63 objects/s recovering 2023-09-23T22:29:48.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10632/125820 objects degraded (8.450%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[223014]: from='osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T22:29:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[223014]: osdmap e348: 8 total, 7 up, 6 in 2023-09-23T22:29:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[223014]: failed to encode map e348 with expected crc 2023-09-23T22:29:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[223014]: from='osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:29:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[223014]: failed to encode map e348 with expected crc 2023-09-23T22:29:48.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[223014]: pgmap v896: 99 pgs: 1 active+recovering+undersized+remapped, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 84 active+clean; 2.6 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 654 op/s; 10632/125820 objects degraded (8.450%); 19998/125820 objects misplaced (15.894%); 0 B/s, 63 objects/s recovering 2023-09-23T22:29:48.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10632/125820 objects degraded (8.450%), 7 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:29:48.660 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:29:49.222 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:49.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:49.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:49.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T22:29:49.329 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:49.401 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:49.405 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:29:49.474 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[223014]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.474 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[223014]: osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150] boot 2023-09-23T22:29:49.474 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[223014]: osdmap e349: 8 total, 8 up, 6 in 2023-09-23T22:29:49.474 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:29:49.474 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[223014]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[223014]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[223014]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[223014]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[215775]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[215775]: osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150] boot 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[215775]: osdmap e349: 8 total, 8 up, 6 in 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[215775]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[215775]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[215775]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:49 smithi026 ceph-mon[215775]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:49 smithi070 ceph-mon[192652]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:49 smithi070 ceph-mon[192652]: osd.2 [v2:172.21.15.26:6818/160909150,v1:172.21.15.26:6819/160909150] boot 2023-09-23T22:29:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:49 smithi070 ceph-mon[192652]: osdmap e349: 8 total, 8 up, 6 in 2023-09-23T22:29:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:49 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T22:29:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:49 smithi070 ceph-mon[192652]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:49 smithi070 ceph-mon[192652]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:49 smithi070 ceph-mon[192652]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:49 smithi070 ceph-mon[192652]: failed to encode map e349 with expected crc 2023-09-23T22:29:49.737 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:29:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:29:49] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-09-23T22:29:49.764 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:49.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:49.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:49.868 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:49.894 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:49.941 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:49.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:49.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:50.051 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:50.083 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:50.208 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:50 smithi070 ceph-mon[192652]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.209 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:50 smithi070 ceph-mon[192652]: osdmap e350: 8 total, 8 up, 6 in 2023-09-23T22:29:50.209 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:50 smithi070 ceph-mon[192652]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.209 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:50 smithi070 ceph-mon[192652]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.209 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:50 smithi070 ceph-mon[192652]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[215775]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[215775]: osdmap e350: 8 total, 8 up, 6 in 2023-09-23T22:29:50.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[215775]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[215775]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[215775]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[215775]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[215775]: pgmap v899: 99 pgs: 1 active+recovery_wait, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 434 op/s; 10632/120606 objects degraded (8.815%); 18972/120606 objects misplaced (15.731%); 0 B/s, 14 objects/s recovering 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[215775]: osdmap e351: 8 total, 8 up, 6 in 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[223014]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[223014]: osdmap e350: 8 total, 8 up, 6 in 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[223014]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[223014]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[223014]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[223014]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[223014]: pgmap v899: 99 pgs: 1 active+recovery_wait, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 434 op/s; 10632/120606 objects degraded (8.815%); 18972/120606 objects misplaced (15.731%); 0 B/s, 14 objects/s recovering 2023-09-23T22:29:50.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:50 smithi026 ceph-mon[223014]: osdmap e351: 8 total, 8 up, 6 in 2023-09-23T22:29:50.480 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:50.499 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:50.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:50.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:50.555 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:50.613 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:29:50.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:50 smithi070 ceph-mon[192652]: failed to encode map e350 with expected crc 2023-09-23T22:29:50.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:50 smithi070 ceph-mon[192652]: pgmap v899: 99 pgs: 1 active+recovery_wait, 7 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 84 active+clean; 2.5 GiB data, 8.6 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 434 op/s; 10632/120606 objects degraded (8.815%); 18972/120606 objects misplaced (15.731%); 0 B/s, 14 objects/s recovering 2023-09-23T22:29:50.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:50 smithi070 ceph-mon[192652]: osdmap e351: 8 total, 8 up, 6 in 2023-09-23T22:29:50.644 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:50.645 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:50.862 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:29:50.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:50.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:29:50.967 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:50.996 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:51.044 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:51.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:51.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:51.150 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:51.189 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:51.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: 7.2 continuing backfill to osd.6 from (268'4264,349'8854] MIN to 349'8854 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: 7.3 continuing backfill to osd.6 from (266'3222,349'8863] MIN to 349'8863 2023-09-23T22:29:51.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:51 smithi070 ceph-mon[192652]: osdmap e352: 8 total, 8 up, 6 in 2023-09-23T22:29:51.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: 7.2 continuing backfill to osd.6 from (268'4264,349'8854] MIN to 349'8854 2023-09-23T22:29:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: 7.3 continuing backfill to osd.6 from (266'3222,349'8863] MIN to 349'8863 2023-09-23T22:29:51.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[223014]: osdmap e352: 8 total, 8 up, 6 in 2023-09-23T22:29:51.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: 7.2 continuing backfill to osd.6 from (268'4264,349'8854] MIN to 349'8854 2023-09-23T22:29:51.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: failed to encode map e351 with expected crc 2023-09-23T22:29:51.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: 7.3 continuing backfill to osd.6 from (266'3222,349'8863] MIN to 349'8863 2023-09-23T22:29:51.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:51 smithi026 ceph-mon[215775]: osdmap e352: 8 total, 8 up, 6 in 2023-09-23T22:29:52.229 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:29:52.353 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:52.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:52.459 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:52.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[223014]: failed to encode map e352 with expected crc 2023-09-23T22:29:52.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[223014]: failed to encode map e352 with expected crc 2023-09-23T22:29:52.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[223014]: 7.c continuing backfill to osd.6 from (268'4097,349'8714] MIN to 349'8714 2023-09-23T22:29:52.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4266708635' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:52.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[223014]: pgmap v902: 99 pgs: 1 remapped, 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovery_wait, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 637 op/s; 3617/118170 objects degraded (3.061%); 18534/118170 objects misplaced (15.684%); 5.5 MiB/s, 154 objects/s recovering 2023-09-23T22:29:52.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[215775]: failed to encode map e352 with expected crc 2023-09-23T22:29:52.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[215775]: failed to encode map e352 with expected crc 2023-09-23T22:29:52.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[215775]: 7.c continuing backfill to osd.6 from (268'4097,349'8714] MIN to 349'8714 2023-09-23T22:29:52.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4266708635' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:52.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:52 smithi026 ceph-mon[215775]: pgmap v902: 99 pgs: 1 remapped, 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovery_wait, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 637 op/s; 3617/118170 objects degraded (3.061%); 18534/118170 objects misplaced (15.684%); 5.5 MiB/s, 154 objects/s recovering 2023-09-23T22:29:52.502 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:52.619 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:52.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:52.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:52 smithi070 ceph-mon[192652]: failed to encode map e352 with expected crc 2023-09-23T22:29:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:52 smithi070 ceph-mon[192652]: failed to encode map e352 with expected crc 2023-09-23T22:29:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:52 smithi070 ceph-mon[192652]: 7.c continuing backfill to osd.6 from (268'4097,349'8714] MIN to 349'8714 2023-09-23T22:29:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4266708635' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:29:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:52 smithi070 ceph-mon[192652]: pgmap v902: 99 pgs: 1 remapped, 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovery_wait, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 637 op/s; 3617/118170 objects degraded (3.061%); 18534/118170 objects misplaced (15.684%); 5.5 MiB/s, 154 objects/s recovering 2023-09-23T22:29:52.724 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:29:52.756 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[223014]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[223014]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[223014]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[223014]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4266708635' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[223014]: osdmap e353: 8 total, 8 up, 6 in 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3617/118170 objects degraded (3.061%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[215775]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[215775]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[215775]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[215775]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4266708635' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:29:53.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[215775]: osdmap e353: 8 total, 8 up, 6 in 2023-09-23T22:29:53.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3617/118170 objects degraded (3.061%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:29:53.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:53 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:29:53.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:53 smithi070 ceph-mon[192652]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:53 smithi070 ceph-mon[192652]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:53 smithi070 ceph-mon[192652]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:53 smithi070 ceph-mon[192652]: failed to encode map e353 with expected crc 2023-09-23T22:29:53.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:53 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4266708635' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:29:53.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:53 smithi070 ceph-mon[192652]: osdmap e353: 8 total, 8 up, 6 in 2023-09-23T22:29:53.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3617/118170 objects degraded (3.061%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:29:54.486 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:29:54.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:54.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:29:54.593 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:54.671 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:54.896 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:54.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:54.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:54.959 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:29:54.994 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:29:55.002 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:29:55.041 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:29:55.072 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:55.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:29:55.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:29:55.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:29:55.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:29:55.182 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:29:55.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:29:55.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:29:55.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:29:55.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:29:55.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:29:55.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:29:55.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:29:55.183 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:29:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:55 smithi070 ceph-mon[192652]: pgmap v904: 99 pgs: 1 remapped, 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovery_wait, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 188 op/s; 3617/118170 objects degraded (3.061%); 18534/118170 objects misplaced (15.684%); 5.1 MiB/s, 130 objects/s recovering 2023-09-23T22:29:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/256642847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:55.483 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:29:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:55 smithi026 ceph-mon[215775]: pgmap v904: 99 pgs: 1 remapped, 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovery_wait, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 188 op/s; 3617/118170 objects degraded (3.061%); 18534/118170 objects misplaced (15.684%); 5.1 MiB/s, 130 objects/s recovering 2023-09-23T22:29:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/256642847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:55.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:55 smithi026 ceph-mon[223014]: pgmap v904: 99 pgs: 1 remapped, 1 remapped+peering, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+undersized+remapped, 1 active+recovery_wait, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 84 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 188 op/s; 3617/118170 objects degraded (3.061%); 18534/118170 objects misplaced (15.684%); 5.1 MiB/s, 130 objects/s recovering 2023-09-23T22:29:55.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:29:55.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:29:55.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/256642847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:29:55.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:55.488 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:29:55.589 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:55.651 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:55.733 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:55.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:55.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:55.837 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:55.865 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:55.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:55.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:55.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:56.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:29:56.099 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:56.205 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:29:56.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:56.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:29:56.314 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:56.376 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:56.530 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:56.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:56.642 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:56.674 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:56.688 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:29:56.719 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:56.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:56.825 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:29:56.847 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:29:56.847 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:29:56.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.847 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.35406.0:144146 7.b e353/334)", 2023-09-23T22:29:56.847 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:56.845746+0000", 2023-09-23T22:29:56.847 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 3.0046e-05, 2023-09-23T22:29:56.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4775000000000003e-05, 2023-09-23T22:29:56.847 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:29:56.847 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.845746+0000", 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.845746+0000", 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.845745+0000", 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.845752+0000", 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1300000000000003e-06 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:29:56.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.845768+0000", 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5313000000000001e-05 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.845776+0000", 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3059999999999998e-06 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:29:56.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:144146 7.b 7:d74176f5:::benchmark_data_smithi026_410749_object53809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:56.820023+0000", 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.025753062, 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025823246000000001, 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 144146 2023-09-23T22:29:56.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.820023+0000", 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.820023+0000", 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.820023+0000", 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.820026+0000", 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3380000000000001e-06 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.820038+0000", 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2082000000000001e-05 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.820045+0000", 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7290000000000003e-06 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.828049+0000", 2023-09-23T22:29:56.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080039050000000004 2023-09-23T22:29:56.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:29:56.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.828182+0000", 2023-09-23T22:29:56.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013275299999999999 2023-09-23T22:29:56.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:29:56.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.828293+0000", 2023-09-23T22:29:56.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111817 2023-09-23T22:29:56.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:29:56.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.839600+0000", 2023-09-23T22:29:56.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011306762999999999 2023-09-23T22:29:56.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:29:56.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:29:56.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:29:56.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:56.839873+0000", 2023-09-23T22:29:56.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000272565 2023-09-23T22:29:56.857 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:29:56.857 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:29:56.857 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:29:56.857 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:29:56.857 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:29:56.857 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 2 2023-09-23T22:29:56.857 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:29:56.953 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:57.084 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:57 smithi070 ceph-mon[192652]: pgmap v905: 99 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 7579/114564 objects degraded (6.616%); 19644/114564 objects misplaced (17.147%); 32 MiB/s, 554 objects/s recovering 2023-09-23T22:29:57.123 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:29:57.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:57.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:29:57.235 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:57.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:57 smithi026 ceph-mon[215775]: pgmap v905: 99 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 7579/114564 objects degraded (6.616%); 19644/114564 objects misplaced (17.147%); 32 MiB/s, 554 objects/s recovering 2023-09-23T22:29:57.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:57 smithi026 ceph-mon[223014]: pgmap v905: 99 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 335 op/s; 7579/114564 objects degraded (6.616%); 19644/114564 objects misplaced (17.147%); 32 MiB/s, 554 objects/s recovering 2023-09-23T22:29:57.280 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:57.376 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:57.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:57.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:57.485 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:29:57.521 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:57.617 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:29:57.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:57.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:29:57.724 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:57.799 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:57.882 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:57.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:57.962 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:29:57.996 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:29:58.032 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:58.117 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:29:58.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:58.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:29:58.131 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T22:29:58.137 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:29:58.225 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:29:58.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3995585061' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:29:58.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:29:58.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7579/114564 objects degraded (6.616%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:29:58.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3995585061' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:29:58.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:29:58.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7579/114564 objects degraded (6.616%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:29:58.276 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:58.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:29:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3995585061' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:29:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:29:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7579/114564 objects degraded (6.616%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:29:58.374 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:29:58.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:58.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:29:58.487 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:29:58.530 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:59.125 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:29:59.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:59.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:29:59.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:59.243 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:29:59.319 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:29:59.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[223014]: pgmap v906: 99 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 7579/114564 objects degraded (6.616%); 19644/114564 objects misplaced (17.147%); 24 MiB/s, 377 objects/s recovering 2023-09-23T22:29:59.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[223014]: failed to encode map e354 with expected crc 2023-09-23T22:29:59.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:29:59.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[223014]: osdmap e354: 8 total, 8 up, 6 in 2023-09-23T22:29:59.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[223014]: failed to encode map e354 with expected crc 2023-09-23T22:29:59.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[215775]: pgmap v906: 99 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 7579/114564 objects degraded (6.616%); 19644/114564 objects misplaced (17.147%); 24 MiB/s, 377 objects/s recovering 2023-09-23T22:29:59.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[215775]: failed to encode map e354 with expected crc 2023-09-23T22:29:59.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:29:59.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[215775]: osdmap e354: 8 total, 8 up, 6 in 2023-09-23T22:29:59.360 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:29:59 smithi026 ceph-mon[215775]: failed to encode map e354 with expected crc 2023-09-23T22:29:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:59 smithi070 ceph-mon[192652]: pgmap v906: 99 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.3 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 164 op/s; 7579/114564 objects degraded (6.616%); 19644/114564 objects misplaced (17.147%); 24 MiB/s, 377 objects/s recovering 2023-09-23T22:29:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:59 smithi070 ceph-mon[192652]: failed to encode map e354 with expected crc 2023-09-23T22:29:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:59 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:29:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:59 smithi070 ceph-mon[192652]: osdmap e354: 8 total, 8 up, 6 in 2023-09-23T22:29:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:29:59 smithi070 ceph-mon[192652]: failed to encode map e354 with expected crc 2023-09-23T22:29:59.544 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:29:59.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:29:59.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:29:59.616 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:29:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:29:59] "GET /metrics HTTP/1.1" 200 36432 "" "Prometheus/2.43.0" 2023-09-23T22:29:59.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:29:59.721 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:00.210 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:00.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:00.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:00.315 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:00.358 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:00.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:00 smithi070 ceph-mon[192652]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 7579/114564 objects degraded (6.616%), 5 pgs degraded 2023-09-23T22:30:00.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:00 smithi070 ceph-mon[192652]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T22:30:00.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:00 smithi070 ceph-mon[192652]: [WRN] PG_DEGRADED: Degraded data redundancy: 7579/114564 objects degraded (6.616%), 5 pgs degraded 2023-09-23T22:30:00.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:00 smithi070 ceph-mon[192652]: pg 2.3 is active+recovery_wait+degraded, acting [5,7,6] 2023-09-23T22:30:00.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:00 smithi070 ceph-mon[192652]: pg 7.0 is active+recovering+undersized+degraded+remapped, acting [5,1] 2023-09-23T22:30:00.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:00 smithi070 ceph-mon[192652]: pg 7.3 is active+recovery_wait+undersized+degraded+remapped, acting [4,3] 2023-09-23T22:30:00.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:00 smithi070 ceph-mon[192652]: pg 7.c is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T22:30:00.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:00 smithi070 ceph-mon[192652]: pg 7.e is active+recovery_wait+degraded+remapped, acting [5,0,3] 2023-09-23T22:30:00.426 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:00.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[215775]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 7579/114564 objects degraded (6.616%), 5 pgs degraded 2023-09-23T22:30:00.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[215775]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T22:30:00.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[215775]: [WRN] PG_DEGRADED: Degraded data redundancy: 7579/114564 objects degraded (6.616%), 5 pgs degraded 2023-09-23T22:30:00.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[215775]: pg 2.3 is active+recovery_wait+degraded, acting [5,7,6] 2023-09-23T22:30:00.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[215775]: pg 7.0 is active+recovering+undersized+degraded+remapped, acting [5,1] 2023-09-23T22:30:00.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[215775]: pg 7.3 is active+recovery_wait+undersized+degraded+remapped, acting [4,3] 2023-09-23T22:30:00.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[215775]: pg 7.c is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T22:30:00.450 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[215775]: pg 7.e is active+recovery_wait+degraded+remapped, acting [5,0,3] 2023-09-23T22:30:00.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[223014]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 7579/114564 objects degraded (6.616%), 5 pgs degraded 2023-09-23T22:30:00.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[223014]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T22:30:00.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[223014]: [WRN] PG_DEGRADED: Degraded data redundancy: 7579/114564 objects degraded (6.616%), 5 pgs degraded 2023-09-23T22:30:00.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[223014]: pg 2.3 is active+recovery_wait+degraded, acting [5,7,6] 2023-09-23T22:30:00.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[223014]: pg 7.0 is active+recovering+undersized+degraded+remapped, acting [5,1] 2023-09-23T22:30:00.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[223014]: pg 7.3 is active+recovery_wait+undersized+degraded+remapped, acting [4,3] 2023-09-23T22:30:00.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[223014]: pg 7.c is active+recovery_wait+undersized+degraded+remapped, acting [0,3] 2023-09-23T22:30:00.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:00 smithi026 ceph-mon[223014]: pg 7.e is active+recovery_wait+degraded+remapped, acting [5,0,3] 2023-09-23T22:30:00.479 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:00.604 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:00.725 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:00.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:00.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:00.838 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:30:00.888 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:00.973 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:30:00.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:00.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:30:01.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:01.176 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:01.299 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:01 smithi026 ceph-mon[223014]: pgmap v908: 99 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 417 op/s; 7579/108156 objects degraded (7.007%); 18612/108156 objects misplaced (17.208%); 22 MiB/s, 345 objects/s recovering 2023-09-23T22:30:01.299 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:01 smithi026 ceph-mon[215775]: pgmap v908: 99 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 417 op/s; 7579/108156 objects degraded (7.007%); 18612/108156 objects misplaced (17.208%); 22 MiB/s, 345 objects/s recovering 2023-09-23T22:30:01.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:01 smithi070 ceph-mon[192652]: pgmap v908: 99 pgs: 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 417 op/s; 7579/108156 objects degraded (7.007%); 18612/108156 objects misplaced (17.208%); 22 MiB/s, 345 objects/s recovering 2023-09-23T22:30:02.148 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:30:02.385 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:02.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1376274039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:30:02.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:02 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:30:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:02.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:02.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1376274039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:30:02.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:02 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:30:02.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:02.576 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:02.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1376274039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:30:02.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:02 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:30:03.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[215775]: pgmap v909: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 7513/104109 objects degraded (7.216%); 17643/104109 objects misplaced (16.947%); 20 MiB/s, 322 objects/s recovering 2023-09-23T22:30:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[215775]: failed to encode map e355 with expected crc 2023-09-23T22:30:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[215775]: failed to encode map e355 with expected crc 2023-09-23T22:30:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:30:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[215775]: osdmap e355: 8 total, 8 up, 6 in 2023-09-23T22:30:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[215775]: failed to encode map e355 with expected crc 2023-09-23T22:30:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7513/104109 objects degraded (7.216%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:30:03.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[223014]: pgmap v909: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 7513/104109 objects degraded (7.216%); 17643/104109 objects misplaced (16.947%); 20 MiB/s, 322 objects/s recovering 2023-09-23T22:30:03.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:03.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[223014]: failed to encode map e355 with expected crc 2023-09-23T22:30:03.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[223014]: failed to encode map e355 with expected crc 2023-09-23T22:30:03.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:30:03.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[223014]: osdmap e355: 8 total, 8 up, 6 in 2023-09-23T22:30:03.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[223014]: failed to encode map e355 with expected crc 2023-09-23T22:30:03.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7513/104109 objects degraded (7.216%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:30:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:03 smithi070 ceph-mon[192652]: pgmap v909: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 504 op/s; 7513/104109 objects degraded (7.216%); 17643/104109 objects misplaced (16.947%); 20 MiB/s, 322 objects/s recovering 2023-09-23T22:30:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:03 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:03 smithi070 ceph-mon[192652]: failed to encode map e355 with expected crc 2023-09-23T22:30:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:03 smithi070 ceph-mon[192652]: failed to encode map e355 with expected crc 2023-09-23T22:30:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:03 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:30:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:03 smithi070 ceph-mon[192652]: osdmap e355: 8 total, 8 up, 6 in 2023-09-23T22:30:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:03 smithi070 ceph-mon[192652]: failed to encode map e355 with expected crc 2023-09-23T22:30:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7513/104109 objects degraded (7.216%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:30:04.045 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:04.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:04.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:04.152 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:04.230 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:05.049 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:05.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:05.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:05.158 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:05.222 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:05.308 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:05.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:05.330 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:05 smithi026 ceph-mon[215775]: pgmap v911: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 7513/104109 objects degraded (7.216%); 17643/104109 objects misplaced (16.947%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T22:30:05.330 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:05 smithi026 ceph-mon[223014]: pgmap v911: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 7513/104109 objects degraded (7.216%); 17643/104109 objects misplaced (16.947%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T22:30:05.418 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:05.494 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:05.566 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:30:05.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:05 smithi070 ceph-mon[192652]: pgmap v911: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 2.1 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 435 op/s; 7513/104109 objects degraded (7.216%); 17643/104109 objects misplaced (16.947%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T22:30:05.626 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:30:05.667 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:30:05.681 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:05.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:05.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:05.709 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:30:05.745 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:30:05.782 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:30:05.796 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:05.833 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:30:05.873 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:30:05.896 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:05.907 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:30:05.907 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:30:05.907 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:30:05.907 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:30:05.907 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:30:05.907 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:30:05.907 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:30:05.907 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:30:06.097 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:30:06.097 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:30:06.160 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:06.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:06.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:06.209 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:30:06.274 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:06.333 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:06 smithi026 ceph-mon[215775]: pgmap v912: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 893 op/s; 7207/93123 objects degraded (7.739%); 15771/93123 objects misplaced (16.936%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T22:30:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2190809136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:06.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/483814256' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:30:06.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:06 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:30:06.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:06 smithi026 ceph-mon[223014]: pgmap v912: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 893 op/s; 7207/93123 objects degraded (7.739%); 15771/93123 objects misplaced (16.936%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T22:30:06.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2190809136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:06.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/483814256' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:30:06.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:06 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:30:06.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:06 smithi070 ceph-mon[192652]: pgmap v912: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 893 op/s; 7207/93123 objects degraded (7.739%); 15771/93123 objects misplaced (16.936%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T22:30:06.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2190809136' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:06.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/483814256' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:30:06.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:06 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:30:06.800 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:30:07.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:07 smithi026 ceph-mon[215775]: failed to encode map e356 with expected crc 2023-09-23T22:30:07.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:07 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:30:07.173 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:07 smithi026 ceph-mon[215775]: osdmap e356: 8 total, 8 up, 6 in 2023-09-23T22:30:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:07 smithi026 ceph-mon[223014]: failed to encode map e356 with expected crc 2023-09-23T22:30:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:07 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:30:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:07 smithi026 ceph-mon[223014]: osdmap e356: 8 total, 8 up, 6 in 2023-09-23T22:30:07.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:07 smithi070 ceph-mon[192652]: failed to encode map e356 with expected crc 2023-09-23T22:30:07.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:07 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:30:07.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:07 smithi070 ceph-mon[192652]: osdmap e356: 8 total, 8 up, 6 in 2023-09-23T22:30:07.660 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:07.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:07.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:07.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:30:07.777 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:07.849 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:07.920 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:07.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:08.024 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:30:08.051 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:08.109 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:30:08.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:08.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:30:08.219 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:08.289 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:08 smithi070 ceph-mon[192652]: pgmap v914: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 7207/93123 objects degraded (7.739%); 15771/93123 objects misplaced (16.936%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T22:30:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7207/93123 objects degraded (7.739%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:08.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:08 smithi026 ceph-mon[215775]: pgmap v914: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 7207/93123 objects degraded (7.739%); 15771/93123 objects misplaced (16.936%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T22:30:08.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7207/93123 objects degraded (7.739%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:08.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:08 smithi026 ceph-mon[223014]: pgmap v914: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 7207/93123 objects degraded (7.739%); 15771/93123 objects misplaced (16.936%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T22:30:08.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7207/93123 objects degraded (7.739%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:08.483 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:08.492 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T22:30:08.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:08.501 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:08.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:08.640 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:08.641 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:08.713 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:08.729 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:08.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:08.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:08.839 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:08.943 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:09.061 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:09.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:09.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:09.168 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:09.199 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:09.237 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:09.314 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:09.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:09.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:09.425 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:09.467 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:09.472 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:09.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:09.545 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:09.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:09.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:09.650 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:09.697 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:30:09.714 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:09.772 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:30:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:30:09] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.43.0" 2023-09-23T22:30:10.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:10 smithi026 ceph-mon[215775]: pgmap v915: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.8 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 724 op/s; 7004/86718 objects degraded (8.077%); 14983/86718 objects misplaced (17.278%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T22:30:10.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:10 smithi026 ceph-mon[223014]: pgmap v915: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.8 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 724 op/s; 7004/86718 objects degraded (8.077%); 14983/86718 objects misplaced (17.278%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T22:30:10.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:10 smithi070 ceph-mon[192652]: pgmap v915: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.8 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 724 op/s; 7004/86718 objects degraded (8.077%); 14983/86718 objects misplaced (17.278%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T22:30:11.234 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:11.281 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:11.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:11.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:11.394 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:30:11.464 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:11.541 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:30:11.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:11.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:30:11.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:11.717 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:12.049 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:12.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:12.163 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:12.250 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:30:12.250 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:30:12.283 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:12.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:12 smithi026 ceph-mon[223014]: pgmap v916: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.7 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 763 op/s; 6788/82647 objects degraded (8.213%); 14114/82647 objects misplaced (17.077%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:30:12.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3300253046' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:12.596 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:30:12.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:12 smithi070 ceph-mon[192652]: pgmap v916: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.7 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 763 op/s; 6788/82647 objects degraded (8.213%); 14114/82647 objects misplaced (17.077%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:30:12.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3300253046' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:12.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:12 smithi026 ceph-mon[215775]: pgmap v916: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.7 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 763 op/s; 6788/82647 objects degraded (8.213%); 14114/82647 objects misplaced (17.077%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T22:30:12.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3300253046' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:30:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3300253046' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:30:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: osdmap e357: 8 total, 8 up, 6 in 2023-09-23T22:30:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6788/82647 objects degraded (8.213%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:30:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3300253046' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:30:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: osdmap e357: 8 total, 8 up, 6 in 2023-09-23T22:30:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6788/82647 objects degraded (8.213%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3300253046' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: osdmap e357: 8 total, 8 up, 6 in 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: failed to encode map e357 with expected crc 2023-09-23T22:30:13.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6788/82647 objects degraded (8.213%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:14.144 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:14.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:14.188 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:30:14.252 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:14.374 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:14.482 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:14.538 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:14.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:14.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:14.654 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:14.717 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:14.913 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:14.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:14.929 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:15.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:15.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:15.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:15 smithi026 ceph-mon[215775]: pgmap v918: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.7 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 6788/82647 objects degraded (8.213%); 14114/82647 objects misplaced (17.077%); 2.1 MiB/s, 32 objects/s recovering 2023-09-23T22:30:15.160 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:15 smithi026 ceph-mon[223014]: pgmap v918: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.7 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 6788/82647 objects degraded (8.213%); 14114/82647 objects misplaced (17.077%); 2.1 MiB/s, 32 objects/s recovering 2023-09-23T22:30:15.275 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:15.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:15.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:15.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:15 smithi070 ceph-mon[192652]: pgmap v918: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.7 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 436 op/s; 6788/82647 objects degraded (8.213%); 14114/82647 objects misplaced (17.077%); 2.1 MiB/s, 32 objects/s recovering 2023-09-23T22:30:15.386 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:15.466 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:15.589 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:15.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:15.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:15.701 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:15.728 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:15.774 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:15.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:15.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:15.877 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:15.964 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:16.083 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:16.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:16.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:16.112 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T22:30:16.203 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:16.286 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:16.352 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/181512201' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:30:16.358 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/181512201' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:30:16.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/181512201' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:30:16.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:30:16.965 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:16.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:16.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:17.071 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:17.080 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:17.115 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T22:30:17.200 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:17.322 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:17.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:17.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:17.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[223014]: pgmap v919: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.5 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 744 op/s; 6167/72261 objects degraded (8.534%); 12626/72261 objects misplaced (17.473%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T22:30:17.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/181512201' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:30:17.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[223014]: osdmap e358: 8 total, 8 up, 6 in 2023-09-23T22:30:17.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[223014]: failed to encode map e358 with expected crc 2023-09-23T22:30:17.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[223014]: failed to encode map e358 with expected crc 2023-09-23T22:30:17.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[223014]: failed to encode map e358 with expected crc 2023-09-23T22:30:17.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[215775]: pgmap v919: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.5 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 744 op/s; 6167/72261 objects degraded (8.534%); 12626/72261 objects misplaced (17.473%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T22:30:17.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/181512201' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:30:17.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[215775]: osdmap e358: 8 total, 8 up, 6 in 2023-09-23T22:30:17.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[215775]: failed to encode map e358 with expected crc 2023-09-23T22:30:17.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[215775]: failed to encode map e358 with expected crc 2023-09-23T22:30:17.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:17 smithi026 ceph-mon[215775]: failed to encode map e358 with expected crc 2023-09-23T22:30:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:17 smithi070 ceph-mon[192652]: pgmap v919: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.5 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 744 op/s; 6167/72261 objects degraded (8.534%); 12626/72261 objects misplaced (17.473%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T22:30:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:17 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/181512201' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:30:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:17 smithi070 ceph-mon[192652]: osdmap e358: 8 total, 8 up, 6 in 2023-09-23T22:30:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:17 smithi070 ceph-mon[192652]: failed to encode map e358 with expected crc 2023-09-23T22:30:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:17 smithi070 ceph-mon[192652]: failed to encode map e358 with expected crc 2023-09-23T22:30:17.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:17 smithi070 ceph-mon[192652]: failed to encode map e358 with expected crc 2023-09-23T22:30:17.426 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:17.500 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:17.579 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:17.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:17.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:17.687 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:17.715 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:30:17.760 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:30:17.764 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:17.802 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:30:17.846 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:30:17.861 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:17.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:17.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:17.888 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:30:17.925 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:30:17.966 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:30:17.971 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:17.972 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:18.007 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:30:18.017 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:18.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:30:18.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:30:18.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:30:18.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:30:18.041 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:30:18.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:30:18.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:30:18.042 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:30:18.108 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:18.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:18.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:18.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:18.308 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:18.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1942240553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:18.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6167/72261 objects degraded (8.534%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:18.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1942240553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:18.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6167/72261 objects degraded (8.534%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:18.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1942240553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:18.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6167/72261 objects degraded (8.534%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:19.196 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:19.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:19.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:19.309 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:19.351 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:19.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:19 smithi026 ceph-mon[215775]: pgmap v921: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.5 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 6167/72261 objects degraded (8.534%); 12626/72261 objects misplaced (17.473%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:30:19.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:19 smithi026 ceph-mon[223014]: pgmap v921: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.5 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 6167/72261 objects degraded (8.534%); 12626/72261 objects misplaced (17.473%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:30:19.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:19 smithi070 ceph-mon[192652]: pgmap v921: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.5 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 601 op/s; 6167/72261 objects degraded (8.534%); 12626/72261 objects misplaced (17.473%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T22:30:19.794 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:19.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:19.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:19.898 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:30:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:30:19] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.43.0" 2023-09-23T22:30:19.909 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:19.962 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:20.006 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:20.035 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:20.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:20.149 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:20.216 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:20.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:20.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:20.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:20.424 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:30:20.479 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:20.550 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:30:20.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:20.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:30:20.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:20.677 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35406.0:160651 7.e e358/335 7:77f8b7b0:::benchmark_data_smithi026_410749_object70314:head v 358'9982, mlcod=270'5218)", 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:20.489100+0000", 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0060754030000000001, 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006107243, 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.678 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.489100+0000", 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.489100+0000", 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.489099+0000", 2023-09-23T22:30:20.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.489109+0000", 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7270000000000002e-06 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.489121+0000", 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1620999999999999e-05 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.489129+0000", 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0010000000000001e-06 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.489171+0000", 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2094000000000001e-05 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:30:20.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.489190+0000", 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8770000000000002e-05 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.35406.0:160653 7.e e358/335 7:72e03d00:::benchmark_data_smithi026_410749_object70316:head v 358'9983, mlcod=270'5218)", 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:20.493430+0000", 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0017458580000000001, 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001845209, 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:30:20.682 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.493430+0000", 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.493430+0000", 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.683 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.493429+0000", 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.493437+0000", 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3820000000000004e-06 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:30:20.684 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.493455+0000", 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7342000000000001e-05 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.493463+0000", 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3080000000000008e-06 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.494045+0000", 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00058178600000000004 2023-09-23T22:30:20.685 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:20.494073+0000", 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7685e-05 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 2 2023-09-23T22:30:20.686 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:30:20.737 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:21.387 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:21 smithi026 ceph-mon[223014]: pgmap v922: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 5966/66051 objects degraded (9.032%); 11806/66051 objects misplaced (17.874%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:30:21.388 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:21 smithi026 ceph-mon[215775]: pgmap v922: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 5966/66051 objects degraded (9.032%); 11806/66051 objects misplaced (17.874%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:30:21.631 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:21 smithi070 ceph-mon[192652]: pgmap v922: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 691 op/s; 5966/66051 objects degraded (9.032%); 11806/66051 objects misplaced (17.874%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:30:21.965 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:21.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:21.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:22.006 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:30:22.073 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:22.128 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:22.225 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:30:22.225 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:22.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:22.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:22.333 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:22.444 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:22.829 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:22.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:22.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:22.934 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:22.974 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:23.070 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:23.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:23.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:23.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:23.280 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:23.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:23 smithi070 ceph-mon[192652]: pgmap v923: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 754 op/s; 5564/62412 objects degraded (8.915%); 11159/62412 objects misplaced (17.880%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:30:23.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 5564/62412 objects degraded (8.915%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:23.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:23 smithi026 ceph-mon[223014]: pgmap v923: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 754 op/s; 5564/62412 objects degraded (8.915%); 11159/62412 objects misplaced (17.880%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:30:23.437 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 5564/62412 objects degraded (8.915%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:23.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:23 smithi026 ceph-mon[215775]: pgmap v923: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 754 op/s; 5564/62412 objects degraded (8.915%); 11159/62412 objects misplaced (17.880%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T22:30:23.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 5564/62412 objects degraded (8.915%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:23.482 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:23.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:23.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:23.599 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:23.639 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:23.656 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:23.698 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:23.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:23.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:23.755 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:23.806 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:23.843 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:23.929 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:23.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:23.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:24.035 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:24.084 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:24.151 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:24.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:24.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:24.189 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:30:24.257 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:30:24.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/666127561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:30:24.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:24 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:30:24.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:24.293 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:24.295 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:24.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/666127561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:30:24.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:24 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:30:24.329 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:24.329 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:24.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/666127561' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:30:24.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:24 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:30:24.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:24.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:24.368 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:30:24.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:24.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:30:24.497 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:24.595 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:25.157 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T22:30:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[215775]: pgmap v924: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 5564/62412 objects degraded (8.915%); 11159/62412 objects misplaced (17.880%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T22:30:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:25.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[215775]: failed to encode map e359 with expected crc 2023-09-23T22:30:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:30:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[215775]: osdmap e359: 8 total, 8 up, 6 in 2023-09-23T22:30:25.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3960259176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:30:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[223014]: pgmap v924: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 5564/62412 objects degraded (8.915%); 11159/62412 objects misplaced (17.880%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T22:30:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[223014]: failed to encode map e359 with expected crc 2023-09-23T22:30:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:30:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[223014]: osdmap e359: 8 total, 8 up, 6 in 2023-09-23T22:30:25.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3960259176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:30:25.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:25 smithi070 ceph-mon[192652]: pgmap v924: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.3 GiB data, 5.7 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 674 op/s; 5564/62412 objects degraded (8.915%); 11159/62412 objects misplaced (17.880%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T22:30:25.630 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:25 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:25.630 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:25 smithi070 ceph-mon[192652]: failed to encode map e359 with expected crc 2023-09-23T22:30:25.630 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:25 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:30:25.630 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:25 smithi070 ceph-mon[192652]: osdmap e359: 8 total, 8 up, 6 in 2023-09-23T22:30:25.630 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3960259176' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:30:26.110 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:26.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:26.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:26.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:26.247 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:26.295 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:26.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:26.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:26.404 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:26.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[223014]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[223014]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3960259176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:30:26.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[223014]: osdmap e360: 8 total, 8 up, 6 in 2023-09-23T22:30:26.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[223014]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[223014]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[223014]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[223014]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[215775]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[215775]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3960259176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:30:26.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[215775]: osdmap e360: 8 total, 8 up, 6 in 2023-09-23T22:30:26.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[215775]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[215775]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[215775]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:26 smithi026 ceph-mon[215775]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.433 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:26 smithi070 ceph-mon[192652]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:26 smithi070 ceph-mon[192652]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3960259176' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:30:26.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:26 smithi070 ceph-mon[192652]: osdmap e360: 8 total, 8 up, 6 in 2023-09-23T22:30:26.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:26 smithi070 ceph-mon[192652]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:26 smithi070 ceph-mon[192652]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:26 smithi070 ceph-mon[192652]: failed to encode map e360 with expected crc 2023-09-23T22:30:26.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:26 smithi070 ceph-mon[192652]: failed to encode map e360 with expected crc 2023-09-23T22:30:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:27 smithi026 ceph-mon[215775]: pgmap v927: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 839 op/s; 4893/52101 objects degraded (9.391%); 9636/52101 objects misplaced (18.495%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T22:30:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:27 smithi026 ceph-mon[215775]: failed to encode map e361 with expected crc 2023-09-23T22:30:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:27 smithi026 ceph-mon[215775]: osdmap e361: 8 total, 8 up, 6 in 2023-09-23T22:30:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:27 smithi026 ceph-mon[223014]: pgmap v927: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 839 op/s; 4893/52101 objects degraded (9.391%); 9636/52101 objects misplaced (18.495%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T22:30:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:27 smithi026 ceph-mon[223014]: failed to encode map e361 with expected crc 2023-09-23T22:30:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:27 smithi026 ceph-mon[223014]: osdmap e361: 8 total, 8 up, 6 in 2023-09-23T22:30:27.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:27 smithi070 ceph-mon[192652]: pgmap v927: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 839 op/s; 4893/52101 objects degraded (9.391%); 9636/52101 objects misplaced (18.495%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T22:30:27.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:27 smithi070 ceph-mon[192652]: failed to encode map e361 with expected crc 2023-09-23T22:30:27.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:27 smithi070 ceph-mon[192652]: osdmap e361: 8 total, 8 up, 6 in 2023-09-23T22:30:27.644 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:30:27.665 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:27.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:27.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:27.780 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:27.829 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:27.913 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:27.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:27.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:28.021 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:28.132 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:28.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4893/52101 objects degraded (9.391%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:28.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4893/52101 objects degraded (9.391%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:28.280 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:28.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:28.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:28.339 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:30:28.339 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:30:28.339 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:30:28.340 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:30:28.340 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:30:28.394 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:28.486 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4893/52101 objects degraded (9.391%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:28.664 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:28.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:28.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:28.724 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:30:28.773 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:28.773 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:30:28.804 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:30:28.844 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:28.846 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:30:28.890 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:30:28.923 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:30:28.971 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:30:28.979 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:28.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:28.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:29.012 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:30:29.021 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:30:29.021 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:30:29.021 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:30:29.021 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:30:29.021 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:30:29.021 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:30:29.022 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:30:29.022 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:30:29.091 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:29.117 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:29.160 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:29.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:29.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:29.267 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:29.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:29 smithi026 ceph-mon[215775]: pgmap v929: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 4893/52101 objects degraded (9.391%); 9636/52101 objects misplaced (18.495%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:30:29.319 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:29 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3853030309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:29.320 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:29 smithi026 ceph-mon[223014]: pgmap v929: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 4893/52101 objects degraded (9.391%); 9636/52101 objects misplaced (18.495%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:30:29.320 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:29 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3853030309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:29.323 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:29.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:29 smithi070 ceph-mon[192652]: pgmap v929: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 1.1 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 572 op/s; 4893/52101 objects degraded (9.391%); 9636/52101 objects misplaced (18.495%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:30:29.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:29 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3853030309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:29.584 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:30:29.678 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:29.718 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:30:29.818 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:29.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:29.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:29.932 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:29.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:30:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:30:29] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2023-09-23T22:30:30.048 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:30.333 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:30.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:30.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:30.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:30 smithi026 ceph-mon[215775]: pgmap v930: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 964 MiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 902 op/s; 4689/46155 objects degraded (10.159%); 8835/46155 objects misplaced (19.142%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:30:30.345 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:30 smithi026 ceph-mon[223014]: pgmap v930: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 964 MiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 902 op/s; 4689/46155 objects degraded (10.159%); 8835/46155 objects misplaced (19.142%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:30:30.443 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:30.530 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:30.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:30 smithi070 ceph-mon[192652]: pgmap v930: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 964 MiB data, 4.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 902 op/s; 4689/46155 objects degraded (10.159%); 8835/46155 objects misplaced (19.142%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:30:30.736 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:30.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:30.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:30.847 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:30.922 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:31.081 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:31.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:31.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:31.196 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:31.222 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:31.269 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:31.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:31.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:31.378 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:31.448 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:31.551 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:31.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:31.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:31.663 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:30:31.688 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:31.692 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:31.743 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:30:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:31.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:30:31.760 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:31.850 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:31.924 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:32.088 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:32.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:32.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:32.203 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:32.274 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:32.407 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:30:32.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:32 smithi026 ceph-mon[215775]: pgmap v931: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 891 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 4304/42648 objects degraded (10.092%); 8133/42648 objects misplaced (19.070%); 6.1 MiB/s, 104 objects/s recovering 2023-09-23T22:30:32.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:32 smithi026 ceph-mon[223014]: pgmap v931: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 891 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 4304/42648 objects degraded (10.092%); 8133/42648 objects misplaced (19.070%); 6.1 MiB/s, 104 objects/s recovering 2023-09-23T22:30:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:32 smithi070 ceph-mon[192652]: pgmap v931: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 891 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 603 op/s; 4304/42648 objects degraded (10.092%); 8133/42648 objects misplaced (19.070%); 6.1 MiB/s, 104 objects/s recovering 2023-09-23T22:30:32.969 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:32.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:32.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:33.076 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:33.109 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:33.690 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:33.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:33.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:33.800 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:33.851 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4304/42648 objects degraded (10.092%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:33.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:33 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3694391572' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:30:33.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:33 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:30:33.868 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:33.955 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:33.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:33.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4304/42648 objects degraded (10.092%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:33 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3694391572' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:30:33.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:33 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:30:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4304/42648 objects degraded (10.092%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:33 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3694391572' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:30:33.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:33 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:30:34.061 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:34.105 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:30:34.149 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:35.030 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:35.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:35.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:35.140 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:35.190 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:35.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T22:30:35.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[215775]: pgmap v932: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 891 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 376 op/s; 4304/42648 objects degraded (10.092%); 8133/42648 objects misplaced (19.070%); 2.8 MiB/s, 50 objects/s recovering 2023-09-23T22:30:35.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:30:35.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[215775]: osdmap e362: 8 total, 8 up, 6 in 2023-09-23T22:30:35.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[215775]: failed to encode map e362 with expected crc 2023-09-23T22:30:35.304 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[215775]: failed to encode map e362 with expected crc 2023-09-23T22:30:35.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[223014]: pgmap v932: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 891 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 376 op/s; 4304/42648 objects degraded (10.092%); 8133/42648 objects misplaced (19.070%); 2.8 MiB/s, 50 objects/s recovering 2023-09-23T22:30:35.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:30:35.304 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[223014]: osdmap e362: 8 total, 8 up, 6 in 2023-09-23T22:30:35.305 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[223014]: failed to encode map e362 with expected crc 2023-09-23T22:30:35.305 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:35 smithi026 ceph-mon[223014]: failed to encode map e362 with expected crc 2023-09-23T22:30:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:35 smithi070 ceph-mon[192652]: pgmap v932: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 891 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 376 op/s; 4304/42648 objects degraded (10.092%); 8133/42648 objects misplaced (19.070%); 2.8 MiB/s, 50 objects/s recovering 2023-09-23T22:30:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:35 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:30:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:35 smithi070 ceph-mon[192652]: osdmap e362: 8 total, 8 up, 6 in 2023-09-23T22:30:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:35 smithi070 ceph-mon[192652]: failed to encode map e362 with expected crc 2023-09-23T22:30:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:35 smithi070 ceph-mon[192652]: failed to encode map e362 with expected crc 2023-09-23T22:30:35.511 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:35.519 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:35.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:35.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:35.577 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:30:35.577 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:30:35.625 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:35.714 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:35.794 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:35.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:35.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:35.900 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:35.977 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:36.108 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:30:36.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[215775]: osdmap e363: 8 total, 8 up, 6 in 2023-09-23T22:30:36.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[215775]: failed to encode map e363 with expected crc 2023-09-23T22:30:36.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[215775]: failed to encode map e363 with expected crc 2023-09-23T22:30:36.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[215775]: failed to encode map e363 with expected crc 2023-09-23T22:30:36.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3405838531' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:36.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:36.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[223014]: osdmap e363: 8 total, 8 up, 6 in 2023-09-23T22:30:36.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[223014]: failed to encode map e363 with expected crc 2023-09-23T22:30:36.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[223014]: failed to encode map e363 with expected crc 2023-09-23T22:30:36.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[223014]: failed to encode map e363 with expected crc 2023-09-23T22:30:36.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3405838531' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:36.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:36.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:36 smithi070 ceph-mon[192652]: osdmap e363: 8 total, 8 up, 6 in 2023-09-23T22:30:36.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:36 smithi070 ceph-mon[192652]: failed to encode map e363 with expected crc 2023-09-23T22:30:36.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:36 smithi070 ceph-mon[192652]: failed to encode map e363 with expected crc 2023-09-23T22:30:36.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:36 smithi070 ceph-mon[192652]: failed to encode map e363 with expected crc 2023-09-23T22:30:36.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:36 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3405838531' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:36.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:36.770 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:36.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:36.856 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:30:36.880 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:36.984 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:37.126 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:37.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:37.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:37.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:37.293 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:37.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:37 smithi070 ceph-mon[192652]: pgmap v935: 99 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 86 active+clean; 684 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 807 op/s; 3691/32709 objects degraded (11.284%); 7021/32709 objects misplaced (21.465%); 2.9 MiB/s, 110 objects/s recovering 2023-09-23T22:30:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:37 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:37 smithi070 ceph-mon[192652]: failed to encode map e364 with expected crc 2023-09-23T22:30:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:37 smithi070 ceph-mon[192652]: failed to encode map e364 with expected crc 2023-09-23T22:30:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:37 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:30:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:37 smithi070 ceph-mon[192652]: osdmap e364: 8 total, 8 up, 6 in 2023-09-23T22:30:37.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[215775]: pgmap v935: 99 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 86 active+clean; 684 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 807 op/s; 3691/32709 objects degraded (11.284%); 7021/32709 objects misplaced (21.465%); 2.9 MiB/s, 110 objects/s recovering 2023-09-23T22:30:37.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:37.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[215775]: failed to encode map e364 with expected crc 2023-09-23T22:30:37.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[215775]: failed to encode map e364 with expected crc 2023-09-23T22:30:37.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:30:37.390 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[215775]: osdmap e364: 8 total, 8 up, 6 in 2023-09-23T22:30:37.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[223014]: pgmap v935: 99 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 86 active+clean; 684 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 807 op/s; 3691/32709 objects degraded (11.284%); 7021/32709 objects misplaced (21.465%); 2.9 MiB/s, 110 objects/s recovering 2023-09-23T22:30:37.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:37.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[223014]: failed to encode map e364 with expected crc 2023-09-23T22:30:37.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[223014]: failed to encode map e364 with expected crc 2023-09-23T22:30:37.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:30:37.390 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:37 smithi026 ceph-mon[223014]: osdmap e364: 8 total, 8 up, 6 in 2023-09-23T22:30:37.398 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:30:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3691/32709 objects degraded (11.284%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:38.422 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:38.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:38.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:38.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:38.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3691/32709 objects degraded (11.284%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:38.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:38.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:38.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:30:38.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:38.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:38.436 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3691/32709 objects degraded (11.284%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:38.531 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:38.669 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:38.794 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:38.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:38.830 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:38.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:38.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:38.988 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:39.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[223014]: pgmap v937: 99 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 86 active+clean; 684 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 551 op/s; 3691/32709 objects degraded (11.284%); 7021/32709 objects misplaced (21.465%); 0 B/s, 76 objects/s recovering 2023-09-23T22:30:39.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:39.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:39.130 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:39.131 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:39 smithi070 ceph-mon[192652]: pgmap v937: 99 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 86 active+clean; 684 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 551 op/s; 3691/32709 objects degraded (11.284%); 7021/32709 objects misplaced (21.465%); 0 B/s, 76 objects/s recovering 2023-09-23T22:30:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:39.463 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:30:39.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[215775]: pgmap v937: 99 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 86 active+clean; 684 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 551 op/s; 3691/32709 objects degraded (11.284%); 7021/32709 objects misplaced (21.465%); 0 B/s, 76 objects/s recovering 2023-09-23T22:30:39.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:39.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:39.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:39.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:39.510 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:30:39.560 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:30:39.607 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:30:39.653 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:30:39.684 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:30:39.740 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:30:39.772 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:30:39.783 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:30:39.783 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:30:39.783 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:30:39.783 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:30:39.783 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:30:39.783 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:30:39.783 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:30:39.783 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:30:39.818 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:30:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:30:39] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.43.0" 2023-09-23T22:30:40.138 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/821082688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:40.276 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:30:40.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/821082688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:40.546 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:40.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:40.557 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/821082688' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:40.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:40.685 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:41.071 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:41.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:41.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:41.176 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:41.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:41.333 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:30:41.333 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:173944 7.4 e364/304 7:2467ddc3:::benchmark_data_smithi026_410749_object83607:head v 364'10984, mlcod=364'10982)", 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:41.331289+0000", 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.00086137500000000005, 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00089209599999999999, 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:30:41.334 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:41.331289+0000", 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:41.331289+0000", 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:41.331289+0000", 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:30:41.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:41.331293+0000", 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4170000000000002e-06 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:41.331303+0000", 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5869999999999999e-06 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:30:41.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:41.331308+0000", 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8130000000000003e-06 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:41.331358+0000", 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0506000000000001e-05 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:30:41.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:41.331381+0000", 2023-09-23T22:30:41.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3198999999999999e-05 2023-09-23T22:30:41.338 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:30:41.338 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:30:41.338 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:30:41.338 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:30:41.338 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T22:30:41.338 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 1 2023-09-23T22:30:41.338 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:30:41.360 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:41 smithi070 ceph-mon[192652]: pgmap v938: 99 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 86 active+clean; 564 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 872 op/s; 3501/26934 objects degraded (12.998%); 6295/26934 objects misplaced (23.372%); 0 B/s, 76 objects/s recovering 2023-09-23T22:30:41.441 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:41.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:41 smithi026 ceph-mon[215775]: pgmap v938: 99 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 86 active+clean; 564 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 872 op/s; 3501/26934 objects degraded (12.998%); 6295/26934 objects misplaced (23.372%); 0 B/s, 76 objects/s recovering 2023-09-23T22:30:41.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:41.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:41.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:41 smithi026 ceph-mon[223014]: pgmap v938: 99 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 86 active+clean; 564 MiB data, 3.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 872 op/s; 3501/26934 objects degraded (12.998%); 6295/26934 objects misplaced (23.372%); 0 B/s, 76 objects/s recovering 2023-09-23T22:30:41.549 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:41.621 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:41.689 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:41.710 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:41.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:41.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:41.815 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:41.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:42.141 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:42.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:42.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:42.257 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:30:42.309 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:42.951 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:30:42.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:42.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:30:43.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:43.074 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:30:43.098 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:43.172 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T22:30:43.269 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:30:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:43 smithi070 ceph-mon[192652]: pgmap v939: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 490 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 861 op/s; 3074/23415 objects degraded (13.128%); 5978/23415 objects misplaced (25.531%); 0 B/s, 123 objects/s recovering 2023-09-23T22:30:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/266399713' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:30:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:43 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:30:43.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3074/23415 objects degraded (13.128%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:43.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:43 smithi026 ceph-mon[215775]: pgmap v939: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 490 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 861 op/s; 3074/23415 objects degraded (13.128%); 5978/23415 objects misplaced (25.531%); 0 B/s, 123 objects/s recovering 2023-09-23T22:30:43.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/266399713' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:30:43.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:43 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:30:43.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3074/23415 objects degraded (13.128%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:43 smithi026 ceph-mon[223014]: pgmap v939: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 490 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 861 op/s; 3074/23415 objects degraded (13.128%); 5978/23415 objects misplaced (25.531%); 0 B/s, 123 objects/s recovering 2023-09-23T22:30:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/266399713' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:30:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:43 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:30:43.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3074/23415 objects degraded (13.128%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:43.494 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:43.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:43.602 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:43.665 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:43.823 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:30:44.072 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:44.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:44.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:44.082 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:30:44.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:30:44.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:44.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:44 smithi026 ceph-mon[215775]: failed to encode map e365 with expected crc 2023-09-23T22:30:44.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:44 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:30:44.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:44 smithi026 ceph-mon[215775]: osdmap e365: 8 total, 8 up, 6 in 2023-09-23T22:30:44.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:44 smithi026 ceph-mon[215775]: failed to encode map e365 with expected crc 2023-09-23T22:30:44.342 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:44.342 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:44 smithi026 ceph-mon[223014]: failed to encode map e365 with expected crc 2023-09-23T22:30:44.342 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:44 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:30:44.342 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:44 smithi026 ceph-mon[223014]: osdmap e365: 8 total, 8 up, 6 in 2023-09-23T22:30:44.342 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:44 smithi026 ceph-mon[223014]: failed to encode map e365 with expected crc 2023-09-23T22:30:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:44 smithi070 ceph-mon[192652]: failed to encode map e365 with expected crc 2023-09-23T22:30:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:44 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:30:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:44 smithi070 ceph-mon[192652]: osdmap e365: 8 total, 8 up, 6 in 2023-09-23T22:30:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:44 smithi070 ceph-mon[192652]: failed to encode map e365 with expected crc 2023-09-23T22:30:44.473 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:44.546 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:44.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:44.557 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:44.658 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:44.747 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:44.903 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:44.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:44.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:45.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:45.099 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:45.236 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:30:45.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:45 smithi026 ceph-mon[215775]: pgmap v941: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 490 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 386 op/s; 3074/23415 objects degraded (13.128%); 5978/23415 objects misplaced (25.531%); 0 B/s, 57 objects/s recovering 2023-09-23T22:30:45.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2585620969' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:30:45.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:45 smithi026 ceph-mon[223014]: pgmap v941: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 490 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 386 op/s; 3074/23415 objects degraded (13.128%); 5978/23415 objects misplaced (25.531%); 0 B/s, 57 objects/s recovering 2023-09-23T22:30:45.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2585620969' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:30:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:45 smithi070 ceph-mon[192652]: pgmap v941: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 490 MiB data, 3.7 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 386 op/s; 3074/23415 objects degraded (13.128%); 5978/23415 objects misplaced (25.531%); 0 B/s, 57 objects/s recovering 2023-09-23T22:30:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2585620969' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:30:46.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:46.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:46.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:46.340 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:46.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:46 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:46.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:46 smithi070 ceph-mon[192652]: failed to encode map e366 with expected crc 2023-09-23T22:30:46.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2585620969' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:30:46.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:46 smithi070 ceph-mon[192652]: osdmap e366: 8 total, 8 up, 6 in 2023-09-23T22:30:46.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:46 smithi070 ceph-mon[192652]: failed to encode map e366 with expected crc 2023-09-23T22:30:46.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:46 smithi070 ceph-mon[192652]: pgmap v943: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 332 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 703 op/s; 2623/15807 objects degraded (16.594%); 5042/15807 objects misplaced (31.897%); 1.5 MiB/s, 153 objects/s recovering 2023-09-23T22:30:46.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:46.447 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:46.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:46.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:46.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:46.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[223014]: failed to encode map e366 with expected crc 2023-09-23T22:30:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2585620969' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:30:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[223014]: osdmap e366: 8 total, 8 up, 6 in 2023-09-23T22:30:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[223014]: failed to encode map e366 with expected crc 2023-09-23T22:30:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[223014]: pgmap v943: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 332 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 703 op/s; 2623/15807 objects degraded (16.594%); 5042/15807 objects misplaced (31.897%); 1.5 MiB/s, 153 objects/s recovering 2023-09-23T22:30:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:30:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[215775]: failed to encode map e366 with expected crc 2023-09-23T22:30:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2585620969' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:30:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[215775]: osdmap e366: 8 total, 8 up, 6 in 2023-09-23T22:30:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[215775]: failed to encode map e366 with expected crc 2023-09-23T22:30:46.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:46 smithi026 ceph-mon[215775]: pgmap v943: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 332 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 703 op/s; 2623/15807 objects degraded (16.594%); 5042/15807 objects misplaced (31.897%); 1.5 MiB/s, 153 objects/s recovering 2023-09-23T22:30:46.552 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:46.624 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:46.725 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:30:46.725 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:30:46.725 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:30:46.725 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:30:46.726 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:30:46.764 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:46.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:46.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:46.870 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:46.923 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:46.998 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:47.678 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:30:47.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:30:47.767 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:30:47.804 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:30:47.834 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:30:47.858 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:47.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:47.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:47.874 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:30:47.911 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:30:47.950 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:30:47.954 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:30:47.954 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:30:47.954 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:30:47.954 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:30:47.954 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:30:47.954 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:30:47.955 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:30:47.955 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:30:47.964 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:48.013 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:48.503 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:48.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:48.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:48.607 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:48.661 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:30:48.676 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:48.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:48 smithi070 ceph-mon[192652]: pgmap v944: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 332 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 2623/15807 objects degraded (16.594%); 5042/15807 objects misplaced (31.897%); 1.5 MiB/s, 153 objects/s recovering 2023-09-23T22:30:48.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2594462930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:48.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 2623/15807 objects degraded (16.594%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:48.842 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:48.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:48.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:48.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:48 smithi026 ceph-mon[223014]: pgmap v944: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 332 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 2623/15807 objects degraded (16.594%); 5042/15807 objects misplaced (31.897%); 1.5 MiB/s, 153 objects/s recovering 2023-09-23T22:30:48.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2594462930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:48.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 2623/15807 objects degraded (16.594%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:48.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:48 smithi026 ceph-mon[215775]: pgmap v944: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 332 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 2623/15807 objects degraded (16.594%); 5042/15807 objects misplaced (31.897%); 1.5 MiB/s, 153 objects/s recovering 2023-09-23T22:30:48.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2594462930' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:48.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 2623/15807 objects degraded (16.594%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:30:48.923 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:30:48.947 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:48.997 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:49.063 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:49.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:49.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:49.171 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:49.239 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:49.261 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:30:49.300 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:49.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:49.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:49.404 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:49.430 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:49.658 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:49.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:49.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:49.768 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:49.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:49.947 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:30:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:30:49] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.43.0" 2023-09-23T22:30:49.955 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:49.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:49.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:50.062 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:50.103 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:50.171 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:50.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:50.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:50.276 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:50.375 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:50.496 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:50.545 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:50.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:50.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:50.655 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:30:50.657 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:30:50.660 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:50.687 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:50.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:50 smithi026 ceph-mon[223014]: pgmap v945: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 229 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 523 op/s; 2467/10845 objects degraded (22.748%); 4275/10845 objects misplaced (39.419%); 2.8 MiB/s, 115 objects/s recovering 2023-09-23T22:30:50.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:50 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:50.948 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:50 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:50.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:50 smithi026 ceph-mon[215775]: pgmap v945: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 229 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 523 op/s; 2467/10845 objects degraded (22.748%); 4275/10845 objects misplaced (39.419%); 2.8 MiB/s, 115 objects/s recovering 2023-09-23T22:30:50.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:50 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:50.948 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:50 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:51.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:50 smithi070 ceph-mon[192652]: pgmap v945: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 229 MiB data, 3.1 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 523 op/s; 2467/10845 objects degraded (22.748%); 4275/10845 objects misplaced (39.419%); 2.8 MiB/s, 115 objects/s recovering 2023-09-23T22:30:51.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:50 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:51.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:50 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:51.185 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:51.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:51.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:51.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:51.322 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:51.706 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:51.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:51.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:51.814 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:51.850 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:30:51.851 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:30:51.859 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:51 smithi070 ceph-mon[192652]: from='client.45767 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:30:51.859 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:51 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:51.859 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:51 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:51.863 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:51.897 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":945,"stamp":"2023-09-23T22:30:49.531995+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":239701181,"num_objects":3615,"num_object_clones":0,"num_object_copies":10845,"num_objects_missing_on_primary":1704,"num_objects_missing":1704,"num_objects_degraded":2467,"num_objects_misplaced":4275,"num_objects_unfound":0,"num_objects_dirty":3615,"num_whiteouts":0,"num_read":6115,"num_read_kb":12434,"num_write":178055,"num_write_kb":5794762,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44779,"num_bytes_recovered":2486881538,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97696,"ondisk_log_size":97696,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":323,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":3245884,"kb_used_data":1111884,"kb_used_omap":15,"kb_used_meta":2133872,"kb_avail":559225028,"statfs":{"total":575970213888,"available":572646428672,"internally_reserved":0,"allocated":1138569216,"data_stored":1128928605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2185085150},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,9,5,6,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-274595840,"num_objects":-4190,"num_object_clones":0,"num_object_copies":-12570,"num_objects_missing_on_primary":-256,"num_objects_missing":-256,"num_objects_degraded":-607,"num_objects_misplaced":-1703,"num_objects_unfound":0,"num_objects_dirty":-4190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4190,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":927,"num_bytes_recovered":23199744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005472"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":392,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108175+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:30:36.108175+0000","last_peered":"2023-09-23T22:30:36.108175+0000","last_clean":"2023-09-23T22:30:36.108175+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:30:36.108175+0000","last_undegraded":"2023-09-23T22:30:36.108175+0000","last_fullsized":"2023-09-23T22:30:36.108175+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":82,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170924+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:30:43.170924+0000","last_peered":"2023-09-23T22:30:43.170924+0000","last_clean":"2023-09-23T22:30:43.170924+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:30:43.170924+0000","last_undegraded":"2023-09-23T22:30:43.170924+0000","last_fullsized":"2023-09-23T22:30:43.170924+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.171270+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:30:43.171270+0000","last_peered":"2023-09-23T22:30:43.171270+0000","last_clean":"2023-09-23T22:30:43.171270+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:30:43.171270+0000","last_undegraded":"2023-09-23T22:30:43.171270+0000","last_fullsized":"2023-09-23T22:30:43.171270+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":244,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164542+0000","last_change":"2023-09-23T22:27:52.097725+0000","last_active":"2023-09-23T22:30:43.164542+0000","last_peered":"2023-09-23T22:30:43.164542+0000","last_clean":"2023-09-23T22:30:43.164542+0000","last_became_active":"2023-09-23T22:27:52.097293+0000","last_became_peered":"2023-09-23T22:27:52.097293+0000","last_unstale":"2023-09-23T22:30:43.164542+0000","last_undegraded":"2023-09-23T22:30:43.164542+0000","last_fullsized":"2023-09-23T22:30:43.164542+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:41.651341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":411,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164188+0000","last_change":"2023-09-23T22:28:05.155218+0000","last_active":"2023-09-23T22:30:43.164188+0000","last_peered":"2023-09-23T22:30:43.164188+0000","last_clean":"2023-09-23T22:30:43.164188+0000","last_became_active":"2023-09-23T22:28:05.154771+0000","last_became_peered":"2023-09-23T22:28:05.154771+0000","last_unstale":"2023-09-23T22:30:43.164188+0000","last_undegraded":"2023-09-23T22:30:43.164188+0000","last_fullsized":"2023-09-23T22:30:43.164188+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:48.395095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.114100+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:30:36.114100+0000","last_peered":"2023-09-23T22:30:36.114100+0000","last_clean":"2023-09-23T22:30:36.114100+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:30:36.114100+0000","last_undegraded":"2023-09-23T22:30:36.114100+0000","last_fullsized":"2023-09-23T22:30:36.114100+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":281,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.167018+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:30:43.167018+0000","last_peered":"2023-09-23T22:30:43.167018+0000","last_clean":"2023-09-23T22:30:43.167018+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:30:43.167018+0000","last_undegraded":"2023-09-23T22:30:43.167018+0000","last_fullsized":"2023-09-23T22:30:43.167018+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.114159+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:30:36.114159+0000","last_peered":"2023-09-23T22:30:36.114159+0000","last_clean":"2023-09-23T22:30:36.114159+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:30:36.114159+0000","last_undegraded":"2023-09-23T22:30:36.114159+0000","last_fullsized":"2023-09-23T22:30:36.114159+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":355,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.107577+0000","last_change":"2023-09-23T22:27:52.091129+0000","last_active":"2023-09-23T22:30:36.107577+0000","last_peered":"2023-09-23T22:30:36.107577+0000","last_clean":"2023-09-23T22:30:36.107577+0000","last_became_active":"2023-09-23T22:27:52.090819+0000","last_became_peered":"2023-09-23T22:27:52.090819+0000","last_unstale":"2023-09-23T22:30:36.107577+0000","last_undegraded":"2023-09-23T22:30:36.107577+0000","last_fullsized":"2023-09-23T22:30:36.107577+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:13.639810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.113145+0000","last_change":"2023-09-23T22:29:01.578787+0000","last_active":"2023-09-23T22:30:36.113145+0000","last_peered":"2023-09-23T22:30:36.113145+0000","last_clean":"2023-09-23T22:30:36.113145+0000","last_became_active":"2023-09-23T22:29:01.578405+0000","last_became_peered":"2023-09-23T22:29:01.578405+0000","last_unstale":"2023-09-23T22:30:36.113145+0000","last_undegraded":"2023-09-23T22:30:36.113145+0000","last_fullsized":"2023-09-23T22:30:36.113145+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:55.201936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":96,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108689+0000","last_change":"2023-09-23T22:27:27.320214+0000","last_active":"2023-09-23T22:30:36.108689+0000","last_peered":"2023-09-23T22:30:36.108689+0000","last_clean":"2023-09-23T22:30:36.108689+0000","last_became_active":"2023-09-23T22:27:27.303456+0000","last_became_peered":"2023-09-23T22:27:27.303456+0000","last_unstale":"2023-09-23T22:30:36.108689+0000","last_undegraded":"2023-09-23T22:30:36.108689+0000","last_fullsized":"2023-09-23T22:30:36.108689+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:17.105540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170523+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:30:43.170523+0000","last_peered":"2023-09-23T22:30:43.170523+0000","last_clean":"2023-09-23T22:30:43.170523+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:30:43.170523+0000","last_undegraded":"2023-09-23T22:30:43.170523+0000","last_fullsized":"2023-09-23T22:30:43.170523+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":429,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108516+0000","last_change":"2023-09-23T22:29:01.673486+0000","last_active":"2023-09-23T22:30:36.108516+0000","last_peered":"2023-09-23T22:30:36.108516+0000","last_clean":"2023-09-23T22:30:36.108516+0000","last_became_active":"2023-09-23T22:29:01.661056+0000","last_became_peered":"2023-09-23T22:29:01.661056+0000","last_unstale":"2023-09-23T22:30:36.108516+0000","last_undegraded":"2023-09-23T22:30:36.108516+0000","last_fullsized":"2023-09-23T22:30:36.108516+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:40.581172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":363,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108996+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:30:36.108996+0000","last_peered":"2023-09-23T22:30:36.108996+0000","last_clean":"2023-09-23T22:30:36.108996+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:30:36.108996+0000","last_undegraded":"2023-09-23T22:30:36.108996+0000","last_fullsized":"2023-09-23T22:30:36.108996+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":340,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.109042+0000","last_change":"2023-09-23T22:29:01.662016+0000","last_active":"2023-09-23T22:30:36.109042+0000","last_peered":"2023-09-23T22:30:36.109042+0000","last_clean":"2023-09-23T22:30:36.109042+0000","last_became_active":"2023-09-23T22:29:01.600346+0000","last_became_peered":"2023-09-23T22:29:01.600346+0000","last_unstale":"2023-09-23T22:30:36.109042+0000","last_undegraded":"2023-09-23T22:30:36.109042+0000","last_fullsized":"2023-09-23T22:30:36.109042+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.205227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":317,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.106931+0000","last_change":"2023-09-23T22:28:41.267094+0000","last_active":"2023-09-23T22:30:36.106931+0000","last_peered":"2023-09-23T22:30:36.106931+0000","last_clean":"2023-09-23T22:30:36.106931+0000","last_became_active":"2023-09-23T22:28:41.266787+0000","last_became_peered":"2023-09-23T22:28:41.266787+0000","last_unstale":"2023-09-23T22:30:36.106931+0000","last_undegraded":"2023-09-23T22:30:36.106931+0000","last_fullsized":"2023-09-23T22:30:36.106931+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:30.013252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":262,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:44.356732+0000","last_change":"2023-09-23T22:27:52.101677+0000","last_active":"2023-09-23T22:30:44.356732+0000","last_peered":"2023-09-23T22:30:44.356732+0000","last_clean":"2023-09-23T22:30:44.356732+0000","last_became_active":"2023-09-23T22:27:52.101483+0000","last_became_peered":"2023-09-23T22:27:52.101483+0000","last_unstale":"2023-09-23T22:30:44.356732+0000","last_undegraded":"2023-09-23T22:30:44.356732+0000","last_fullsized":"2023-09-23T22:30:44.356732+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:01.772629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":364,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.107152+0000","last_change":"2023-09-23T22:29:01.588394+0000","last_active":"2023-09-23T22:30:36.107152+0000","last_peered":"2023-09-23T22:30:36.107152+0000","last_clean":"2023-09-23T22:30:36.107152+0000","last_became_active":"2023-09-23T22:29:01.587588+0000","last_became_peered":"2023-09-23T22:29:01.587588+0000","last_unstale":"2023-09-23T22:30:36.107152+0000","last_undegraded":"2023-09-23T22:30:36.107152+0000","last_fullsized":"2023-09-23T22:30:36.107152+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:12.141014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":298,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108369+0000","last_change":"2023-09-23T22:29:01.604689+0000","last_active":"2023-09-23T22:30:36.108369+0000","last_peered":"2023-09-23T22:30:36.108369+0000","last_clean":"2023-09-23T22:30:36.108369+0000","last_became_active":"2023-09-23T22:29:01.603739+0000","last_became_peered":"2023-09-23T22:29:01.603739+0000","last_unstale":"2023-09-23T22:30:36.108369+0000","last_undegraded":"2023-09-23T22:30:36.108369+0000","last_fullsized":"2023-09-23T22:30:36.108369+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:26.149229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":252,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164619+0000","last_change":"2023-09-23T22:29:01.575934+0000","last_active":"2023-09-23T22:30:43.164619+0000","last_peered":"2023-09-23T22:30:43.164619+0000","last_clean":"2023-09-23T22:30:43.164619+0000","last_became_active":"2023-09-23T22:29:01.575404+0000","last_became_peered":"2023-09-23T22:29:01.575404+0000","last_unstale":"2023-09-23T22:30:43.164619+0000","last_undegraded":"2023-09-23T22:30:43.164619+0000","last_fullsized":"2023-09-23T22:30:43.164619+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:38.753379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":56,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.114079+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:30:36.114079+0000","last_peered":"2023-09-23T22:30:36.114079+0000","last_clean":"2023-09-23T22:30:36.114079+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:30:36.114079+0000","last_undegraded":"2023-09-23T22:30:36.114079+0000","last_fullsized":"2023-09-23T22:30:36.114079+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.114016+0000","last_change":"2023-09-23T22:29:01.583368+0000","last_active":"2023-09-23T22:30:36.114016+0000","last_peered":"2023-09-23T22:30:36.114016+0000","last_clean":"2023-09-23T22:30:36.114016+0000","last_became_active":"2023-09-23T22:29:01.582967+0000","last_became_peered":"2023-09-23T22:29:01.582967+0000","last_unstale":"2023-09-23T22:30:36.114016+0000","last_undegraded":"2023-09-23T22:30:36.114016+0000","last_fullsized":"2023-09-23T22:30:36.114016+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:48.275429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"366'11096","reported_seq":18844,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:48.922912+0000","last_change":"2023-09-23T22:29:10.081764+0000","last_active":"2023-09-23T22:30:48.922912+0000","last_peered":"2023-09-23T22:30:48.922912+0000","last_clean":"2023-09-23T22:30:48.922912+0000","last_became_active":"2023-09-23T22:29:10.067355+0000","last_became_peered":"2023-09-23T22:29:10.067355+0000","last_unstale":"2023-09-23T22:30:48.922912+0000","last_undegraded":"2023-09-23T22:30:48.922912+0000","last_fullsized":"2023-09-23T22:30:48.922912+0000","mapping_epoch":334,"log_start":"280'6004","ondisk_log_start":"280'6004","created":260,"last_epoch_clean":335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5092,"log_dups_size":0,"ondisk_log_size":5092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.970345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11534336,"num_objects":176,"num_object_clones":0,"num_object_copies":528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11096,"num_write_kb":360704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3784,"num_bytes_recovered":247988224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":360,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.109927+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:30:36.109927+0000","last_peered":"2023-09-23T22:30:36.109927+0000","last_clean":"2023-09-23T22:30:36.109927+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:30:36.109927+0000","last_undegraded":"2023-09-23T22:30:36.109927+0000","last_fullsized":"2023-09-23T22:30:36.109927+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":357,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.106162+0000","last_change":"2023-09-23T22:29:01.597976+0000","last_active":"2023-09-23T22:30:36.106162+0000","last_peered":"2023-09-23T22:30:36.106162+0000","last_clean":"2023-09-23T22:30:36.106162+0000","last_became_active":"2023-09-23T22:29:01.597479+0000","last_became_peered":"2023-09-23T22:29:01.597479+0000","last_unstale":"2023-09-23T22:30:36.106162+0000","last_undegraded":"2023-09-23T22:30:36.106162+0000","last_fullsized":"2023-09-23T22:30:36.106162+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:55.990001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"365'11177","reported_seq":15368,"reported_epoch":365,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:44.981017+0000","last_change":"2023-09-23T22:29:17.154270+0000","last_active":"2023-09-23T22:30:44.981017+0000","last_peered":"2023-09-23T22:30:44.981017+0000","last_clean":"2023-09-23T22:26:06.848974+0000","last_became_active":"2023-09-23T22:28:06.779572+0000","last_became_peered":"2023-09-23T22:28:06.779572+0000","last_unstale":"2023-09-23T22:30:44.981017+0000","last_undegraded":"2023-09-23T22:30:44.981017+0000","last_fullsized":"2023-09-23T22:30:44.981017+0000","mapping_epoch":311,"log_start":"273'5323","ondisk_log_start":"273'5323","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5854,"log_dups_size":0,"ondisk_log_size":5854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:46.168583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21692416,"num_objects":331,"num_object_clones":0,"num_object_copies":993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":331,"num_objects_unfound":0,"num_objects_dirty":331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11177,"num_write_kb":368256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2023,"num_bytes_recovered":30081024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,3],"avail_no_missing":["5","1","2","3"],"object_location_counts":[{"shards":"1,3,5","objects":331}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":302,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.109891+0000","last_change":"2023-09-23T22:27:52.154449+0000","last_active":"2023-09-23T22:30:36.109891+0000","last_peered":"2023-09-23T22:30:36.109891+0000","last_clean":"2023-09-23T22:30:36.109891+0000","last_became_active":"2023-09-23T22:27:52.096263+0000","last_became_peered":"2023-09-23T22:27:52.096263+0000","last_unstale":"2023-09-23T22:30:36.109891+0000","last_undegraded":"2023-09-23T22:30:36.109891+0000","last_fullsized":"2023-09-23T22:30:36.109891+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:02.582951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":392,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108263+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:30:36.108263+0000","last_peered":"2023-09-23T22:30:36.108263+0000","last_clean":"2023-09-23T22:30:36.108263+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:30:36.108263+0000","last_undegraded":"2023-09-23T22:30:36.108263+0000","last_fullsized":"2023-09-23T22:30:36.108263+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"366'11265","reported_seq":11401,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:49.501728+0000","last_change":"2023-09-23T22:27:53.074369+0000","last_active":"2023-09-23T22:30:49.501728+0000","last_peered":"2023-09-23T22:30:49.501728+0000","last_clean":"2023-09-23T22:30:49.501728+0000","last_became_active":"2023-09-23T22:27:53.039034+0000","last_became_peered":"2023-09-23T22:27:53.039034+0000","last_unstale":"2023-09-23T22:30:49.501728+0000","last_undegraded":"2023-09-23T22:30:49.501728+0000","last_fullsized":"2023-09-23T22:30:49.501728+0000","mapping_epoch":305,"log_start":"269'4969","ondisk_log_start":"269'4969","created":260,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6296,"log_dups_size":0,"ondisk_log_size":6296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:46.830216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11599872,"num_objects":177,"num_object_clones":0,"num_object_copies":531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11265,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":318,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.107366+0000","last_change":"2023-09-23T22:29:01.633048+0000","last_active":"2023-09-23T22:30:36.107366+0000","last_peered":"2023-09-23T22:30:36.107366+0000","last_clean":"2023-09-23T22:30:36.107366+0000","last_became_active":"2023-09-23T22:29:01.587680+0000","last_became_peered":"2023-09-23T22:29:01.587680+0000","last_unstale":"2023-09-23T22:30:36.107366+0000","last_undegraded":"2023-09-23T22:30:36.107366+0000","last_fullsized":"2023-09-23T22:30:36.107366+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.830912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.113134+0000","last_change":"2023-09-23T22:29:01.593596+0000","last_active":"2023-09-23T22:30:36.113134+0000","last_peered":"2023-09-23T22:30:36.113134+0000","last_clean":"2023-09-23T22:30:36.113134+0000","last_became_active":"2023-09-23T22:29:01.593303+0000","last_became_peered":"2023-09-23T22:29:01.593303+0000","last_unstale":"2023-09-23T22:30:36.113134+0000","last_undegraded":"2023-09-23T22:30:36.113134+0000","last_fullsized":"2023-09-23T22:30:36.113134+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:02.980860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"365'10684","reported_seq":22180,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:44.810191+0000","last_change":"2023-09-23T22:28:15.149660+0000","last_active":"2023-09-23T22:30:44.810191+0000","last_peered":"2023-09-23T22:30:44.810191+0000","last_clean":"2023-09-23T22:30:44.810191+0000","last_became_active":"2023-09-23T22:28:15.134552+0000","last_became_peered":"2023-09-23T22:28:15.134552+0000","last_unstale":"2023-09-23T22:30:44.810191+0000","last_undegraded":"2023-09-23T22:30:44.810191+0000","last_fullsized":"2023-09-23T22:30:44.810191+0000","mapping_epoch":315,"log_start":"274'5108","ondisk_log_start":"274'5108","created":260,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5576,"log_dups_size":0,"ondisk_log_size":5576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:52.196467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20185088,"num_objects":308,"num_object_clones":0,"num_object_copies":924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10684,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5677,"num_bytes_recovered":372047872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":395,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.107040+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:30:36.107040+0000","last_peered":"2023-09-23T22:30:36.107040+0000","last_clean":"2023-09-23T22:30:36.107040+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:30:36.107040+0000","last_undegraded":"2023-09-23T22:30:36.107040+0000","last_fullsized":"2023-09-23T22:30:36.107040+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":251,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:44.356813+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:30:44.356813+0000","last_peered":"2023-09-23T22:30:44.356813+0000","last_clean":"2023-09-23T22:30:44.356813+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:30:44.356813+0000","last_undegraded":"2023-09-23T22:30:44.356813+0000","last_fullsized":"2023-09-23T22:30:44.356813+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"366'10996","reported_seq":15272,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:49.349743+0000","last_change":"2023-09-23T22:29:53.222638+0000","last_active":"2023-09-23T22:30:49.349743+0000","last_peered":"2023-09-23T22:30:49.349743+0000","last_clean":"2023-09-23T22:27:26.103428+0000","last_became_active":"2023-09-23T22:27:28.277815+0000","last_became_peered":"2023-09-23T22:27:28.277815+0000","last_unstale":"2023-09-23T22:30:49.349743+0000","last_undegraded":"2023-09-23T22:30:49.349743+0000","last_fullsized":"2023-09-23T22:30:49.349743+0000","mapping_epoch":298,"log_start":"269'4665","ondisk_log_start":"269'4665","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6331,"log_dups_size":0,"ondisk_log_size":6331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:57.836786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11141120,"num_objects":170,"num_object_clones":0,"num_object_copies":510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":43,"num_objects_unfound":0,"num_objects_dirty":170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10996,"num_write_kb":357312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2069,"num_bytes_recovered":135593984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":170}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":690,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.106145+0000","last_change":"2023-09-23T22:29:04.057613+0000","last_active":"2023-09-23T22:30:36.106145+0000","last_peered":"2023-09-23T22:30:36.106145+0000","last_clean":"2023-09-23T22:30:36.106145+0000","last_became_active":"2023-09-23T22:29:04.057082+0000","last_became_peered":"2023-09-23T22:29:04.057082+0000","last_unstale":"2023-09-23T22:30:36.106145+0000","last_undegraded":"2023-09-23T22:30:36.106145+0000","last_fullsized":"2023-09-23T22:30:36.106145+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:24.948720+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":197,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164838+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:30:43.164838+0000","last_peered":"2023-09-23T22:30:43.164838+0000","last_clean":"2023-09-23T22:30:43.164838+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:30:43.164838+0000","last_undegraded":"2023-09-23T22:30:43.164838+0000","last_fullsized":"2023-09-23T22:30:43.164838+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":373,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108445+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:30:36.108445+0000","last_peered":"2023-09-23T22:30:36.108445+0000","last_clean":"2023-09-23T22:30:36.108445+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:30:36.108445+0000","last_undegraded":"2023-09-23T22:30:36.108445+0000","last_fullsized":"2023-09-23T22:30:36.108445+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"365'10956","reported_seq":13886,"reported_epoch":365,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:30:45.036482+0000","last_change":"2023-09-23T22:29:11.163110+0000","last_active":"2023-09-23T22:30:45.036482+0000","last_peered":"2023-09-23T22:30:45.036482+0000","last_clean":"2023-09-23T22:26:06.765343+0000","last_became_active":"2023-09-23T22:29:11.143204+0000","last_became_peered":"2023-09-23T22:29:11.143204+0000","last_unstale":"2023-09-23T22:30:45.036482+0000","last_undegraded":"2023-09-23T22:29:11.140016+0000","last_fullsized":"2023-09-23T22:30:45.036482+0000","mapping_epoch":335,"log_start":"270'5184","ondisk_log_start":"270'5184","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5772,"log_dups_size":0,"ondisk_log_size":5772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:14.098578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21364736,"num_objects":326,"num_object_clones":0,"num_object_copies":978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":326,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10956,"num_write_kb":361024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1394,"num_bytes_recovered":91357184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,3],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,2,5","objects":325},{"shards":"0,3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":746,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.106316+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:30:36.106316+0000","last_peered":"2023-09-23T22:30:36.106316+0000","last_clean":"2023-09-23T22:30:36.106316+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:30:36.106316+0000","last_undegraded":"2023-09-23T22:30:36.106316+0000","last_fullsized":"2023-09-23T22:30:36.106316+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.113332+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:30:36.113332+0000","last_peered":"2023-09-23T22:30:36.113332+0000","last_clean":"2023-09-23T22:30:36.113332+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:30:36.113332+0000","last_undegraded":"2023-09-23T22:30:36.113332+0000","last_fullsized":"2023-09-23T22:30:36.113332+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":400,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108526+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:30:36.108526+0000","last_peered":"2023-09-23T22:30:36.108526+0000","last_clean":"2023-09-23T22:30:36.108526+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:30:36.108526+0000","last_undegraded":"2023-09-23T22:30:36.108526+0000","last_fullsized":"2023-09-23T22:30:36.108526+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"366'11133","reported_seq":19328,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:49.515180+0000","last_change":"2023-09-23T22:29:02.735621+0000","last_active":"2023-09-23T22:30:49.515180+0000","last_peered":"2023-09-23T22:30:49.515180+0000","last_clean":"2023-09-23T22:29:00.425895+0000","last_became_active":"2023-09-23T22:29:02.721422+0000","last_became_peered":"2023-09-23T22:29:02.721422+0000","last_unstale":"2023-09-23T22:30:49.515180+0000","last_undegraded":"2023-09-23T22:30:49.515180+0000","last_fullsized":"2023-09-23T22:30:49.515180+0000","mapping_epoch":329,"log_start":"269'4841","ondisk_log_start":"269'4841","created":260,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6292,"log_dups_size":0,"ondisk_log_size":6292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:41.454007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11534360,"num_objects":177,"num_object_clones":0,"num_object_copies":531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":177,"num_objects_unfound":0,"num_objects_dirty":177,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11133,"num_write_kb":361857,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4012,"num_bytes_recovered":262864920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[0,1,7],"avail_no_missing":["0","1","3","7"],"object_location_counts":[{"shards":"0,1,7","objects":177}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":346,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108085+0000","last_change":"2023-09-23T22:28:05.062902+0000","last_active":"2023-09-23T22:30:36.108085+0000","last_peered":"2023-09-23T22:30:36.108085+0000","last_clean":"2023-09-23T22:30:36.108085+0000","last_became_active":"2023-09-23T22:28:05.062607+0000","last_became_peered":"2023-09-23T22:28:05.062607+0000","last_unstale":"2023-09-23T22:30:36.108085+0000","last_undegraded":"2023-09-23T22:30:36.108085+0000","last_fullsized":"2023-09-23T22:30:36.108085+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:21.888780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":381,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.109797+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:30:36.109797+0000","last_peered":"2023-09-23T22:30:36.109797+0000","last_clean":"2023-09-23T22:30:36.109797+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:30:36.109797+0000","last_undegraded":"2023-09-23T22:30:36.109797+0000","last_fullsized":"2023-09-23T22:30:36.109797+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"366'10917","reported_seq":14112,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:30:49.518077+0000","last_change":"2023-09-23T22:29:53.222339+0000","last_active":"2023-09-23T22:30:49.518077+0000","last_peered":"2023-09-23T22:30:49.518077+0000","last_clean":"2023-09-23T22:26:06.845528+0000","last_became_active":"2023-09-23T22:29:52.034520+0000","last_became_peered":"2023-09-23T22:29:52.034520+0000","last_unstale":"2023-09-23T22:30:49.518077+0000","last_undegraded":"2023-09-23T22:29:52.019265+0000","last_fullsized":"2023-09-23T22:29:51.188771+0000","mapping_epoch":351,"log_start":"269'4597","ondisk_log_start":"269'4597","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6320,"log_dups_size":0,"ondisk_log_size":6320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:22.488491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10420224,"num_objects":159,"num_object_clones":0,"num_object_copies":477,"num_objects_missing_on_primary":1704,"num_objects_missing":1704,"num_objects_degraded":443,"num_objects_misplaced":1420,"num_objects_unfound":0,"num_objects_dirty":159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10917,"num_write_kb":354432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1518,"num_bytes_recovered":99418112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[0,3],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"0,3","objects":1},{"shards":"2,3,7","objects":158}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":719,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.106514+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:30:36.106514+0000","last_peered":"2023-09-23T22:30:36.106514+0000","last_clean":"2023-09-23T22:30:36.106514+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:30:36.106514+0000","last_undegraded":"2023-09-23T22:30:36.106514+0000","last_fullsized":"2023-09-23T22:30:36.106514+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170171+0000","last_change":"2023-09-23T22:27:52.106215+0000","last_active":"2023-09-23T22:30:43.170171+0000","last_peered":"2023-09-23T22:30:43.170171+0000","last_clean":"2023-09-23T22:30:43.170171+0000","last_became_active":"2023-09-23T22:27:52.104315+0000","last_became_peered":"2023-09-23T22:27:52.104315+0000","last_unstale":"2023-09-23T22:30:43.170171+0000","last_undegraded":"2023-09-23T22:30:43.170171+0000","last_fullsized":"2023-09-23T22:30:43.170171+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:44.360992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":73,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.169928+0000","last_change":"2023-09-23T22:27:52.096512+0000","last_active":"2023-09-23T22:30:43.169928+0000","last_peered":"2023-09-23T22:30:43.169928+0000","last_clean":"2023-09-23T22:30:43.169928+0000","last_became_active":"2023-09-23T22:27:52.096242+0000","last_became_peered":"2023-09-23T22:27:52.096242+0000","last_unstale":"2023-09-23T22:30:43.169928+0000","last_undegraded":"2023-09-23T22:30:43.169928+0000","last_fullsized":"2023-09-23T22:30:43.169928+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:12.364350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"365'10867","reported_seq":26790,"reported_epoch":365,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:44.980570+0000","last_change":"2023-09-23T22:29:49.257546+0000","last_active":"2023-09-23T22:30:44.980570+0000","last_peered":"2023-09-23T22:30:44.980570+0000","last_clean":"2023-09-23T22:29:00.251578+0000","last_became_active":"2023-09-23T22:29:02.721609+0000","last_became_peered":"2023-09-23T22:29:02.721609+0000","last_unstale":"2023-09-23T22:30:44.980570+0000","last_undegraded":"2023-09-23T22:30:44.980570+0000","last_fullsized":"2023-09-23T22:30:44.980570+0000","mapping_epoch":329,"log_start":"275'5220","ondisk_log_start":"275'5220","created":260,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5647,"log_dups_size":0,"ondisk_log_size":5647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:37.536476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18153472,"num_objects":277,"num_object_clones":0,"num_object_copies":831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":169,"num_objects_unfound":0,"num_objects_dirty":277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10867,"num_write_kb":356608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6911,"num_bytes_recovered":452919296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,1,0],"avail_no_missing":["4","0","1","3","7"],"object_location_counts":[{"shards":"0,1,4","objects":277}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":610,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.114083+0000","last_change":"2023-09-23T22:29:04.057506+0000","last_active":"2023-09-23T22:30:36.114083+0000","last_peered":"2023-09-23T22:30:36.114083+0000","last_clean":"2023-09-23T22:30:36.114083+0000","last_became_active":"2023-09-23T22:29:04.057207+0000","last_became_peered":"2023-09-23T22:29:04.057207+0000","last_unstale":"2023-09-23T22:30:36.114083+0000","last_undegraded":"2023-09-23T22:30:36.114083+0000","last_fullsized":"2023-09-23T22:30:36.114083+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:47.781486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:47.781486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":692,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.172682+0000","last_change":"2023-09-23T22:28:41.357575+0000","last_active":"2023-09-23T22:30:43.172682+0000","last_peered":"2023-09-23T22:30:43.172682+0000","last_clean":"2023-09-23T22:30:43.172682+0000","last_became_active":"2023-09-23T22:28:41.356971+0000","last_became_peered":"2023-09-23T22:28:41.356971+0000","last_unstale":"2023-09-23T22:30:43.172682+0000","last_undegraded":"2023-09-23T22:30:43.172682+0000","last_fullsized":"2023-09-23T22:30:43.172682+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.810917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":422,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.107068+0000","last_change":"2023-09-23T22:29:01.608081+0000","last_active":"2023-09-23T22:30:36.107068+0000","last_peered":"2023-09-23T22:30:36.107068+0000","last_clean":"2023-09-23T22:30:36.107068+0000","last_became_active":"2023-09-23T22:29:01.607726+0000","last_became_peered":"2023-09-23T22:29:01.607726+0000","last_unstale":"2023-09-23T22:30:36.107068+0000","last_undegraded":"2023-09-23T22:30:36.107068+0000","last_fullsized":"2023-09-23T22:30:36.107068+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:30.179258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170374+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:30:43.170374+0000","last_peered":"2023-09-23T22:30:43.170374+0000","last_clean":"2023-09-23T22:30:43.170374+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:30:43.170374+0000","last_undegraded":"2023-09-23T22:30:43.170374+0000","last_fullsized":"2023-09-23T22:30:43.170374+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"366'11052","reported_seq":11209,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:49.514189+0000","last_change":"2023-09-23T22:29:01.683197+0000","last_active":"2023-09-23T22:30:49.514189+0000","last_peered":"2023-09-23T22:30:49.514189+0000","last_clean":"2023-09-23T22:27:11.662680+0000","last_became_active":"2023-09-23T22:29:01.670427+0000","last_became_peered":"2023-09-23T22:29:01.670427+0000","last_unstale":"2023-09-23T22:30:49.514189+0000","last_undegraded":"2023-09-23T22:30:49.514189+0000","last_fullsized":"2023-09-23T22:30:49.514189+0000","mapping_epoch":328,"log_start":"269'4747","ondisk_log_start":"269'4747","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6305,"log_dups_size":0,"ondisk_log_size":6305,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:30.348815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9437184,"num_objects":144,"num_object_clones":0,"num_object_copies":432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":144,"num_objects_unfound":0,"num_objects_dirty":144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11052,"num_write_kb":358272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[0,1,3],"avail_no_missing":["0","1","3"],"object_location_counts":[{"shards":"0,1,3","objects":144}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":709,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.106317+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:30:36.106317+0000","last_peered":"2023-09-23T22:30:36.106317+0000","last_clean":"2023-09-23T22:30:36.106317+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:30:36.106317+0000","last_undegraded":"2023-09-23T22:30:36.106317+0000","last_fullsized":"2023-09-23T22:30:36.106317+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1243,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.106637+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:30:36.106637+0000","last_peered":"2023-09-23T22:30:36.106637+0000","last_clean":"2023-09-23T22:30:36.106637+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:30:36.106637+0000","last_undegraded":"2023-09-23T22:30:36.106637+0000","last_fullsized":"2023-09-23T22:30:36.106637+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":404,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108944+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:30:36.108944+0000","last_peered":"2023-09-23T22:30:36.108944+0000","last_clean":"2023-09-23T22:30:36.108944+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:30:36.108944+0000","last_undegraded":"2023-09-23T22:30:36.108944+0000","last_fullsized":"2023-09-23T22:30:36.108944+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.113381+0000","last_change":"2023-09-23T22:29:01.593644+0000","last_active":"2023-09-23T22:30:36.113381+0000","last_peered":"2023-09-23T22:30:36.113381+0000","last_clean":"2023-09-23T22:30:36.113381+0000","last_became_active":"2023-09-23T22:29:01.593326+0000","last_became_peered":"2023-09-23T22:29:01.593326+0000","last_unstale":"2023-09-23T22:30:36.113381+0000","last_undegraded":"2023-09-23T22:30:36.113381+0000","last_fullsized":"2023-09-23T22:30:36.113381+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:52.921796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.4","version":"366'11241","reported_seq":11363,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:49.306123+0000","last_change":"2023-09-23T22:27:52.125206+0000","last_active":"2023-09-23T22:30:49.306123+0000","last_peered":"2023-09-23T22:30:49.306123+0000","last_clean":"2023-09-23T22:30:49.306123+0000","last_became_active":"2023-09-23T22:27:52.107584+0000","last_became_peered":"2023-09-23T22:27:52.107584+0000","last_unstale":"2023-09-23T22:30:49.306123+0000","last_undegraded":"2023-09-23T22:30:49.306123+0000","last_fullsized":"2023-09-23T22:30:49.306123+0000","mapping_epoch":304,"log_start":"269'4966","ondisk_log_start":"269'4966","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6275,"log_dups_size":0,"ondisk_log_size":6275,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:23.737567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11468800,"num_objects":175,"num_object_clones":0,"num_object_copies":525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11241,"num_write_kb":365312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":698,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.166174+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:30:43.166174+0000","last_peered":"2023-09-23T22:30:43.166174+0000","last_clean":"2023-09-23T22:30:43.166174+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:30:43.166174+0000","last_undegraded":"2023-09-23T22:30:43.166174+0000","last_fullsized":"2023-09-23T22:30:43.166174+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":582,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165622+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:30:43.165622+0000","last_peered":"2023-09-23T22:30:43.165622+0000","last_clean":"2023-09-23T22:30:43.165622+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:30:43.165622+0000","last_undegraded":"2023-09-23T22:30:43.165622+0000","last_fullsized":"2023-09-23T22:30:43.165622+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.114153+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:30:36.114153+0000","last_peered":"2023-09-23T22:30:36.114153+0000","last_clean":"2023-09-23T22:30:36.114153+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:30:36.114153+0000","last_undegraded":"2023-09-23T22:30:36.114153+0000","last_fullsized":"2023-09-23T22:30:36.114153+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.169667+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:30:43.169667+0000","last_peered":"2023-09-23T22:30:43.169667+0000","last_clean":"2023-09-23T22:30:43.169667+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:30:43.169667+0000","last_undegraded":"2023-09-23T22:30:43.169667+0000","last_fullsized":"2023-09-23T22:30:43.169667+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"366'11122","reported_seq":13173,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:49.235331+0000","last_change":"2023-09-23T22:29:51.202545+0000","last_active":"2023-09-23T22:30:49.235331+0000","last_peered":"2023-09-23T22:30:49.235331+0000","last_clean":"2023-09-23T22:27:26.256729+0000","last_became_active":"2023-09-23T22:27:28.279658+0000","last_became_peered":"2023-09-23T22:27:28.279658+0000","last_unstale":"2023-09-23T22:30:49.235331+0000","last_undegraded":"2023-09-23T22:30:49.235331+0000","last_fullsized":"2023-09-23T22:30:49.235331+0000","mapping_epoch":298,"log_start":"269'4810","ondisk_log_start":"269'4810","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6312,"log_dups_size":0,"ondisk_log_size":6312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:04.471284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11927552,"num_objects":182,"num_object_clones":0,"num_object_copies":546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":111,"num_objects_unfound":0,"num_objects_dirty":182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11122,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":957,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,3],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":182}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1061,"reported_epoch":364,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:30:36.106529+0000","last_change":"2023-09-23T22:29:50.198353+0000","last_active":"2023-09-23T22:30:36.106529+0000","last_peered":"2023-09-23T22:30:36.106529+0000","last_clean":"2023-09-23T22:27:56.036441+0000","last_became_active":"2023-09-23T22:29:50.198096+0000","last_became_peered":"2023-09-23T22:29:50.198096+0000","last_unstale":"2023-09-23T22:30:36.106529+0000","last_undegraded":"2023-09-23T22:29:50.186847+0000","last_fullsized":"2023-09-23T22:30:36.106529+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:48.105235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":732,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164276+0000","last_change":"2023-09-23T22:29:04.055480+0000","last_active":"2023-09-23T22:30:43.164276+0000","last_peered":"2023-09-23T22:30:43.164276+0000","last_clean":"2023-09-23T22:30:43.164276+0000","last_became_active":"2023-09-23T22:29:04.055134+0000","last_became_peered":"2023-09-23T22:29:04.055134+0000","last_unstale":"2023-09-23T22:30:43.164276+0000","last_undegraded":"2023-09-23T22:30:43.164276+0000","last_fullsized":"2023-09-23T22:30:43.164276+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:59.018639+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":246,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.106589+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:30:36.106589+0000","last_peered":"2023-09-23T22:30:36.106589+0000","last_clean":"2023-09-23T22:30:36.106589+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:30:36.106589+0000","last_undegraded":"2023-09-23T22:30:36.106589+0000","last_fullsized":"2023-09-23T22:30:36.106589+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.113644+0000","last_change":"2023-09-23T22:29:01.594509+0000","last_active":"2023-09-23T22:30:36.113644+0000","last_peered":"2023-09-23T22:30:36.113644+0000","last_clean":"2023-09-23T22:30:36.113644+0000","last_became_active":"2023-09-23T22:29:01.594249+0000","last_became_peered":"2023-09-23T22:29:01.594249+0000","last_unstale":"2023-09-23T22:30:36.113644+0000","last_undegraded":"2023-09-23T22:30:36.113644+0000","last_fullsized":"2023-09-23T22:30:36.113644+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:14.972304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"366'11359","reported_seq":18335,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:49.515133+0000","last_change":"2023-09-23T22:29:49.247679+0000","last_active":"2023-09-23T22:30:49.515133+0000","last_peered":"2023-09-23T22:30:49.515133+0000","last_clean":"2023-09-23T22:30:49.515133+0000","last_became_active":"2023-09-23T22:29:46.177429+0000","last_became_peered":"2023-09-23T22:29:46.177429+0000","last_unstale":"2023-09-23T22:30:49.515133+0000","last_undegraded":"2023-09-23T22:30:49.515133+0000","last_fullsized":"2023-09-23T22:30:49.515133+0000","mapping_epoch":346,"log_start":"269'5043","ondisk_log_start":"269'5043","created":260,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6316,"log_dups_size":0,"ondisk_log_size":6316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:01.079734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10944512,"num_objects":167,"num_object_clones":0,"num_object_copies":501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11359,"num_write_kb":368832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":116916224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":587,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.113419+0000","last_change":"2023-09-23T22:29:04.055972+0000","last_active":"2023-09-23T22:30:36.113419+0000","last_peered":"2023-09-23T22:30:36.113419+0000","last_clean":"2023-09-23T22:30:36.113419+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:30:36.113419+0000","last_undegraded":"2023-09-23T22:30:36.113419+0000","last_fullsized":"2023-09-23T22:30:36.113419+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":716,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170417+0000","last_change":"2023-09-23T22:27:52.122905+0000","last_active":"2023-09-23T22:30:43.170417+0000","last_peered":"2023-09-23T22:30:43.170417+0000","last_clean":"2023-09-23T22:30:43.170417+0000","last_became_active":"2023-09-23T22:27:52.116583+0000","last_became_peered":"2023-09-23T22:27:52.116583+0000","last_unstale":"2023-09-23T22:30:43.170417+0000","last_undegraded":"2023-09-23T22:30:43.170417+0000","last_fullsized":"2023-09-23T22:30:43.170417+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:21.421519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":260,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165686+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:30:43.165686+0000","last_peered":"2023-09-23T22:30:43.165686+0000","last_clean":"2023-09-23T22:30:43.165686+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:30:43.165686+0000","last_undegraded":"2023-09-23T22:30:43.165686+0000","last_fullsized":"2023-09-23T22:30:43.165686+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":244,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164616+0000","last_change":"2023-09-23T22:28:05.169578+0000","last_active":"2023-09-23T22:30:43.164616+0000","last_peered":"2023-09-23T22:30:43.164616+0000","last_clean":"2023-09-23T22:30:43.164616+0000","last_became_active":"2023-09-23T22:28:05.169263+0000","last_became_peered":"2023-09-23T22:28:05.169263+0000","last_unstale":"2023-09-23T22:30:43.164616+0000","last_undegraded":"2023-09-23T22:30:43.164616+0000","last_fullsized":"2023-09-23T22:30:43.164616+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:39.446410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"365'11080","reported_seq":22758,"reported_epoch":365,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:30:45.042597+0000","last_change":"2023-09-23T22:29:49.258974+0000","last_active":"2023-09-23T22:30:45.042597+0000","last_peered":"2023-09-23T22:30:45.042597+0000","last_clean":"2023-09-23T22:27:11.623396+0000","last_became_active":"2023-09-23T22:29:02.885320+0000","last_became_peered":"2023-09-23T22:29:02.885320+0000","last_unstale":"2023-09-23T22:30:45.042597+0000","last_undegraded":"2023-09-23T22:29:02.608430+0000","last_fullsized":"2023-09-23T22:29:02.560725+0000","mapping_epoch":329,"log_start":"274'5265","ondisk_log_start":"274'5265","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5815,"log_dups_size":0,"ondisk_log_size":5815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:13.200109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22544384,"num_objects":344,"num_object_clones":0,"num_object_copies":1032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1385,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11080,"num_write_kb":365568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5575,"num_bytes_recovered":240779264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,5","objects":-1041}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":715,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.113557+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:30:36.113557+0000","last_peered":"2023-09-23T22:30:36.113557+0000","last_clean":"2023-09-23T22:30:36.113557+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:30:36.113557+0000","last_undegraded":"2023-09-23T22:30:36.113557+0000","last_fullsized":"2023-09-23T22:30:36.113557+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":260,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165227+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:30:43.165227+0000","last_peered":"2023-09-23T22:30:43.165227+0000","last_clean":"2023-09-23T22:30:43.165227+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:30:43.165227+0000","last_undegraded":"2023-09-23T22:30:43.165227+0000","last_fullsized":"2023-09-23T22:30:43.165227+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":291,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.107105+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:30:36.107105+0000","last_peered":"2023-09-23T22:30:36.107105+0000","last_clean":"2023-09-23T22:30:36.107105+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:30:36.107105+0000","last_undegraded":"2023-09-23T22:30:36.107105+0000","last_fullsized":"2023-09-23T22:30:36.107105+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":224,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164662+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:30:43.164662+0000","last_peered":"2023-09-23T22:30:43.164662+0000","last_clean":"2023-09-23T22:30:43.164662+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:30:43.164662+0000","last_undegraded":"2023-09-23T22:30:43.164662+0000","last_fullsized":"2023-09-23T22:30:43.164662+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"365'11021","reported_seq":12783,"reported_epoch":365,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:30:44.980667+0000","last_change":"2023-09-23T22:29:50.361405+0000","last_active":"2023-09-23T22:30:44.980667+0000","last_peered":"2023-09-23T22:30:44.980667+0000","last_clean":"2023-09-23T22:26:06.872687+0000","last_became_active":"2023-09-23T22:29:50.249230+0000","last_became_peered":"2023-09-23T22:29:50.249230+0000","last_unstale":"2023-09-23T22:30:44.980667+0000","last_undegraded":"2023-09-23T22:29:50.243725+0000","last_fullsized":"2023-09-23T22:29:50.191587+0000","mapping_epoch":350,"log_start":"276'5422","ondisk_log_start":"276'5422","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5599,"log_dups_size":0,"ondisk_log_size":5599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:47.073843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19857408,"num_objects":303,"num_object_clones":0,"num_object_copies":909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":303,"num_objects_misplaced":1708,"num_objects_unfound":0,"num_objects_dirty":303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11021,"num_write_kb":362368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":793,"num_bytes_recovered":32702464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,3],"avail_no_missing":["4","1","3"],"object_location_counts":[{"shards":"2,3,4","objects":200},{"shards":"3,4","objects":103}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":719,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108356+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:30:36.108356+0000","last_peered":"2023-09-23T22:30:36.108356+0000","last_clean":"2023-09-23T22:30:36.108356+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:30:36.108356+0000","last_undegraded":"2023-09-23T22:30:36.108356+0000","last_fullsized":"2023-09-23T22:30:36.108356+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":184,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.166837+0000","last_change":"2023-09-23T22:27:27.333140+0000","last_active":"2023-09-23T22:30:43.166837+0000","last_peered":"2023-09-23T22:30:43.166837+0000","last_clean":"2023-09-23T22:30:43.166837+0000","last_became_active":"2023-09-23T22:27:27.287527+0000","last_became_peered":"2023-09-23T22:27:27.287527+0000","last_unstale":"2023-09-23T22:30:43.166837+0000","last_undegraded":"2023-09-23T22:30:43.166837+0000","last_fullsized":"2023-09-23T22:30:43.166837+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:51.648586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":861,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.169789+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:30:43.169789+0000","last_peered":"2023-09-23T22:30:43.169789+0000","last_clean":"2023-09-23T22:30:43.169789+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:30:43.169789+0000","last_undegraded":"2023-09-23T22:30:43.169789+0000","last_fullsized":"2023-09-23T22:30:43.169789+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":296,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164160+0000","last_change":"2023-09-23T22:27:52.136692+0000","last_active":"2023-09-23T22:30:43.164160+0000","last_peered":"2023-09-23T22:30:43.164160+0000","last_clean":"2023-09-23T22:30:43.164160+0000","last_became_active":"2023-09-23T22:27:52.136397+0000","last_became_peered":"2023-09-23T22:27:52.136397+0000","last_unstale":"2023-09-23T22:30:43.164160+0000","last_undegraded":"2023-09-23T22:30:43.164160+0000","last_fullsized":"2023-09-23T22:30:43.164160+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.036231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.2","version":"366'11116","reported_seq":24074,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:49.015720+0000","last_change":"2023-09-23T22:29:50.220538+0000","last_active":"2023-09-23T22:30:49.015720+0000","last_peered":"2023-09-23T22:30:49.015720+0000","last_clean":"2023-09-23T22:29:00.398116+0000","last_became_active":"2023-09-23T22:29:50.197351+0000","last_became_peered":"2023-09-23T22:29:50.197351+0000","last_unstale":"2023-09-23T22:30:49.015720+0000","last_undegraded":"2023-09-23T22:30:49.015720+0000","last_fullsized":"2023-09-23T22:30:49.015720+0000","mapping_epoch":350,"log_start":"280'6024","ondisk_log_start":"280'6024","created":260,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5092,"log_dups_size":0,"ondisk_log_size":5092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:17.462990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11272192,"num_objects":172,"num_object_clones":0,"num_object_copies":516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":172,"num_objects_unfound":0,"num_objects_dirty":172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11116,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6378,"num_bytes_recovered":318308352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[1,7,0],"avail_no_missing":["1","0","2","3","7"],"object_location_counts":[{"shards":"0,1,7","objects":172}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":611,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.107129+0000","last_change":"2023-09-23T22:29:51.194726+0000","last_active":"2023-09-23T22:30:36.107129+0000","last_peered":"2023-09-23T22:30:36.107129+0000","last_clean":"2023-09-23T22:30:36.107129+0000","last_became_active":"2023-09-23T22:29:51.194253+0000","last_became_peered":"2023-09-23T22:29:51.194253+0000","last_unstale":"2023-09-23T22:30:36.107129+0000","last_undegraded":"2023-09-23T22:30:36.107129+0000","last_fullsized":"2023-09-23T22:30:36.107129+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:45.734686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.734686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1144050,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":56,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.114011+0000","last_change":"2023-09-23T22:29:09.070063+0000","last_active":"2023-09-23T22:30:36.114011+0000","last_peered":"2023-09-23T22:30:36.114011+0000","last_clean":"2023-09-23T22:30:36.114011+0000","last_became_active":"2023-09-23T22:29:01.584585+0000","last_became_peered":"2023-09-23T22:29:01.584585+0000","last_unstale":"2023-09-23T22:30:36.114011+0000","last_undegraded":"2023-09-23T22:30:36.114011+0000","last_fullsized":"2023-09-23T22:30:36.114011+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:44.762021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":229,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164466+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:30:43.164466+0000","last_peered":"2023-09-23T22:30:43.164466+0000","last_clean":"2023-09-23T22:30:43.164466+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:30:43.164466+0000","last_undegraded":"2023-09-23T22:30:43.164466+0000","last_fullsized":"2023-09-23T22:30:43.164466+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":338,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108153+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:30:36.108153+0000","last_peered":"2023-09-23T22:30:36.108153+0000","last_clean":"2023-09-23T22:30:36.108153+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:30:36.108153+0000","last_undegraded":"2023-09-23T22:30:36.108153+0000","last_fullsized":"2023-09-23T22:30:36.108153+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":16,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164579+0000","last_change":"2023-09-23T22:30:26.181266+0000","last_active":"2023-09-23T22:30:43.164579+0000","last_peered":"2023-09-23T22:30:43.164579+0000","last_clean":"2023-09-23T22:30:43.164579+0000","last_became_active":"2023-09-23T22:30:26.180875+0000","last_became_peered":"2023-09-23T22:30:26.180875+0000","last_unstale":"2023-09-23T22:30:43.164579+0000","last_undegraded":"2023-09-23T22:30:43.164579+0000","last_fullsized":"2023-09-23T22:30:43.164579+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:11.776290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":116,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170624+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:30:43.170624+0000","last_peered":"2023-09-23T22:30:43.170624+0000","last_clean":"2023-09-23T22:30:43.170624+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:30:43.170624+0000","last_undegraded":"2023-09-23T22:30:43.170624+0000","last_fullsized":"2023-09-23T22:30:43.170624+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":355,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.106987+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:30:36.106987+0000","last_peered":"2023-09-23T22:30:36.106987+0000","last_clean":"2023-09-23T22:30:36.106987+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:30:36.106987+0000","last_undegraded":"2023-09-23T22:30:36.106987+0000","last_fullsized":"2023-09-23T22:30:36.106987+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":380,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.109084+0000","last_change":"2023-09-23T22:28:05.114400+0000","last_active":"2023-09-23T22:30:36.109084+0000","last_peered":"2023-09-23T22:30:36.109084+0000","last_clean":"2023-09-23T22:30:36.109084+0000","last_became_active":"2023-09-23T22:28:05.114095+0000","last_became_peered":"2023-09-23T22:28:05.114095+0000","last_unstale":"2023-09-23T22:30:36.109084+0000","last_undegraded":"2023-09-23T22:30:36.109084+0000","last_fullsized":"2023-09-23T22:30:36.109084+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:23.836996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":380,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.107142+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:30:36.107142+0000","last_peered":"2023-09-23T22:30:36.107142+0000","last_clean":"2023-09-23T22:30:36.107142+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:30:36.107142+0000","last_undegraded":"2023-09-23T22:30:36.107142+0000","last_fullsized":"2023-09-23T22:30:36.107142+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":229,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.108101+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:30:36.108101+0000","last_peered":"2023-09-23T22:30:36.108101+0000","last_clean":"2023-09-23T22:30:36.108101+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:30:36.108101+0000","last_undegraded":"2023-09-23T22:30:36.108101+0000","last_fullsized":"2023-09-23T22:30:36.108101+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":46,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.113510+0000","last_change":"2023-09-23T22:29:01.593820+0000","last_active":"2023-09-23T22:30:36.113510+0000","last_peered":"2023-09-23T22:30:36.113510+0000","last_clean":"2023-09-23T22:30:36.113510+0000","last_became_active":"2023-09-23T22:29:01.593176+0000","last_became_peered":"2023-09-23T22:29:01.593176+0000","last_unstale":"2023-09-23T22:30:36.113510+0000","last_undegraded":"2023-09-23T22:30:36.113510+0000","last_fullsized":"2023-09-23T22:30:36.113510+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:33.824095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":365,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165783+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:30:43.165783+0000","last_peered":"2023-09-23T22:30:43.165783+0000","last_clean":"2023-09-23T22:30:43.165783+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:30:43.165783+0000","last_undegraded":"2023-09-23T22:30:43.165783+0000","last_fullsized":"2023-09-23T22:30:43.165783+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":303,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.166941+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:30:43.166941+0000","last_peered":"2023-09-23T22:30:43.166941+0000","last_clean":"2023-09-23T22:30:43.166941+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:30:43.166941+0000","last_undegraded":"2023-09-23T22:30:43.166941+0000","last_fullsized":"2023-09-23T22:30:43.166941+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":423,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T22:30:36.107934+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:30:36.107934+0000","last_peered":"2023-09-23T22:30:36.107934+0000","last_clean":"2023-09-23T22:30:36.107934+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:30:36.107934+0000","last_undegraded":"2023-09-23T22:30:36.107934+0000","last_fullsized":"2023-09-23T22:30:36.107934+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":235077656,"num_objects":3588,"num_object_clones":0,"num_object_copies":10764,"num_objects_missing_on_primary":1704,"num_objects_missing":1704,"num_objects_degraded":2457,"num_objects_misplaced":4275,"num_objects_unfound":0,"num_objects_dirty":3588,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":177082,"num_write_kb":5781377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44504,"num_bytes_recovered":2463629336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2276081664,"data_stored":2276065376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94894,"ondisk_log_size":94894,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":26454,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":423698,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":132,"num_read_kb":249,"num_write":66,"num_write_kb":660,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2254133,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":177,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234316,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":634648,"kb_used_data":312592,"kb_used_omap":0,"kb_used_meta":322048,"kb_avail":93110504,"statfs":{"total":95995035648,"available":95345156096,"internally_reserved":0,"allocated":320094208,"data_stored":318698119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":329777152},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":6.1479999999999997,"15min":4.1870000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.0519999999999996,"5min":220.66,"15min":220.66},"last":2.3159999999999998},{"interface":"front","average":{"1min":1.5049999999999999,"5min":6.3280000000000003,"15min":4.5540000000000003},"min":{"1min":0.29099999999999998,"5min":0.246,"15min":0.217},"max":{"1min":5.6799999999999997,"5min":220.59899999999999,"15min":220.59899999999999},"last":0.245}]},{"osd":1,"last update":"Sat Sep 23 22:30:29 2023","interfaces":[{"interface":"back","average":{"1min":4.9500000000000002,"5min":4.9409999999999998,"15min":4.5590000000000002},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.23799999999999999},"max":{"1min":76.984999999999999,"5min":144.77500000000001,"15min":144.77500000000001},"last":1.46},{"interface":"front","average":{"1min":5.2080000000000002,"5min":5.2359999999999998,"15min":4.8239999999999998},"min":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.24299999999999999},"max":{"1min":79.069999999999993,"5min":144.751,"15min":144.751},"last":0.31900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:28:58 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":1.5049999999999999,"15min":3.931},"min":{"1min":0.317,"5min":0.29999999999999999,"15min":0.24399999999999999},"max":{"1min":1.165,"5min":9.0310000000000006,"15min":125.101},"last":2.3399999999999999},{"interface":"front","average":{"1min":0.65100000000000002,"5min":1.4690000000000001,"15min":3.7959999999999998},"min":{"1min":0.34000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.2999999999999998,"5min":8.3140000000000001,"15min":126.021},"last":1.534}]},{"osd":3,"last update":"Sat Sep 23 22:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":4.5060000000000002,"15min":3.6850000000000001},"min":{"1min":0.29399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.4260000000000002,"5min":219.90700000000001,"15min":219.90700000000001},"last":1.486},{"interface":"front","average":{"1min":2.1059999999999999,"5min":5.1550000000000002,"15min":3.944},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.234,"5min":219.976,"15min":219.976},"last":0.34499999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.52900000000000003,"15min":1.379},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23499999999999999},"max":{"1min":3.9020000000000001,"5min":3.9020000000000001,"15min":88.543000000000006},"last":1.0800000000000001},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.497,"15min":1.3089999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.9950000000000001,"5min":3.9950000000000001,"15min":88.495999999999995},"last":2.2789999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.48899999999999999,"15min":1.2809999999999999},"min":{"1min":0.28999999999999998,"5min":0.249,"15min":0.24299999999999999},"max":{"1min":1.8129999999999999,"5min":1.825,"15min":93.816000000000003},"last":1.425},{"interface":"front","average":{"1min":0.876,"5min":0.625,"15min":1.5580000000000001},"min":{"1min":0.35999999999999999,"5min":0.309,"15min":0.27300000000000002},"max":{"1min":4.2560000000000002,"5min":4.2560000000000002,"15min":95.579999999999998},"last":0.94699999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.54000000000000004,"15min":0.749},"min":{"1min":0.30499999999999999,"5min":0.29499999999999998,"15min":0.28999999999999998},"max":{"1min":2.0129999999999999,"5min":2.0779999999999998,"15min":9.0139999999999993},"last":1.5089999999999999},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.61499999999999999,"15min":0.98899999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":4.2859999999999996,"5min":4.2859999999999996,"15min":13.362},"last":0.39300000000000002}]}]},{"osd":6,"up_from":46,"seq":197568495952,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":167836,"kb_used_data":61716,"kb_used_omap":0,"kb_used_meta":106112,"kb_avail":93577316,"statfs":{"total":95995035648,"available":95823171584,"internally_reserved":0,"allocated":63197184,"data_stored":61798303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":108658688},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:17 2023","interfaces":[{"interface":"back","average":{"1min":5.298,"5min":2.2309999999999999,"15min":2.1160000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":55.206000000000003,"5min":55.206000000000003,"15min":108.773},"last":213.23699999999999},{"interface":"front","average":{"1min":6.0060000000000002,"5min":2.6219999999999999,"15min":3.1429999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.21099999999999999},"max":{"1min":55.375999999999998,"5min":55.375999999999998,"15min":135.45599999999999},"last":213.304}]},{"osd":1,"last update":"Sat Sep 23 22:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":2.1030000000000002,"15min":3.6320000000000001},"min":{"1min":0.27800000000000002,"5min":0.26000000000000001,"15min":0.22800000000000001},"max":{"1min":6.8380000000000001,"5min":80.430000000000007,"15min":129.964},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.129,"5min":2.6640000000000001,"15min":3.9449999999999998},"min":{"1min":0.35699999999999998,"5min":0.254,"15min":0.254},"max":{"1min":6.9249999999999998,"5min":84.192999999999998,"15min":131.18600000000001},"last":0.51000000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:28:58 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":1.3740000000000001,"15min":2.6459999999999999},"min":{"1min":0.29499999999999998,"5min":0.28499999999999998,"15min":0.23899999999999999},"max":{"1min":2.153,"5min":10.225,"15min":177.46700000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.94399999999999995,"5min":1.413,"15min":3.1699999999999999},"min":{"1min":0.33700000000000002,"5min":0.309,"15min":0.27700000000000002},"max":{"1min":3.012,"5min":10.093,"15min":177.226},"last":0.54300000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.77,"15min":3.4159999999999999},"min":{"1min":0.36799999999999999,"5min":0.316,"15min":0.311},"max":{"1min":5.7640000000000002,"5min":42.523000000000003,"15min":125.125},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.631,"5min":1.9359999999999999,"15min":3.2410000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":7.4589999999999996,"5min":39.926000000000002,"15min":76.406999999999996},"last":0.40899999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:29:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":1.474,"15min":2.4470000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.7679999999999998,"5min":67.483999999999995,"15min":305.30200000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.60499999999999998,"5min":1.524,"15min":2.427},"min":{"1min":0.29299999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.6459999999999999,"5min":67.814999999999998,"15min":305.40300000000002},"last":0.33600000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:29:43 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.47699999999999998,"15min":0.995},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.8340000000000001,"5min":2.3069999999999999,"15min":67.045000000000002},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53100000000000003,"15min":1.0780000000000001},"min":{"1min":0.30099999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.6850000000000001,"5min":5.5910000000000002,"15min":66.802000000000007},"last":0.36699999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:30:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.60199999999999998,"15min":1.024},"min":{"1min":0.33800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":3.1850000000000001,"5min":3.7509999999999999,"15min":82.801000000000002},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.64300000000000002,"15min":1.0069999999999999},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.6989999999999998,"5min":5.6989999999999998,"15min":83.052999999999997},"last":0.497}]}]},{"osd":1,"up_from":98,"seq":420906795225,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":678704,"kb_used_data":140520,"kb_used_omap":7,"kb_used_meta":538168,"kb_avail":93066448,"statfs":{"total":95995035648,"available":95300042752,"internally_reserved":0,"allocated":143892480,"data_stored":141892178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":551084404},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5219999999999998,"5min":4.9199999999999999,"15min":3.9049999999999998},"min":{"1min":0.24399999999999999,"5min":0.216,"15min":0.189},"max":{"1min":18.045000000000002,"5min":90.923000000000002,"15min":196.095},"last":0.499},{"interface":"front","average":{"1min":4.5499999999999998,"5min":5.4550000000000001,"15min":4.5899999999999999},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.20799999999999999},"max":{"1min":14.1,"5min":92.093000000000004,"15min":196.85300000000001},"last":0.59699999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:28:54 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":1.3819999999999999,"15min":1.613},"min":{"1min":0.252,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":1.998,"5min":10.898999999999999,"15min":33.902000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.626,"5min":2.6899999999999999,"15min":2.4860000000000002},"min":{"1min":0.31,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":2.6339999999999999,"5min":28.911999999999999,"15min":34.781999999999996},"last":0.53200000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":4.2590000000000003,"15min":4.452},"min":{"1min":0.33900000000000002,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":9.3089999999999993,"5min":88.340000000000003,"15min":160.499},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.5179999999999998,"5min":3.8109999999999999,"15min":4.2389999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.223},"max":{"1min":8.1590000000000007,"5min":91.781000000000006,"15min":160.465},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":2.2389999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.0470000000000002,"5min":9.6539999999999999,"15min":234.79900000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.694,"5min":1.7370000000000001,"15min":2.8210000000000002},"min":{"1min":0.39900000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.093,"5min":9.093,"15min":234.81700000000001},"last":0.70799999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5110000000000001,"5min":2.0659999999999998,"15min":2.3940000000000001},"min":{"1min":0.40300000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.2759999999999998,"5min":10.973000000000001,"15min":196.79900000000001},"last":0.745},{"interface":"front","average":{"1min":1.389,"5min":1.6499999999999999,"15min":2.2869999999999999},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1200000000000001,"5min":7.8120000000000003,"15min":196.68299999999999},"last":0.52000000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.8089999999999999,"15min":1.7949999999999999},"min":{"1min":0.41399999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.7779999999999996,"5min":8.5540000000000003,"15min":16.314},"last":9.7159999999999993},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.377,"15min":1.3080000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":3.4729999999999999,"5min":9.6910000000000007,"15min":11.385999999999999},"last":0.437}]},{"osd":7,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4910000000000001,"5min":1.9099999999999999,"15min":1.9350000000000001},"min":{"1min":0.44900000000000001,"5min":0.39500000000000002,"15min":0.32900000000000001},"max":{"1min":8.0660000000000007,"5min":8.6059999999999999,"15min":16.265000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.3700000000000001,"15min":1.3220000000000001},"min":{"1min":0.439,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.165,"5min":9.7129999999999992,"15min":12.045999999999999},"last":0.55100000000000005}]}]},{"osd":0,"up_from":93,"seq":399431958754,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":848436,"kb_used_data":283244,"kb_used_omap":8,"kb_used_meta":565175,"kb_avail":92896716,"statfs":{"total":95995035648,"available":95126237184,"internally_reserved":0,"allocated":290041856,"data_stored":287979098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":578740074},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,3,4,3,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:30:32 2023","interfaces":[{"interface":"back","average":{"1min":6.282,"5min":4.0720000000000001,"15min":3.851},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":89.439999999999998,"5min":89.439999999999998,"15min":169.77199999999999},"last":4.3390000000000004},{"interface":"front","average":{"1min":8.2460000000000004,"5min":4.0750000000000002,"15min":4.0979999999999999},"min":{"1min":0.26400000000000001,"5min":0.218,"15min":0.20300000000000001},"max":{"1min":89.287999999999997,"5min":89.287999999999997,"15min":169.52099999999999},"last":2.4510000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:28:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":2.726,"15min":3.4550000000000001},"min":{"1min":0.314,"5min":0.21299999999999999,"15min":0.20799999999999999},"max":{"1min":4.2930000000000001,"5min":65.245000000000005,"15min":206.495},"last":2.2130000000000001},{"interface":"front","average":{"1min":0.77900000000000003,"5min":2.9900000000000002,"15min":4.1280000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.22800000000000001},"max":{"1min":4.1920000000000002,"5min":67.736000000000004,"15min":207.34399999999999},"last":3.605}]},{"osd":3,"last update":"Sat Sep 23 22:30:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.149,"15min":3.6070000000000002},"min":{"1min":0.23200000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.234,"5min":15.404,"15min":146.87299999999999},"last":3.5169999999999999},{"interface":"front","average":{"1min":3.9460000000000002,"5min":2.335,"15min":3.7709999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":22.303000000000001,"5min":22.303000000000001,"15min":145.48099999999999},"last":2.1829999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.7729999999999999,"15min":2.3780000000000001},"min":{"1min":0.35299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.1430000000000007,"5min":10.837999999999999,"15min":82.825000000000003},"last":6.7649999999999997},{"interface":"front","average":{"1min":1.411,"5min":1.23,"15min":1.6020000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.0220000000000002,"5min":10.08,"15min":75.930999999999997},"last":3.5699999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":2.0249999999999999,"15min":2.839},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":13.564,"5min":13.564,"15min":93.685000000000002},"last":6.7910000000000004},{"interface":"front","average":{"1min":1.768,"5min":1.3779999999999999,"15min":2.081},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.486000000000001,"5min":13.486000000000001,"15min":93.546000000000006},"last":3.548}]},{"osd":6,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.9059999999999999,"15min":2.1240000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.063000000000001,"5min":12.063000000000001,"15min":16.366},"last":6.7249999999999996},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.3440000000000001,"15min":1.474},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":5.3209999999999997,"5min":10.134,"15min":22.263000000000002},"last":3.5870000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":1.5740000000000001,"15min":2.048},"min":{"1min":0.45500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.6430000000000007,"5min":9.6430000000000007,"15min":32.773000000000003},"last":2.2559999999999998},{"interface":"front","average":{"1min":2.609,"5min":1.847,"15min":2.286},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":8.2240000000000002,"5min":10.901,"15min":23.265999999999998},"last":10.666}]}]},{"osd":2,"up_from":0,"seq":1498943586317,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468171,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019221,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":472408,"kb_used_data":191312,"kb_used_omap":0,"kb_used_meta":281024,"kb_avail":93272744,"statfs":{"total":95995035648,"available":95511289856,"internally_reserved":0,"allocated":195903488,"data_stored":194508172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":287768576},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":4.2300000000000004,"15min":3.5699999999999998},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.9559999999999995,"5min":104.252,"15min":150.63499999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.3399999999999999,"5min":4.5119999999999996,"15min":3.7080000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.9740000000000002,"5min":104.294,"15min":150.672},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":6.2060000000000004,"5min":3.9550000000000001,"15min":3.1869999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.21299999999999999},"max":{"1min":81.122,"5min":95.114000000000004,"15min":107.346},"last":0.30399999999999999},{"interface":"front","average":{"1min":6.1959999999999997,"5min":3.9399999999999999,"15min":3.528},"min":{"1min":0.26700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":81.090999999999994,"5min":95.222999999999999,"15min":107.33499999999999},"last":1.105}]},{"osd":2,"last update":"Sat Sep 23 22:28:56 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":1.1439999999999999,"15min":2.4980000000000002},"min":{"1min":0.29199999999999998,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":0.57499999999999996,"5min":10.609,"15min":138.06100000000001},"last":3.137},{"interface":"front","average":{"1min":0.89000000000000001,"5min":1.6639999999999999,"15min":2.8180000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28199999999999997},"max":{"1min":4.4740000000000002,"5min":10.677,"15min":138.15199999999999},"last":3.153}]},{"osd":3,"last update":"Sat Sep 23 22:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":4.0629999999999997,"15min":2.4630000000000001},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.0940000000000003,"5min":102.896,"15min":102.896},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.2040000000000002,"5min":4.2220000000000004,"15min":2.5739999999999998},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":14.318,"5min":112.667,"15min":112.667},"last":1.21}]},{"osd":5,"last update":"Sat Sep 23 22:30:39 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":0.76800000000000002,"15min":1.0029999999999999},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.407,"5min":13.407,"15min":28.399000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.7709999999999999,"5min":0.79200000000000004,"15min":1.036},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":14.212,"5min":14.212,"15min":28.350999999999999},"last":0.31900000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.76200000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.251},"max":{"1min":1.179,"5min":1.179,"15min":7.8550000000000004},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.50600000000000001,"15min":0.75600000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.5940000000000001,"5min":2.0289999999999999,"15min":8.5210000000000008},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.59499999999999997,"15min":0.88600000000000001},"min":{"1min":0.33100000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3179999999999996,"5min":4.3179999999999996,"15min":8.577},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.4119999999999999,"5min":0.69099999999999995,"15min":0.88700000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9319999999999999,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":0.504}]}]},{"osd":5,"up_from":38,"seq":163208757589,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":443852,"kb_used_data":122500,"kb_used_omap":0,"kb_used_meta":321344,"kb_avail":93301300,"statfs":{"total":95995035648,"available":95540531200,"internally_reserved":0,"allocated":125440000,"data_stored":124052735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":329056256},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:14 2023","interfaces":[{"interface":"back","average":{"1min":3.4399999999999999,"5min":7.1689999999999996,"15min":4.3899999999999997},"min":{"1min":0.23899999999999999,"5min":0.22,"15min":0.20499999999999999},"max":{"1min":16.109999999999999,"5min":163.54499999999999,"15min":163.54499999999999},"last":3.2490000000000001},{"interface":"front","average":{"1min":1.7709999999999999,"5min":6.5010000000000003,"15min":4.2089999999999996},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":5.7949999999999999,"5min":162.81800000000001,"15min":162.81800000000001},"last":0.46000000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":3.9329999999999998,"15min":3.8740000000000001},"min":{"1min":0.38100000000000001,"5min":0.27900000000000003,"15min":0.23799999999999999},"max":{"1min":7.7779999999999996,"5min":113.514,"15min":113.514},"last":1.101},{"interface":"front","average":{"1min":3.1240000000000001,"5min":3.9990000000000001,"15min":3.8860000000000001},"min":{"1min":0.33100000000000002,"5min":0.255,"15min":0.252},"max":{"1min":7.6319999999999997,"5min":113.42700000000001,"15min":113.42700000000001},"last":1.0900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:28:57 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.1890000000000001,"15min":3.7130000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.27700000000000002},"max":{"1min":6.8600000000000003,"5min":10.441000000000001,"15min":104.254},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.484,"15min":4.0519999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.26900000000000002},"max":{"1min":3.6539999999999999,"5min":8.3729999999999993,"15min":104.217},"last":3.379}]},{"osd":3,"last update":"Sat Sep 23 22:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":5.883,"15min":4.1040000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.246},"max":{"1min":9.3369999999999997,"5min":132.59,"15min":132.59},"last":3.452},{"interface":"front","average":{"1min":3.024,"5min":6.0949999999999998,"15min":4.1040000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":7.6479999999999997,"5min":129.381,"15min":129.381},"last":0.441}]},{"osd":4,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.9610000000000001,"15min":1.528},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.24099999999999999},"max":{"1min":3.7240000000000002,"5min":126.679,"15min":126.679},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.879,"5min":1.98,"15min":1.5589999999999999},"min":{"1min":0.309,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":2.7610000000000001,"5min":126.78100000000001,"15min":126.78100000000001},"last":3.2839999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:29:46 2023","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.53200000000000003,"15min":1.173},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.9590000000000001,"5min":3.9590000000000001,"15min":114.83499999999999},"last":3.3130000000000002},{"interface":"front","average":{"1min":1.004,"5min":0.63300000000000001,"15min":1.1100000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":6.1779999999999999,"5min":6.1779999999999999,"15min":110.905},"last":3.6930000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.624,"15min":0.91600000000000004},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":4.1120000000000001,"5min":4.827,"15min":8.8030000000000008},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.58999999999999997,"15min":0.77900000000000003},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.308},"max":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":7.6609999999999996},"last":1.8029999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":265818112,"data_stored":265814040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":116854784,"data_stored":116850712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":158859264,"data_stored":158859264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1083969536,"data_stored":1083965464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":178061312,"data_stored":178061312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":114360320,"data_stored":114360320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":50266112,"data_stored":50266112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":307892224,"data_stored":307888152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:30:51.905 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:30:51.925 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:51.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:51.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:51 smithi026 ceph-mon[215775]: from='client.45767 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:30:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:51 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:51.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:51 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:51.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:51 smithi026 ceph-mon[223014]: from='client.45767 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:30:51.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:51 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:51.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:51 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:52.005 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:30:52.034 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:52.151 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:52.342 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:52.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:52.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:52.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:30:52.476 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:52.518 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:30:52.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:52.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:30:52.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:52.650 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:52.941 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:52.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:52 smithi026 ceph-mon[215775]: pgmap v946: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 166 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 619 op/s; 1730/7842 objects degraded (22.061%); 4027/7842 objects misplaced (51.352%); 4.1 MiB/s, 214 objects/s recovering 2023-09-23T22:30:52.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:30:52.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:30:52.950 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:52.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:52 smithi026 ceph-mon[223014]: pgmap v946: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 166 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 619 op/s; 1730/7842 objects degraded (22.061%); 4027/7842 objects misplaced (51.352%); 4.1 MiB/s, 214 objects/s recovering 2023-09-23T22:30:52.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:30:52.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:30:52.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:52.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:52.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:53.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:30:53.057 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:52 smithi070 ceph-mon[192652]: pgmap v946: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 166 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 619 op/s; 1730/7842 objects degraded (22.061%); 4027/7842 objects misplaced (51.352%); 4.1 MiB/s, 214 objects/s recovering 2023-09-23T22:30:53.058 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:30:53.058 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:30:53.058 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:30:53.059 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:30:53.098 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:53.128 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:53.179 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:30:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:53.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:30:53.196 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:53.296 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:30:53.356 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:53.451 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:30:53.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:53.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:30:53.560 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:30:53.632 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:53.686 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:30:53.835 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:30:53.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:53.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:30:53.940 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:53.957 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 1730/7842 objects degraded (22.061%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:30:53.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 1730/7842 objects degraded (22.061%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:30:53.994 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 1730/7842 objects degraded (22.061%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:30:54.307 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:54.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:54.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:54.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 90334 objects 2023-09-23T22:30:54.412 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :269.086 2023-09-23T22:30:54.416 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:54.443 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:54.490 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:54.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:54.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:54.593 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:54.622 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:54.803 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:54 smithi070 ceph-mon[192652]: pgmap v947: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 166 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 518 op/s; 1730/7842 objects degraded (22.061%); 4027/7842 objects misplaced (51.352%); 3.4 MiB/s, 179 objects/s recovering 2023-09-23T22:30:54.803 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:54.804 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2127353261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:54.804 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:54 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:54 smithi026 ceph-mon[223014]: pgmap v947: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 166 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 518 op/s; 1730/7842 objects degraded (22.061%); 4027/7842 objects misplaced (51.352%); 3.4 MiB/s, 179 objects/s recovering 2023-09-23T22:30:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2127353261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:54 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:54 smithi026 ceph-mon[215775]: pgmap v947: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 166 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 518 op/s; 1730/7842 objects degraded (22.061%); 4027/7842 objects misplaced (51.352%); 3.4 MiB/s, 179 objects/s recovering 2023-09-23T22:30:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:30:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2127353261' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:54.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:54 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:30:55.086 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T22:30:55.086 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T22:30:55.118 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:30:55.322 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:55.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:55.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:55.426 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:55.458 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:55.672 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:55.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:55.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:55.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:55.807 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:55.957 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:55.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:55.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:55.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:30:55.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[215775]: failed to encode map e367 with expected crc 2023-09-23T22:30:55.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[215775]: failed to encode map e367 with expected crc 2023-09-23T22:30:55.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:30:55.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[215775]: osdmap e367: 8 total, 8 up, 6 in 2023-09-23T22:30:55.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[215775]: failed to encode map e367 with expected crc 2023-09-23T22:30:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:30:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[223014]: failed to encode map e367 with expected crc 2023-09-23T22:30:55.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[223014]: failed to encode map e367 with expected crc 2023-09-23T22:30:55.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:30:55.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[223014]: osdmap e367: 8 total, 8 up, 6 in 2023-09-23T22:30:55.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:55 smithi026 ceph-mon[223014]: failed to encode map e367 with expected crc 2023-09-23T22:30:56.002 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:30:56.002 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:30:56.023 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:55 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:30:56.023 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:55 smithi070 ceph-mon[192652]: failed to encode map e367 with expected crc 2023-09-23T22:30:56.023 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:55 smithi070 ceph-mon[192652]: failed to encode map e367 with expected crc 2023-09-23T22:30:56.024 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:55 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:30:56.024 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:55 smithi070 ceph-mon[192652]: osdmap e367: 8 total, 8 up, 6 in 2023-09-23T22:30:56.024 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:55 smithi070 ceph-mon[192652]: failed to encode map e367 with expected crc 2023-09-23T22:30:56.061 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:56.094 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:56.248 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:56.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:56.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:56.352 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:56.384 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:56.432 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:56.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:56.536 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:30:56.566 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:56.569 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:56.740 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:30:56.743 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:30:56.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:56.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:30:56.761 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:30:56.780 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:30:56.782 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:30:56.782 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:30:56.803 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:30:56.827 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:30:56.842 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:30:56.847 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:30:56.865 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:30:56.882 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:56.889 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:30:56.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:30:56.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:30:56.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:30:56.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:30:56.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:30:56.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:30:56.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:30:56.891 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:30:57.033 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:30:57.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:57.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:30:57.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[223014]: pgmap v949: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 5.5 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 539/135 objects degraded (399.259%); 2978/135 objects misplaced (2205.926%); 4.0 MiB/s, 206 objects/s recovering 2023-09-23T22:30:57.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[223014]: failed to encode map e368 with expected crc 2023-09-23T22:30:57.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[223014]: failed to encode map e368 with expected crc 2023-09-23T22:30:57.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[223014]: osdmap e368: 8 total, 8 up, 6 in 2023-09-23T22:30:57.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[223014]: failed to encode map e368 with expected crc 2023-09-23T22:30:57.123 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1840451937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:57.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[215775]: pgmap v949: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 5.5 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 539/135 objects degraded (399.259%); 2978/135 objects misplaced (2205.926%); 4.0 MiB/s, 206 objects/s recovering 2023-09-23T22:30:57.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[215775]: failed to encode map e368 with expected crc 2023-09-23T22:30:57.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[215775]: failed to encode map e368 with expected crc 2023-09-23T22:30:57.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[215775]: osdmap e368: 8 total, 8 up, 6 in 2023-09-23T22:30:57.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[215775]: failed to encode map e368 with expected crc 2023-09-23T22:30:57.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1840451937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:57.136 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:30:57.168 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:57.345 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:30:57.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:57.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:30:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:57 smithi070 ceph-mon[192652]: pgmap v949: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 5.5 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 539/135 objects degraded (399.259%); 2978/135 objects misplaced (2205.926%); 4.0 MiB/s, 206 objects/s recovering 2023-09-23T22:30:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:57 smithi070 ceph-mon[192652]: failed to encode map e368 with expected crc 2023-09-23T22:30:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:57 smithi070 ceph-mon[192652]: failed to encode map e368 with expected crc 2023-09-23T22:30:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:57 smithi070 ceph-mon[192652]: osdmap e368: 8 total, 8 up, 6 in 2023-09-23T22:30:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:57 smithi070 ceph-mon[192652]: failed to encode map e368 with expected crc 2023-09-23T22:30:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1840451937' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:30:57.449 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:57.477 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:57.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:57.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:57.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:57.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:57.703 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:58.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:30:58.347 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":950,"stamp":"2023-09-23T22:30:56.112861+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5803173,"num_objects":45,"num_object_clones":0,"num_object_copies":135,"num_objects_missing_on_primary":1546,"num_objects_missing":1546,"num_objects_degraded":539,"num_objects_misplaced":2978,"num_objects_unfound":0,"num_objects_dirty":45,"num_whiteouts":0,"num_read":6115,"num_read_kb":12434,"num_write":181625,"num_write_kb":5794762,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46684,"num_bytes_recovered":2518535426,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97566,"ondisk_log_size":97566,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":323,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2591592,"kb_used_data":453816,"kb_used_omap":15,"kb_used_meta":2137648,"kb_avail":559879320,"statfs":{"total":575970213888,"available":573316423680,"internally_reserved":0,"allocated":464707584,"data_stored":455055357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2188951774},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-342294552,"num_objects":-5224,"num_object_clones":0,"num_object_copies":-15672,"num_objects_missing_on_primary":-313,"num_objects_missing":-313,"num_objects_degraded":-2084,"num_objects_misplaced":-2064,"num_objects_unfound":0,"num_objects_dirty":-5224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5224,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2062,"num_bytes_recovered":41943040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.581309"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":393,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164029+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:30:43.164029+0000","last_peered":"2023-09-23T22:30:43.164029+0000","last_clean":"2023-09-23T22:30:43.164029+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:30:43.164029+0000","last_undegraded":"2023-09-23T22:30:43.164029+0000","last_fullsized":"2023-09-23T22:30:43.164029+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":82,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170924+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:30:43.170924+0000","last_peered":"2023-09-23T22:30:43.170924+0000","last_clean":"2023-09-23T22:30:43.170924+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:30:43.170924+0000","last_undegraded":"2023-09-23T22:30:43.170924+0000","last_fullsized":"2023-09-23T22:30:43.170924+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.171270+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:30:43.171270+0000","last_peered":"2023-09-23T22:30:43.171270+0000","last_clean":"2023-09-23T22:30:43.171270+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:30:43.171270+0000","last_undegraded":"2023-09-23T22:30:43.171270+0000","last_fullsized":"2023-09-23T22:30:43.171270+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":244,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164542+0000","last_change":"2023-09-23T22:27:52.097725+0000","last_active":"2023-09-23T22:30:43.164542+0000","last_peered":"2023-09-23T22:30:43.164542+0000","last_clean":"2023-09-23T22:30:43.164542+0000","last_became_active":"2023-09-23T22:27:52.097293+0000","last_became_peered":"2023-09-23T22:27:52.097293+0000","last_unstale":"2023-09-23T22:30:43.164542+0000","last_undegraded":"2023-09-23T22:30:43.164542+0000","last_fullsized":"2023-09-23T22:30:43.164542+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:41.651341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":411,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164188+0000","last_change":"2023-09-23T22:28:05.155218+0000","last_active":"2023-09-23T22:30:43.164188+0000","last_peered":"2023-09-23T22:30:43.164188+0000","last_clean":"2023-09-23T22:30:43.164188+0000","last_became_active":"2023-09-23T22:28:05.154771+0000","last_became_peered":"2023-09-23T22:28:05.154771+0000","last_unstale":"2023-09-23T22:30:43.164188+0000","last_undegraded":"2023-09-23T22:30:43.164188+0000","last_fullsized":"2023-09-23T22:30:43.164188+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:48.395095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.176077+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:30:43.176077+0000","last_peered":"2023-09-23T22:30:43.176077+0000","last_clean":"2023-09-23T22:30:43.176077+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:30:43.176077+0000","last_undegraded":"2023-09-23T22:30:43.176077+0000","last_fullsized":"2023-09-23T22:30:43.176077+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":281,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.167018+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:30:43.167018+0000","last_peered":"2023-09-23T22:30:43.167018+0000","last_clean":"2023-09-23T22:30:43.167018+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:30:43.167018+0000","last_undegraded":"2023-09-23T22:30:43.167018+0000","last_fullsized":"2023-09-23T22:30:43.167018+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.173752+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:30:43.173752+0000","last_peered":"2023-09-23T22:30:43.173752+0000","last_clean":"2023-09-23T22:30:43.173752+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:30:43.173752+0000","last_undegraded":"2023-09-23T22:30:43.173752+0000","last_fullsized":"2023-09-23T22:30:43.173752+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":356,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170227+0000","last_change":"2023-09-23T22:27:52.091129+0000","last_active":"2023-09-23T22:30:43.170227+0000","last_peered":"2023-09-23T22:30:43.170227+0000","last_clean":"2023-09-23T22:30:43.170227+0000","last_became_active":"2023-09-23T22:27:52.090819+0000","last_became_peered":"2023-09-23T22:27:52.090819+0000","last_unstale":"2023-09-23T22:30:43.170227+0000","last_undegraded":"2023-09-23T22:30:43.170227+0000","last_fullsized":"2023-09-23T22:30:43.170227+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:13.639810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.173939+0000","last_change":"2023-09-23T22:29:01.578787+0000","last_active":"2023-09-23T22:30:43.173939+0000","last_peered":"2023-09-23T22:30:43.173939+0000","last_clean":"2023-09-23T22:30:43.173939+0000","last_became_active":"2023-09-23T22:29:01.578405+0000","last_became_peered":"2023-09-23T22:29:01.578405+0000","last_unstale":"2023-09-23T22:30:43.173939+0000","last_undegraded":"2023-09-23T22:30:43.173939+0000","last_fullsized":"2023-09-23T22:30:43.173939+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:55.201936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":97,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165992+0000","last_change":"2023-09-23T22:27:27.320214+0000","last_active":"2023-09-23T22:30:43.165992+0000","last_peered":"2023-09-23T22:30:43.165992+0000","last_clean":"2023-09-23T22:30:43.165992+0000","last_became_active":"2023-09-23T22:27:27.303456+0000","last_became_peered":"2023-09-23T22:27:27.303456+0000","last_unstale":"2023-09-23T22:30:43.165992+0000","last_undegraded":"2023-09-23T22:30:43.165992+0000","last_fullsized":"2023-09-23T22:30:43.165992+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:17.105540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170523+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:30:43.170523+0000","last_peered":"2023-09-23T22:30:43.170523+0000","last_clean":"2023-09-23T22:30:43.170523+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:30:43.170523+0000","last_undegraded":"2023-09-23T22:30:43.170523+0000","last_fullsized":"2023-09-23T22:30:43.170523+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":430,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163632+0000","last_change":"2023-09-23T22:29:01.673486+0000","last_active":"2023-09-23T22:30:43.163632+0000","last_peered":"2023-09-23T22:30:43.163632+0000","last_clean":"2023-09-23T22:30:43.163632+0000","last_became_active":"2023-09-23T22:29:01.661056+0000","last_became_peered":"2023-09-23T22:29:01.661056+0000","last_unstale":"2023-09-23T22:30:43.163632+0000","last_undegraded":"2023-09-23T22:30:43.163632+0000","last_fullsized":"2023-09-23T22:30:43.163632+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:40.581172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":364,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164436+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:30:43.164436+0000","last_peered":"2023-09-23T22:30:43.164436+0000","last_clean":"2023-09-23T22:30:43.164436+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:30:43.164436+0000","last_undegraded":"2023-09-23T22:30:43.164436+0000","last_fullsized":"2023-09-23T22:30:43.164436+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":341,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164480+0000","last_change":"2023-09-23T22:29:01.662016+0000","last_active":"2023-09-23T22:30:43.164480+0000","last_peered":"2023-09-23T22:30:43.164480+0000","last_clean":"2023-09-23T22:30:43.164480+0000","last_became_active":"2023-09-23T22:29:01.600346+0000","last_became_peered":"2023-09-23T22:29:01.600346+0000","last_unstale":"2023-09-23T22:30:43.164480+0000","last_undegraded":"2023-09-23T22:30:43.164480+0000","last_fullsized":"2023-09-23T22:30:43.164480+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.205227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":318,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168207+0000","last_change":"2023-09-23T22:28:41.267094+0000","last_active":"2023-09-23T22:30:43.168207+0000","last_peered":"2023-09-23T22:30:43.168207+0000","last_clean":"2023-09-23T22:30:43.168207+0000","last_became_active":"2023-09-23T22:28:41.266787+0000","last_became_peered":"2023-09-23T22:28:41.266787+0000","last_unstale":"2023-09-23T22:30:43.168207+0000","last_undegraded":"2023-09-23T22:30:43.168207+0000","last_fullsized":"2023-09-23T22:30:43.168207+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:30.013252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":262,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:44.356732+0000","last_change":"2023-09-23T22:27:52.101677+0000","last_active":"2023-09-23T22:30:44.356732+0000","last_peered":"2023-09-23T22:30:44.356732+0000","last_clean":"2023-09-23T22:30:44.356732+0000","last_became_active":"2023-09-23T22:27:52.101483+0000","last_became_peered":"2023-09-23T22:27:52.101483+0000","last_unstale":"2023-09-23T22:30:44.356732+0000","last_undegraded":"2023-09-23T22:30:44.356732+0000","last_fullsized":"2023-09-23T22:30:44.356732+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:01.772629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":365,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.169148+0000","last_change":"2023-09-23T22:29:01.588394+0000","last_active":"2023-09-23T22:30:43.169148+0000","last_peered":"2023-09-23T22:30:43.169148+0000","last_clean":"2023-09-23T22:30:43.169148+0000","last_became_active":"2023-09-23T22:29:01.587588+0000","last_became_peered":"2023-09-23T22:29:01.587588+0000","last_unstale":"2023-09-23T22:30:43.169148+0000","last_undegraded":"2023-09-23T22:30:43.169148+0000","last_fullsized":"2023-09-23T22:30:43.169148+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:12.141014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":299,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163573+0000","last_change":"2023-09-23T22:29:01.604689+0000","last_active":"2023-09-23T22:30:43.163573+0000","last_peered":"2023-09-23T22:30:43.163573+0000","last_clean":"2023-09-23T22:30:43.163573+0000","last_became_active":"2023-09-23T22:29:01.603739+0000","last_became_peered":"2023-09-23T22:29:01.603739+0000","last_unstale":"2023-09-23T22:30:43.163573+0000","last_undegraded":"2023-09-23T22:30:43.163573+0000","last_fullsized":"2023-09-23T22:30:43.163573+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:26.149229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":252,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164619+0000","last_change":"2023-09-23T22:29:01.575934+0000","last_active":"2023-09-23T22:30:43.164619+0000","last_peered":"2023-09-23T22:30:43.164619+0000","last_clean":"2023-09-23T22:30:43.164619+0000","last_became_active":"2023-09-23T22:29:01.575404+0000","last_became_peered":"2023-09-23T22:29:01.575404+0000","last_unstale":"2023-09-23T22:30:43.164619+0000","last_undegraded":"2023-09-23T22:30:43.164619+0000","last_fullsized":"2023-09-23T22:30:43.164619+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:38.753379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":57,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.173674+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:30:43.173674+0000","last_peered":"2023-09-23T22:30:43.173674+0000","last_clean":"2023-09-23T22:30:43.173674+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:30:43.173674+0000","last_undegraded":"2023-09-23T22:30:43.173674+0000","last_fullsized":"2023-09-23T22:30:43.173674+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.175979+0000","last_change":"2023-09-23T22:29:01.583368+0000","last_active":"2023-09-23T22:30:43.175979+0000","last_peered":"2023-09-23T22:30:43.175979+0000","last_clean":"2023-09-23T22:30:43.175979+0000","last_became_active":"2023-09-23T22:29:01.582967+0000","last_became_peered":"2023-09-23T22:29:01.582967+0000","last_unstale":"2023-09-23T22:30:43.175979+0000","last_undegraded":"2023-09-23T22:30:43.175979+0000","last_fullsized":"2023-09-23T22:30:43.175979+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:48.275429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.b","version":"366'11262","reported_seq":19010,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:54.101812+0000","last_change":"2023-09-23T22:29:10.081764+0000","last_active":"2023-09-23T22:30:54.101812+0000","last_peered":"2023-09-23T22:30:54.101812+0000","last_clean":"2023-09-23T22:30:54.101812+0000","last_became_active":"2023-09-23T22:29:10.067355+0000","last_became_peered":"2023-09-23T22:29:10.067355+0000","last_unstale":"2023-09-23T22:30:54.101812+0000","last_undegraded":"2023-09-23T22:30:54.101812+0000","last_fullsized":"2023-09-23T22:30:54.101812+0000","mapping_epoch":334,"log_start":"286'6204","ondisk_log_start":"286'6204","created":260,"last_epoch_clean":335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5058,"log_dups_size":0,"ondisk_log_size":5058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:45.970345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":655360,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11262,"num_write_kb":360704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3784,"num_bytes_recovered":247988224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":361,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163882+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:30:43.163882+0000","last_peered":"2023-09-23T22:30:43.163882+0000","last_clean":"2023-09-23T22:30:43.163882+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:30:43.163882+0000","last_undegraded":"2023-09-23T22:30:43.163882+0000","last_fullsized":"2023-09-23T22:30:43.163882+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":358,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170250+0000","last_change":"2023-09-23T22:29:01.597976+0000","last_active":"2023-09-23T22:30:43.170250+0000","last_peered":"2023-09-23T22:30:43.170250+0000","last_clean":"2023-09-23T22:30:43.170250+0000","last_became_active":"2023-09-23T22:29:01.597479+0000","last_became_peered":"2023-09-23T22:29:01.597479+0000","last_unstale":"2023-09-23T22:30:43.170250+0000","last_undegraded":"2023-09-23T22:30:43.170250+0000","last_fullsized":"2023-09-23T22:30:43.170250+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:55.990001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"366'11508","reported_seq":15700,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:54.363814+0000","last_change":"2023-09-23T22:29:17.154270+0000","last_active":"2023-09-23T22:30:54.363814+0000","last_peered":"2023-09-23T22:30:54.363814+0000","last_clean":"2023-09-23T22:26:06.848974+0000","last_became_active":"2023-09-23T22:28:06.779572+0000","last_became_peered":"2023-09-23T22:28:06.779572+0000","last_unstale":"2023-09-23T22:30:54.363814+0000","last_undegraded":"2023-09-23T22:30:54.363814+0000","last_fullsized":"2023-09-23T22:30:54.363814+0000","mapping_epoch":311,"log_start":"278'5723","ondisk_log_start":"278'5723","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5785,"log_dups_size":0,"ondisk_log_size":5785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:46.168583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11508,"num_write_kb":368256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2023,"num_bytes_recovered":30081024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,3],"avail_no_missing":["5","1","2","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":303,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163494+0000","last_change":"2023-09-23T22:27:52.154449+0000","last_active":"2023-09-23T22:30:43.163494+0000","last_peered":"2023-09-23T22:30:43.163494+0000","last_clean":"2023-09-23T22:30:43.163494+0000","last_became_active":"2023-09-23T22:27:52.096263+0000","last_became_peered":"2023-09-23T22:27:52.096263+0000","last_unstale":"2023-09-23T22:30:43.163494+0000","last_undegraded":"2023-09-23T22:30:43.163494+0000","last_fullsized":"2023-09-23T22:30:43.163494+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:02.582951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":393,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163495+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:30:43.163495+0000","last_peered":"2023-09-23T22:30:43.163495+0000","last_clean":"2023-09-23T22:30:43.163495+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:30:43.163495+0000","last_undegraded":"2023-09-23T22:30:43.163495+0000","last_fullsized":"2023-09-23T22:30:43.163495+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"366'11442","reported_seq":11578,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:54.383648+0000","last_change":"2023-09-23T22:27:53.074369+0000","last_active":"2023-09-23T22:30:54.383648+0000","last_peered":"2023-09-23T22:30:54.383648+0000","last_clean":"2023-09-23T22:30:54.383648+0000","last_became_active":"2023-09-23T22:27:53.039034+0000","last_became_peered":"2023-09-23T22:27:53.039034+0000","last_unstale":"2023-09-23T22:30:54.383648+0000","last_undegraded":"2023-09-23T22:30:54.383648+0000","last_fullsized":"2023-09-23T22:30:54.383648+0000","mapping_epoch":305,"log_start":"270'5169","ondisk_log_start":"270'5169","created":260,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6273,"log_dups_size":0,"ondisk_log_size":6273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:46.830216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11442,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":319,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.169230+0000","last_change":"2023-09-23T22:29:01.633048+0000","last_active":"2023-09-23T22:30:43.169230+0000","last_peered":"2023-09-23T22:30:43.169230+0000","last_clean":"2023-09-23T22:30:43.169230+0000","last_became_active":"2023-09-23T22:29:01.587680+0000","last_became_peered":"2023-09-23T22:29:01.587680+0000","last_unstale":"2023-09-23T22:30:43.169230+0000","last_undegraded":"2023-09-23T22:30:43.169230+0000","last_fullsized":"2023-09-23T22:30:43.169230+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.830912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.175047+0000","last_change":"2023-09-23T22:29:01.593596+0000","last_active":"2023-09-23T22:30:43.175047+0000","last_peered":"2023-09-23T22:30:43.175047+0000","last_clean":"2023-09-23T22:30:43.175047+0000","last_became_active":"2023-09-23T22:29:01.593303+0000","last_became_peered":"2023-09-23T22:29:01.593303+0000","last_unstale":"2023-09-23T22:30:43.175047+0000","last_undegraded":"2023-09-23T22:30:43.175047+0000","last_fullsized":"2023-09-23T22:30:43.175047+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:02.980860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"366'10992","reported_seq":22489,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:54.360659+0000","last_change":"2023-09-23T22:28:15.149660+0000","last_active":"2023-09-23T22:30:54.360659+0000","last_peered":"2023-09-23T22:30:54.360659+0000","last_clean":"2023-09-23T22:30:54.360659+0000","last_became_active":"2023-09-23T22:28:15.134552+0000","last_became_peered":"2023-09-23T22:28:15.134552+0000","last_unstale":"2023-09-23T22:30:54.360659+0000","last_undegraded":"2023-09-23T22:30:54.360659+0000","last_fullsized":"2023-09-23T22:30:54.360659+0000","mapping_epoch":315,"log_start":"278'5408","ondisk_log_start":"278'5408","created":260,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5584,"log_dups_size":0,"ondisk_log_size":5584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:52.196467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10992,"num_write_kb":351744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5677,"num_bytes_recovered":372047872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":396,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168424+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:30:43.168424+0000","last_peered":"2023-09-23T22:30:43.168424+0000","last_clean":"2023-09-23T22:30:43.168424+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:30:43.168424+0000","last_undegraded":"2023-09-23T22:30:43.168424+0000","last_fullsized":"2023-09-23T22:30:43.168424+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":251,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:44.356813+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:30:44.356813+0000","last_peered":"2023-09-23T22:30:44.356813+0000","last_clean":"2023-09-23T22:30:44.356813+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:30:44.356813+0000","last_undegraded":"2023-09-23T22:30:44.356813+0000","last_fullsized":"2023-09-23T22:30:44.356813+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"366'11166","reported_seq":15442,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:54.310911+0000","last_change":"2023-09-23T22:29:53.222638+0000","last_active":"2023-09-23T22:30:54.310911+0000","last_peered":"2023-09-23T22:30:54.310911+0000","last_clean":"2023-09-23T22:27:26.103428+0000","last_became_active":"2023-09-23T22:27:28.277815+0000","last_became_peered":"2023-09-23T22:27:28.277815+0000","last_unstale":"2023-09-23T22:30:54.310911+0000","last_undegraded":"2023-09-23T22:30:54.310911+0000","last_fullsized":"2023-09-23T22:30:54.310911+0000","mapping_epoch":298,"log_start":"269'4865","ondisk_log_start":"269'4865","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6301,"log_dups_size":0,"ondisk_log_size":6301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:57.836786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11166,"num_write_kb":357312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2069,"num_bytes_recovered":135593984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,3],"avail_no_missing":["0","3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":691,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.167718+0000","last_change":"2023-09-23T22:29:04.057613+0000","last_active":"2023-09-23T22:30:43.167718+0000","last_peered":"2023-09-23T22:30:43.167718+0000","last_clean":"2023-09-23T22:30:43.167718+0000","last_became_active":"2023-09-23T22:29:04.057082+0000","last_became_peered":"2023-09-23T22:29:04.057082+0000","last_unstale":"2023-09-23T22:30:43.167718+0000","last_undegraded":"2023-09-23T22:30:43.167718+0000","last_fullsized":"2023-09-23T22:30:43.167718+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:24.948720+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":197,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164838+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:30:43.164838+0000","last_peered":"2023-09-23T22:30:43.164838+0000","last_clean":"2023-09-23T22:30:43.164838+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:30:43.164838+0000","last_undegraded":"2023-09-23T22:30:43.164838+0000","last_fullsized":"2023-09-23T22:30:43.164838+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":374,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163572+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:30:43.163572+0000","last_peered":"2023-09-23T22:30:43.163572+0000","last_clean":"2023-09-23T22:30:43.163572+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:30:43.163572+0000","last_undegraded":"2023-09-23T22:30:43.163572+0000","last_fullsized":"2023-09-23T22:30:43.163572+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"366'11282","reported_seq":14863,"reported_epoch":366,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:30:54.403475+0000","last_change":"2023-09-23T22:29:11.163110+0000","last_active":"2023-09-23T22:30:54.403475+0000","last_peered":"2023-09-23T22:30:54.403475+0000","last_clean":"2023-09-23T22:26:06.765343+0000","last_became_active":"2023-09-23T22:29:11.143204+0000","last_became_peered":"2023-09-23T22:29:11.143204+0000","last_unstale":"2023-09-23T22:30:54.403475+0000","last_undegraded":"2023-09-23T22:29:11.140016+0000","last_fullsized":"2023-09-23T22:30:54.403475+0000","mapping_epoch":335,"log_start":"276'5484","ondisk_log_start":"276'5484","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5798,"log_dups_size":0,"ondisk_log_size":5798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:14.098578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11282,"num_write_kb":361024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1719,"num_bytes_recovered":112656384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,3],"avail_no_missing":["5","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":747,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170332+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:30:43.170332+0000","last_peered":"2023-09-23T22:30:43.170332+0000","last_clean":"2023-09-23T22:30:43.170332+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:30:43.170332+0000","last_undegraded":"2023-09-23T22:30:43.170332+0000","last_fullsized":"2023-09-23T22:30:43.170332+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.175623+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:30:43.175623+0000","last_peered":"2023-09-23T22:30:43.175623+0000","last_clean":"2023-09-23T22:30:43.175623+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:30:43.175623+0000","last_undegraded":"2023-09-23T22:30:43.175623+0000","last_fullsized":"2023-09-23T22:30:43.175623+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":401,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165833+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:30:43.165833+0000","last_peered":"2023-09-23T22:30:43.165833+0000","last_clean":"2023-09-23T22:30:43.165833+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:30:43.165833+0000","last_undegraded":"2023-09-23T22:30:43.165833+0000","last_fullsized":"2023-09-23T22:30:43.165833+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"366'11310","reported_seq":19506,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:54.380175+0000","last_change":"2023-09-23T22:29:02.735621+0000","last_active":"2023-09-23T22:30:54.380175+0000","last_peered":"2023-09-23T22:30:54.380175+0000","last_clean":"2023-09-23T22:29:00.425895+0000","last_became_active":"2023-09-23T22:29:02.721422+0000","last_became_peered":"2023-09-23T22:29:02.721422+0000","last_unstale":"2023-09-23T22:30:54.380175+0000","last_undegraded":"2023-09-23T22:30:54.380175+0000","last_fullsized":"2023-09-23T22:30:54.380175+0000","mapping_epoch":329,"log_start":"269'5041","ondisk_log_start":"269'5041","created":260,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6269,"log_dups_size":0,"ondisk_log_size":6269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:41.454007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11310,"num_write_kb":361857,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4012,"num_bytes_recovered":262864920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[0,1,7],"avail_no_missing":["0","1","3","5","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":347,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163328+0000","last_change":"2023-09-23T22:28:05.062902+0000","last_active":"2023-09-23T22:30:43.163328+0000","last_peered":"2023-09-23T22:30:43.163328+0000","last_clean":"2023-09-23T22:30:43.163328+0000","last_became_active":"2023-09-23T22:28:05.062607+0000","last_became_peered":"2023-09-23T22:28:05.062607+0000","last_unstale":"2023-09-23T22:30:43.163328+0000","last_undegraded":"2023-09-23T22:30:43.163328+0000","last_fullsized":"2023-09-23T22:30:43.163328+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:21.888780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":382,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163331+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:30:43.163331+0000","last_peered":"2023-09-23T22:30:43.163331+0000","last_clean":"2023-09-23T22:30:43.163331+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:30:43.163331+0000","last_undegraded":"2023-09-23T22:30:43.163331+0000","last_fullsized":"2023-09-23T22:30:43.163331+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"366'11076","reported_seq":14587,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:30:54.407807+0000","last_change":"2023-09-23T22:29:53.222339+0000","last_active":"2023-09-23T22:30:54.407807+0000","last_peered":"2023-09-23T22:30:54.407807+0000","last_clean":"2023-09-23T22:26:06.845528+0000","last_became_active":"2023-09-23T22:29:52.034520+0000","last_became_peered":"2023-09-23T22:29:52.034520+0000","last_unstale":"2023-09-23T22:30:54.407807+0000","last_undegraded":"2023-09-23T22:29:52.019265+0000","last_fullsized":"2023-09-23T22:29:51.188771+0000","mapping_epoch":351,"log_start":"269'4797","ondisk_log_start":"269'4797","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6279,"log_dups_size":0,"ondisk_log_size":6279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:22.488491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":1546,"num_objects_missing":1546,"num_objects_degraded":284,"num_objects_misplaced":1262,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11076,"num_write_kb":354432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1676,"num_bytes_recovered":109772800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[0,3],"avail_no_missing":["1","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":720,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168369+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:30:43.168369+0000","last_peered":"2023-09-23T22:30:43.168369+0000","last_clean":"2023-09-23T22:30:43.168369+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:30:43.168369+0000","last_undegraded":"2023-09-23T22:30:43.168369+0000","last_fullsized":"2023-09-23T22:30:43.168369+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170171+0000","last_change":"2023-09-23T22:27:52.106215+0000","last_active":"2023-09-23T22:30:43.170171+0000","last_peered":"2023-09-23T22:30:43.170171+0000","last_clean":"2023-09-23T22:30:43.170171+0000","last_became_active":"2023-09-23T22:27:52.104315+0000","last_became_peered":"2023-09-23T22:27:52.104315+0000","last_unstale":"2023-09-23T22:30:43.170171+0000","last_undegraded":"2023-09-23T22:30:43.170171+0000","last_fullsized":"2023-09-23T22:30:43.170171+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:44.360992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":73,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.169928+0000","last_change":"2023-09-23T22:27:52.096512+0000","last_active":"2023-09-23T22:30:43.169928+0000","last_peered":"2023-09-23T22:30:43.169928+0000","last_clean":"2023-09-23T22:30:43.169928+0000","last_became_active":"2023-09-23T22:27:52.096242+0000","last_became_peered":"2023-09-23T22:27:52.096242+0000","last_unstale":"2023-09-23T22:30:43.169928+0000","last_undegraded":"2023-09-23T22:30:43.169928+0000","last_fullsized":"2023-09-23T22:30:43.169928+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:12.364350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"366'11144","reported_seq":27069,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:54.349481+0000","last_change":"2023-09-23T22:29:49.257546+0000","last_active":"2023-09-23T22:30:54.349481+0000","last_peered":"2023-09-23T22:30:54.349481+0000","last_clean":"2023-09-23T22:29:00.251578+0000","last_became_active":"2023-09-23T22:29:02.721609+0000","last_became_peered":"2023-09-23T22:29:02.721609+0000","last_unstale":"2023-09-23T22:30:54.349481+0000","last_undegraded":"2023-09-23T22:30:54.349481+0000","last_fullsized":"2023-09-23T22:30:54.349481+0000","mapping_epoch":329,"log_start":"278'5520","ondisk_log_start":"278'5520","created":260,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5624,"log_dups_size":0,"ondisk_log_size":5624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:37.536476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11144,"num_write_kb":356608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6911,"num_bytes_recovered":452919296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,1,0],"avail_no_missing":["4","0","1","3","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":611,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.176036+0000","last_change":"2023-09-23T22:29:04.057506+0000","last_active":"2023-09-23T22:30:43.176036+0000","last_peered":"2023-09-23T22:30:43.176036+0000","last_clean":"2023-09-23T22:30:43.176036+0000","last_became_active":"2023-09-23T22:29:04.057207+0000","last_became_peered":"2023-09-23T22:29:04.057207+0000","last_unstale":"2023-09-23T22:30:43.176036+0000","last_undegraded":"2023-09-23T22:30:43.176036+0000","last_fullsized":"2023-09-23T22:30:43.176036+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:47.781486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:47.781486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":692,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.172682+0000","last_change":"2023-09-23T22:28:41.357575+0000","last_active":"2023-09-23T22:30:43.172682+0000","last_peered":"2023-09-23T22:30:43.172682+0000","last_clean":"2023-09-23T22:30:43.172682+0000","last_became_active":"2023-09-23T22:28:41.356971+0000","last_became_peered":"2023-09-23T22:28:41.356971+0000","last_unstale":"2023-09-23T22:30:43.172682+0000","last_undegraded":"2023-09-23T22:30:43.172682+0000","last_fullsized":"2023-09-23T22:30:43.172682+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.810917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":423,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168987+0000","last_change":"2023-09-23T22:29:01.608081+0000","last_active":"2023-09-23T22:30:43.168987+0000","last_peered":"2023-09-23T22:30:43.168987+0000","last_clean":"2023-09-23T22:30:43.168987+0000","last_became_active":"2023-09-23T22:29:01.607726+0000","last_became_peered":"2023-09-23T22:29:01.607726+0000","last_unstale":"2023-09-23T22:30:43.168987+0000","last_undegraded":"2023-09-23T22:30:43.168987+0000","last_fullsized":"2023-09-23T22:30:43.168987+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:30.179258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170374+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:30:43.170374+0000","last_peered":"2023-09-23T22:30:43.170374+0000","last_clean":"2023-09-23T22:30:43.170374+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:30:43.170374+0000","last_undegraded":"2023-09-23T22:30:43.170374+0000","last_fullsized":"2023-09-23T22:30:43.170374+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"366'11196","reported_seq":11353,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:54.350426+0000","last_change":"2023-09-23T22:29:01.683197+0000","last_active":"2023-09-23T22:30:54.350426+0000","last_peered":"2023-09-23T22:30:54.350426+0000","last_clean":"2023-09-23T22:27:11.662680+0000","last_became_active":"2023-09-23T22:29:01.670427+0000","last_became_peered":"2023-09-23T22:29:01.670427+0000","last_unstale":"2023-09-23T22:30:54.350426+0000","last_undegraded":"2023-09-23T22:30:54.350426+0000","last_fullsized":"2023-09-23T22:30:54.350426+0000","mapping_epoch":328,"log_start":"269'4847","ondisk_log_start":"269'4847","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6349,"log_dups_size":0,"ondisk_log_size":6349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:30.348815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11196,"num_write_kb":358272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[0,1,3],"avail_no_missing":["0","1","3","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":710,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168054+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:30:43.168054+0000","last_peered":"2023-09-23T22:30:43.168054+0000","last_clean":"2023-09-23T22:30:43.168054+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:30:43.168054+0000","last_undegraded":"2023-09-23T22:30:43.168054+0000","last_fullsized":"2023-09-23T22:30:43.168054+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1244,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168537+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:30:43.168537+0000","last_peered":"2023-09-23T22:30:43.168537+0000","last_clean":"2023-09-23T22:30:43.168537+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:30:43.168537+0000","last_undegraded":"2023-09-23T22:30:43.168537+0000","last_fullsized":"2023-09-23T22:30:43.168537+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":405,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164396+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:30:43.164396+0000","last_peered":"2023-09-23T22:30:43.164396+0000","last_clean":"2023-09-23T22:30:43.164396+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:30:43.164396+0000","last_undegraded":"2023-09-23T22:30:43.164396+0000","last_fullsized":"2023-09-23T22:30:43.164396+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.174288+0000","last_change":"2023-09-23T22:29:01.593644+0000","last_active":"2023-09-23T22:30:43.174288+0000","last_peered":"2023-09-23T22:30:43.174288+0000","last_clean":"2023-09-23T22:30:43.174288+0000","last_became_active":"2023-09-23T22:29:01.593326+0000","last_became_peered":"2023-09-23T22:29:01.593326+0000","last_unstale":"2023-09-23T22:30:43.174288+0000","last_undegraded":"2023-09-23T22:30:43.174288+0000","last_fullsized":"2023-09-23T22:30:43.174288+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:52.921796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.4","version":"366'11416","reported_seq":11538,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:54.384492+0000","last_change":"2023-09-23T22:27:52.125206+0000","last_active":"2023-09-23T22:30:54.384492+0000","last_peered":"2023-09-23T22:30:54.384492+0000","last_clean":"2023-09-23T22:30:54.384492+0000","last_became_active":"2023-09-23T22:27:52.107584+0000","last_became_peered":"2023-09-23T22:27:52.107584+0000","last_unstale":"2023-09-23T22:30:54.384492+0000","last_undegraded":"2023-09-23T22:30:54.384492+0000","last_fullsized":"2023-09-23T22:30:54.384492+0000","mapping_epoch":304,"log_start":"269'5066","ondisk_log_start":"269'5066","created":260,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6350,"log_dups_size":0,"ondisk_log_size":6350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:23.737567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11416,"num_write_kb":365312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":698,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.166174+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:30:43.166174+0000","last_peered":"2023-09-23T22:30:43.166174+0000","last_clean":"2023-09-23T22:30:43.166174+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:30:43.166174+0000","last_undegraded":"2023-09-23T22:30:43.166174+0000","last_fullsized":"2023-09-23T22:30:43.166174+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":582,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165622+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:30:43.165622+0000","last_peered":"2023-09-23T22:30:43.165622+0000","last_clean":"2023-09-23T22:30:43.165622+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:30:43.165622+0000","last_undegraded":"2023-09-23T22:30:43.165622+0000","last_fullsized":"2023-09-23T22:30:43.165622+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.173797+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:30:43.173797+0000","last_peered":"2023-09-23T22:30:43.173797+0000","last_clean":"2023-09-23T22:30:43.173797+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:30:43.173797+0000","last_undegraded":"2023-09-23T22:30:43.173797+0000","last_fullsized":"2023-09-23T22:30:43.173797+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.169667+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:30:43.169667+0000","last_peered":"2023-09-23T22:30:43.169667+0000","last_clean":"2023-09-23T22:30:43.169667+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:30:43.169667+0000","last_undegraded":"2023-09-23T22:30:43.169667+0000","last_fullsized":"2023-09-23T22:30:43.169667+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"366'11304","reported_seq":13355,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:54.395771+0000","last_change":"2023-09-23T22:29:51.202545+0000","last_active":"2023-09-23T22:30:54.395771+0000","last_peered":"2023-09-23T22:30:54.395771+0000","last_clean":"2023-09-23T22:27:26.256729+0000","last_became_active":"2023-09-23T22:27:28.279658+0000","last_became_peered":"2023-09-23T22:27:28.279658+0000","last_unstale":"2023-09-23T22:30:54.395771+0000","last_undegraded":"2023-09-23T22:30:54.395771+0000","last_fullsized":"2023-09-23T22:30:54.395771+0000","mapping_epoch":298,"log_start":"269'5010","ondisk_log_start":"269'5010","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6294,"log_dups_size":0,"ondisk_log_size":6294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:04.471284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11304,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":957,"num_bytes_recovered":62652416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,3],"avail_no_missing":["0","3","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1062,"reported_epoch":365,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T22:30:43.168291+0000","last_change":"2023-09-23T22:29:50.198353+0000","last_active":"2023-09-23T22:30:43.168291+0000","last_peered":"2023-09-23T22:30:43.168291+0000","last_clean":"2023-09-23T22:27:56.036441+0000","last_became_active":"2023-09-23T22:29:50.198096+0000","last_became_peered":"2023-09-23T22:29:50.198096+0000","last_unstale":"2023-09-23T22:30:43.168291+0000","last_undegraded":"2023-09-23T22:29:50.186847+0000","last_fullsized":"2023-09-23T22:30:43.168291+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":275,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:48.105235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":732,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164276+0000","last_change":"2023-09-23T22:29:04.055480+0000","last_active":"2023-09-23T22:30:43.164276+0000","last_peered":"2023-09-23T22:30:43.164276+0000","last_clean":"2023-09-23T22:30:43.164276+0000","last_became_active":"2023-09-23T22:29:04.055134+0000","last_became_peered":"2023-09-23T22:29:04.055134+0000","last_unstale":"2023-09-23T22:30:43.164276+0000","last_undegraded":"2023-09-23T22:30:43.164276+0000","last_fullsized":"2023-09-23T22:30:43.164276+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:59.018639+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":247,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168451+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:30:43.168451+0000","last_peered":"2023-09-23T22:30:43.168451+0000","last_clean":"2023-09-23T22:30:43.168451+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:30:43.168451+0000","last_undegraded":"2023-09-23T22:30:43.168451+0000","last_fullsized":"2023-09-23T22:30:43.168451+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.175546+0000","last_change":"2023-09-23T22:29:01.594509+0000","last_active":"2023-09-23T22:30:43.175546+0000","last_peered":"2023-09-23T22:30:43.175546+0000","last_clean":"2023-09-23T22:30:43.175546+0000","last_became_active":"2023-09-23T22:29:01.594249+0000","last_became_peered":"2023-09-23T22:29:01.594249+0000","last_unstale":"2023-09-23T22:30:43.175546+0000","last_undegraded":"2023-09-23T22:30:43.175546+0000","last_fullsized":"2023-09-23T22:30:43.175546+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:14.972304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"366'11526","reported_seq":18502,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:54.391746+0000","last_change":"2023-09-23T22:29:49.247679+0000","last_active":"2023-09-23T22:30:54.391746+0000","last_peered":"2023-09-23T22:30:54.391746+0000","last_clean":"2023-09-23T22:30:54.391746+0000","last_became_active":"2023-09-23T22:29:46.177429+0000","last_became_peered":"2023-09-23T22:29:46.177429+0000","last_unstale":"2023-09-23T22:30:54.391746+0000","last_undegraded":"2023-09-23T22:30:54.391746+0000","last_fullsized":"2023-09-23T22:30:54.391746+0000","mapping_epoch":346,"log_start":"270'5243","ondisk_log_start":"270'5243","created":260,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":6283,"log_dups_size":0,"ondisk_log_size":6283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:01.079734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11526,"num_write_kb":368832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3412,"num_bytes_recovered":116916224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":588,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.177984+0000","last_change":"2023-09-23T22:29:04.055972+0000","last_active":"2023-09-23T22:30:43.177984+0000","last_peered":"2023-09-23T22:30:43.177984+0000","last_clean":"2023-09-23T22:30:43.177984+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:30:43.177984+0000","last_undegraded":"2023-09-23T22:30:43.177984+0000","last_fullsized":"2023-09-23T22:30:43.177984+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":716,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170417+0000","last_change":"2023-09-23T22:27:52.122905+0000","last_active":"2023-09-23T22:30:43.170417+0000","last_peered":"2023-09-23T22:30:43.170417+0000","last_clean":"2023-09-23T22:30:43.170417+0000","last_became_active":"2023-09-23T22:27:52.116583+0000","last_became_peered":"2023-09-23T22:27:52.116583+0000","last_unstale":"2023-09-23T22:30:43.170417+0000","last_undegraded":"2023-09-23T22:30:43.170417+0000","last_fullsized":"2023-09-23T22:30:43.170417+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:21.421519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":260,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165686+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:30:43.165686+0000","last_peered":"2023-09-23T22:30:43.165686+0000","last_clean":"2023-09-23T22:30:43.165686+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:30:43.165686+0000","last_undegraded":"2023-09-23T22:30:43.165686+0000","last_fullsized":"2023-09-23T22:30:43.165686+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":244,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164616+0000","last_change":"2023-09-23T22:28:05.169578+0000","last_active":"2023-09-23T22:30:43.164616+0000","last_peered":"2023-09-23T22:30:43.164616+0000","last_clean":"2023-09-23T22:30:43.164616+0000","last_became_active":"2023-09-23T22:28:05.169263+0000","last_became_peered":"2023-09-23T22:28:05.169263+0000","last_unstale":"2023-09-23T22:30:43.164616+0000","last_undegraded":"2023-09-23T22:30:43.164616+0000","last_fullsized":"2023-09-23T22:30:43.164616+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:39.446410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"366'11424","reported_seq":25949,"reported_epoch":366,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:30:55.050679+0000","last_change":"2023-09-23T22:29:49.258974+0000","last_active":"2023-09-23T22:30:55.050679+0000","last_peered":"2023-09-23T22:30:55.050679+0000","last_clean":"2023-09-23T22:27:11.623396+0000","last_became_active":"2023-09-23T22:29:02.885320+0000","last_became_peered":"2023-09-23T22:29:02.885320+0000","last_unstale":"2023-09-23T22:30:55.050679+0000","last_undegraded":"2023-09-23T22:29:02.608430+0000","last_fullsized":"2023-09-23T22:29:02.560725+0000","mapping_epoch":329,"log_start":"276'5565","ondisk_log_start":"276'5565","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5859,"log_dups_size":0,"ondisk_log_size":5859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:13.200109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":245,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11424,"num_write_kb":365568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6997,"num_bytes_recovered":240779264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":["5","1","3"],"object_location_counts":[{"shards":"1,5","objects":-245}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":716,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.175650+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:30:43.175650+0000","last_peered":"2023-09-23T22:30:43.175650+0000","last_clean":"2023-09-23T22:30:43.175650+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:30:43.175650+0000","last_undegraded":"2023-09-23T22:30:43.175650+0000","last_fullsized":"2023-09-23T22:30:43.175650+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":260,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165227+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:30:43.165227+0000","last_peered":"2023-09-23T22:30:43.165227+0000","last_clean":"2023-09-23T22:30:43.165227+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:30:43.165227+0000","last_undegraded":"2023-09-23T22:30:43.165227+0000","last_fullsized":"2023-09-23T22:30:43.165227+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":292,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168517+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:30:43.168517+0000","last_peered":"2023-09-23T22:30:43.168517+0000","last_clean":"2023-09-23T22:30:43.168517+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:30:43.168517+0000","last_undegraded":"2023-09-23T22:30:43.168517+0000","last_fullsized":"2023-09-23T22:30:43.168517+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":224,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164662+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:30:43.164662+0000","last_peered":"2023-09-23T22:30:43.164662+0000","last_clean":"2023-09-23T22:30:43.164662+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:30:43.164662+0000","last_undegraded":"2023-09-23T22:30:43.164662+0000","last_fullsized":"2023-09-23T22:30:43.164662+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"366'11324","reported_seq":13087,"reported_epoch":366,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:30:54.361397+0000","last_change":"2023-09-23T22:29:50.361405+0000","last_active":"2023-09-23T22:30:54.361397+0000","last_peered":"2023-09-23T22:30:54.361397+0000","last_clean":"2023-09-23T22:26:06.872687+0000","last_became_active":"2023-09-23T22:29:50.249230+0000","last_became_peered":"2023-09-23T22:29:50.249230+0000","last_unstale":"2023-09-23T22:30:54.361397+0000","last_undegraded":"2023-09-23T22:29:50.243725+0000","last_fullsized":"2023-09-23T22:29:50.191587+0000","mapping_epoch":350,"log_start":"279'5722","ondisk_log_start":"279'5722","created":260,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5602,"log_dups_size":0,"ondisk_log_size":5602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:47.073843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1708,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11324,"num_write_kb":362368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":793,"num_bytes_recovered":32702464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,3],"avail_no_missing":["4","1","3","6"],"object_location_counts":[{"shards":"3,4","objects":-200}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":720,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.165684+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:30:43.165684+0000","last_peered":"2023-09-23T22:30:43.165684+0000","last_clean":"2023-09-23T22:30:43.165684+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:30:43.165684+0000","last_undegraded":"2023-09-23T22:30:43.165684+0000","last_fullsized":"2023-09-23T22:30:43.165684+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":184,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.166837+0000","last_change":"2023-09-23T22:27:27.333140+0000","last_active":"2023-09-23T22:30:43.166837+0000","last_peered":"2023-09-23T22:30:43.166837+0000","last_clean":"2023-09-23T22:30:43.166837+0000","last_became_active":"2023-09-23T22:27:27.287527+0000","last_became_peered":"2023-09-23T22:27:27.287527+0000","last_unstale":"2023-09-23T22:30:43.166837+0000","last_undegraded":"2023-09-23T22:30:43.166837+0000","last_fullsized":"2023-09-23T22:30:43.166837+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:51.648586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":861,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.169789+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:30:43.169789+0000","last_peered":"2023-09-23T22:30:43.169789+0000","last_clean":"2023-09-23T22:30:43.169789+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:30:43.169789+0000","last_undegraded":"2023-09-23T22:30:43.169789+0000","last_fullsized":"2023-09-23T22:30:43.169789+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":296,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164160+0000","last_change":"2023-09-23T22:27:52.136692+0000","last_active":"2023-09-23T22:30:43.164160+0000","last_peered":"2023-09-23T22:30:43.164160+0000","last_clean":"2023-09-23T22:30:43.164160+0000","last_became_active":"2023-09-23T22:27:52.136397+0000","last_became_peered":"2023-09-23T22:27:52.136397+0000","last_unstale":"2023-09-23T22:30:43.164160+0000","last_undegraded":"2023-09-23T22:30:43.164160+0000","last_fullsized":"2023-09-23T22:30:43.164160+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.036231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.2","version":"366'11280","reported_seq":24238,"reported_epoch":366,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:30:54.056164+0000","last_change":"2023-09-23T22:29:50.220538+0000","last_active":"2023-09-23T22:30:54.056164+0000","last_peered":"2023-09-23T22:30:54.056164+0000","last_clean":"2023-09-23T22:29:00.398116+0000","last_became_active":"2023-09-23T22:29:50.197351+0000","last_became_peered":"2023-09-23T22:29:50.197351+0000","last_unstale":"2023-09-23T22:30:54.056164+0000","last_undegraded":"2023-09-23T22:30:54.056164+0000","last_fullsized":"2023-09-23T22:30:54.056164+0000","mapping_epoch":350,"log_start":"285'6224","ondisk_log_start":"285'6224","created":260,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:24:45.961568+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.961568+0000","objects_scrubbed":0,"log_size":5056,"log_dups_size":0,"ondisk_log_size":5056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:01:17.462990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":524288,"num_objects":8,"num_object_clones":0,"num_object_copies":24,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":8,"num_objects_unfound":0,"num_objects_dirty":8,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11280,"num_write_kb":361216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6378,"num_bytes_recovered":318308352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[1,7,0],"avail_no_missing":["1","0","2","3","7"],"object_location_counts":[{"shards":"0,1,7","objects":8}],"blocked_by":[],"up_primary":6,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":612,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168648+0000","last_change":"2023-09-23T22:29:51.194726+0000","last_active":"2023-09-23T22:30:43.168648+0000","last_peered":"2023-09-23T22:30:43.168648+0000","last_clean":"2023-09-23T22:30:43.168648+0000","last_became_active":"2023-09-23T22:29:51.194253+0000","last_became_peered":"2023-09-23T22:29:51.194253+0000","last_unstale":"2023-09-23T22:30:43.168648+0000","last_undegraded":"2023-09-23T22:30:43.168648+0000","last_fullsized":"2023-09-23T22:30:43.168648+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:45.734686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.734686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1144050,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":57,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.175915+0000","last_change":"2023-09-23T22:29:09.070063+0000","last_active":"2023-09-23T22:30:43.175915+0000","last_peered":"2023-09-23T22:30:43.175915+0000","last_clean":"2023-09-23T22:30:43.175915+0000","last_became_active":"2023-09-23T22:29:01.584585+0000","last_became_peered":"2023-09-23T22:29:01.584585+0000","last_unstale":"2023-09-23T22:30:43.175915+0000","last_undegraded":"2023-09-23T22:30:43.175915+0000","last_fullsized":"2023-09-23T22:30:43.175915+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:44.762021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":229,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164466+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:30:43.164466+0000","last_peered":"2023-09-23T22:30:43.164466+0000","last_clean":"2023-09-23T22:30:43.164466+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:30:43.164466+0000","last_undegraded":"2023-09-23T22:30:43.164466+0000","last_fullsized":"2023-09-23T22:30:43.164466+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":339,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163393+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:30:43.163393+0000","last_peered":"2023-09-23T22:30:43.163393+0000","last_clean":"2023-09-23T22:30:43.163393+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:30:43.163393+0000","last_undegraded":"2023-09-23T22:30:43.163393+0000","last_fullsized":"2023-09-23T22:30:43.163393+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":16,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164579+0000","last_change":"2023-09-23T22:30:26.181266+0000","last_active":"2023-09-23T22:30:43.164579+0000","last_peered":"2023-09-23T22:30:43.164579+0000","last_clean":"2023-09-23T22:30:43.164579+0000","last_became_active":"2023-09-23T22:30:26.180875+0000","last_became_peered":"2023-09-23T22:30:26.180875+0000","last_unstale":"2023-09-23T22:30:43.164579+0000","last_undegraded":"2023-09-23T22:30:43.164579+0000","last_fullsized":"2023-09-23T22:30:43.164579+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:11.776290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":116,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.170624+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:30:43.170624+0000","last_peered":"2023-09-23T22:30:43.170624+0000","last_clean":"2023-09-23T22:30:43.170624+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:30:43.170624+0000","last_undegraded":"2023-09-23T22:30:43.170624+0000","last_fullsized":"2023-09-23T22:30:43.170624+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":356,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.168338+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:30:43.168338+0000","last_peered":"2023-09-23T22:30:43.168338+0000","last_clean":"2023-09-23T22:30:43.168338+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:30:43.168338+0000","last_undegraded":"2023-09-23T22:30:43.168338+0000","last_fullsized":"2023-09-23T22:30:43.168338+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":381,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.164522+0000","last_change":"2023-09-23T22:28:05.114400+0000","last_active":"2023-09-23T22:30:43.164522+0000","last_peered":"2023-09-23T22:30:43.164522+0000","last_clean":"2023-09-23T22:30:43.164522+0000","last_became_active":"2023-09-23T22:28:05.114095+0000","last_became_peered":"2023-09-23T22:28:05.114095+0000","last_unstale":"2023-09-23T22:30:43.164522+0000","last_undegraded":"2023-09-23T22:30:43.164522+0000","last_fullsized":"2023-09-23T22:30:43.164522+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:23.836996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":381,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.169073+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:30:43.169073+0000","last_peered":"2023-09-23T22:30:43.169073+0000","last_clean":"2023-09-23T22:30:43.169073+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:30:43.169073+0000","last_undegraded":"2023-09-23T22:30:43.169073+0000","last_fullsized":"2023-09-23T22:30:43.169073+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":230,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163941+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:30:43.163941+0000","last_peered":"2023-09-23T22:30:43.163941+0000","last_clean":"2023-09-23T22:30:43.163941+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:30:43.163941+0000","last_undegraded":"2023-09-23T22:30:43.163941+0000","last_fullsized":"2023-09-23T22:30:43.163941+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":47,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.175560+0000","last_change":"2023-09-23T22:29:01.593820+0000","last_active":"2023-09-23T22:30:43.175560+0000","last_peered":"2023-09-23T22:30:43.175560+0000","last_clean":"2023-09-23T22:30:43.175560+0000","last_became_active":"2023-09-23T22:29:01.593176+0000","last_became_peered":"2023-09-23T22:29:01.593176+0000","last_unstale":"2023-09-23T22:30:43.175560+0000","last_undegraded":"2023-09-23T22:30:43.175560+0000","last_fullsized":"2023-09-23T22:30:43.175560+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:33.824095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":366,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T22:30:45.208375+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:30:45.208375+0000","last_peered":"2023-09-23T22:30:45.208375+0000","last_clean":"2023-09-23T22:30:45.208375+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:30:45.208375+0000","last_undegraded":"2023-09-23T22:30:45.208375+0000","last_fullsized":"2023-09-23T22:30:45.208375+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":303,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.166941+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:30:43.166941+0000","last_peered":"2023-09-23T22:30:43.166941+0000","last_clean":"2023-09-23T22:30:43.166941+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:30:43.166941+0000","last_undegraded":"2023-09-23T22:30:43.166941+0000","last_fullsized":"2023-09-23T22:30:43.166941+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":424,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T22:30:43.163275+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:30:43.163275+0000","last_peered":"2023-09-23T22:30:43.163275+0000","last_clean":"2023-09-23T22:30:43.163275+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:30:43.163275+0000","last_undegraded":"2023-09-23T22:30:43.163275+0000","last_fullsized":"2023-09-23T22:30:43.163275+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1179648,"num_objects":18,"num_object_clones":0,"num_object_copies":54,"num_objects_missing_on_primary":1546,"num_objects_missing":1546,"num_objects_degraded":529,"num_objects_misplaced":2978,"num_objects_unfound":0,"num_objects_dirty":18,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":180652,"num_write_kb":5781377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":46409,"num_bytes_recovered":2495283224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1486823424,"data_stored":1486815280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94764,"ondisk_log_size":94764,"up":48,"acting":45,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":26454,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":423698,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":132,"num_read_kb":249,"num_write":66,"num_write_kb":660,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2254133,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":177,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234318,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":482272,"kb_used_data":161368,"kb_used_omap":0,"kb_used_meta":320896,"kb_avail":93262880,"statfs":{"total":95995035648,"available":95501189120,"internally_reserved":0,"allocated":165240832,"data_stored":163844893,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":328597504},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":6.1479999999999997,"15min":4.1870000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.0519999999999996,"5min":220.66,"15min":220.66},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.5049999999999999,"5min":6.3280000000000003,"15min":4.5540000000000003},"min":{"1min":0.29099999999999998,"5min":0.246,"15min":0.217},"max":{"1min":5.6799999999999997,"5min":220.59899999999999,"15min":220.59899999999999},"last":8.0920000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:30:29 2023","interfaces":[{"interface":"back","average":{"1min":4.9500000000000002,"5min":4.9409999999999998,"15min":4.5590000000000002},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.23799999999999999},"max":{"1min":76.984999999999999,"5min":144.77500000000001,"15min":144.77500000000001},"last":6.1399999999999997},{"interface":"front","average":{"1min":5.2080000000000002,"5min":5.2359999999999998,"15min":4.8239999999999998},"min":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.24299999999999999},"max":{"1min":79.069999999999993,"5min":144.751,"15min":144.751},"last":7.8879999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7279999999999998,"5min":5.7279999999999998,"15min":5.7279999999999998},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.0529999999999999,"5min":3.0529999999999999,"15min":3.0529999999999999},"last":1.323}]},{"osd":3,"last update":"Sat Sep 23 22:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":4.5060000000000002,"15min":3.6850000000000001},"min":{"1min":0.29399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.4260000000000002,"5min":219.90700000000001,"15min":219.90700000000001},"last":1.2909999999999999},{"interface":"front","average":{"1min":2.1059999999999999,"5min":5.1550000000000002,"15min":3.944},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.234,"5min":219.976,"15min":219.976},"last":2.1760000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.52900000000000003,"15min":1.379},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23499999999999999},"max":{"1min":3.9020000000000001,"5min":3.9020000000000001,"15min":88.543000000000006},"last":8.0180000000000007},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.497,"15min":1.3089999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.9950000000000001,"5min":3.9950000000000001,"15min":88.495999999999995},"last":0.28599999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.48899999999999999,"15min":1.2809999999999999},"min":{"1min":0.28999999999999998,"5min":0.249,"15min":0.24299999999999999},"max":{"1min":1.8129999999999999,"5min":1.825,"15min":93.816000000000003},"last":1.514},{"interface":"front","average":{"1min":0.876,"5min":0.625,"15min":1.5580000000000001},"min":{"1min":0.35999999999999999,"5min":0.309,"15min":0.27300000000000002},"max":{"1min":4.2560000000000002,"5min":4.2560000000000002,"15min":95.579999999999998},"last":8.0459999999999994}]},{"osd":6,"last update":"Sat Sep 23 22:30:02 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.54000000000000004,"15min":0.749},"min":{"1min":0.30499999999999999,"5min":0.29499999999999998,"15min":0.28999999999999998},"max":{"1min":2.0129999999999999,"5min":2.0779999999999998,"15min":9.0139999999999993},"last":6.0880000000000001},{"interface":"front","average":{"1min":0.85299999999999998,"5min":0.61499999999999999,"15min":0.98899999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":4.2859999999999996,"5min":4.2859999999999996,"15min":13.362},"last":1.365}]}]},{"osd":6,"up_from":46,"seq":197568495954,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":124264,"kb_used_data":9632,"kb_used_omap":0,"kb_used_meta":114624,"kb_avail":93620888,"statfs":{"total":95995035648,"available":95867789312,"internally_reserved":0,"allocated":9863168,"data_stored":8460365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":117374976},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:17 2023","interfaces":[{"interface":"back","average":{"1min":5.298,"5min":2.2309999999999999,"15min":2.1160000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":55.206000000000003,"5min":55.206000000000003,"15min":108.773},"last":7.4800000000000004},{"interface":"front","average":{"1min":6.0060000000000002,"5min":2.6219999999999999,"15min":3.1429999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.21099999999999999},"max":{"1min":55.375999999999998,"5min":55.375999999999998,"15min":135.45599999999999},"last":2.9180000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":2.1030000000000002,"15min":3.6320000000000001},"min":{"1min":0.27800000000000002,"5min":0.26000000000000001,"15min":0.22800000000000001},"max":{"1min":6.8380000000000001,"5min":80.430000000000007,"15min":129.964},"last":6.5839999999999996},{"interface":"front","average":{"1min":2.129,"5min":2.6640000000000001,"15min":3.9449999999999998},"min":{"1min":0.35699999999999998,"5min":0.254,"15min":0.254},"max":{"1min":6.9249999999999998,"5min":84.192999999999998,"15min":131.18600000000001},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:30:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":1.9039999999999999,"15min":1.9039999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.6559999999999997,"5min":6.6559999999999997,"15min":6.6559999999999997},"last":2.7879999999999998},{"interface":"front","average":{"1min":2.1499999999999999,"5min":2.1499999999999999,"15min":2.1499999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":9.7319999999999993,"5min":9.7319999999999993,"15min":9.7319999999999993},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.77,"15min":3.4159999999999999},"min":{"1min":0.36799999999999999,"5min":0.316,"15min":0.311},"max":{"1min":5.7640000000000002,"5min":42.523000000000003,"15min":125.125},"last":3.6429999999999998},{"interface":"front","average":{"1min":1.631,"5min":1.9359999999999999,"15min":3.2410000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":7.4589999999999996,"5min":39.926000000000002,"15min":76.406999999999996},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:30:48 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":0.65900000000000003,"15min":2.5129999999999999},"min":{"1min":0.309,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.8650000000000002,"5min":6.8650000000000002,"15min":305.30200000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.62,"15min":2.4630000000000001},"min":{"1min":0.30399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":5.5979999999999999,"5min":5.5979999999999999,"15min":305.40300000000002},"last":3.5030000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:30:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":0.68799999999999994,"15min":1.056},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.6660000000000004,"5min":5.6660000000000004,"15min":67.045000000000002},"last":2.7389999999999999},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.63800000000000001,"15min":1.107},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.0170000000000003,"5min":5.5910000000000002,"15min":66.802000000000007},"last":0.32600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:30:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.60199999999999998,"15min":1.024},"min":{"1min":0.33800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":3.1850000000000001,"5min":3.7509999999999999,"15min":82.801000000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.64300000000000002,"15min":1.0069999999999999},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.6989999999999998,"5min":5.6989999999999998,"15min":83.052999999999997},"last":2.7669999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795226,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":565296,"kb_used_data":39400,"kb_used_omap":7,"kb_used_meta":525880,"kb_avail":93179856,"statfs":{"total":95995035648,"available":95416172544,"internally_reserved":0,"allocated":40345600,"data_stored":38345298,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":538501492},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5219999999999998,"5min":4.9199999999999999,"15min":3.9049999999999998},"min":{"1min":0.24399999999999999,"5min":0.216,"15min":0.189},"max":{"1min":18.045000000000002,"5min":90.923000000000002,"15min":196.095},"last":0.85999999999999999},{"interface":"front","average":{"1min":4.5499999999999998,"5min":5.4550000000000001,"15min":4.5899999999999999},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.20799999999999999},"max":{"1min":14.1,"5min":92.093000000000004,"15min":196.85300000000001},"last":12.375999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:30:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.8250000000000002,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.371,"5min":2.371,"15min":2.371},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.2349999999999994,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":1.4850000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":4.2590000000000003,"15min":4.452},"min":{"1min":0.33900000000000002,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":9.3089999999999993,"5min":88.340000000000003,"15min":160.499},"last":2.327},{"interface":"front","average":{"1min":2.5179999999999998,"5min":3.8109999999999999,"15min":4.2389999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.223},"max":{"1min":8.1590000000000007,"5min":91.781000000000006,"15min":160.465},"last":4.6379999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":2.2389999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.0470000000000002,"5min":9.6539999999999999,"15min":234.79900000000001},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.694,"5min":1.7370000000000001,"15min":2.8210000000000002},"min":{"1min":0.39900000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.093,"5min":9.093,"15min":234.81700000000001},"last":1.526}]},{"osd":5,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5110000000000001,"5min":2.0659999999999998,"15min":2.3940000000000001},"min":{"1min":0.40300000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.2759999999999998,"5min":10.973000000000001,"15min":196.79900000000001},"last":2.2970000000000002},{"interface":"front","average":{"1min":1.389,"5min":1.6499999999999999,"15min":2.2869999999999999},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1200000000000001,"5min":7.8120000000000003,"15min":196.68299999999999},"last":4.7729999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.8089999999999999,"15min":1.7949999999999999},"min":{"1min":0.41399999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.7779999999999996,"5min":8.5540000000000003,"15min":16.314},"last":5.0529999999999999},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.377,"15min":1.3080000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":3.4729999999999999,"5min":9.6910000000000007,"15min":11.385999999999999},"last":0.82899999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4910000000000001,"5min":1.9099999999999999,"15min":1.9350000000000001},"min":{"1min":0.44900000000000001,"5min":0.39500000000000002,"15min":0.32900000000000001},"max":{"1min":8.0660000000000007,"5min":8.6059999999999999,"15min":16.265000000000001},"last":4.8710000000000004},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.3700000000000001,"15min":1.3220000000000001},"min":{"1min":0.439,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.165,"5min":9.7129999999999992,"15min":12.045999999999999},"last":0.51400000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958755,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":734004,"kb_used_data":181100,"kb_used_omap":8,"kb_used_meta":552887,"kb_avail":93011148,"statfs":{"total":95995035648,"available":95243415552,"internally_reserved":0,"allocated":185446400,"data_stored":183383642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":566157162},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:30:32 2023","interfaces":[{"interface":"back","average":{"1min":6.282,"5min":4.0720000000000001,"15min":3.851},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":89.439999999999998,"5min":89.439999999999998,"15min":169.77199999999999},"last":0.26100000000000001},{"interface":"front","average":{"1min":8.2460000000000004,"5min":4.0750000000000002,"15min":4.0979999999999999},"min":{"1min":0.26400000000000001,"5min":0.218,"15min":0.20300000000000001},"max":{"1min":89.287999999999997,"5min":89.287999999999997,"15min":169.52099999999999},"last":0.375}]},{"osd":2,"last update":"Sat Sep 23 22:30:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":3.1960000000000002,"15min":3.1960000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":22.263000000000002,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.1920000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.504}]},{"osd":3,"last update":"Sat Sep 23 22:30:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.149,"15min":3.6070000000000002},"min":{"1min":0.23200000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.234,"5min":15.404,"15min":146.87299999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":3.9460000000000002,"5min":2.335,"15min":3.7709999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":22.303000000000001,"5min":22.303000000000001,"15min":145.48099999999999},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.7729999999999999,"15min":2.3780000000000001},"min":{"1min":0.35299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.1430000000000007,"5min":10.837999999999999,"15min":82.825000000000003},"last":3.2909999999999999},{"interface":"front","average":{"1min":1.411,"5min":1.23,"15min":1.6020000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.0220000000000002,"5min":10.08,"15min":75.930999999999997},"last":0.95499999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":2.0249999999999999,"15min":2.839},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":13.564,"5min":13.564,"15min":93.685000000000002},"last":0.79400000000000004},{"interface":"front","average":{"1min":1.768,"5min":1.3779999999999999,"15min":2.081},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.486000000000001,"5min":13.486000000000001,"15min":93.546000000000006},"last":2.617}]},{"osd":6,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.9059999999999999,"15min":2.1240000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.063000000000001,"5min":12.063000000000001,"15min":16.366},"last":2.29},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.3440000000000001,"15min":1.474},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":5.3209999999999997,"5min":10.134,"15min":22.263000000000002},"last":2.3889999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":1.5740000000000001,"15min":2.048},"min":{"1min":0.45500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.6430000000000007,"5min":9.6430000000000007,"15min":32.773000000000003},"last":1.137},{"interface":"front","average":{"1min":2.609,"5min":1.847,"15min":2.286},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":8.2240000000000002,"5min":10.901,"15min":23.265999999999998},"last":1.121}]}]},{"osd":2,"up_from":0,"seq":1498943586318,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468173,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019223,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":328548,"kb_used_data":53596,"kb_used_omap":0,"kb_used_meta":274880,"kb_avail":93416604,"statfs":{"total":95995035648,"available":95658602496,"internally_reserved":0,"allocated":54882304,"data_stored":53483066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":281477120},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":4.2300000000000004,"15min":3.5699999999999998},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.9559999999999995,"5min":104.252,"15min":150.63499999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":2.3399999999999999,"5min":4.5119999999999996,"15min":3.7080000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.9740000000000002,"5min":104.294,"15min":150.672},"last":2.161}]},{"osd":1,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":6.2060000000000004,"5min":3.9550000000000001,"15min":3.1869999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.21299999999999999},"max":{"1min":81.122,"5min":95.114000000000004,"15min":107.346},"last":0.32600000000000001},{"interface":"front","average":{"1min":6.1959999999999997,"5min":3.9399999999999999,"15min":3.528},"min":{"1min":0.26700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":81.090999999999994,"5min":95.222999999999999,"15min":107.33499999999999},"last":2.2690000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:30:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1699999999999999,"5min":2.1699999999999999,"15min":2.1699999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6790000000000003,"5min":7.6790000000000003,"15min":7.6790000000000003},"last":2.2050000000000001},{"interface":"front","average":{"1min":1.54,"5min":1.54,"15min":1.54},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.6379999999999999,"5min":4.6379999999999999,"15min":4.6379999999999999},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":4.0629999999999997,"15min":2.4630000000000001},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.0940000000000003,"5min":102.896,"15min":102.896},"last":5.7119999999999997},{"interface":"front","average":{"1min":2.2040000000000002,"5min":4.2220000000000004,"15min":2.5739999999999998},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":14.318,"5min":112.667,"15min":112.667},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 22:30:39 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":0.76800000000000002,"15min":1.0029999999999999},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.407,"5min":13.407,"15min":28.399000000000001},"last":0.253},{"interface":"front","average":{"1min":1.7709999999999999,"5min":0.79200000000000004,"15min":1.036},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":14.212,"5min":14.212,"15min":28.350999999999999},"last":0.443}]},{"osd":6,"last update":"Sat Sep 23 22:29:51 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.76200000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.251},"max":{"1min":1.179,"5min":1.179,"15min":7.8550000000000004},"last":0.38},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.50600000000000001,"15min":0.75600000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.5940000000000001,"5min":2.0289999999999999,"15min":8.5210000000000008},"last":2.1880000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.59499999999999997,"15min":0.88600000000000001},"min":{"1min":0.33100000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3179999999999996,"5min":4.3179999999999996,"15min":8.577},"last":2.2389999999999999},{"interface":"front","average":{"1min":1.4119999999999999,"5min":0.69099999999999995,"15min":0.88700000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9319999999999999,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":0.46200000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757591,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357208,"kb_used_data":8720,"kb_used_omap":0,"kb_used_meta":348480,"kb_avail":93387944,"statfs":{"total":95995035648,"available":95629254656,"internally_reserved":0,"allocated":8929280,"data_stored":7538093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":356843520},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:14 2023","interfaces":[{"interface":"back","average":{"1min":3.4399999999999999,"5min":7.1689999999999996,"15min":4.3899999999999997},"min":{"1min":0.23899999999999999,"5min":0.22,"15min":0.20499999999999999},"max":{"1min":16.109999999999999,"5min":163.54499999999999,"15min":163.54499999999999},"last":3.1659999999999999},{"interface":"front","average":{"1min":1.7709999999999999,"5min":6.5010000000000003,"15min":4.2089999999999996},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":5.7949999999999999,"5min":162.81800000000001,"15min":162.81800000000001},"last":15.609999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":3.9329999999999998,"15min":3.8740000000000001},"min":{"1min":0.38100000000000001,"5min":0.27900000000000003,"15min":0.23799999999999999},"max":{"1min":7.7779999999999996,"5min":113.514,"15min":113.514},"last":7.8109999999999999},{"interface":"front","average":{"1min":3.1240000000000001,"5min":3.9990000000000001,"15min":3.8860000000000001},"min":{"1min":0.33100000000000002,"5min":0.255,"15min":0.252},"max":{"1min":7.6319999999999997,"5min":113.42700000000001,"15min":113.42700000000001},"last":7.8540000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:30:50 2023","interfaces":[{"interface":"back","average":{"1min":2.411,"5min":2.411,"15min":2.411},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":10.451000000000001,"5min":10.451000000000001,"15min":10.451000000000001},"last":2.4750000000000001},{"interface":"front","average":{"1min":2.5289999999999999,"5min":2.5289999999999999,"15min":2.5289999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9569999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":2.4140000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":5.883,"15min":4.1040000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.246},"max":{"1min":9.3369999999999997,"5min":132.59,"15min":132.59},"last":4.8369999999999997},{"interface":"front","average":{"1min":3.024,"5min":6.0949999999999998,"15min":4.1040000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":7.6479999999999997,"5min":129.381,"15min":129.381},"last":2.3980000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.9610000000000001,"15min":1.528},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.24099999999999999},"max":{"1min":3.7240000000000002,"5min":126.679,"15min":126.679},"last":1.6379999999999999},{"interface":"front","average":{"1min":0.879,"5min":1.98,"15min":1.5589999999999999},"min":{"1min":0.309,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":2.7610000000000001,"5min":126.78100000000001,"15min":126.78100000000001},"last":1.7030000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:30:46 2023","interfaces":[{"interface":"back","average":{"1min":4.9059999999999997,"5min":1.4039999999999999,"15min":1.462},"min":{"1min":0.36499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":70.581999999999994,"5min":70.581999999999994,"15min":114.83499999999999},"last":1.7989999999999999},{"interface":"front","average":{"1min":5.6050000000000004,"5min":1.6599999999999999,"15min":1.4430000000000001},"min":{"1min":0.33600000000000002,"5min":0.248,"15min":0.248},"max":{"1min":71.093000000000004,"5min":71.093000000000004,"15min":110.905},"last":0.373}]},{"osd":7,"last update":"Sat Sep 23 22:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.624,"15min":0.91600000000000004},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":4.1120000000000001,"5min":4.827,"15min":8.8030000000000008},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.58999999999999997,"15min":0.77900000000000003},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.308},"max":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":7.6609999999999996},"last":1.669}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":157417472,"data_stored":157417472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8196096,"data_stored":8192024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":158859264,"data_stored":158859264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":960040960,"data_stored":960036888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":41549824,"data_stored":41549824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":160759808,"data_stored":160759808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:30:58.354 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:30:58.378 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[223014]: from='client.35832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[223014]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[223014]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[223014]: osdmap e369: 8 total, 8 up, 6 in 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[223014]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[223014]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[223014]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[223014]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 539/135 objects degraded (399.259%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[215775]: from='client.35832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:30:58.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[215775]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[215775]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[215775]: osdmap e369: 8 total, 8 up, 6 in 2023-09-23T22:30:58.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[215775]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[215775]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[215775]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[215775]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 539/135 objects degraded (399.259%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:30:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:58 smithi070 ceph-mon[192652]: from='client.35832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:30:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:58 smithi070 ceph-mon[192652]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:58 smithi070 ceph-mon[192652]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:58 smithi070 ceph-mon[192652]: osdmap e369: 8 total, 8 up, 6 in 2023-09-23T22:30:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:58 smithi070 ceph-mon[192652]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:58 smithi070 ceph-mon[192652]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:58 smithi070 ceph-mon[192652]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:58 smithi070 ceph-mon[192652]: failed to encode map e369 with expected crc 2023-09-23T22:30:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 539/135 objects degraded (399.259%), 5 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:30:58.781 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:58.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:58.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:58.840 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:30:58.885 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:30:58.895 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:58.912 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:58.916 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:30:58.956 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:30:58.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:58.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:30:59.063 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:59.098 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:59.181 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:30:59.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:30:59.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:30:59.213 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:30:59.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:30:59.318 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:30:59.415 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:30:59 smithi070 ceph-mon[192652]: pgmap v952: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 5.5 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 539/135 objects degraded (399.259%); 2978/135 objects misplaced (2205.926%); 3.0 MiB/s, 172 objects/s recovering 2023-09-23T22:30:59.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:30:59 smithi026 ceph-mon[215775]: pgmap v952: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 5.5 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 539/135 objects degraded (399.259%); 2978/135 objects misplaced (2205.926%); 3.0 MiB/s, 172 objects/s recovering 2023-09-23T22:30:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:30:59 smithi026 ceph-mon[223014]: pgmap v952: 99 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 5.5 MiB data, 2.5 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 427 op/s; 539/135 objects degraded (399.259%); 2978/135 objects misplaced (2205.926%); 3.0 MiB/s, 172 objects/s recovering 2023-09-23T22:30:59.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:30:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:30:59] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2023-09-23T22:31:00.182 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_2' removed 2023-09-23T22:31:00.226 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:00.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:00.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:00.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:00 smithi070 ceph-mon[192652]: failed to encode map e370 with expected crc 2023-09-23T22:31:00.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:00 smithi070 ceph-mon[192652]: osdmap e370: 8 total, 8 up, 6 in 2023-09-23T22:31:00.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:00 smithi070 ceph-mon[192652]: failed to encode map e370 with expected crc 2023-09-23T22:31:00.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:00 smithi070 ceph-mon[192652]: failed to encode map e370 with expected crc 2023-09-23T22:31:00.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1043484839' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:31:00.391 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:00 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:31:00.405 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:00.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[223014]: failed to encode map e370 with expected crc 2023-09-23T22:31:00.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[223014]: osdmap e370: 8 total, 8 up, 6 in 2023-09-23T22:31:00.429 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[223014]: failed to encode map e370 with expected crc 2023-09-23T22:31:00.429 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[223014]: failed to encode map e370 with expected crc 2023-09-23T22:31:00.429 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1043484839' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:31:00.429 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:31:00.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[215775]: failed to encode map e370 with expected crc 2023-09-23T22:31:00.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[215775]: osdmap e370: 8 total, 8 up, 6 in 2023-09-23T22:31:00.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[215775]: failed to encode map e370 with expected crc 2023-09-23T22:31:00.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[215775]: failed to encode map e370 with expected crc 2023-09-23T22:31:00.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1043484839' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:31:00.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:00 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:31:00.454 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:00.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:00.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:00.560 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:00.587 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:00.684 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:00.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:00.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:00.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:00.820 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:00.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:31:00.868 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:31:00.868 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:31:00.868 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:31:00.868 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:31:00.869 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:31:00.870 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:00.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:00.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:00.973 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:01.001 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:01.094 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:01.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:01.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:01.141 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[223014]: pgmap v954: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.4 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 255/81 objects degraded (314.815%); 1708/81 objects misplaced (2108.642%); 0 B/s, 46 objects/s recovering 2023-09-23T22:31:01.199 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:01.230 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:01.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:01 smithi070 ceph-mon[192652]: pgmap v954: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.4 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 255/81 objects degraded (314.815%); 1708/81 objects misplaced (2108.642%); 0 B/s, 46 objects/s recovering 2023-09-23T22:31:01.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:01 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:31:01.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:01 smithi070 ceph-mon[192652]: osdmap e371: 8 total, 8 up, 6 in 2023-09-23T22:31:01.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:01 smithi070 ceph-mon[192652]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:01 smithi070 ceph-mon[192652]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:01 smithi070 ceph-mon[192652]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:01 smithi070 ceph-mon[192652]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[215775]: pgmap v954: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.4 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 4 op/s; 255/81 objects degraded (314.815%); 1708/81 objects misplaced (2108.642%); 0 B/s, 46 objects/s recovering 2023-09-23T22:31:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:31:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[215775]: osdmap e371: 8 total, 8 up, 6 in 2023-09-23T22:31:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[215775]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[215775]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[215775]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[215775]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:31:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[223014]: osdmap e371: 8 total, 8 up, 6 in 2023-09-23T22:31:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[223014]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[223014]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[223014]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:01 smithi026 ceph-mon[223014]: failed to encode map e371 with expected crc 2023-09-23T22:31:01.598 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:31:01.598 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:31:01.599 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T22:31:01.599 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T22:31:01.648 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:31:01.708 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:01.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:01.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:01.804 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:01.812 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:01.838 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:01.883 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:01.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:01.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:01.988 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:01.992 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:31:02.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:02.059 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:02.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:02.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:02.151 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:31:02.163 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:02.194 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:02.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/482877369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:02.280 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:02 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:02.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/482877369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:02.280 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:02 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:02.435 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/482877369' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:02.435 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:02 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:02.538 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:31:02.538 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 10/81 objects degraded (12.346%), 1 pg degraded","count":1},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1246,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":372,"num_osds":8,"num_up_osds":8,"osd_up_since":1695508188,"num_in_osds":6,"osd_in_since":1695508140,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":82},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":83,"num_pools":4,"num_objects":27,"data_bytes":4623525,"bytes_used":2458513408,"bytes_avail":573511700480,"bytes_total":575970213888,"degraded_objects":10,"degraded_total":81,"degraded_ratio":0.12345679012345678},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":24,"modified":"2023-09-23T22:30:47.531589+0000","services":{}},"progress_events":{"886520b3-41de-411e-bb18-7e9b4ebd0d21":{"message":"Global Recovery Event (4m)\n [========================....] (remaining: 37s)","progress":0.87878787517547607,"add_to_ceph_s":true},"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:31:02.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:02.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:02.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:03.058 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:03.089 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:03.179 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:03.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:03.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:03.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[223014]: pgmap v956: 83 pgs: 1 active+recovery_wait+degraded, 82 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 10/81 objects degraded (12.346%) 2023-09-23T22:31:03.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:03.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[223014]: failed to encode map e372 with expected crc 2023-09-23T22:31:03.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[223014]: failed to encode map e372 with expected crc 2023-09-23T22:31:03.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:31:03.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[223014]: osdmap e372: 8 total, 8 up, 6 in 2023-09-23T22:31:03.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/981887884' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:31:03.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10/81 objects degraded (12.346%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:31:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[215775]: pgmap v956: 83 pgs: 1 active+recovery_wait+degraded, 82 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 10/81 objects degraded (12.346%) 2023-09-23T22:31:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:03.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[215775]: failed to encode map e372 with expected crc 2023-09-23T22:31:03.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[215775]: failed to encode map e372 with expected crc 2023-09-23T22:31:03.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:31:03.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[215775]: osdmap e372: 8 total, 8 up, 6 in 2023-09-23T22:31:03.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/981887884' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:31:03.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10/81 objects degraded (12.346%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:31:03.282 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:03.313 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:03.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:03 smithi070 ceph-mon[192652]: pgmap v956: 83 pgs: 1 active+recovery_wait+degraded, 82 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 10/81 objects degraded (12.346%) 2023-09-23T22:31:03.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:03 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:03.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:03 smithi070 ceph-mon[192652]: failed to encode map e372 with expected crc 2023-09-23T22:31:03.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:03 smithi070 ceph-mon[192652]: failed to encode map e372 with expected crc 2023-09-23T22:31:03.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:03 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:31:03.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:03 smithi070 ceph-mon[192652]: osdmap e372: 8 total, 8 up, 6 in 2023-09-23T22:31:03.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/981887884' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:31:03.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10/81 objects degraded (12.346%), 1 pg degraded (PG_DEGRADED) 2023-09-23T22:31:03.721 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:31:03.721 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:31:03.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:31:03.721 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:31:03.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.721 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:51738 7.e 7:7a19a68c:::benchmark_data_smithi026_410749_object51737:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e266)", 2023-09-23T22:31:03.721 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:29.524484+0000", 2023-09-23T22:31:03.721 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 334.19351392499999, 2023-09-23T22:31:03.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.222769941, 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 51738 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524484+0000", 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524484+0000", 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524483+0000", 2023-09-23T22:31:03.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524552+0000", 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8769000000000005e-05 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524565+0000", 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3443e-05 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524571+0000", 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5049999999999998e-06 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524622+0000", 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.083e-05 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524754+0000", 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013249400000000001 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.524892+0000", 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000138221 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.526723+0000", 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018302489999999999 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.620491+0000", 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.093768588 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.747112+0000", 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.126621083 2023-09-23T22:31:03.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.747215+0000", 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103001 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:29.747254+0000", 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9026999999999998e-05 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.728 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:55106 7.e 7:71164a53:::benchmark_data_smithi026_410749_object55105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e267)", 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:25:32.988479+0000", 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 330.72951968799998, 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18959135999999999, 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.729 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 55106 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988479+0000", 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988479+0000", 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988478+0000", 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988536+0000", 2023-09-23T22:31:03.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7941e-05 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988543+0000", 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3370000000000002e-06 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.988547+0000", 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8600000000000003e-06 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.990260+0000", 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017131119999999999 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.990359+0000", 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9418000000000003e-05 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.990481+0000", 2023-09-23T22:31:03.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121428 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.992382+0000", 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001900936 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:32.994279+0000", 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018973060000000001 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:33.177986+0000", 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18370663400000001 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:33.178030+0000", 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4264000000000002e-05 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:25:33.178070+0000", 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9923999999999999e-05 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:85118 7.0 7:0c960c48:::benchmark_data_smithi026_410749_object85117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:13.151705+0000", 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 290.56629280200002, 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26973130099999998, 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.736 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 85118 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151705+0000", 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151705+0000", 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151704+0000", 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151740+0000", 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5531999999999998e-05 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151753+0000", 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2944e-05 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151760+0000", 2023-09-23T22:31:03.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7109999999999998e-06 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151794+0000", 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4319000000000001e-05 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151894+0000", 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100226 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.152000+0000", 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010639799999999999 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.156285+0000", 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042846309999999997 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.164218+0000", 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0079328939999999994 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421339+0000", 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.25712146299999999 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421396+0000", 2023-09-23T22:31:03.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6419999999999999e-05 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421437+0000", 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0874999999999997e-05 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:85122 7.0 7:0bdfca06:::benchmark_data_smithi026_410749_object85121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T22:31:03.743 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:26:13.151774+0000", 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 290.56622461699999, 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26980743699999998, 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 85122 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151774+0000", 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151774+0000", 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151773+0000", 2023-09-23T22:31:03.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151806+0000", 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2521999999999998e-05 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151814+0000", 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9039999999999992e-06 2023-09-23T22:31:03.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.151818+0000", 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9239999999999998e-06 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.154034+0000", 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022159889999999998 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.154134+0000", 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9874e-05 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.154240+0000", 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105707 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.156464+0000", 2023-09-23T22:31:03.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022235610000000002 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.164263+0000", 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007799851 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421490+0000", 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.25722648399999998 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421543+0000", 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3010000000000002e-05 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:26:13.421581+0000", 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8149999999999999e-05 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:110124 7.0 7:04a13c4f:::benchmark_data_smithi026_410749_object19787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e294)", 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:27:14.974937+0000", 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 228.74306079600001, 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18275385999999999, 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.751 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 110124 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.974937+0000", 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.974937+0000", 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.974936+0000", 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.974944+0000", 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1689999999999992e-06 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.974970+0000", 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5641000000000001e-05 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.974977+0000", 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6850000000000001e-06 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.975026+0000", 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8350000000000003e-05 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.975152+0000", 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012605099999999999 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.975249+0000", 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7052000000000005e-05 2023-09-23T22:31:03.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.978344+0000", 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030952140000000002 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:14.983404+0000", 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050595600000000003 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:15.154533+0000", 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.171128857 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:15.154597+0000", 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4766999999999998e-05 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:15.157691+0000", 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030940469999999999 2023-09-23T22:31:03.757 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:111799 7.0 7:01e97c7a:::benchmark_data_smithi026_410749_object21462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:27:19.116660+0000", 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 224.60133826000001, 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23504631300000001, 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.758 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 111799 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116660+0000", 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116660+0000", 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116658+0000", 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116668+0000", 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8840000000000006e-06 2023-09-23T22:31:03.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116691+0000", 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2724000000000001e-05 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116700+0000", 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1139999999999995e-06 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116756+0000", 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6657000000000003e-05 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116890+0000", 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000133394 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:31:03.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.117006+0000", 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000116006 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.123814+0000", 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006808644 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.147657+0000", 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.023842642000000001 2023-09-23T22:31:03.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.349064+0000", 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.201407105 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.349100+0000", 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5772999999999998e-05 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.351706+0000", 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026062960000000001 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:111801 7.0 7:0db77900:::benchmark_data_smithi026_410749_object21464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:27:19.116719+0000", 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 224.60127885399999, 2023-09-23T22:31:03.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26494647999999998, 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 111801 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116719+0000", 2023-09-23T22:31:03.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116719+0000", 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116719+0000", 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116723+0000", 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9519999999999996e-06 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116742+0000", 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8609000000000001e-05 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.116747+0000", 2023-09-23T22:31:03.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0799999999999996e-06 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.120678+0000", 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039311540000000001 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.120851+0000", 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017375499999999999 2023-09-23T22:31:03.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.120987+0000", 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013525399999999999 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.123925+0000", 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029381960000000001 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.147708+0000", 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.023782936000000001 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.379146+0000", 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23143855799999999 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.379212+0000", 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5988000000000006e-05 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.381666+0000", 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002453462 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:111808 7.0 7:0e5a98b7:::benchmark_data_smithi026_410749_object21471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:27:19.127950+0000", 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 224.59004872200001, 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.255720371, 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 111808 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.773 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.127950+0000", 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.127950+0000", 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.127943+0000", 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999938 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.127956+0000", 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2979999999999999e-05 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.127974+0000", 2023-09-23T22:31:03.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.732e-05 2023-09-23T22:31:03.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.127981+0000", 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3939999999999998e-06 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.128043+0000", 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1902999999999993e-05 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.128223+0000", 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017972299999999999 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.128331+0000", 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010849999999999999 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.130770+0000", 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024384139999999999 2023-09-23T22:31:03.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.147739+0000", 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016968846999999999 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.381743+0000", 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23400421299999999 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.381805+0000", 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2687999999999993e-05 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:27:19.383670+0000", 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001864447 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:120698 7.a 7:5c2bb614:::benchmark_data_smithi026_410749_object30361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:29:17.634205+0000", 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 106.08379371300001, 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17895103200000001, 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.782 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 120698 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.634205+0000", 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.634205+0000", 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.634203+0000", 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.634210+0000", 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9330000000000004e-06 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.634225+0000", 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5491000000000001e-05 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.634232+0000", 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5799999999999997e-06 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.634293+0000", 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0952999999999997e-05 2023-09-23T22:31:03.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.634426+0000", 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013301900000000001 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3,4", 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.634554+0000", 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012789600000000001 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.640214+0000", 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056601940000000003 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.642121+0000", 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019071750000000001 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.643042+0000", 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00092099599999999999 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.812971+0000", 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.169929473 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.813032+0000", 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0899000000000001e-05 2023-09-23T22:31:03.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:17.813156+0000", 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012328299999999999 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:122060 7.0 7:0f169407:::benchmark_data_smithi026_410749_object31723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T22:31:03.789 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:29:19.762845+0000", 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 103.955153451, 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22079342299999999, 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 122060 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.762845+0000", 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.762845+0000", 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.762843+0000", 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.762852+0000", 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8599999999999999e-06 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.762874+0000", 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2466000000000001e-05 2023-09-23T22:31:03.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.762885+0000", 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0696999999999999e-05 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.763029+0000", 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014421200000000001 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.763386+0000", 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000356349 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.763560+0000", 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017482399999999999 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.771035+0000", 2023-09-23T22:31:03.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0074747989999999999 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.771058+0000", 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2650000000000002e-05 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.981257+0000", 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.21019858499999999 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.981322+0000", 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5335999999999999e-05 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:19.983638+0000", 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002316392 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:131789 7.e 7:7949edda:::benchmark_data_smithi026_410749_object41452:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:29:33.851729+0000", 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 89.866269255999995, 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.24289037799999999, 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 131789 2023-09-23T22:31:03.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.851729+0000", 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.851729+0000", 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.851727+0000", 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.851734+0000", 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4860000000000002e-06 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.851744+0000", 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0693000000000001e-05 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.851751+0000", 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.365e-06 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.851814+0000", 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3713000000000005e-05 2023-09-23T22:31:03.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.851935+0000", 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120493 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.852041+0000", 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106578 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.855033+0000", 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002991793 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.856511+0000", 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014781110000000001 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.866664+0000", 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010152178 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:34.093974+0000", 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22731000200000001 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:34.094044+0000", 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0188e-05 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:34.094619+0000", 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00057566300000000002 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:131794 7.e 7:7c38fa95:::benchmark_data_smithi026_410749_object41457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e344)", 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:29:33.859506+0000", 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 89.858491764000007, 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.24398457400000001, 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 131794 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.805 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.859506+0000", 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.859506+0000", 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.859505+0000", 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.859512+0000", 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0199999999999997e-06 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.859525+0000", 2023-09-23T22:31:03.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3400999999999999e-05 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.859530+0000", 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1059999999999999e-06 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.859576+0000", 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5389000000000002e-05 2023-09-23T22:31:03.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.859692+0000", 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011595999999999999 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.859792+0000", 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010051 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.862172+0000", 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002379393 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.862187+0000", 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5679e-05 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:33.866697+0000", 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045095140000000001 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:34.103015+0000", 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23631788100000001 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:34.103051+0000", 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5879e-05 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:34.103491+0000", 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044046999999999999 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:143477 7.0 7:0e0ec3ce:::benchmark_data_smithi026_410749_object53140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:29:55.725825+0000", 2023-09-23T22:31:03.812 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 67.992173030000004, 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17748339199999999, 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 143477 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.725825+0000", 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.725825+0000", 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.725824+0000", 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:03.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.725832+0000", 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0639999999999994e-06 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.725848+0000", 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6328e-05 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.725855+0000", 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4200000000000001e-06 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.725905+0000", 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9966999999999998e-05 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.726044+0000", 2023-09-23T22:31:03.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013835799999999999 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.726157+0000", 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112957 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.729076+0000", 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029190779999999999 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.735806+0000", 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067297870000000001 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.900885+0000", 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16507946600000001 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.900924+0000", 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8671999999999998e-05 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:55.903309+0000", 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023849140000000001 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:159377 7.e 7:70af855d:::benchmark_data_smithi026_410749_object69040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:18.575315+0000", 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 45.142683478999999, 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18900946800000001, 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 159377 2023-09-23T22:31:03.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.575315+0000", 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.575315+0000", 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.575314+0000", 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.575319+0000", 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2850000000000002e-06 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.575331+0000", 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1987000000000001e-05 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.575337+0000", 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5520000000000001e-06 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.575397+0000", 2023-09-23T22:31:03.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9988000000000002e-05 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.575534+0000", 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000136842 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.575668+0000", 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013365100000000001 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.578636+0000", 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029681059999999999 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.580060+0000", 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014240990000000001 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.580589+0000", 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000529106 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.763668+0000", 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18307868299999999 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.763736+0000", 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7749000000000005e-05 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:18.764324+0000", 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00058854000000000003 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.827 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:173141 7.0 7:0e49c921:::benchmark_data_smithi026_410749_object82804:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:39.830867+0000", 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 23.887131584999999, 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.24111913400000001, 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 173141 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830867+0000", 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830867+0000", 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830866+0000", 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830869+0000", 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0570000000000001e-06 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830879+0000", 2023-09-23T22:31:03.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3870000000000001e-06 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830883+0000", 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7849999999999999e-06 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830938+0000", 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4487000000000002e-05 2023-09-23T22:31:03.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.831096+0000", 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015795499999999999 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.831290+0000", 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019433499999999999 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.835998+0000", 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047081010000000001 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.837792+0000", 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001794096 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.071088+0000", 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.233295424 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.071154+0000", 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6507000000000005e-05 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.071986+0000", 2023-09-23T22:31:03.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00083142899999999996 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:173142 7.0 7:0ca5667f:::benchmark_data_smithi026_410749_object82805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:39.830900+0000", 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 23.887098243000001, 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.24168465, 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.835 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 173142 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830900+0000", 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830900+0000", 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830900+0000", 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830902+0000", 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7889999999999999e-06 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830909+0000", 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4829999999999996e-06 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.830913+0000", 2023-09-23T22:31:03.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2570000000000001e-06 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.833804+0000", 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028910189999999999 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.833916+0000", 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011202 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.834031+0000", 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011454400000000001 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.836087+0000", 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020561970000000001 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.837872+0000", 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017847150000000001 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.072042+0000", 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.234170506 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.072101+0000", 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8483000000000002e-05 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.072585+0000", 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048402100000000001 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:174714 7.e 7:7849021d:::benchmark_data_smithi026_410749_object84377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:42.797651+0000", 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 20.920347748000001, 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.27930596200000002, 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 174714 2023-09-23T22:31:03.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.797651+0000", 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.797651+0000", 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.797649+0000", 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.797656+0000", 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0049999999999998e-06 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.797670+0000", 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3889e-05 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.797676+0000", 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7689999999999999e-06 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.806755+0000", 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0090787539999999996 2023-09-23T22:31:03.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.807185+0000", 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00042948299999999998 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.816304+0000", 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0091196479999999993 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.816376+0000", 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1388000000000002e-05 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.816480+0000", 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010485899999999999 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.822624+0000", 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061440030000000003 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.823198+0000", 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00057356300000000002 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.824725+0000", 2023-09-23T22:31:03.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001526962 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:43.076314+0000", 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.25158919400000002 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:43.076387+0000", 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3046000000000005e-05 2023-09-23T22:31:03.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:43.076956+0000", 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00056921399999999996 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:174722 7.e 7:7e2b2386:::benchmark_data_smithi026_410749_object84385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:42.811784+0000", 2023-09-23T22:31:03.851 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 20.906213825999998, 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.273787642, 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 174722 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.811784+0000", 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.811784+0000", 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.811783+0000", 2023-09-23T22:31:03.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.811787+0000", 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0999999999999997e-06 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.811797+0000", 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6260000000000007e-06 2023-09-23T22:31:03.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.811806+0000", 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3219999999999999e-06 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.812057+0000", 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025043500000000002 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.812564+0000", 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00050701800000000003 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.819607+0000", 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0070436630000000004 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.819671+0000", 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3847e-05 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.819819+0000", 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014788299999999999 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.823256+0000", 2023-09-23T22:31:03.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034369800000000001 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.823493+0000", 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000236548 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.824755+0000", 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001262771 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:43.084914+0000", 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26015880400000002 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:43.084990+0000", 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6074000000000004e-05 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:43.085572+0000", 2023-09-23T22:31:03.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00058179200000000003 2023-09-23T22:31:03.860 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.860 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.860 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.860 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:174725 7.e 7:7c3a9da7:::benchmark_data_smithi026_410749_object84388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-23T22:31:03.860 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:42.816540+0000", 2023-09-23T22:31:03.860 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 20.901458648999999, 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26951963099999998, 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 174725 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.816540+0000", 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.816540+0000", 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.816538+0000", 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.816544+0000", 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5990000000000001e-06 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.816554+0000", 2023-09-23T22:31:03.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6579999999999997e-06 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.816559+0000", 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.711e-06 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.822670+0000", 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061109650000000003 2023-09-23T22:31:03.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.823135+0000", 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00046461000000000001 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.826798+0000", 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036629100000000001 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.826861+0000", 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2941999999999995e-05 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.826953+0000", 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2756000000000004e-05 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.829368+0000", 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024143630000000001 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.829379+0000", 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0971e-05 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:42.830491+0000", 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011126980000000001 2023-09-23T22:31:03.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:43.085640+0000", 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.255148284 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:43.085702+0000", 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1927999999999994e-05 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:43.086059+0000", 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000357606 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:175959 7.e 7:72adf532:::benchmark_data_smithi026_410749_object85622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:45.638869+0000", 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 18.079129574, 2023-09-23T22:31:03.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19342552199999999, 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 175959 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.638869+0000", 2023-09-23T22:31:03.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.638869+0000", 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.638867+0000", 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:03.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.638873+0000", 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9440000000000002e-06 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.638887+0000", 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3419e-05 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.638892+0000", 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6699999999999999e-06 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.638943+0000", 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0936999999999997e-05 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.639342+0000", 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039891200000000002 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.645642+0000", 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062998790000000004 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:31:03.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.645697+0000", 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5423000000000001e-05 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,1,3", 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.645776+0000", 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8338999999999997e-05 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.648438+0000", 2023-09-23T22:31:03.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026625590000000001 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.648480+0000", 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1254000000000002e-05 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.650009+0000", 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015298040000000001 2023-09-23T22:31:03.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.831745+0000", 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18173568900000001 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.831802+0000", 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6690000000000001e-05 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:31:03.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:31:03.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:31:03.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:45.832294+0000", 2023-09-23T22:31:03.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00049240299999999998 2023-09-23T22:31:03.878 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.878 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.878 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.878 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:31:03.878 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:31:03.878 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:31:03.880 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T22:31:03.880 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:31:03.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:03.881 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:03.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:03.935 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:03.982 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:04.019 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:04.071 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:04.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:04.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:04.175 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:04.201 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:04 smithi070 ceph-mon[192652]: failed to encode map e372 with expected crc 2023-09-23T22:31:04.204 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:04.286 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:04 smithi026 ceph-mon[223014]: failed to encode map e372 with expected crc 2023-09-23T22:31:04.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:04 smithi026 ceph-mon[215775]: failed to encode map e372 with expected crc 2023-09-23T22:31:04.376 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:04.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:04.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:04.437 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:31:04.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:04.494 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:31:04.508 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:04.518 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:31:04.539 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:31:04.550 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:04.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:04.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:04.560 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:31:04.580 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:31:04.599 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:31:04.618 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:31:04.637 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:31:04.638 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:31:04.639 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:31:04.639 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:31:04.639 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:31:04.639 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:31:04.639 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:31:04.639 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:31:04.640 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:31:04.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:04.659 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:04.691 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:04.792 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:31:05.105 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:05.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:05.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:05.209 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:05.238 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:05.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:05 smithi070 ceph-mon[192652]: pgmap v958: 83 pgs: 1 active+recovery_wait+degraded, 82 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 10/81 objects degraded (12.346%) 2023-09-23T22:31:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:05 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2557816057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:05.399 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:31:05.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:05 smithi026 ceph-mon[215775]: pgmap v958: 83 pgs: 1 active+recovery_wait+degraded, 82 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 10/81 objects degraded (12.346%) 2023-09-23T22:31:05.402 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:05 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2557816057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:05.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:05 smithi026 ceph-mon[223014]: pgmap v958: 83 pgs: 1 active+recovery_wait+degraded, 82 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 10/81 objects degraded (12.346%) 2023-09-23T22:31:05.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:05 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2557816057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:05.436 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:05.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:05.539 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:05.566 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:05.805 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:05.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:05.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:05.909 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:05.944 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:05.996 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:05.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:06.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:06.100 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:06.128 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:06.174 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:06.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:06.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:06.278 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:06.308 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:06 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/81 objects degraded (12.346%), 1 pg degraded) 2023-09-23T22:31:06.308 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:06 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/81 objects degraded (12.346%), 1 pg degraded) 2023-09-23T22:31:06.311 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:06.375 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:31:06.376 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:31:06.403 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:06 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 10/81 objects degraded (12.346%), 1 pg degraded) 2023-09-23T22:31:06.408 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:06.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:06.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:06.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:06.540 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:06.769 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:06.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:06.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:06.807 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:06.874 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:06.904 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:06.964 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:06.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:06.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:07.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:07.097 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:31:07.098 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:07.196 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:07.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:07.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:07.217 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_3' created 2023-09-23T22:31:07.300 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:07.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:07 smithi026 ceph-mon[215775]: pgmap v959: 83 pgs: 83 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:07.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3748810213' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T22:31:07.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:07 smithi026 ceph-mon[223014]: pgmap v959: 83 pgs: 83 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:07.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3748810213' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T22:31:07.337 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:07 smithi070 ceph-mon[192652]: pgmap v959: 83 pgs: 83 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:07.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3748810213' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T22:31:07.447 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:31:07.846 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:31:07.885 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:07.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:07.989 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:08.017 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:08.212 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[223014]: failed to encode map e373 with expected crc 2023-09-23T22:31:08.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[223014]: failed to encode map e373 with expected crc 2023-09-23T22:31:08.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3748810213' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T22:31:08.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[223014]: osdmap e373: 8 total, 8 up, 6 in 2023-09-23T22:31:08.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[223014]: pgmap v961: 99 pgs: 16 unknown, 83 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-23T22:31:08.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[215775]: failed to encode map e373 with expected crc 2023-09-23T22:31:08.213 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[215775]: failed to encode map e373 with expected crc 2023-09-23T22:31:08.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3748810213' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T22:31:08.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[215775]: osdmap e373: 8 total, 8 up, 6 in 2023-09-23T22:31:08.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:08 smithi026 ceph-mon[215775]: pgmap v961: 99 pgs: 16 unknown, 83 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-23T22:31:08.279 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T22:31:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:08 smithi070 ceph-mon[192652]: failed to encode map e373 with expected crc 2023-09-23T22:31:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:08 smithi070 ceph-mon[192652]: failed to encode map e373 with expected crc 2023-09-23T22:31:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:08 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3748810213' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T22:31:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:08 smithi070 ceph-mon[192652]: osdmap e373: 8 total, 8 up, 6 in 2023-09-23T22:31:08.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:08 smithi070 ceph-mon[192652]: pgmap v961: 99 pgs: 16 unknown, 83 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-23T22:31:08.590 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:08.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:08.695 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:08.725 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:08.781 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:08.790 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:08.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:08.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:08.876 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:31:08.878 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:31:08.894 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:08.925 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:08.930 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:09.082 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:31:09.082 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:31:09.135 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:09.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:09.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:09.240 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:09.271 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:09.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[223014]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[223014]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[223014]: osdmap e374: 8 total, 8 up, 6 in 2023-09-23T22:31:09.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[223014]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[223014]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[223014]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[223014]: from='client.45815 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:31:09.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:09.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:09.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[215775]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[215775]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[215775]: osdmap e374: 8 total, 8 up, 6 in 2023-09-23T22:31:09.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[215775]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[215775]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[215775]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[215775]: from='client.45815 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:31:09.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:09.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:09.519 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:09.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:09.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:09.623 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:09 smithi070 ceph-mon[192652]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:09 smithi070 ceph-mon[192652]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:09 smithi070 ceph-mon[192652]: osdmap e374: 8 total, 8 up, 6 in 2023-09-23T22:31:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:09 smithi070 ceph-mon[192652]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:09 smithi070 ceph-mon[192652]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:09 smithi070 ceph-mon[192652]: failed to encode map e374 with expected crc 2023-09-23T22:31:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:09 smithi070 ceph-mon[192652]: from='client.45815 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:31:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:09.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:09.653 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:09.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:31:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:31:09] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2023-09-23T22:31:09.795 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:31:10.052 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:10.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:10.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:10.140 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:31:10.156 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:10.189 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:10.228 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:31:10.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[215775]: osdmap e375: 8 total, 8 up, 6 in 2023-09-23T22:31:10.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4173621364' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[215775]: pgmap v964: 99 pgs: 9 unknown, 90 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-23T22:31:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[215775]: failed to encode map e375 with expected crc 2023-09-23T22:31:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[215775]: failed to encode map e375 with expected crc 2023-09-23T22:31:10.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[223014]: osdmap e375: 8 total, 8 up, 6 in 2023-09-23T22:31:10.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4173621364' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:10.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:10.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[223014]: pgmap v964: 99 pgs: 9 unknown, 90 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-23T22:31:10.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[223014]: failed to encode map e375 with expected crc 2023-09-23T22:31:10.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:10 smithi026 ceph-mon[223014]: failed to encode map e375 with expected crc 2023-09-23T22:31:10.538 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:10 smithi070 ceph-mon[192652]: osdmap e375: 8 total, 8 up, 6 in 2023-09-23T22:31:10.538 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4173621364' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:10.538 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:10.538 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:10 smithi070 ceph-mon[192652]: pgmap v964: 99 pgs: 9 unknown, 90 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-23T22:31:10.538 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:10 smithi070 ceph-mon[192652]: failed to encode map e375 with expected crc 2023-09-23T22:31:10.538 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:10 smithi070 ceph-mon[192652]: failed to encode map e375 with expected crc 2023-09-23T22:31:11.108 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":962,"stamp":"2023-09-23T22:31:08.211204+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4623525,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":6114,"num_read_kb":12433,"num_write":973,"num_write_kb":13385,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":285,"num_bytes_recovered":23252202,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2802,"ondisk_log_size":2802,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":256,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2401880,"kb_used_data":220840,"kb_used_omap":15,"kb_used_meta":2180976,"kb_avail":560069032,"statfs":{"total":575970213888,"available":573510688768,"internally_reserved":0,"allocated":226140160,"data_stored":216342097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2233319646},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":46,"apply_latency_ms":46,"commit_latency_ns":46000000,"apply_latency_ns":46000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.671167"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":399,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.148614+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:31:00.148614+0000","last_peered":"2023-09-23T22:31:00.148614+0000","last_clean":"2023-09-23T22:31:00.148614+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:31:00.148614+0000","last_undegraded":"2023-09-23T22:31:00.148614+0000","last_fullsized":"2023-09-23T22:31:00.148614+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":88,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.144528+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:31:00.144528+0000","last_peered":"2023-09-23T22:31:00.144528+0000","last_clean":"2023-09-23T22:31:00.144528+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:31:00.144528+0000","last_undegraded":"2023-09-23T22:31:00.144528+0000","last_fullsized":"2023-09-23T22:31:00.144528+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":88,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.147318+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:31:00.147318+0000","last_peered":"2023-09-23T22:31:00.147318+0000","last_clean":"2023-09-23T22:31:00.147318+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:31:00.147318+0000","last_undegraded":"2023-09-23T22:31:00.147318+0000","last_fullsized":"2023-09-23T22:31:00.147318+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":250,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.292875+0000","last_change":"2023-09-23T22:27:52.097725+0000","last_active":"2023-09-23T22:31:00.292875+0000","last_peered":"2023-09-23T22:31:00.292875+0000","last_clean":"2023-09-23T22:31:00.292875+0000","last_became_active":"2023-09-23T22:27:52.097293+0000","last_became_peered":"2023-09-23T22:27:52.097293+0000","last_unstale":"2023-09-23T22:31:00.292875+0000","last_undegraded":"2023-09-23T22:31:00.292875+0000","last_fullsized":"2023-09-23T22:31:00.292875+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:41.651341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":417,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.283804+0000","last_change":"2023-09-23T22:28:05.155218+0000","last_active":"2023-09-23T22:31:00.283804+0000","last_peered":"2023-09-23T22:31:00.283804+0000","last_clean":"2023-09-23T22:31:00.283804+0000","last_became_active":"2023-09-23T22:28:05.154771+0000","last_became_peered":"2023-09-23T22:28:05.154771+0000","last_unstale":"2023-09-23T22:31:00.283804+0000","last_undegraded":"2023-09-23T22:31:00.283804+0000","last_fullsized":"2023-09-23T22:31:00.283804+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:48.395095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281412+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:31:00.281412+0000","last_peered":"2023-09-23T22:31:00.281412+0000","last_clean":"2023-09-23T22:31:00.281412+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:31:00.281412+0000","last_undegraded":"2023-09-23T22:31:00.281412+0000","last_fullsized":"2023-09-23T22:31:00.281412+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":288,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:31:02.604643+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:31:02.604643+0000","last_peered":"2023-09-23T22:31:02.604643+0000","last_clean":"2023-09-23T22:31:02.604643+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:31:02.604643+0000","last_undegraded":"2023-09-23T22:31:02.604643+0000","last_fullsized":"2023-09-23T22:31:02.604643+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281270+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:31:00.281270+0000","last_peered":"2023-09-23T22:31:00.281270+0000","last_clean":"2023-09-23T22:31:00.281270+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:31:00.281270+0000","last_undegraded":"2023-09-23T22:31:00.281270+0000","last_fullsized":"2023-09-23T22:31:00.281270+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":362,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.289054+0000","last_change":"2023-09-23T22:27:52.091129+0000","last_active":"2023-09-23T22:31:00.289054+0000","last_peered":"2023-09-23T22:31:00.289054+0000","last_clean":"2023-09-23T22:31:00.289054+0000","last_became_active":"2023-09-23T22:27:52.090819+0000","last_became_peered":"2023-09-23T22:27:52.090819+0000","last_unstale":"2023-09-23T22:31:00.289054+0000","last_undegraded":"2023-09-23T22:31:00.289054+0000","last_fullsized":"2023-09-23T22:31:00.289054+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:13.639810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281317+0000","last_change":"2023-09-23T22:29:01.578787+0000","last_active":"2023-09-23T22:31:00.281317+0000","last_peered":"2023-09-23T22:31:00.281317+0000","last_clean":"2023-09-23T22:31:00.281317+0000","last_became_active":"2023-09-23T22:29:01.578405+0000","last_became_peered":"2023-09-23T22:29:01.578405+0000","last_unstale":"2023-09-23T22:31:00.281317+0000","last_undegraded":"2023-09-23T22:31:00.281317+0000","last_fullsized":"2023-09-23T22:31:00.281317+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:55.201936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":103,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.148911+0000","last_change":"2023-09-23T22:27:27.320214+0000","last_active":"2023-09-23T22:31:00.148911+0000","last_peered":"2023-09-23T22:31:00.148911+0000","last_clean":"2023-09-23T22:31:00.148911+0000","last_became_active":"2023-09-23T22:27:27.303456+0000","last_became_peered":"2023-09-23T22:27:27.303456+0000","last_unstale":"2023-09-23T22:31:00.148911+0000","last_undegraded":"2023-09-23T22:31:00.148911+0000","last_fullsized":"2023-09-23T22:31:00.148911+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:17.105540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":20,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.148329+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:31:00.148329+0000","last_peered":"2023-09-23T22:31:00.148329+0000","last_clean":"2023-09-23T22:31:00.148329+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:31:00.148329+0000","last_undegraded":"2023-09-23T22:31:00.148329+0000","last_fullsized":"2023-09-23T22:31:00.148329+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":436,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149364+0000","last_change":"2023-09-23T22:29:01.673486+0000","last_active":"2023-09-23T22:31:00.149364+0000","last_peered":"2023-09-23T22:31:00.149364+0000","last_clean":"2023-09-23T22:31:00.149364+0000","last_became_active":"2023-09-23T22:29:01.661056+0000","last_became_peered":"2023-09-23T22:29:01.661056+0000","last_unstale":"2023-09-23T22:31:00.149364+0000","last_undegraded":"2023-09-23T22:31:00.149364+0000","last_fullsized":"2023-09-23T22:31:00.149364+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:40.581172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":370,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149859+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:31:00.149859+0000","last_peered":"2023-09-23T22:31:00.149859+0000","last_clean":"2023-09-23T22:31:00.149859+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:31:00.149859+0000","last_undegraded":"2023-09-23T22:31:00.149859+0000","last_fullsized":"2023-09-23T22:31:00.149859+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":347,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149436+0000","last_change":"2023-09-23T22:29:01.662016+0000","last_active":"2023-09-23T22:31:00.149436+0000","last_peered":"2023-09-23T22:31:00.149436+0000","last_clean":"2023-09-23T22:31:00.149436+0000","last_became_active":"2023-09-23T22:29:01.600346+0000","last_became_peered":"2023-09-23T22:29:01.600346+0000","last_unstale":"2023-09-23T22:31:00.149436+0000","last_undegraded":"2023-09-23T22:31:00.149436+0000","last_fullsized":"2023-09-23T22:31:00.149436+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.205227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":324,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.288013+0000","last_change":"2023-09-23T22:28:41.267094+0000","last_active":"2023-09-23T22:31:00.288013+0000","last_peered":"2023-09-23T22:31:00.288013+0000","last_clean":"2023-09-23T22:31:00.288013+0000","last_became_active":"2023-09-23T22:28:41.266787+0000","last_became_peered":"2023-09-23T22:28:41.266787+0000","last_unstale":"2023-09-23T22:31:00.288013+0000","last_undegraded":"2023-09-23T22:31:00.288013+0000","last_fullsized":"2023-09-23T22:31:00.288013+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:30.013252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":268,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.284972+0000","last_change":"2023-09-23T22:27:52.101677+0000","last_active":"2023-09-23T22:31:00.284972+0000","last_peered":"2023-09-23T22:31:00.284972+0000","last_clean":"2023-09-23T22:31:00.284972+0000","last_became_active":"2023-09-23T22:27:52.101483+0000","last_became_peered":"2023-09-23T22:27:52.101483+0000","last_unstale":"2023-09-23T22:31:00.284972+0000","last_undegraded":"2023-09-23T22:31:00.284972+0000","last_fullsized":"2023-09-23T22:31:00.284972+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:01.772629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":371,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.290765+0000","last_change":"2023-09-23T22:29:01.588394+0000","last_active":"2023-09-23T22:31:00.290765+0000","last_peered":"2023-09-23T22:31:00.290765+0000","last_clean":"2023-09-23T22:31:00.290765+0000","last_became_active":"2023-09-23T22:29:01.587588+0000","last_became_peered":"2023-09-23T22:29:01.587588+0000","last_unstale":"2023-09-23T22:31:00.290765+0000","last_undegraded":"2023-09-23T22:31:00.290765+0000","last_fullsized":"2023-09-23T22:31:00.290765+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:12.141014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":305,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149342+0000","last_change":"2023-09-23T22:29:01.604689+0000","last_active":"2023-09-23T22:31:00.149342+0000","last_peered":"2023-09-23T22:31:00.149342+0000","last_clean":"2023-09-23T22:31:00.149342+0000","last_became_active":"2023-09-23T22:29:01.603739+0000","last_became_peered":"2023-09-23T22:29:01.603739+0000","last_unstale":"2023-09-23T22:31:00.149342+0000","last_undegraded":"2023-09-23T22:31:00.149342+0000","last_fullsized":"2023-09-23T22:31:00.149342+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:26.149229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":259,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T22:31:02.604570+0000","last_change":"2023-09-23T22:29:01.575934+0000","last_active":"2023-09-23T22:31:02.604570+0000","last_peered":"2023-09-23T22:31:02.604570+0000","last_clean":"2023-09-23T22:31:02.604570+0000","last_became_active":"2023-09-23T22:29:01.575404+0000","last_became_peered":"2023-09-23T22:29:01.575404+0000","last_unstale":"2023-09-23T22:31:02.604570+0000","last_undegraded":"2023-09-23T22:31:02.604570+0000","last_fullsized":"2023-09-23T22:31:02.604570+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:38.753379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":63,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.280743+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:31:00.280743+0000","last_peered":"2023-09-23T22:31:00.280743+0000","last_clean":"2023-09-23T22:31:00.280743+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:31:00.280743+0000","last_undegraded":"2023-09-23T22:31:00.280743+0000","last_fullsized":"2023-09-23T22:31:00.280743+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281239+0000","last_change":"2023-09-23T22:29:01.583368+0000","last_active":"2023-09-23T22:31:00.281239+0000","last_peered":"2023-09-23T22:31:00.281239+0000","last_clean":"2023-09-23T22:31:00.281239+0000","last_became_active":"2023-09-23T22:29:01.582967+0000","last_became_peered":"2023-09-23T22:29:01.582967+0000","last_unstale":"2023-09-23T22:31:00.281239+0000","last_undegraded":"2023-09-23T22:31:00.281239+0000","last_fullsized":"2023-09-23T22:31:00.281239+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:48.275429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":367,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149798+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:31:00.149798+0000","last_peered":"2023-09-23T22:31:00.149798+0000","last_clean":"2023-09-23T22:31:00.149798+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:31:00.149798+0000","last_undegraded":"2023-09-23T22:31:00.149798+0000","last_fullsized":"2023-09-23T22:31:00.149798+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":364,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.282866+0000","last_change":"2023-09-23T22:29:01.597976+0000","last_active":"2023-09-23T22:31:00.282866+0000","last_peered":"2023-09-23T22:31:00.282866+0000","last_clean":"2023-09-23T22:31:00.282866+0000","last_became_active":"2023-09-23T22:29:01.597479+0000","last_became_peered":"2023-09-23T22:29:01.597479+0000","last_unstale":"2023-09-23T22:31:00.282866+0000","last_undegraded":"2023-09-23T22:31:00.282866+0000","last_fullsized":"2023-09-23T22:31:00.282866+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:55.990001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":309,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149230+0000","last_change":"2023-09-23T22:27:52.154449+0000","last_active":"2023-09-23T22:31:00.149230+0000","last_peered":"2023-09-23T22:31:00.149230+0000","last_clean":"2023-09-23T22:31:00.149230+0000","last_became_active":"2023-09-23T22:27:52.096263+0000","last_became_peered":"2023-09-23T22:27:52.096263+0000","last_unstale":"2023-09-23T22:31:00.149230+0000","last_undegraded":"2023-09-23T22:31:00.149230+0000","last_fullsized":"2023-09-23T22:31:00.149230+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:02.582951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":399,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149258+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:31:00.149258+0000","last_peered":"2023-09-23T22:31:00.149258+0000","last_clean":"2023-09-23T22:31:00.149258+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:31:00.149258+0000","last_undegraded":"2023-09-23T22:31:00.149258+0000","last_fullsized":"2023-09-23T22:31:00.149258+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":325,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.290792+0000","last_change":"2023-09-23T22:29:01.633048+0000","last_active":"2023-09-23T22:31:00.290792+0000","last_peered":"2023-09-23T22:31:00.290792+0000","last_clean":"2023-09-23T22:31:00.290792+0000","last_became_active":"2023-09-23T22:29:01.587680+0000","last_became_peered":"2023-09-23T22:29:01.587680+0000","last_unstale":"2023-09-23T22:31:00.290792+0000","last_undegraded":"2023-09-23T22:31:00.290792+0000","last_fullsized":"2023-09-23T22:31:00.290792+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.830912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.280691+0000","last_change":"2023-09-23T22:29:01.593596+0000","last_active":"2023-09-23T22:31:00.280691+0000","last_peered":"2023-09-23T22:31:00.280691+0000","last_clean":"2023-09-23T22:31:00.280691+0000","last_became_active":"2023-09-23T22:29:01.593303+0000","last_became_peered":"2023-09-23T22:29:01.593303+0000","last_unstale":"2023-09-23T22:31:00.280691+0000","last_undegraded":"2023-09-23T22:31:00.280691+0000","last_fullsized":"2023-09-23T22:31:00.280691+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:02.980860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":402,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.287677+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:31:00.287677+0000","last_peered":"2023-09-23T22:31:00.287677+0000","last_clean":"2023-09-23T22:31:00.287677+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:31:00.287677+0000","last_undegraded":"2023-09-23T22:31:00.287677+0000","last_fullsized":"2023-09-23T22:31:00.287677+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":257,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.284799+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:31:00.284799+0000","last_peered":"2023-09-23T22:31:00.284799+0000","last_clean":"2023-09-23T22:31:00.284799+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:31:00.284799+0000","last_undegraded":"2023-09-23T22:31:00.284799+0000","last_fullsized":"2023-09-23T22:31:00.284799+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":697,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.287131+0000","last_change":"2023-09-23T22:29:04.057613+0000","last_active":"2023-09-23T22:31:00.287131+0000","last_peered":"2023-09-23T22:31:00.287131+0000","last_clean":"2023-09-23T22:31:00.287131+0000","last_became_active":"2023-09-23T22:29:04.057082+0000","last_became_peered":"2023-09-23T22:29:04.057082+0000","last_unstale":"2023-09-23T22:31:00.287131+0000","last_undegraded":"2023-09-23T22:31:00.287131+0000","last_fullsized":"2023-09-23T22:31:00.287131+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:24.948720+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":203,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.284879+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:31:00.284879+0000","last_peered":"2023-09-23T22:31:00.284879+0000","last_clean":"2023-09-23T22:31:00.284879+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:31:00.284879+0000","last_undegraded":"2023-09-23T22:31:00.284879+0000","last_fullsized":"2023-09-23T22:31:00.284879+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":380,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149277+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:31:00.149277+0000","last_peered":"2023-09-23T22:31:00.149277+0000","last_clean":"2023-09-23T22:31:00.149277+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:31:00.149277+0000","last_undegraded":"2023-09-23T22:31:00.149277+0000","last_fullsized":"2023-09-23T22:31:00.149277+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":753,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.287075+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:31:00.287075+0000","last_peered":"2023-09-23T22:31:00.287075+0000","last_clean":"2023-09-23T22:31:00.287075+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:31:00.287075+0000","last_undegraded":"2023-09-23T22:31:00.287075+0000","last_fullsized":"2023-09-23T22:31:00.287075+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.280819+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:31:00.280819+0000","last_peered":"2023-09-23T22:31:00.280819+0000","last_clean":"2023-09-23T22:31:00.280819+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:31:00.280819+0000","last_undegraded":"2023-09-23T22:31:00.280819+0000","last_fullsized":"2023-09-23T22:31:00.280819+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":407,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.148766+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:31:00.148766+0000","last_peered":"2023-09-23T22:31:00.148766+0000","last_clean":"2023-09-23T22:31:00.148766+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:31:00.148766+0000","last_undegraded":"2023-09-23T22:31:00.148766+0000","last_fullsized":"2023-09-23T22:31:00.148766+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":353,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149018+0000","last_change":"2023-09-23T22:28:05.062902+0000","last_active":"2023-09-23T22:31:00.149018+0000","last_peered":"2023-09-23T22:31:00.149018+0000","last_clean":"2023-09-23T22:31:00.149018+0000","last_became_active":"2023-09-23T22:28:05.062607+0000","last_became_peered":"2023-09-23T22:28:05.062607+0000","last_unstale":"2023-09-23T22:31:00.149018+0000","last_undegraded":"2023-09-23T22:31:00.149018+0000","last_fullsized":"2023-09-23T22:31:00.149018+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:21.888780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":388,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149029+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:31:00.149029+0000","last_peered":"2023-09-23T22:31:00.149029+0000","last_clean":"2023-09-23T22:31:00.149029+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:31:00.149029+0000","last_undegraded":"2023-09-23T22:31:00.149029+0000","last_fullsized":"2023-09-23T22:31:00.149029+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":726,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.289812+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:31:00.289812+0000","last_peered":"2023-09-23T22:31:00.289812+0000","last_clean":"2023-09-23T22:31:00.289812+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:31:00.289812+0000","last_undegraded":"2023-09-23T22:31:00.289812+0000","last_fullsized":"2023-09-23T22:31:00.289812+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":79,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.142022+0000","last_change":"2023-09-23T22:27:52.106215+0000","last_active":"2023-09-23T22:31:00.142022+0000","last_peered":"2023-09-23T22:31:00.142022+0000","last_clean":"2023-09-23T22:31:00.142022+0000","last_became_active":"2023-09-23T22:27:52.104315+0000","last_became_peered":"2023-09-23T22:27:52.104315+0000","last_unstale":"2023-09-23T22:31:00.142022+0000","last_undegraded":"2023-09-23T22:31:00.142022+0000","last_fullsized":"2023-09-23T22:31:00.142022+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:44.360992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":79,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.142816+0000","last_change":"2023-09-23T22:27:52.096512+0000","last_active":"2023-09-23T22:31:00.142816+0000","last_peered":"2023-09-23T22:31:00.142816+0000","last_clean":"2023-09-23T22:31:00.142816+0000","last_became_active":"2023-09-23T22:27:52.096242+0000","last_became_peered":"2023-09-23T22:27:52.096242+0000","last_unstale":"2023-09-23T22:31:00.142816+0000","last_undegraded":"2023-09-23T22:31:00.142816+0000","last_fullsized":"2023-09-23T22:31:00.142816+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:12.364350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":617,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281334+0000","last_change":"2023-09-23T22:29:04.057506+0000","last_active":"2023-09-23T22:31:00.281334+0000","last_peered":"2023-09-23T22:31:00.281334+0000","last_clean":"2023-09-23T22:31:00.281334+0000","last_became_active":"2023-09-23T22:29:04.057207+0000","last_became_peered":"2023-09-23T22:29:04.057207+0000","last_unstale":"2023-09-23T22:31:00.281334+0000","last_undegraded":"2023-09-23T22:31:00.281334+0000","last_fullsized":"2023-09-23T22:31:00.281334+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:47.781486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:47.781486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":698,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.284440+0000","last_change":"2023-09-23T22:28:41.357575+0000","last_active":"2023-09-23T22:31:00.284440+0000","last_peered":"2023-09-23T22:31:00.284440+0000","last_clean":"2023-09-23T22:31:00.284440+0000","last_became_active":"2023-09-23T22:28:41.356971+0000","last_became_peered":"2023-09-23T22:28:41.356971+0000","last_unstale":"2023-09-23T22:31:00.284440+0000","last_undegraded":"2023-09-23T22:31:00.284440+0000","last_fullsized":"2023-09-23T22:31:00.284440+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.810917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":429,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.290709+0000","last_change":"2023-09-23T22:29:01.608081+0000","last_active":"2023-09-23T22:31:00.290709+0000","last_peered":"2023-09-23T22:31:00.290709+0000","last_clean":"2023-09-23T22:31:00.290709+0000","last_became_active":"2023-09-23T22:29:01.607726+0000","last_became_peered":"2023-09-23T22:29:01.607726+0000","last_unstale":"2023-09-23T22:31:00.290709+0000","last_undegraded":"2023-09-23T22:31:00.290709+0000","last_fullsized":"2023-09-23T22:31:00.290709+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:30.179258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":79,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.147719+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:31:00.147719+0000","last_peered":"2023-09-23T22:31:00.147719+0000","last_clean":"2023-09-23T22:31:00.147719+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:31:00.147719+0000","last_undegraded":"2023-09-23T22:31:00.147719+0000","last_fullsized":"2023-09-23T22:31:00.147719+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":716,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.287909+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:31:00.287909+0000","last_peered":"2023-09-23T22:31:00.287909+0000","last_clean":"2023-09-23T22:31:00.287909+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:31:00.287909+0000","last_undegraded":"2023-09-23T22:31:00.287909+0000","last_fullsized":"2023-09-23T22:31:00.287909+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1250,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.289971+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:31:00.289971+0000","last_peered":"2023-09-23T22:31:00.289971+0000","last_clean":"2023-09-23T22:31:00.289971+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:31:00.289971+0000","last_undegraded":"2023-09-23T22:31:00.289971+0000","last_fullsized":"2023-09-23T22:31:00.289971+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":411,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149148+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:31:00.149148+0000","last_peered":"2023-09-23T22:31:00.149148+0000","last_clean":"2023-09-23T22:31:00.149148+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:31:00.149148+0000","last_undegraded":"2023-09-23T22:31:00.149148+0000","last_fullsized":"2023-09-23T22:31:00.149148+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281274+0000","last_change":"2023-09-23T22:29:01.593644+0000","last_active":"2023-09-23T22:31:00.281274+0000","last_peered":"2023-09-23T22:31:00.281274+0000","last_clean":"2023-09-23T22:31:00.281274+0000","last_became_active":"2023-09-23T22:29:01.593326+0000","last_became_peered":"2023-09-23T22:29:01.593326+0000","last_unstale":"2023-09-23T22:31:00.281274+0000","last_undegraded":"2023-09-23T22:31:00.281274+0000","last_fullsized":"2023-09-23T22:31:00.281274+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:52.921796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":704,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.284260+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:31:00.284260+0000","last_peered":"2023-09-23T22:31:00.284260+0000","last_clean":"2023-09-23T22:31:00.284260+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:31:00.284260+0000","last_undegraded":"2023-09-23T22:31:00.284260+0000","last_fullsized":"2023-09-23T22:31:00.284260+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":588,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.286818+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:31:00.286818+0000","last_peered":"2023-09-23T22:31:00.286818+0000","last_clean":"2023-09-23T22:31:00.286818+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:31:00.286818+0000","last_undegraded":"2023-09-23T22:31:00.286818+0000","last_fullsized":"2023-09-23T22:31:00.286818+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281161+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:31:00.281161+0000","last_peered":"2023-09-23T22:31:00.281161+0000","last_clean":"2023-09-23T22:31:00.281161+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:31:00.281161+0000","last_undegraded":"2023-09-23T22:31:00.281161+0000","last_fullsized":"2023-09-23T22:31:00.281161+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":20,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.142542+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:31:00.142542+0000","last_peered":"2023-09-23T22:31:00.142542+0000","last_clean":"2023-09-23T22:31:00.142542+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:31:00.142542+0000","last_undegraded":"2023-09-23T22:31:00.142542+0000","last_fullsized":"2023-09-23T22:31:00.142542+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1092,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.354175+0000","last_change":"2023-09-23T22:31:00.353818+0000","last_active":"2023-09-23T22:31:00.354175+0000","last_peered":"2023-09-23T22:31:00.354175+0000","last_clean":"2023-09-23T22:31:00.354175+0000","last_became_active":"2023-09-23T22:29:50.198096+0000","last_became_peered":"2023-09-23T22:29:50.198096+0000","last_unstale":"2023-09-23T22:31:00.354175+0000","last_undegraded":"2023-09-23T22:31:00.354175+0000","last_fullsized":"2023-09-23T22:31:00.354175+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:48.105235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":738,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.283931+0000","last_change":"2023-09-23T22:29:04.055480+0000","last_active":"2023-09-23T22:31:00.283931+0000","last_peered":"2023-09-23T22:31:00.283931+0000","last_clean":"2023-09-23T22:31:00.283931+0000","last_became_active":"2023-09-23T22:29:04.055134+0000","last_became_peered":"2023-09-23T22:29:04.055134+0000","last_unstale":"2023-09-23T22:31:00.283931+0000","last_undegraded":"2023-09-23T22:31:00.283931+0000","last_fullsized":"2023-09-23T22:31:00.283931+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:59.018639+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":253,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.289891+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:31:00.289891+0000","last_peered":"2023-09-23T22:31:00.289891+0000","last_clean":"2023-09-23T22:31:00.289891+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:31:00.289891+0000","last_undegraded":"2023-09-23T22:31:00.289891+0000","last_fullsized":"2023-09-23T22:31:00.289891+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281502+0000","last_change":"2023-09-23T22:29:01.594509+0000","last_active":"2023-09-23T22:31:00.281502+0000","last_peered":"2023-09-23T22:31:00.281502+0000","last_clean":"2023-09-23T22:31:00.281502+0000","last_became_active":"2023-09-23T22:29:01.594249+0000","last_became_peered":"2023-09-23T22:29:01.594249+0000","last_unstale":"2023-09-23T22:31:00.281502+0000","last_undegraded":"2023-09-23T22:31:00.281502+0000","last_fullsized":"2023-09-23T22:31:00.281502+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:14.972304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":594,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281852+0000","last_change":"2023-09-23T22:29:04.055972+0000","last_active":"2023-09-23T22:31:00.281852+0000","last_peered":"2023-09-23T22:31:00.281852+0000","last_clean":"2023-09-23T22:31:00.281852+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:31:00.281852+0000","last_undegraded":"2023-09-23T22:31:00.281852+0000","last_fullsized":"2023-09-23T22:31:00.281852+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":722,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.142119+0000","last_change":"2023-09-23T22:27:52.122905+0000","last_active":"2023-09-23T22:31:00.142119+0000","last_peered":"2023-09-23T22:31:00.142119+0000","last_clean":"2023-09-23T22:31:00.142119+0000","last_became_active":"2023-09-23T22:27:52.116583+0000","last_became_peered":"2023-09-23T22:27:52.116583+0000","last_unstale":"2023-09-23T22:31:00.142119+0000","last_undegraded":"2023-09-23T22:31:00.142119+0000","last_fullsized":"2023-09-23T22:31:00.142119+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:21.421519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":266,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.286908+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:31:00.286908+0000","last_peered":"2023-09-23T22:31:00.286908+0000","last_clean":"2023-09-23T22:31:00.286908+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:31:00.286908+0000","last_undegraded":"2023-09-23T22:31:00.286908+0000","last_fullsized":"2023-09-23T22:31:00.286908+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":250,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.292959+0000","last_change":"2023-09-23T22:28:05.169578+0000","last_active":"2023-09-23T22:31:00.292959+0000","last_peered":"2023-09-23T22:31:00.292959+0000","last_clean":"2023-09-23T22:31:00.292959+0000","last_became_active":"2023-09-23T22:28:05.169263+0000","last_became_peered":"2023-09-23T22:28:05.169263+0000","last_unstale":"2023-09-23T22:31:00.292959+0000","last_undegraded":"2023-09-23T22:31:00.292959+0000","last_fullsized":"2023-09-23T22:31:00.292959+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:39.446410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":722,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.280726+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:31:00.280726+0000","last_peered":"2023-09-23T22:31:00.280726+0000","last_clean":"2023-09-23T22:31:00.280726+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:31:00.280726+0000","last_undegraded":"2023-09-23T22:31:00.280726+0000","last_fullsized":"2023-09-23T22:31:00.280726+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":266,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.292080+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:31:00.292080+0000","last_peered":"2023-09-23T22:31:00.292080+0000","last_clean":"2023-09-23T22:31:00.292080+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:31:00.292080+0000","last_undegraded":"2023-09-23T22:31:00.292080+0000","last_fullsized":"2023-09-23T22:31:00.292080+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":298,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.287772+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:31:00.287772+0000","last_peered":"2023-09-23T22:31:00.287772+0000","last_clean":"2023-09-23T22:31:00.287772+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:31:00.287772+0000","last_undegraded":"2023-09-23T22:31:00.287772+0000","last_fullsized":"2023-09-23T22:31:00.287772+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":230,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.284727+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:31:00.284727+0000","last_peered":"2023-09-23T22:31:00.284727+0000","last_clean":"2023-09-23T22:31:00.284727+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:31:00.284727+0000","last_undegraded":"2023-09-23T22:31:00.284727+0000","last_fullsized":"2023-09-23T22:31:00.284727+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":726,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.148622+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:31:00.148622+0000","last_peered":"2023-09-23T22:31:00.148622+0000","last_clean":"2023-09-23T22:31:00.148622+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:31:00.148622+0000","last_undegraded":"2023-09-23T22:31:00.148622+0000","last_fullsized":"2023-09-23T22:31:00.148622+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":190,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.298874+0000","last_change":"2023-09-23T22:27:27.333140+0000","last_active":"2023-09-23T22:31:00.298874+0000","last_peered":"2023-09-23T22:31:00.298874+0000","last_clean":"2023-09-23T22:31:00.298874+0000","last_became_active":"2023-09-23T22:27:27.287527+0000","last_became_peered":"2023-09-23T22:27:27.287527+0000","last_unstale":"2023-09-23T22:31:00.298874+0000","last_undegraded":"2023-09-23T22:31:00.298874+0000","last_fullsized":"2023-09-23T22:31:00.298874+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:51.648586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":867,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.144349+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:31:00.144349+0000","last_peered":"2023-09-23T22:31:00.144349+0000","last_clean":"2023-09-23T22:31:00.144349+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:31:00.144349+0000","last_undegraded":"2023-09-23T22:31:00.144349+0000","last_fullsized":"2023-09-23T22:31:00.144349+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":302,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.283699+0000","last_change":"2023-09-23T22:27:52.136692+0000","last_active":"2023-09-23T22:31:00.283699+0000","last_peered":"2023-09-23T22:31:00.283699+0000","last_clean":"2023-09-23T22:31:00.283699+0000","last_became_active":"2023-09-23T22:27:52.136397+0000","last_became_peered":"2023-09-23T22:27:52.136397+0000","last_unstale":"2023-09-23T22:31:00.283699+0000","last_undegraded":"2023-09-23T22:31:00.283699+0000","last_fullsized":"2023-09-23T22:31:00.283699+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.036231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T22:31:07.200270+0000","last_change":"2023-09-23T22:31:07.200270+0000","last_active":"2023-09-23T22:31:07.200270+0000","last_peered":"2023-09-23T22:31:07.200270+0000","last_clean":"2023-09-23T22:31:07.200270+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T22:31:07.200270+0000","last_undegraded":"2023-09-23T22:31:07.200270+0000","last_fullsized":"2023-09-23T22:31:07.200270+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":618,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.287936+0000","last_change":"2023-09-23T22:29:51.194726+0000","last_active":"2023-09-23T22:31:00.287936+0000","last_peered":"2023-09-23T22:31:00.287936+0000","last_clean":"2023-09-23T22:31:00.287936+0000","last_became_active":"2023-09-23T22:29:51.194253+0000","last_became_peered":"2023-09-23T22:29:51.194253+0000","last_unstale":"2023-09-23T22:31:00.287936+0000","last_undegraded":"2023-09-23T22:31:00.287936+0000","last_fullsized":"2023-09-23T22:31:00.287936+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:45.734686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.734686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1144050,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":63,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.281147+0000","last_change":"2023-09-23T22:29:09.070063+0000","last_active":"2023-09-23T22:31:00.281147+0000","last_peered":"2023-09-23T22:31:00.281147+0000","last_clean":"2023-09-23T22:31:00.281147+0000","last_became_active":"2023-09-23T22:29:01.584585+0000","last_became_peered":"2023-09-23T22:29:01.584585+0000","last_unstale":"2023-09-23T22:31:00.281147+0000","last_undegraded":"2023-09-23T22:31:00.281147+0000","last_fullsized":"2023-09-23T22:31:00.281147+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:44.762021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":235,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.292245+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:31:00.292245+0000","last_peered":"2023-09-23T22:31:00.292245+0000","last_clean":"2023-09-23T22:31:00.292245+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:31:00.292245+0000","last_undegraded":"2023-09-23T22:31:00.292245+0000","last_fullsized":"2023-09-23T22:31:00.292245+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":345,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149104+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:31:00.149104+0000","last_peered":"2023-09-23T22:31:00.149104+0000","last_clean":"2023-09-23T22:31:00.149104+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:31:00.149104+0000","last_undegraded":"2023-09-23T22:31:00.149104+0000","last_fullsized":"2023-09-23T22:31:00.149104+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":22,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.284636+0000","last_change":"2023-09-23T22:30:26.181266+0000","last_active":"2023-09-23T22:31:00.284636+0000","last_peered":"2023-09-23T22:31:00.284636+0000","last_clean":"2023-09-23T22:31:00.284636+0000","last_became_active":"2023-09-23T22:30:26.180875+0000","last_became_peered":"2023-09-23T22:30:26.180875+0000","last_unstale":"2023-09-23T22:31:00.284636+0000","last_undegraded":"2023-09-23T22:31:00.284636+0000","last_fullsized":"2023-09-23T22:31:00.284636+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:11.776290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":122,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.142513+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:31:00.142513+0000","last_peered":"2023-09-23T22:31:00.142513+0000","last_clean":"2023-09-23T22:31:00.142513+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:31:00.142513+0000","last_undegraded":"2023-09-23T22:31:00.142513+0000","last_fullsized":"2023-09-23T22:31:00.142513+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":362,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.287591+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:31:00.287591+0000","last_peered":"2023-09-23T22:31:00.287591+0000","last_clean":"2023-09-23T22:31:00.287591+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:31:00.287591+0000","last_undegraded":"2023-09-23T22:31:00.287591+0000","last_fullsized":"2023-09-23T22:31:00.287591+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":387,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.149556+0000","last_change":"2023-09-23T22:28:05.114400+0000","last_active":"2023-09-23T22:31:00.149556+0000","last_peered":"2023-09-23T22:31:00.149556+0000","last_clean":"2023-09-23T22:31:00.149556+0000","last_became_active":"2023-09-23T22:28:05.114095+0000","last_became_peered":"2023-09-23T22:28:05.114095+0000","last_unstale":"2023-09-23T22:31:00.149556+0000","last_undegraded":"2023-09-23T22:31:00.149556+0000","last_fullsized":"2023-09-23T22:31:00.149556+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:23.836996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":387,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.290703+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:31:00.290703+0000","last_peered":"2023-09-23T22:31:00.290703+0000","last_clean":"2023-09-23T22:31:00.290703+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:31:00.290703+0000","last_undegraded":"2023-09-23T22:31:00.290703+0000","last_fullsized":"2023-09-23T22:31:00.290703+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":236,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.148519+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:31:00.148519+0000","last_peered":"2023-09-23T22:31:00.148519+0000","last_clean":"2023-09-23T22:31:00.148519+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:31:00.148519+0000","last_undegraded":"2023-09-23T22:31:00.148519+0000","last_fullsized":"2023-09-23T22:31:00.148519+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":53,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.280672+0000","last_change":"2023-09-23T22:29:01.593820+0000","last_active":"2023-09-23T22:31:00.280672+0000","last_peered":"2023-09-23T22:31:00.280672+0000","last_clean":"2023-09-23T22:31:00.280672+0000","last_became_active":"2023-09-23T22:29:01.593176+0000","last_became_peered":"2023-09-23T22:29:01.593176+0000","last_unstale":"2023-09-23T22:31:00.280672+0000","last_undegraded":"2023-09-23T22:31:00.280672+0000","last_fullsized":"2023-09-23T22:31:00.280672+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:33.824095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":372,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.286981+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:31:00.286981+0000","last_peered":"2023-09-23T22:31:00.286981+0000","last_clean":"2023-09-23T22:31:00.286981+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:31:00.286981+0000","last_undegraded":"2023-09-23T22:31:00.286981+0000","last_fullsized":"2023-09-23T22:31:00.286981+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":309,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.298952+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:31:00.298952+0000","last_peered":"2023-09-23T22:31:00.298952+0000","last_clean":"2023-09-23T22:31:00.298952+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:31:00.298952+0000","last_undegraded":"2023-09-23T22:31:00.298952+0000","last_fullsized":"2023-09-23T22:31:00.298952+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":430,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T22:31:00.148365+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:31:00.148365+0000","last_peered":"2023-09-23T22:31:00.148365+0000","last_clean":"2023-09-23T22:31:00.148365+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:31:00.148365+0000","last_undegraded":"2023-09-23T22:31:00.148365+0000","last_fullsized":"2023-09-23T22:31:00.148365+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":26454,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":423698,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":132,"num_read_kb":249,"num_write":66,"num_write_kb":660,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2254133,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":187,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234320,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":374948,"kb_used_data":53532,"kb_used_omap":0,"kb_used_meta":321408,"kb_avail":93370204,"statfs":{"total":95995035648,"available":95611088896,"internally_reserved":0,"allocated":54816768,"data_stored":53400210,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":329121792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":6.1479999999999997,"15min":4.1870000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.0519999999999996,"5min":220.66,"15min":220.66},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.5049999999999999,"5min":6.3280000000000003,"15min":4.5540000000000003},"min":{"1min":0.29099999999999998,"5min":0.246,"15min":0.217},"max":{"1min":5.6799999999999997,"5min":220.59899999999999,"15min":220.59899999999999},"last":0.622}]},{"osd":1,"last update":"Sat Sep 23 22:30:29 2023","interfaces":[{"interface":"back","average":{"1min":4.9500000000000002,"5min":4.9409999999999998,"15min":4.5590000000000002},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.23799999999999999},"max":{"1min":76.984999999999999,"5min":144.77500000000001,"15min":144.77500000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":5.2080000000000002,"5min":5.2359999999999998,"15min":4.8239999999999998},"min":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.24299999999999999},"max":{"1min":79.069999999999993,"5min":144.751,"15min":144.751},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7279999999999998,"5min":5.7279999999999998,"15min":5.7279999999999998},"last":0.65600000000000003},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.0529999999999999,"5min":3.0529999999999999,"15min":3.0529999999999999},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":4.5060000000000002,"15min":3.6850000000000001},"min":{"1min":0.29399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.4260000000000002,"5min":219.90700000000001,"15min":219.90700000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.1059999999999999,"5min":5.1550000000000002,"15min":3.944},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.234,"5min":219.976,"15min":219.976},"last":0.67900000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:31:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":0.69699999999999995,"15min":1.4319999999999999},"min":{"1min":0.27900000000000003,"5min":0.24299999999999999,"15min":0.23499999999999999},"max":{"1min":8.0180000000000007,"5min":8.0180000000000007,"15min":88.543000000000006},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.64500000000000002,"15min":1.349},"min":{"1min":0.27700000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.7649999999999997,"5min":5.7649999999999997,"15min":88.495999999999995},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.57199999999999995,"15min":1.3080000000000001},"min":{"1min":0.31,"5min":0.249,"15min":0.24299999999999999},"max":{"1min":2.4750000000000001,"5min":2.4750000000000001,"15min":93.816000000000003},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.5840000000000001,"5min":0.82699999999999996,"15min":1.6220000000000001},"min":{"1min":0.39100000000000001,"5min":0.309,"15min":0.27300000000000002},"max":{"1min":8.0459999999999994,"5min":8.0459999999999994,"15min":95.579999999999998},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:31:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.65600000000000003,"15min":0.78400000000000003},"min":{"1min":0.33400000000000002,"5min":0.29499999999999998,"15min":0.28999999999999998},"max":{"1min":6.0880000000000001,"5min":6.0880000000000001,"15min":9.0139999999999993},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.224,"5min":0.74199999999999999,"15min":1.028},"min":{"1min":0.36399999999999999,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":5.109,"5min":5.109,"15min":13.362},"last":0.70299999999999996}]}]},{"osd":6,"up_from":46,"seq":197568495956,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":130348,"kb_used_data":8804,"kb_used_omap":0,"kb_used_meta":121536,"kb_avail":93614804,"statfs":{"total":95995035648,"available":95861559296,"internally_reserved":0,"allocated":9015296,"data_stored":7591874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":124452864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:17 2023","interfaces":[{"interface":"back","average":{"1min":5.298,"5min":2.2309999999999999,"15min":2.1160000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":55.206000000000003,"5min":55.206000000000003,"15min":108.773},"last":0.36699999999999999},{"interface":"front","average":{"1min":6.0060000000000002,"5min":2.6219999999999999,"15min":3.1429999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.21099999999999999},"max":{"1min":55.375999999999998,"5min":55.375999999999998,"15min":135.45599999999999},"last":0.40799999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":2.1030000000000002,"15min":3.6320000000000001},"min":{"1min":0.27800000000000002,"5min":0.26000000000000001,"15min":0.22800000000000001},"max":{"1min":6.8380000000000001,"5min":80.430000000000007,"15min":129.964},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.129,"5min":2.6640000000000001,"15min":3.9449999999999998},"min":{"1min":0.35699999999999998,"5min":0.254,"15min":0.254},"max":{"1min":6.9249999999999998,"5min":84.192999999999998,"15min":131.18600000000001},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:30:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":1.9039999999999999,"15min":1.9039999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.6559999999999997,"5min":6.6559999999999997,"15min":6.6559999999999997},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.1499999999999999,"5min":2.1499999999999999,"15min":2.1499999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":9.7319999999999993,"5min":9.7319999999999993,"15min":9.7319999999999993},"last":0.58799999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.77,"15min":3.4159999999999999},"min":{"1min":0.36799999999999999,"5min":0.316,"15min":0.311},"max":{"1min":5.7640000000000002,"5min":42.523000000000003,"15min":125.125},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.631,"5min":1.9359999999999999,"15min":3.2410000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":7.4589999999999996,"5min":39.926000000000002,"15min":76.406999999999996},"last":0.55100000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:30:48 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":0.65900000000000003,"15min":2.5129999999999999},"min":{"1min":0.309,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.8650000000000002,"5min":6.8650000000000002,"15min":305.30200000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.62,"15min":2.4630000000000001},"min":{"1min":0.30399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":5.5979999999999999,"5min":5.5979999999999999,"15min":305.40300000000002},"last":0.496}]},{"osd":5,"last update":"Sat Sep 23 22:30:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":0.68799999999999994,"15min":1.056},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.6660000000000004,"5min":5.6660000000000004,"15min":67.045000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.63800000000000001,"15min":1.107},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.0170000000000003,"5min":5.5910000000000002,"15min":66.802000000000007},"last":0.48099999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:30:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.60199999999999998,"15min":1.024},"min":{"1min":0.33800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":3.1850000000000001,"5min":3.7509999999999999,"15min":82.801000000000002},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.64300000000000002,"15min":1.0069999999999999},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.6989999999999998,"5min":5.6989999999999998,"15min":83.052999999999997},"last":0.54200000000000004}]}]},{"osd":1,"up_from":98,"seq":420906795228,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":556928,"kb_used_data":8888,"kb_used_omap":7,"kb_used_meta":548024,"kb_avail":93188224,"statfs":{"total":95995035648,"available":95424741376,"internally_reserved":0,"allocated":9101312,"data_stored":7069330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":561176948},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5219999999999998,"5min":4.9199999999999999,"15min":3.9049999999999998},"min":{"1min":0.24399999999999999,"5min":0.216,"15min":0.189},"max":{"1min":18.045000000000002,"5min":90.923000000000002,"15min":196.095},"last":0.45300000000000001},{"interface":"front","average":{"1min":4.5499999999999998,"5min":5.4550000000000001,"15min":4.5899999999999999},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.20799999999999999},"max":{"1min":14.1,"5min":92.093000000000004,"15min":196.85300000000001},"last":2.823}]},{"osd":2,"last update":"Sat Sep 23 22:30:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.8250000000000002,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.371,"5min":2.371,"15min":2.371},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.2349999999999994,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":1.629}]},{"osd":3,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":4.2590000000000003,"15min":4.452},"min":{"1min":0.33900000000000002,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":9.3089999999999993,"5min":88.340000000000003,"15min":160.499},"last":4.8540000000000001},{"interface":"front","average":{"1min":2.5179999999999998,"5min":3.8109999999999999,"15min":4.2389999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.223},"max":{"1min":8.1590000000000007,"5min":91.781000000000006,"15min":160.465},"last":4.8769999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":2.2389999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.0470000000000002,"5min":9.6539999999999999,"15min":234.79900000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.694,"5min":1.7370000000000001,"15min":2.8210000000000002},"min":{"1min":0.39900000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.093,"5min":9.093,"15min":234.81700000000001},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5110000000000001,"5min":2.0659999999999998,"15min":2.3940000000000001},"min":{"1min":0.40300000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.2759999999999998,"5min":10.973000000000001,"15min":196.79900000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.389,"5min":1.6499999999999999,"15min":2.2869999999999999},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1200000000000001,"5min":7.8120000000000003,"15min":196.68299999999999},"last":1.8740000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.8089999999999999,"15min":1.7949999999999999},"min":{"1min":0.41399999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.7779999999999996,"5min":8.5540000000000003,"15min":16.314},"last":1.98},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.377,"15min":1.3080000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":3.4729999999999999,"5min":9.6910000000000007,"15min":11.385999999999999},"last":0.64900000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4910000000000001,"5min":1.9099999999999999,"15min":1.9350000000000001},"min":{"1min":0.44900000000000001,"5min":0.39500000000000002,"15min":0.32900000000000001},"max":{"1min":8.0660000000000007,"5min":8.6059999999999999,"15min":16.265000000000001},"last":1.948},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.3700000000000001,"15min":1.3220000000000001},"min":{"1min":0.439,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.165,"5min":9.7129999999999992,"15min":12.045999999999999},"last":0.68999999999999995}]}]},{"osd":0,"up_from":93,"seq":399431958757,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":692228,"kb_used_data":113084,"kb_used_omap":8,"kb_used_meta":579127,"kb_avail":93052924,"statfs":{"total":95995035648,"available":95286194176,"internally_reserved":0,"allocated":115798016,"data_stored":113703578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":593026922},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:30:32 2023","interfaces":[{"interface":"back","average":{"1min":6.282,"5min":4.0720000000000001,"15min":3.851},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":89.439999999999998,"5min":89.439999999999998,"15min":169.77199999999999},"last":0.20499999999999999},{"interface":"front","average":{"1min":8.2460000000000004,"5min":4.0750000000000002,"15min":4.0979999999999999},"min":{"1min":0.26400000000000001,"5min":0.218,"15min":0.20300000000000001},"max":{"1min":89.287999999999997,"5min":89.287999999999997,"15min":169.52099999999999},"last":0.25600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:30:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":3.1960000000000002,"15min":3.1960000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":22.263000000000002,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.1920000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.27300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:30:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.149,"15min":3.6070000000000002},"min":{"1min":0.23200000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.234,"5min":15.404,"15min":146.87299999999999},"last":0.30399999999999999},{"interface":"front","average":{"1min":3.9460000000000002,"5min":2.335,"15min":3.7709999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":22.303000000000001,"5min":22.303000000000001,"15min":145.48099999999999},"last":0.316}]},{"osd":4,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6019999999999999,"5min":1.7729999999999999,"15min":2.3780000000000001},"min":{"1min":0.35299999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":8.1430000000000007,"5min":10.837999999999999,"15min":82.825000000000003},"last":0.503},{"interface":"front","average":{"1min":1.411,"5min":1.23,"15min":1.6020000000000001},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.0220000000000002,"5min":10.08,"15min":75.930999999999997},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":2.0249999999999999,"15min":2.839},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":13.564,"5min":13.564,"15min":93.685000000000002},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.768,"5min":1.3779999999999999,"15min":2.081},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.486000000000001,"5min":13.486000000000001,"15min":93.546000000000006},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.9059999999999999,"15min":2.1240000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":12.063000000000001,"5min":12.063000000000001,"15min":16.366},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.5469999999999999,"5min":1.3440000000000001,"15min":1.474},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":5.3209999999999997,"5min":10.134,"15min":22.263000000000002},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:30:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":1.5740000000000001,"15min":2.048},"min":{"1min":0.45500000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":9.6430000000000007,"5min":9.6430000000000007,"15min":32.773000000000003},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.609,"5min":1.847,"15min":2.286},"min":{"1min":0.34200000000000003,"5min":0.307,"15min":0.307},"max":{"1min":8.2240000000000002,"5min":10.901,"15min":23.265999999999998},"last":0.436}]}]},{"osd":2,"up_from":0,"seq":1498943586321,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468175,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019225,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":304808,"kb_used_data":27744,"kb_used_omap":0,"kb_used_meta":277056,"kb_avail":93440344,"statfs":{"total":95995035648,"available":95682912256,"internally_reserved":0,"allocated":28409856,"data_stored":26989999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":283705344},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":4.2300000000000004,"15min":3.5699999999999998},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.9559999999999995,"5min":104.252,"15min":150.63499999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.3399999999999999,"5min":4.5119999999999996,"15min":3.7080000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.9740000000000002,"5min":104.294,"15min":150.672},"last":0.36699999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":6.2060000000000004,"5min":3.9550000000000001,"15min":3.1869999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.21299999999999999},"max":{"1min":81.122,"5min":95.114000000000004,"15min":107.346},"last":0.33200000000000002},{"interface":"front","average":{"1min":6.1959999999999997,"5min":3.9399999999999999,"15min":3.528},"min":{"1min":0.26700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":81.090999999999994,"5min":95.222999999999999,"15min":107.33499999999999},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:30:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1699999999999999,"5min":2.1699999999999999,"15min":2.1699999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6790000000000003,"5min":7.6790000000000003,"15min":7.6790000000000003},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.54,"5min":1.54,"15min":1.54},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.6379999999999999,"5min":4.6379999999999999,"15min":4.6379999999999999},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":4.0629999999999997,"15min":2.4630000000000001},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.0940000000000003,"5min":102.896,"15min":102.896},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.2040000000000002,"5min":4.2220000000000004,"15min":2.5739999999999998},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":14.318,"5min":112.667,"15min":112.667},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:30:39 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":0.76800000000000002,"15min":1.0029999999999999},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.407,"5min":13.407,"15min":28.399000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.7709999999999999,"5min":0.79200000000000004,"15min":1.036},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":14.212,"5min":14.212,"15min":28.350999999999999},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:30:55 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":0.66000000000000003,"15min":0.82099999999999995},"min":{"1min":0.32300000000000001,"5min":0.26400000000000001,"15min":0.251},"max":{"1min":6.0110000000000001,"5min":6.0110000000000001,"15min":7.8550000000000004},"last":0.433},{"interface":"front","average":{"1min":1.3440000000000001,"5min":0.65900000000000003,"15min":0.80500000000000005},"min":{"1min":0.35599999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.0149999999999997,"5min":5.0149999999999997,"15min":8.5210000000000008},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.59499999999999997,"15min":0.88600000000000001},"min":{"1min":0.33100000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3179999999999996,"5min":4.3179999999999996,"15min":8.577},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.4119999999999999,"5min":0.69099999999999995,"15min":0.88700000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9319999999999999,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":0.49399999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757593,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":342620,"kb_used_data":8788,"kb_used_omap":0,"kb_used_meta":333824,"kb_avail":93402532,"statfs":{"total":95995035648,"available":95644192768,"internally_reserved":0,"allocated":8998912,"data_stored":7587106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":341835776},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:14 2023","interfaces":[{"interface":"back","average":{"1min":3.4399999999999999,"5min":7.1689999999999996,"15min":4.3899999999999997},"min":{"1min":0.23899999999999999,"5min":0.22,"15min":0.20499999999999999},"max":{"1min":16.109999999999999,"5min":163.54499999999999,"15min":163.54499999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.7709999999999999,"5min":6.5010000000000003,"15min":4.2089999999999996},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":5.7949999999999999,"5min":162.81800000000001,"15min":162.81800000000001},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":3.9329999999999998,"15min":3.8740000000000001},"min":{"1min":0.38100000000000001,"5min":0.27900000000000003,"15min":0.23799999999999999},"max":{"1min":7.7779999999999996,"5min":113.514,"15min":113.514},"last":0.35599999999999998},{"interface":"front","average":{"1min":3.1240000000000001,"5min":3.9990000000000001,"15min":3.8860000000000001},"min":{"1min":0.33100000000000002,"5min":0.255,"15min":0.252},"max":{"1min":7.6319999999999997,"5min":113.42700000000001,"15min":113.42700000000001},"last":0.54000000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:30:50 2023","interfaces":[{"interface":"back","average":{"1min":2.411,"5min":2.411,"15min":2.411},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":10.451000000000001,"5min":10.451000000000001,"15min":10.451000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.5289999999999999,"5min":2.5289999999999999,"15min":2.5289999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9569999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.55500000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":5.883,"15min":4.1040000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.246},"max":{"1min":9.3369999999999997,"5min":132.59,"15min":132.59},"last":0.51300000000000001},{"interface":"front","average":{"1min":3.024,"5min":6.0949999999999998,"15min":4.1040000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":7.6479999999999997,"5min":129.381,"15min":129.381},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.9610000000000001,"15min":1.528},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.24099999999999999},"max":{"1min":3.7240000000000002,"5min":126.679,"15min":126.679},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.879,"5min":1.98,"15min":1.5589999999999999},"min":{"1min":0.309,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":2.7610000000000001,"5min":126.78100000000001,"15min":126.78100000000001},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:30:46 2023","interfaces":[{"interface":"back","average":{"1min":4.9059999999999997,"5min":1.4039999999999999,"15min":1.462},"min":{"1min":0.36499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":70.581999999999994,"5min":70.581999999999994,"15min":114.83499999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":5.6050000000000004,"5min":1.6599999999999999,"15min":1.4430000000000001},"min":{"1min":0.33600000000000002,"5min":0.248,"15min":0.248},"max":{"1min":71.093000000000004,"5min":71.093000000000004,"15min":110.905},"last":0.52900000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.624,"15min":0.91600000000000004},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":4.1120000000000001,"5min":4.827,"15min":8.8030000000000008},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.58999999999999997,"15min":0.77900000000000003},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.308},"max":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":7.6609999999999996},"last":0.67500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:31:11.242 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:11.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:11.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:11.345 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:11.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[215775]: failed to encode map e375 with expected crc 2023-09-23T22:31:11.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[215775]: failed to encode map e375 with expected crc 2023-09-23T22:31:11.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:11.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[215775]: failed to encode map e376 with expected crc 2023-09-23T22:31:11.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[215775]: failed to encode map e376 with expected crc 2023-09-23T22:31:11.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:31:11.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[215775]: osdmap e376: 8 total, 8 up, 6 in 2023-09-23T22:31:11.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[223014]: failed to encode map e375 with expected crc 2023-09-23T22:31:11.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[223014]: failed to encode map e375 with expected crc 2023-09-23T22:31:11.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:11.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[223014]: failed to encode map e376 with expected crc 2023-09-23T22:31:11.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[223014]: failed to encode map e376 with expected crc 2023-09-23T22:31:11.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:31:11.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:11 smithi026 ceph-mon[223014]: osdmap e376: 8 total, 8 up, 6 in 2023-09-23T22:31:11.374 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:11.498 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:11 smithi070 ceph-mon[192652]: failed to encode map e375 with expected crc 2023-09-23T22:31:11.498 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:11 smithi070 ceph-mon[192652]: failed to encode map e375 with expected crc 2023-09-23T22:31:11.498 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:11 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:11.499 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:11 smithi070 ceph-mon[192652]: failed to encode map e376 with expected crc 2023-09-23T22:31:11.499 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:11 smithi070 ceph-mon[192652]: failed to encode map e376 with expected crc 2023-09-23T22:31:11.499 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:31:11.499 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:11 smithi070 ceph-mon[192652]: osdmap e376: 8 total, 8 up, 6 in 2023-09-23T22:31:11.709 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:31:11.709 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:31:11.774 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:11.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:11.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:11.814 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:31:11.832 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:31:11.853 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:31:11.873 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:31:11.878 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:11.892 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:31:11.910 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:11.911 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:31:11.930 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:31:11.948 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:31:11.949 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:31:11.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:31:11.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:31:11.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:31:11.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:31:11.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:31:11.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:31:11.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:31:11.953 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:11.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:11.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:12.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:12.088 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:12.292 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:12.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:12.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:12.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[223014]: failed to encode map e375 with expected crc 2023-09-23T22:31:12.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[223014]: failed to encode map e375 with expected crc 2023-09-23T22:31:12.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[223014]: failed to encode map e376 with expected crc 2023-09-23T22:31:12.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[223014]: failed to encode map e376 with expected crc 2023-09-23T22:31:12.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[223014]: pgmap v966: 99 pgs: 99 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:12.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/107740819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:12.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[215775]: failed to encode map e375 with expected crc 2023-09-23T22:31:12.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[215775]: failed to encode map e375 with expected crc 2023-09-23T22:31:12.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[215775]: failed to encode map e376 with expected crc 2023-09-23T22:31:12.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[215775]: failed to encode map e376 with expected crc 2023-09-23T22:31:12.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[215775]: pgmap v966: 99 pgs: 99 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:12.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/107740819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:12.425 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:12.443 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:31:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:12 smithi070 ceph-mon[192652]: failed to encode map e375 with expected crc 2023-09-23T22:31:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:12 smithi070 ceph-mon[192652]: failed to encode map e375 with expected crc 2023-09-23T22:31:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:12 smithi070 ceph-mon[192652]: failed to encode map e376 with expected crc 2023-09-23T22:31:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:12 smithi070 ceph-mon[192652]: failed to encode map e376 with expected crc 2023-09-23T22:31:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:12 smithi070 ceph-mon[192652]: pgmap v966: 99 pgs: 99 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/107740819' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:12.709 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:12.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:12.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:12.785 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:31:12.813 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:12.841 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:12.923 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:12.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:12.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:13.026 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:13.054 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:13.096 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:13.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:13.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:13.200 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:13.234 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:13.236 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T22:31:13.284 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:13.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:13.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:13.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3691152958' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:13.388 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:13.414 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:13.458 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:13.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:13.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:13.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3691152958' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:13.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3691152958' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:31:13.562 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:13.589 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:13.634 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:13.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:13.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:13.738 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:13.765 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:13.874 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:31:13.969 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:13.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:13.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:14.077 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:14.104 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:14.115 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:31:14.199 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:14.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:14.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:14.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[215775]: failed to encode map e377 with expected crc 2023-09-23T22:31:14.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[215775]: failed to encode map e377 with expected crc 2023-09-23T22:31:14.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3691152958' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:31:14.255 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:14.302 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:14.331 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:14.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:14 smithi070 ceph-mon[192652]: failed to encode map e377 with expected crc 2023-09-23T22:31:14.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:14 smithi070 ceph-mon[192652]: failed to encode map e377 with expected crc 2023-09-23T22:31:14.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:14 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3691152958' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:31:14.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:14 smithi070 ceph-mon[192652]: osdmap e377: 8 total, 8 up, 6 in 2023-09-23T22:31:14.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:14 smithi070 ceph-mon[192652]: pgmap v968: 99 pgs: 99 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:14.382 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:14.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:14.401 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:31:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:31:14.402 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:31:14.402 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:31:14.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[223014]: failed to encode map e377 with expected crc 2023-09-23T22:31:14.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[223014]: failed to encode map e377 with expected crc 2023-09-23T22:31:14.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3691152958' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:31:14.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[223014]: osdmap e377: 8 total, 8 up, 6 in 2023-09-23T22:31:14.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[223014]: pgmap v968: 99 pgs: 99 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:14.486 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:14.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[215775]: osdmap e377: 8 total, 8 up, 6 in 2023-09-23T22:31:14.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:14 smithi026 ceph-mon[215775]: pgmap v968: 99 pgs: 99 active+clean; 4.4 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:14.514 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:14.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:14.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:14.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:14.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:14.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:14.763 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:31:14.777 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:14.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:14.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:14.881 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:14.908 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:14.960 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:14.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:14.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:14.968 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T22:31:15.005 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:31:15.005 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:31:15.064 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:15.088 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T22:31:15.088 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:31:15.088 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_497658 2023-09-23T22:31:15.089 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:31:15.089 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:31:15.094 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:15.226 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:31:15.244 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:15.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:15.355 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:15.386 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:15.419 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:15.502 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:15.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:15.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:15.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:15.620 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:15.655 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:15.834 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:15.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:15.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:15.908 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:31:15.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:15.999 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:16.089 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 3128 3112 194.489 194.5 0.00960203 0.00512643 2023-09-23T22:31:16.129 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:16.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:16.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:16.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:16.287 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:16.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:16 smithi070 ceph-mon[192652]: pgmap v969: 99 pgs: 99 active+clean; 4.4 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:16 smithi026 ceph-mon[223014]: pgmap v969: 99 pgs: 99 active+clean; 4.4 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:16 smithi026 ceph-mon[215775]: pgmap v969: 99 pgs: 99 active+clean; 4.4 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:17.089 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 5976 5960 186.236 178 0.00491938 0.00535917 2023-09-23T22:31:17.617 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:31:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:17 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2090309654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:31:17.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:17 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2090309654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:31:17.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:17 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2090309654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:31:18.089 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 7869 7853 163.592 118.312 0.00520436 0.0061049 2023-09-23T22:31:18.477 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:18.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:18.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:18.581 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:18.615 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:18 smithi070 ceph-mon[192652]: pgmap v970: 99 pgs: 99 active+clean; 4.4 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:18 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:18 smithi070 ceph-mon[192652]: failed to encode map e378 with expected crc 2023-09-23T22:31:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2090309654' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:31:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:18 smithi070 ceph-mon[192652]: osdmap e378: 8 total, 8 up, 6 in 2023-09-23T22:31:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:18 smithi070 ceph-mon[192652]: failed to encode map e378 with expected crc 2023-09-23T22:31:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:18 smithi070 ceph-mon[192652]: failed to encode map e378 with expected crc 2023-09-23T22:31:18.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[223014]: pgmap v970: 99 pgs: 99 active+clean; 4.4 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:18.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:18.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[223014]: failed to encode map e378 with expected crc 2023-09-23T22:31:18.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2090309654' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:31:18.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[223014]: osdmap e378: 8 total, 8 up, 6 in 2023-09-23T22:31:18.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[223014]: failed to encode map e378 with expected crc 2023-09-23T22:31:18.879 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[223014]: failed to encode map e378 with expected crc 2023-09-23T22:31:18.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[215775]: pgmap v970: 99 pgs: 99 active+clean; 4.4 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:31:18.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:18.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[215775]: failed to encode map e378 with expected crc 2023-09-23T22:31:18.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2090309654' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:31:18.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[215775]: osdmap e378: 8 total, 8 up, 6 in 2023-09-23T22:31:18.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[215775]: failed to encode map e378 with expected crc 2023-09-23T22:31:18.880 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:18 smithi026 ceph-mon[215775]: failed to encode map e378 with expected crc 2023-09-23T22:31:19.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 9642 9626 150.373 110.812 0.00256515 0.00664346 2023-09-23T22:31:19.120 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:19.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:19.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:19.234 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:19.302 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:19.314 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:31:19.314 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:31:19.314 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:31:19.314 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:31:19.314 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:31:19.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:19.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:19.610 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:19.711 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:19.780 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:19.878 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:19.880 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:31:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:31:19] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2023-09-23T22:31:19.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:19.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:19.904 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:31:19.937 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:31:19.947 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:31:19.950 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:31:19.972 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:31:19.983 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:20.008 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:31:20.039 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:20.048 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:31:20.081 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:31:20.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 12202 12186 152.295 160 0.0067844 0.00655966 2023-09-23T22:31:20.110 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:31:20.144 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:31:20.146 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:31:20.146 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:31:20.146 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:31:20.146 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:31:20.147 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:31:20.147 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:31:20.147 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:31:20.147 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:31:20.158 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:20.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:20.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:20.265 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:20.307 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:20.385 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:20.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:20.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:20.490 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:20.550 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:20 smithi070 ceph-mon[192652]: 1.7 scrub starts 2023-09-23T22:31:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:20 smithi070 ceph-mon[192652]: 1.7 scrub ok 2023-09-23T22:31:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:20 smithi070 ceph-mon[192652]: pgmap v972: 99 pgs: 99 active+clean; 284 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 35 MiB/s wr, 558 op/s 2023-09-23T22:31:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:20 smithi070 ceph-mon[192652]: 1.4 scrub starts 2023-09-23T22:31:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:20 smithi070 ceph-mon[192652]: 1.4 scrub ok 2023-09-23T22:31:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3960266327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:20 smithi070 ceph-mon[192652]: from='client.35907 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:31:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[215775]: 1.7 scrub starts 2023-09-23T22:31:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[215775]: 1.7 scrub ok 2023-09-23T22:31:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[215775]: pgmap v972: 99 pgs: 99 active+clean; 284 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 35 MiB/s wr, 558 op/s 2023-09-23T22:31:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[215775]: 1.4 scrub starts 2023-09-23T22:31:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[215775]: 1.4 scrub ok 2023-09-23T22:31:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3960266327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:20.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[215775]: from='client.35907 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:31:20.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[223014]: 1.7 scrub starts 2023-09-23T22:31:20.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[223014]: 1.7 scrub ok 2023-09-23T22:31:20.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:20.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[223014]: pgmap v972: 99 pgs: 99 active+clean; 284 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 35 MiB/s wr, 558 op/s 2023-09-23T22:31:20.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[223014]: 1.4 scrub starts 2023-09-23T22:31:20.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[223014]: 1.4 scrub ok 2023-09-23T22:31:20.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3960266327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:20.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:20 smithi026 ceph-mon[223014]: from='client.35907 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:31:21.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 13889 13873 144.484 105.438 0.00887889 0.00691261 2023-09-23T22:31:21.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:31:21.151 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:31:21.656 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:21.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:21.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:21.725 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:31:21.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:21.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:21 smithi026 ceph-mon[223014]: 1.3 scrub starts 2023-09-23T22:31:21.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:21 smithi026 ceph-mon[223014]: 1.3 scrub ok 2023-09-23T22:31:21.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:21 smithi026 ceph-mon[215775]: 1.3 scrub starts 2023-09-23T22:31:21.794 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:21 smithi026 ceph-mon[215775]: 1.3 scrub ok 2023-09-23T22:31:21.798 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:21.877 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:21 smithi070 ceph-mon[192652]: 1.3 scrub starts 2023-09-23T22:31:21.878 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:21 smithi070 ceph-mon[192652]: 1.3 scrub ok 2023-09-23T22:31:21.907 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":972,"stamp":"2023-09-23T22:31:19.544617+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":297438373,"num_objects":4495,"num_object_clones":0,"num_object_copies":13485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4495,"num_whiteouts":0,"num_read":6114,"num_read_kb":12433,"num_write":5441,"num_write_kb":299337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":285,"num_bytes_recovered":23252202,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7270,"ondisk_log_size":7270,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2827132,"kb_used_data":649804,"kb_used_omap":15,"kb_used_meta":2177264,"kb_avail":559643780,"statfs":{"total":575970213888,"available":573075230720,"internally_reserved":0,"allocated":665399296,"data_stored":655562123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2229518558},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":292814848,"num_objects":4468,"num_object_clones":0,"num_object_copies":13404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4468,"num_write_kb":285952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002104"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":404,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226206+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:31:10.226206+0000","last_peered":"2023-09-23T22:31:10.226206+0000","last_clean":"2023-09-23T22:31:10.226206+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:31:10.226206+0000","last_undegraded":"2023-09-23T22:31:10.226206+0000","last_fullsized":"2023-09-23T22:31:10.226206+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":94,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680536+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:31:13.680536+0000","last_peered":"2023-09-23T22:31:13.680536+0000","last_clean":"2023-09-23T22:31:13.680536+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:31:13.680536+0000","last_undegraded":"2023-09-23T22:31:13.680536+0000","last_fullsized":"2023-09-23T22:31:13.680536+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":94,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680357+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:31:13.680357+0000","last_peered":"2023-09-23T22:31:13.680357+0000","last_clean":"2023-09-23T22:31:13.680357+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:31:13.680357+0000","last_undegraded":"2023-09-23T22:31:13.680357+0000","last_fullsized":"2023-09-23T22:31:13.680357+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":257,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:14.527040+0000","last_change":"2023-09-23T22:27:52.097725+0000","last_active":"2023-09-23T22:31:14.527040+0000","last_peered":"2023-09-23T22:31:14.527040+0000","last_clean":"2023-09-23T22:31:14.527040+0000","last_became_active":"2023-09-23T22:27:52.097293+0000","last_became_peered":"2023-09-23T22:27:52.097293+0000","last_unstale":"2023-09-23T22:31:14.527040+0000","last_undegraded":"2023-09-23T22:31:14.527040+0000","last_fullsized":"2023-09-23T22:31:14.527040+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:41.651341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":423,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463105+0000","last_change":"2023-09-23T22:28:05.155218+0000","last_active":"2023-09-23T22:31:13.463105+0000","last_peered":"2023-09-23T22:31:13.463105+0000","last_clean":"2023-09-23T22:31:13.463105+0000","last_became_active":"2023-09-23T22:28:05.154771+0000","last_became_peered":"2023-09-23T22:28:05.154771+0000","last_unstale":"2023-09-23T22:31:13.463105+0000","last_undegraded":"2023-09-23T22:31:13.463105+0000","last_fullsized":"2023-09-23T22:31:13.463105+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:48.395095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.821016+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:31:10.821016+0000","last_peered":"2023-09-23T22:31:10.821016+0000","last_clean":"2023-09-23T22:31:10.821016+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:31:10.821016+0000","last_undegraded":"2023-09-23T22:31:10.821016+0000","last_fullsized":"2023-09-23T22:31:10.821016+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":294,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463639+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:31:13.463639+0000","last_peered":"2023-09-23T22:31:13.463639+0000","last_clean":"2023-09-23T22:31:13.463639+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:31:13.463639+0000","last_undegraded":"2023-09-23T22:31:13.463639+0000","last_fullsized":"2023-09-23T22:31:13.463639+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820357+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:31:10.820357+0000","last_peered":"2023-09-23T22:31:10.820357+0000","last_clean":"2023-09-23T22:31:10.820357+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:31:10.820357+0000","last_undegraded":"2023-09-23T22:31:10.820357+0000","last_fullsized":"2023-09-23T22:31:10.820357+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":367,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819526+0000","last_change":"2023-09-23T22:27:52.091129+0000","last_active":"2023-09-23T22:31:10.819526+0000","last_peered":"2023-09-23T22:31:10.819526+0000","last_clean":"2023-09-23T22:31:10.819526+0000","last_became_active":"2023-09-23T22:27:52.090819+0000","last_became_peered":"2023-09-23T22:27:52.090819+0000","last_unstale":"2023-09-23T22:31:10.819526+0000","last_undegraded":"2023-09-23T22:31:10.819526+0000","last_fullsized":"2023-09-23T22:31:10.819526+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:13.639810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820640+0000","last_change":"2023-09-23T22:29:01.578787+0000","last_active":"2023-09-23T22:31:10.820640+0000","last_peered":"2023-09-23T22:31:10.820640+0000","last_clean":"2023-09-23T22:31:10.820640+0000","last_became_active":"2023-09-23T22:29:01.578405+0000","last_became_peered":"2023-09-23T22:29:01.578405+0000","last_unstale":"2023-09-23T22:31:10.820640+0000","last_undegraded":"2023-09-23T22:31:10.820640+0000","last_fullsized":"2023-09-23T22:31:10.820640+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:55.201936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":108,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226334+0000","last_change":"2023-09-23T22:27:27.320214+0000","last_active":"2023-09-23T22:31:10.226334+0000","last_peered":"2023-09-23T22:31:10.226334+0000","last_clean":"2023-09-23T22:31:10.226334+0000","last_became_active":"2023-09-23T22:27:27.303456+0000","last_became_peered":"2023-09-23T22:27:27.303456+0000","last_unstale":"2023-09-23T22:31:10.226334+0000","last_undegraded":"2023-09-23T22:31:10.226334+0000","last_fullsized":"2023-09-23T22:31:10.226334+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:17.105540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":26,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680587+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:31:13.680587+0000","last_peered":"2023-09-23T22:31:13.680587+0000","last_clean":"2023-09-23T22:31:13.680587+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:31:13.680587+0000","last_undegraded":"2023-09-23T22:31:13.680587+0000","last_fullsized":"2023-09-23T22:31:13.680587+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":441,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226204+0000","last_change":"2023-09-23T22:29:01.673486+0000","last_active":"2023-09-23T22:31:10.226204+0000","last_peered":"2023-09-23T22:31:10.226204+0000","last_clean":"2023-09-23T22:31:10.226204+0000","last_became_active":"2023-09-23T22:29:01.661056+0000","last_became_peered":"2023-09-23T22:29:01.661056+0000","last_unstale":"2023-09-23T22:31:10.226204+0000","last_undegraded":"2023-09-23T22:31:10.226204+0000","last_fullsized":"2023-09-23T22:31:10.226204+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:40.581172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":375,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226098+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:31:10.226098+0000","last_peered":"2023-09-23T22:31:10.226098+0000","last_clean":"2023-09-23T22:31:10.226098+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:31:10.226098+0000","last_undegraded":"2023-09-23T22:31:10.226098+0000","last_fullsized":"2023-09-23T22:31:10.226098+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":352,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226184+0000","last_change":"2023-09-23T22:29:01.662016+0000","last_active":"2023-09-23T22:31:10.226184+0000","last_peered":"2023-09-23T22:31:10.226184+0000","last_clean":"2023-09-23T22:31:10.226184+0000","last_became_active":"2023-09-23T22:29:01.600346+0000","last_became_peered":"2023-09-23T22:29:01.600346+0000","last_unstale":"2023-09-23T22:31:10.226184+0000","last_undegraded":"2023-09-23T22:31:10.226184+0000","last_fullsized":"2023-09-23T22:31:10.226184+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.205227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":329,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819458+0000","last_change":"2023-09-23T22:28:41.267094+0000","last_active":"2023-09-23T22:31:10.819458+0000","last_peered":"2023-09-23T22:31:10.819458+0000","last_clean":"2023-09-23T22:31:10.819458+0000","last_became_active":"2023-09-23T22:28:41.266787+0000","last_became_peered":"2023-09-23T22:28:41.266787+0000","last_unstale":"2023-09-23T22:31:10.819458+0000","last_undegraded":"2023-09-23T22:31:10.819458+0000","last_fullsized":"2023-09-23T22:31:10.819458+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:30.013252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":274,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463758+0000","last_change":"2023-09-23T22:27:52.101677+0000","last_active":"2023-09-23T22:31:13.463758+0000","last_peered":"2023-09-23T22:31:13.463758+0000","last_clean":"2023-09-23T22:31:13.463758+0000","last_became_active":"2023-09-23T22:27:52.101483+0000","last_became_peered":"2023-09-23T22:27:52.101483+0000","last_unstale":"2023-09-23T22:31:13.463758+0000","last_undegraded":"2023-09-23T22:31:13.463758+0000","last_fullsized":"2023-09-23T22:31:13.463758+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:01.772629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":376,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819477+0000","last_change":"2023-09-23T22:29:01.588394+0000","last_active":"2023-09-23T22:31:10.819477+0000","last_peered":"2023-09-23T22:31:10.819477+0000","last_clean":"2023-09-23T22:31:10.819477+0000","last_became_active":"2023-09-23T22:29:01.587588+0000","last_became_peered":"2023-09-23T22:29:01.587588+0000","last_unstale":"2023-09-23T22:31:10.819477+0000","last_undegraded":"2023-09-23T22:31:10.819477+0000","last_fullsized":"2023-09-23T22:31:10.819477+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:12.141014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":310,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226017+0000","last_change":"2023-09-23T22:29:01.604689+0000","last_active":"2023-09-23T22:31:10.226017+0000","last_peered":"2023-09-23T22:31:10.226017+0000","last_clean":"2023-09-23T22:31:10.226017+0000","last_became_active":"2023-09-23T22:29:01.603739+0000","last_became_peered":"2023-09-23T22:29:01.603739+0000","last_unstale":"2023-09-23T22:31:10.226017+0000","last_undegraded":"2023-09-23T22:31:10.226017+0000","last_fullsized":"2023-09-23T22:31:10.226017+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:26.149229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":265,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463723+0000","last_change":"2023-09-23T22:29:01.575934+0000","last_active":"2023-09-23T22:31:13.463723+0000","last_peered":"2023-09-23T22:31:13.463723+0000","last_clean":"2023-09-23T22:31:13.463723+0000","last_became_active":"2023-09-23T22:29:01.575404+0000","last_became_peered":"2023-09-23T22:29:01.575404+0000","last_unstale":"2023-09-23T22:31:13.463723+0000","last_undegraded":"2023-09-23T22:31:13.463723+0000","last_fullsized":"2023-09-23T22:31:13.463723+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:38.753379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":68,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820275+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:31:10.820275+0000","last_peered":"2023-09-23T22:31:10.820275+0000","last_clean":"2023-09-23T22:31:10.820275+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:31:10.820275+0000","last_undegraded":"2023-09-23T22:31:10.820275+0000","last_fullsized":"2023-09-23T22:31:10.820275+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820940+0000","last_change":"2023-09-23T22:29:01.583368+0000","last_active":"2023-09-23T22:31:10.820940+0000","last_peered":"2023-09-23T22:31:10.820940+0000","last_clean":"2023-09-23T22:31:10.820940+0000","last_became_active":"2023-09-23T22:29:01.582967+0000","last_became_peered":"2023-09-23T22:29:01.582967+0000","last_unstale":"2023-09-23T22:31:10.820940+0000","last_undegraded":"2023-09-23T22:31:10.820940+0000","last_fullsized":"2023-09-23T22:31:10.820940+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:48.275429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819388+0000","last_change":"2023-09-23T22:31:08.226781+0000","last_active":"2023-09-23T22:31:10.819388+0000","last_peered":"2023-09-23T22:31:10.819388+0000","last_clean":"2023-09-23T22:31:10.819388+0000","last_became_active":"2023-09-23T22:31:08.226511+0000","last_became_peered":"2023-09-23T22:31:08.226511+0000","last_unstale":"2023-09-23T22:31:10.819388+0000","last_undegraded":"2023-09-23T22:31:10.819388+0000","last_fullsized":"2023-09-23T22:31:10.819388+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:03.735360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":372,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226513+0000","last_change":"2023-09-23T22:22:44.335630+0000","last_active":"2023-09-23T22:31:10.226513+0000","last_peered":"2023-09-23T22:31:10.226513+0000","last_clean":"2023-09-23T22:31:10.226513+0000","last_became_active":"2023-09-23T22:22:44.321839+0000","last_became_peered":"2023-09-23T22:22:44.321839+0000","last_unstale":"2023-09-23T22:31:10.226513+0000","last_undegraded":"2023-09-23T22:31:10.226513+0000","last_fullsized":"2023-09-23T22:31:10.226513+0000","mapping_epoch":239,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":240,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:48.640123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":132,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":369,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819025+0000","last_change":"2023-09-23T22:29:01.597976+0000","last_active":"2023-09-23T22:31:10.819025+0000","last_peered":"2023-09-23T22:31:10.819025+0000","last_clean":"2023-09-23T22:31:10.819025+0000","last_became_active":"2023-09-23T22:29:01.597479+0000","last_became_peered":"2023-09-23T22:29:01.597479+0000","last_unstale":"2023-09-23T22:31:10.819025+0000","last_undegraded":"2023-09-23T22:31:10.819025+0000","last_fullsized":"2023-09-23T22:31:10.819025+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:55.990001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820769+0000","last_change":"2023-09-23T22:31:08.224376+0000","last_active":"2023-09-23T22:31:10.820769+0000","last_peered":"2023-09-23T22:31:10.820769+0000","last_clean":"2023-09-23T22:31:10.820769+0000","last_became_active":"2023-09-23T22:31:08.224102+0000","last_became_peered":"2023-09-23T22:31:08.224102+0000","last_unstale":"2023-09-23T22:31:10.820769+0000","last_undegraded":"2023-09-23T22:31:10.820769+0000","last_fullsized":"2023-09-23T22:31:10.820769+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:46.647034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":314,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226488+0000","last_change":"2023-09-23T22:27:52.154449+0000","last_active":"2023-09-23T22:31:10.226488+0000","last_peered":"2023-09-23T22:31:10.226488+0000","last_clean":"2023-09-23T22:31:10.226488+0000","last_became_active":"2023-09-23T22:27:52.096263+0000","last_became_peered":"2023-09-23T22:27:52.096263+0000","last_unstale":"2023-09-23T22:31:10.226488+0000","last_undegraded":"2023-09-23T22:31:10.226488+0000","last_fullsized":"2023-09-23T22:31:10.226488+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:02.582951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":404,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.225959+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:31:10.225959+0000","last_peered":"2023-09-23T22:31:10.225959+0000","last_clean":"2023-09-23T22:31:10.225959+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:31:10.225959+0000","last_undegraded":"2023-09-23T22:31:10.225959+0000","last_fullsized":"2023-09-23T22:31:10.225959+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"378'693","reported_seq":707,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:31:19.524388+0000","last_change":"2023-09-23T22:31:08.221475+0000","last_active":"2023-09-23T22:31:19.524388+0000","last_peered":"2023-09-23T22:31:19.524388+0000","last_clean":"2023-09-23T22:31:19.524388+0000","last_became_active":"2023-09-23T22:31:08.221128+0000","last_became_peered":"2023-09-23T22:31:08.221128+0000","last_unstale":"2023-09-23T22:31:19.524388+0000","last_undegraded":"2023-09-23T22:31:19.524388+0000","last_fullsized":"2023-09-23T22:31:19.524388+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":693,"log_dups_size":0,"ondisk_log_size":693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:47.211013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45416448,"num_objects":693,"num_object_clones":0,"num_object_copies":2079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":693,"num_write_kb":44352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":330,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819541+0000","last_change":"2023-09-23T22:29:01.633048+0000","last_active":"2023-09-23T22:31:10.819541+0000","last_peered":"2023-09-23T22:31:10.819541+0000","last_clean":"2023-09-23T22:31:10.819541+0000","last_became_active":"2023-09-23T22:29:01.587680+0000","last_became_peered":"2023-09-23T22:29:01.587680+0000","last_unstale":"2023-09-23T22:31:10.819541+0000","last_undegraded":"2023-09-23T22:31:10.819541+0000","last_fullsized":"2023-09-23T22:31:10.819541+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.830912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820747+0000","last_change":"2023-09-23T22:29:01.593596+0000","last_active":"2023-09-23T22:31:10.820747+0000","last_peered":"2023-09-23T22:31:10.820747+0000","last_clean":"2023-09-23T22:31:10.820747+0000","last_became_active":"2023-09-23T22:29:01.593303+0000","last_became_peered":"2023-09-23T22:29:01.593303+0000","last_unstale":"2023-09-23T22:31:10.820747+0000","last_undegraded":"2023-09-23T22:31:10.820747+0000","last_fullsized":"2023-09-23T22:31:10.820747+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:02.980860+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820806+0000","last_change":"2023-09-23T22:31:08.225591+0000","last_active":"2023-09-23T22:31:10.820806+0000","last_peered":"2023-09-23T22:31:10.820806+0000","last_clean":"2023-09-23T22:31:10.820806+0000","last_became_active":"2023-09-23T22:31:08.225286+0000","last_became_peered":"2023-09-23T22:31:08.225286+0000","last_unstale":"2023-09-23T22:31:10.820806+0000","last_undegraded":"2023-09-23T22:31:10.820806+0000","last_fullsized":"2023-09-23T22:31:10.820806+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:14.895634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":407,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819621+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:31:10.819621+0000","last_peered":"2023-09-23T22:31:10.819621+0000","last_clean":"2023-09-23T22:31:10.819621+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:31:10.819621+0000","last_undegraded":"2023-09-23T22:31:10.819621+0000","last_fullsized":"2023-09-23T22:31:10.819621+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":263,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463422+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:31:13.463422+0000","last_peered":"2023-09-23T22:31:13.463422+0000","last_clean":"2023-09-23T22:31:13.463422+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:31:13.463422+0000","last_undegraded":"2023-09-23T22:31:13.463422+0000","last_fullsized":"2023-09-23T22:31:13.463422+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819002+0000","last_change":"2023-09-23T22:31:08.226199+0000","last_active":"2023-09-23T22:31:10.819002+0000","last_peered":"2023-09-23T22:31:10.819002+0000","last_clean":"2023-09-23T22:31:10.819002+0000","last_became_active":"2023-09-23T22:31:08.225930+0000","last_became_peered":"2023-09-23T22:31:08.225930+0000","last_unstale":"2023-09-23T22:31:10.819002+0000","last_undegraded":"2023-09-23T22:31:10.819002+0000","last_fullsized":"2023-09-23T22:31:10.819002+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:16.294443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":702,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819103+0000","last_change":"2023-09-23T22:29:04.057613+0000","last_active":"2023-09-23T22:31:10.819103+0000","last_peered":"2023-09-23T22:31:10.819103+0000","last_clean":"2023-09-23T22:31:10.819103+0000","last_became_active":"2023-09-23T22:29:04.057082+0000","last_became_peered":"2023-09-23T22:29:04.057082+0000","last_unstale":"2023-09-23T22:31:10.819103+0000","last_undegraded":"2023-09-23T22:31:10.819103+0000","last_fullsized":"2023-09-23T22:31:10.819103+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:24.948720+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":209,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463701+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:31:13.463701+0000","last_peered":"2023-09-23T22:31:13.463701+0000","last_clean":"2023-09-23T22:31:13.463701+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:31:13.463701+0000","last_undegraded":"2023-09-23T22:31:13.463701+0000","last_fullsized":"2023-09-23T22:31:13.463701+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":385,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226116+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:31:10.226116+0000","last_peered":"2023-09-23T22:31:10.226116+0000","last_clean":"2023-09-23T22:31:10.226116+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:31:10.226116+0000","last_undegraded":"2023-09-23T22:31:10.226116+0000","last_fullsized":"2023-09-23T22:31:10.226116+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"378'600","reported_seq":614,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:31:19.091459+0000","last_change":"2023-09-23T22:31:08.227045+0000","last_active":"2023-09-23T22:31:19.091459+0000","last_peered":"2023-09-23T22:31:19.091459+0000","last_clean":"2023-09-23T22:31:19.091459+0000","last_became_active":"2023-09-23T22:31:08.226758+0000","last_became_peered":"2023-09-23T22:31:08.226758+0000","last_unstale":"2023-09-23T22:31:19.091459+0000","last_undegraded":"2023-09-23T22:31:19.091459+0000","last_fullsized":"2023-09-23T22:31:19.091459+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":600,"log_dups_size":0,"ondisk_log_size":600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:55.118636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39321600,"num_objects":600,"num_object_clones":0,"num_object_copies":1800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":600,"num_write_kb":38400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":758,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819111+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:31:10.819111+0000","last_peered":"2023-09-23T22:31:10.819111+0000","last_clean":"2023-09-23T22:31:10.819111+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:31:10.819111+0000","last_undegraded":"2023-09-23T22:31:10.819111+0000","last_fullsized":"2023-09-23T22:31:10.819111+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820644+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:31:10.820644+0000","last_peered":"2023-09-23T22:31:10.820644+0000","last_clean":"2023-09-23T22:31:10.820644+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:31:10.820644+0000","last_undegraded":"2023-09-23T22:31:10.820644+0000","last_fullsized":"2023-09-23T22:31:10.820644+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":412,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226223+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:31:10.226223+0000","last_peered":"2023-09-23T22:31:10.226223+0000","last_clean":"2023-09-23T22:31:10.226223+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:31:10.226223+0000","last_undegraded":"2023-09-23T22:31:10.226223+0000","last_fullsized":"2023-09-23T22:31:10.226223+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.225803+0000","last_change":"2023-09-23T22:31:08.225049+0000","last_active":"2023-09-23T22:31:10.225803+0000","last_peered":"2023-09-23T22:31:10.225803+0000","last_clean":"2023-09-23T22:31:10.225803+0000","last_became_active":"2023-09-23T22:31:08.224765+0000","last_became_peered":"2023-09-23T22:31:08.224765+0000","last_unstale":"2023-09-23T22:31:10.225803+0000","last_undegraded":"2023-09-23T22:31:10.225803+0000","last_fullsized":"2023-09-23T22:31:10.225803+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.842171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":358,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.225862+0000","last_change":"2023-09-23T22:28:05.062902+0000","last_active":"2023-09-23T22:31:10.225862+0000","last_peered":"2023-09-23T22:31:10.225862+0000","last_clean":"2023-09-23T22:31:10.225862+0000","last_became_active":"2023-09-23T22:28:05.062607+0000","last_became_peered":"2023-09-23T22:28:05.062607+0000","last_unstale":"2023-09-23T22:31:10.225862+0000","last_undegraded":"2023-09-23T22:31:10.225862+0000","last_fullsized":"2023-09-23T22:31:10.225862+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:21.888780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":393,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226448+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:31:10.226448+0000","last_peered":"2023-09-23T22:31:10.226448+0000","last_clean":"2023-09-23T22:31:10.226448+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:31:10.226448+0000","last_undegraded":"2023-09-23T22:31:10.226448+0000","last_fullsized":"2023-09-23T22:31:10.226448+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"378'627","reported_seq":640,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:31:19.110474+0000","last_change":"2023-09-23T22:31:08.226959+0000","last_active":"2023-09-23T22:31:19.110474+0000","last_peered":"2023-09-23T22:31:19.110474+0000","last_clean":"2023-09-23T22:31:19.110474+0000","last_became_active":"2023-09-23T22:31:08.226652+0000","last_became_peered":"2023-09-23T22:31:08.226652+0000","last_unstale":"2023-09-23T22:31:19.110474+0000","last_undegraded":"2023-09-23T22:31:19.110474+0000","last_fullsized":"2023-09-23T22:31:19.110474+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":627,"log_dups_size":0,"ondisk_log_size":627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:22.758103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41091072,"num_objects":627,"num_object_clones":0,"num_object_copies":1881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":627,"num_write_kb":40128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":731,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819188+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:31:10.819188+0000","last_peered":"2023-09-23T22:31:10.819188+0000","last_clean":"2023-09-23T22:31:10.819188+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:31:10.819188+0000","last_undegraded":"2023-09-23T22:31:10.819188+0000","last_fullsized":"2023-09-23T22:31:10.819188+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":85,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680323+0000","last_change":"2023-09-23T22:27:52.106215+0000","last_active":"2023-09-23T22:31:13.680323+0000","last_peered":"2023-09-23T22:31:13.680323+0000","last_clean":"2023-09-23T22:31:13.680323+0000","last_became_active":"2023-09-23T22:27:52.104315+0000","last_became_peered":"2023-09-23T22:27:52.104315+0000","last_unstale":"2023-09-23T22:31:13.680323+0000","last_undegraded":"2023-09-23T22:31:13.680323+0000","last_fullsized":"2023-09-23T22:31:13.680323+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:44.360992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":85,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680910+0000","last_change":"2023-09-23T22:27:52.096512+0000","last_active":"2023-09-23T22:31:13.680910+0000","last_peered":"2023-09-23T22:31:13.680910+0000","last_clean":"2023-09-23T22:31:13.680910+0000","last_became_active":"2023-09-23T22:27:52.096242+0000","last_became_peered":"2023-09-23T22:27:52.096242+0000","last_unstale":"2023-09-23T22:31:13.680910+0000","last_undegraded":"2023-09-23T22:31:13.680910+0000","last_fullsized":"2023-09-23T22:31:13.680910+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:12.364350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"378'680","reported_seq":694,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:31:19.509595+0000","last_change":"2023-09-23T22:31:08.222016+0000","last_active":"2023-09-23T22:31:19.509595+0000","last_peered":"2023-09-23T22:31:19.509595+0000","last_clean":"2023-09-23T22:31:19.509595+0000","last_became_active":"2023-09-23T22:31:08.221777+0000","last_became_peered":"2023-09-23T22:31:08.221777+0000","last_unstale":"2023-09-23T22:31:19.509595+0000","last_undegraded":"2023-09-23T22:31:19.509595+0000","last_fullsized":"2023-09-23T22:31:19.509595+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":680,"log_dups_size":0,"ondisk_log_size":680,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:42.779843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44564480,"num_objects":680,"num_object_clones":0,"num_object_copies":2040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":680,"num_write_kb":43520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":622,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820994+0000","last_change":"2023-09-23T22:29:04.057506+0000","last_active":"2023-09-23T22:31:10.820994+0000","last_peered":"2023-09-23T22:31:10.820994+0000","last_clean":"2023-09-23T22:31:10.820994+0000","last_became_active":"2023-09-23T22:29:04.057207+0000","last_became_peered":"2023-09-23T22:29:04.057207+0000","last_unstale":"2023-09-23T22:31:10.820994+0000","last_undegraded":"2023-09-23T22:31:10.820994+0000","last_fullsized":"2023-09-23T22:31:10.820994+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:47.781486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:47.781486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":704,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.462937+0000","last_change":"2023-09-23T22:28:41.357575+0000","last_active":"2023-09-23T22:31:13.462937+0000","last_peered":"2023-09-23T22:31:13.462937+0000","last_clean":"2023-09-23T22:31:13.462937+0000","last_became_active":"2023-09-23T22:28:41.356971+0000","last_became_peered":"2023-09-23T22:28:41.356971+0000","last_unstale":"2023-09-23T22:31:13.462937+0000","last_undegraded":"2023-09-23T22:31:13.462937+0000","last_fullsized":"2023-09-23T22:31:13.462937+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:59.810917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":434,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819377+0000","last_change":"2023-09-23T22:29:01.608081+0000","last_active":"2023-09-23T22:31:10.819377+0000","last_peered":"2023-09-23T22:31:10.819377+0000","last_clean":"2023-09-23T22:31:10.819377+0000","last_became_active":"2023-09-23T22:29:01.607726+0000","last_became_peered":"2023-09-23T22:29:01.607726+0000","last_unstale":"2023-09-23T22:31:10.819377+0000","last_undegraded":"2023-09-23T22:31:10.819377+0000","last_fullsized":"2023-09-23T22:31:10.819377+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:30.179258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":85,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680518+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:31:13.680518+0000","last_peered":"2023-09-23T22:31:13.680518+0000","last_clean":"2023-09-23T22:31:13.680518+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:31:13.680518+0000","last_undegraded":"2023-09-23T22:31:13.680518+0000","last_fullsized":"2023-09-23T22:31:13.680518+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.818941+0000","last_change":"2023-09-23T22:31:08.226112+0000","last_active":"2023-09-23T22:31:10.818941+0000","last_peered":"2023-09-23T22:31:10.818941+0000","last_clean":"2023-09-23T22:31:10.818941+0000","last_became_active":"2023-09-23T22:31:08.225844+0000","last_became_peered":"2023-09-23T22:31:08.225844+0000","last_unstale":"2023-09-23T22:31:10.818941+0000","last_undegraded":"2023-09-23T22:31:10.818941+0000","last_fullsized":"2023-09-23T22:31:10.818941+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:29.797934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":721,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819281+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:31:10.819281+0000","last_peered":"2023-09-23T22:31:10.819281+0000","last_clean":"2023-09-23T22:31:10.819281+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:31:10.819281+0000","last_undegraded":"2023-09-23T22:31:10.819281+0000","last_fullsized":"2023-09-23T22:31:10.819281+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1255,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819283+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:31:10.819283+0000","last_peered":"2023-09-23T22:31:10.819283+0000","last_clean":"2023-09-23T22:31:10.819283+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:31:10.819283+0000","last_undegraded":"2023-09-23T22:31:10.819283+0000","last_fullsized":"2023-09-23T22:31:10.819283+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":416,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226018+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:31:10.226018+0000","last_peered":"2023-09-23T22:31:10.226018+0000","last_clean":"2023-09-23T22:31:10.226018+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:31:10.226018+0000","last_undegraded":"2023-09-23T22:31:10.226018+0000","last_fullsized":"2023-09-23T22:31:10.226018+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820721+0000","last_change":"2023-09-23T22:29:01.593644+0000","last_active":"2023-09-23T22:31:10.820721+0000","last_peered":"2023-09-23T22:31:10.820721+0000","last_clean":"2023-09-23T22:31:10.820721+0000","last_became_active":"2023-09-23T22:29:01.593326+0000","last_became_peered":"2023-09-23T22:29:01.593326+0000","last_unstale":"2023-09-23T22:31:10.820721+0000","last_undegraded":"2023-09-23T22:31:10.820721+0000","last_fullsized":"2023-09-23T22:31:10.820721+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:52.921796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.225809+0000","last_change":"2023-09-23T22:31:08.222523+0000","last_active":"2023-09-23T22:31:10.225809+0000","last_peered":"2023-09-23T22:31:10.225809+0000","last_clean":"2023-09-23T22:31:10.225809+0000","last_became_active":"2023-09-23T22:31:08.222276+0000","last_became_peered":"2023-09-23T22:31:08.222276+0000","last_unstale":"2023-09-23T22:31:10.225809+0000","last_undegraded":"2023-09-23T22:31:10.225809+0000","last_fullsized":"2023-09-23T22:31:10.225809+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:15.583891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":710,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463763+0000","last_change":"2023-09-23T22:22:47.352720+0000","last_active":"2023-09-23T22:31:13.463763+0000","last_peered":"2023-09-23T22:31:13.463763+0000","last_clean":"2023-09-23T22:31:13.463763+0000","last_became_active":"2023-09-23T22:22:47.352341+0000","last_became_peered":"2023-09-23T22:22:47.352341+0000","last_unstale":"2023-09-23T22:31:13.463763+0000","last_undegraded":"2023-09-23T22:31:13.463763+0000","last_fullsized":"2023-09-23T22:31:13.463763+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:27.937907+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":594,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463160+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:31:13.463160+0000","last_peered":"2023-09-23T22:31:13.463160+0000","last_clean":"2023-09-23T22:31:13.463160+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:31:13.463160+0000","last_undegraded":"2023-09-23T22:31:13.463160+0000","last_fullsized":"2023-09-23T22:31:13.463160+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820366+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:31:10.820366+0000","last_peered":"2023-09-23T22:31:10.820366+0000","last_clean":"2023-09-23T22:31:10.820366+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:31:10.820366+0000","last_undegraded":"2023-09-23T22:31:10.820366+0000","last_fullsized":"2023-09-23T22:31:10.820366+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":26,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680760+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:31:13.680760+0000","last_peered":"2023-09-23T22:31:13.680760+0000","last_clean":"2023-09-23T22:31:13.680760+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:31:13.680760+0000","last_undegraded":"2023-09-23T22:31:13.680760+0000","last_fullsized":"2023-09-23T22:31:13.680760+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820297+0000","last_change":"2023-09-23T22:31:08.227185+0000","last_active":"2023-09-23T22:31:10.820297+0000","last_peered":"2023-09-23T22:31:10.820297+0000","last_clean":"2023-09-23T22:31:10.820297+0000","last_became_active":"2023-09-23T22:31:08.226918+0000","last_became_peered":"2023-09-23T22:31:08.226918+0000","last_unstale":"2023-09-23T22:31:10.820297+0000","last_undegraded":"2023-09-23T22:31:10.820297+0000","last_fullsized":"2023-09-23T22:31:10.820297+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:54.257936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1097,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819315+0000","last_change":"2023-09-23T22:31:00.353818+0000","last_active":"2023-09-23T22:31:10.819315+0000","last_peered":"2023-09-23T22:31:10.819315+0000","last_clean":"2023-09-23T22:31:10.819315+0000","last_became_active":"2023-09-23T22:29:50.198096+0000","last_became_peered":"2023-09-23T22:29:50.198096+0000","last_unstale":"2023-09-23T22:31:10.819315+0000","last_undegraded":"2023-09-23T22:31:10.819315+0000","last_fullsized":"2023-09-23T22:31:10.819315+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:48.105235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":745,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:14.526917+0000","last_change":"2023-09-23T22:29:04.055480+0000","last_active":"2023-09-23T22:31:14.526917+0000","last_peered":"2023-09-23T22:31:14.526917+0000","last_clean":"2023-09-23T22:31:14.526917+0000","last_became_active":"2023-09-23T22:29:04.055134+0000","last_became_peered":"2023-09-23T22:29:04.055134+0000","last_unstale":"2023-09-23T22:31:14.526917+0000","last_undegraded":"2023-09-23T22:31:14.526917+0000","last_fullsized":"2023-09-23T22:31:14.526917+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:59.018639+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,6],"acting":[0,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":258,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819241+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:31:10.819241+0000","last_peered":"2023-09-23T22:31:10.819241+0000","last_clean":"2023-09-23T22:31:10.819241+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:31:10.819241+0000","last_undegraded":"2023-09-23T22:31:10.819241+0000","last_fullsized":"2023-09-23T22:31:10.819241+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820903+0000","last_change":"2023-09-23T22:29:01.594509+0000","last_active":"2023-09-23T22:31:10.820903+0000","last_peered":"2023-09-23T22:31:10.820903+0000","last_clean":"2023-09-23T22:31:10.820903+0000","last_became_active":"2023-09-23T22:29:01.594249+0000","last_became_peered":"2023-09-23T22:29:01.594249+0000","last_unstale":"2023-09-23T22:31:10.820903+0000","last_undegraded":"2023-09-23T22:31:10.820903+0000","last_fullsized":"2023-09-23T22:31:10.820903+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:14.972304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"378'590","reported_seq":604,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:31:19.104967+0000","last_change":"2023-09-23T22:31:08.223664+0000","last_active":"2023-09-23T22:31:19.104967+0000","last_peered":"2023-09-23T22:31:19.104967+0000","last_clean":"2023-09-23T22:31:19.104967+0000","last_became_active":"2023-09-23T22:31:08.223323+0000","last_became_peered":"2023-09-23T22:31:08.223323+0000","last_unstale":"2023-09-23T22:31:19.104967+0000","last_undegraded":"2023-09-23T22:31:19.104967+0000","last_fullsized":"2023-09-23T22:31:19.104967+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":590,"log_dups_size":0,"ondisk_log_size":590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:38.536861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38666240,"num_objects":590,"num_object_clones":0,"num_object_copies":1770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":590,"num_write_kb":37760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":599,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820714+0000","last_change":"2023-09-23T22:29:04.055972+0000","last_active":"2023-09-23T22:31:10.820714+0000","last_peered":"2023-09-23T22:31:10.820714+0000","last_clean":"2023-09-23T22:31:10.820714+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:31:10.820714+0000","last_undegraded":"2023-09-23T22:31:10.820714+0000","last_fullsized":"2023-09-23T22:31:10.820714+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.683826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.683826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":728,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680357+0000","last_change":"2023-09-23T22:27:52.122905+0000","last_active":"2023-09-23T22:31:13.680357+0000","last_peered":"2023-09-23T22:31:13.680357+0000","last_clean":"2023-09-23T22:31:13.680357+0000","last_became_active":"2023-09-23T22:27:52.116583+0000","last_became_peered":"2023-09-23T22:27:52.116583+0000","last_unstale":"2023-09-23T22:31:13.680357+0000","last_undegraded":"2023-09-23T22:31:13.680357+0000","last_fullsized":"2023-09-23T22:31:13.680357+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:21.421519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":272,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463207+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:31:13.463207+0000","last_peered":"2023-09-23T22:31:13.463207+0000","last_clean":"2023-09-23T22:31:13.463207+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:31:13.463207+0000","last_undegraded":"2023-09-23T22:31:13.463207+0000","last_fullsized":"2023-09-23T22:31:13.463207+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":256,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463676+0000","last_change":"2023-09-23T22:28:05.169578+0000","last_active":"2023-09-23T22:31:13.463676+0000","last_peered":"2023-09-23T22:31:13.463676+0000","last_clean":"2023-09-23T22:31:13.463676+0000","last_became_active":"2023-09-23T22:28:05.169263+0000","last_became_peered":"2023-09-23T22:28:05.169263+0000","last_unstale":"2023-09-23T22:31:13.463676+0000","last_undegraded":"2023-09-23T22:31:13.463676+0000","last_fullsized":"2023-09-23T22:31:13.463676+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:14:39.446410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"378'584","reported_seq":598,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:31:19.092736+0000","last_change":"2023-09-23T22:31:08.224259+0000","last_active":"2023-09-23T22:31:19.092736+0000","last_peered":"2023-09-23T22:31:19.092736+0000","last_clean":"2023-09-23T22:31:19.092736+0000","last_became_active":"2023-09-23T22:31:08.224062+0000","last_became_peered":"2023-09-23T22:31:08.224062+0000","last_unstale":"2023-09-23T22:31:19.092736+0000","last_undegraded":"2023-09-23T22:31:19.092736+0000","last_fullsized":"2023-09-23T22:31:19.092736+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":584,"log_dups_size":0,"ondisk_log_size":584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:18.330753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38273024,"num_objects":584,"num_object_clones":0,"num_object_copies":1752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":584,"num_write_kb":37376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":727,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820719+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:31:10.820719+0000","last_peered":"2023-09-23T22:31:10.820719+0000","last_clean":"2023-09-23T22:31:10.820719+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:31:10.820719+0000","last_undegraded":"2023-09-23T22:31:10.820719+0000","last_fullsized":"2023-09-23T22:31:10.820719+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":272,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463480+0000","last_change":"2023-09-23T22:18:57.718538+0000","last_active":"2023-09-23T22:31:13.463480+0000","last_peered":"2023-09-23T22:31:13.463480+0000","last_clean":"2023-09-23T22:31:13.463480+0000","last_became_active":"2023-09-23T22:18:57.718132+0000","last_became_peered":"2023-09-23T22:18:57.718132+0000","last_unstale":"2023-09-23T22:31:13.463480+0000","last_undegraded":"2023-09-23T22:31:13.463480+0000","last_fullsized":"2023-09-23T22:31:13.463480+0000","mapping_epoch":182,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":183,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:59:24.390768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":303,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819702+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:31:10.819702+0000","last_peered":"2023-09-23T22:31:10.819702+0000","last_clean":"2023-09-23T22:31:10.819702+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:31:10.819702+0000","last_undegraded":"2023-09-23T22:31:10.819702+0000","last_fullsized":"2023-09-23T22:31:10.819702+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":236,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463478+0000","last_change":"2023-09-23T22:17:15.725207+0000","last_active":"2023-09-23T22:31:13.463478+0000","last_peered":"2023-09-23T22:31:13.463478+0000","last_clean":"2023-09-23T22:31:13.463478+0000","last_became_active":"2023-09-23T22:17:15.724746+0000","last_became_peered":"2023-09-23T22:17:15.724746+0000","last_unstale":"2023-09-23T22:31:13.463478+0000","last_undegraded":"2023-09-23T22:31:13.463478+0000","last_fullsized":"2023-09-23T22:31:13.463478+0000","mapping_epoch":158,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":159,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:10.044242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"378'694","reported_seq":708,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T22:31:19.516658+0000","last_change":"2023-09-23T22:31:08.223746+0000","last_active":"2023-09-23T22:31:19.516658+0000","last_peered":"2023-09-23T22:31:19.516658+0000","last_clean":"2023-09-23T22:31:19.516658+0000","last_became_active":"2023-09-23T22:31:08.223473+0000","last_became_peered":"2023-09-23T22:31:08.223473+0000","last_unstale":"2023-09-23T22:31:19.516658+0000","last_undegraded":"2023-09-23T22:31:19.516658+0000","last_fullsized":"2023-09-23T22:31:19.516658+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":694,"log_dups_size":0,"ondisk_log_size":694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:50:35.931090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45481984,"num_objects":694,"num_object_clones":0,"num_object_copies":2082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":694,"num_write_kb":44416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":731,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226106+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:31:10.226106+0000","last_peered":"2023-09-23T22:31:10.226106+0000","last_clean":"2023-09-23T22:31:10.226106+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:31:10.226106+0000","last_undegraded":"2023-09-23T22:31:10.226106+0000","last_fullsized":"2023-09-23T22:31:10.226106+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":196,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463548+0000","last_change":"2023-09-23T22:27:27.333140+0000","last_active":"2023-09-23T22:31:13.463548+0000","last_peered":"2023-09-23T22:31:13.463548+0000","last_clean":"2023-09-23T22:31:13.463548+0000","last_became_active":"2023-09-23T22:27:27.287527+0000","last_became_peered":"2023-09-23T22:27:27.287527+0000","last_unstale":"2023-09-23T22:31:13.463548+0000","last_undegraded":"2023-09-23T22:31:13.463548+0000","last_fullsized":"2023-09-23T22:31:13.463548+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:51.648586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":873,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680886+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:31:13.680886+0000","last_peered":"2023-09-23T22:31:13.680886+0000","last_clean":"2023-09-23T22:31:13.680886+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:31:13.680886+0000","last_undegraded":"2023-09-23T22:31:13.680886+0000","last_fullsized":"2023-09-23T22:31:13.680886+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":308,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463095+0000","last_change":"2023-09-23T22:27:52.136692+0000","last_active":"2023-09-23T22:31:13.463095+0000","last_peered":"2023-09-23T22:31:13.463095+0000","last_clean":"2023-09-23T22:31:13.463095+0000","last_became_active":"2023-09-23T22:27:52.136397+0000","last_became_peered":"2023-09-23T22:27:52.136397+0000","last_unstale":"2023-09-23T22:31:13.463095+0000","last_undegraded":"2023-09-23T22:31:13.463095+0000","last_fullsized":"2023-09-23T22:31:13.463095+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:23.036231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":12,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819356+0000","last_change":"2023-09-23T22:31:08.228309+0000","last_active":"2023-09-23T22:31:10.819356+0000","last_peered":"2023-09-23T22:31:10.819356+0000","last_clean":"2023-09-23T22:31:10.819356+0000","last_became_active":"2023-09-23T22:31:08.227973+0000","last_became_peered":"2023-09-23T22:31:08.227973+0000","last_unstale":"2023-09-23T22:31:10.819356+0000","last_undegraded":"2023-09-23T22:31:10.819356+0000","last_fullsized":"2023-09-23T22:31:10.819356+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:44.717425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":623,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819781+0000","last_change":"2023-09-23T22:29:51.194726+0000","last_active":"2023-09-23T22:31:10.819781+0000","last_peered":"2023-09-23T22:31:10.819781+0000","last_clean":"2023-09-23T22:31:10.819781+0000","last_became_active":"2023-09-23T22:29:51.194253+0000","last_became_peered":"2023-09-23T22:29:51.194253+0000","last_unstale":"2023-09-23T22:31:10.819781+0000","last_undegraded":"2023-09-23T22:31:10.819781+0000","last_fullsized":"2023-09-23T22:31:10.819781+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:24:45.734686+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:24:45.734686+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1144050,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":68,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820910+0000","last_change":"2023-09-23T22:29:09.070063+0000","last_active":"2023-09-23T22:31:10.820910+0000","last_peered":"2023-09-23T22:31:10.820910+0000","last_clean":"2023-09-23T22:31:10.820910+0000","last_became_active":"2023-09-23T22:29:01.584585+0000","last_became_peered":"2023-09-23T22:29:01.584585+0000","last_unstale":"2023-09-23T22:31:10.820910+0000","last_undegraded":"2023-09-23T22:31:10.820910+0000","last_fullsized":"2023-09-23T22:31:10.820910+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:44.762021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":241,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463592+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:31:13.463592+0000","last_peered":"2023-09-23T22:31:13.463592+0000","last_clean":"2023-09-23T22:31:13.463592+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:31:13.463592+0000","last_undegraded":"2023-09-23T22:31:13.463592+0000","last_fullsized":"2023-09-23T22:31:13.463592+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":350,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.225863+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:31:10.225863+0000","last_peered":"2023-09-23T22:31:10.225863+0000","last_clean":"2023-09-23T22:31:10.225863+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:31:10.225863+0000","last_undegraded":"2023-09-23T22:31:10.225863+0000","last_fullsized":"2023-09-23T22:31:10.225863+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":28,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463355+0000","last_change":"2023-09-23T22:30:26.181266+0000","last_active":"2023-09-23T22:31:13.463355+0000","last_peered":"2023-09-23T22:31:13.463355+0000","last_clean":"2023-09-23T22:31:13.463355+0000","last_became_active":"2023-09-23T22:30:26.180875+0000","last_became_peered":"2023-09-23T22:30:26.180875+0000","last_unstale":"2023-09-23T22:31:13.463355+0000","last_undegraded":"2023-09-23T22:31:13.463355+0000","last_fullsized":"2023-09-23T22:31:13.463355+0000","mapping_epoch":360,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":361,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:11.776290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":128,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.680659+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:31:13.680659+0000","last_peered":"2023-09-23T22:31:13.680659+0000","last_clean":"2023-09-23T22:31:13.680659+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:31:13.680659+0000","last_undegraded":"2023-09-23T22:31:13.680659+0000","last_fullsized":"2023-09-23T22:31:13.680659+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":367,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819534+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:31:10.819534+0000","last_peered":"2023-09-23T22:31:10.819534+0000","last_clean":"2023-09-23T22:31:10.819534+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:31:10.819534+0000","last_undegraded":"2023-09-23T22:31:10.819534+0000","last_fullsized":"2023-09-23T22:31:10.819534+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":392,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226228+0000","last_change":"2023-09-23T22:28:05.114400+0000","last_active":"2023-09-23T22:31:10.226228+0000","last_peered":"2023-09-23T22:31:10.226228+0000","last_clean":"2023-09-23T22:31:10.226228+0000","last_became_active":"2023-09-23T22:28:05.114095+0000","last_became_peered":"2023-09-23T22:28:05.114095+0000","last_unstale":"2023-09-23T22:31:10.226228+0000","last_undegraded":"2023-09-23T22:31:10.226228+0000","last_fullsized":"2023-09-23T22:31:10.226228+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:23.836996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":392,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.819456+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:31:10.819456+0000","last_peered":"2023-09-23T22:31:10.819456+0000","last_clean":"2023-09-23T22:31:10.819456+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:31:10.819456+0000","last_undegraded":"2023-09-23T22:31:10.819456+0000","last_fullsized":"2023-09-23T22:31:10.819456+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":241,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.226102+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:31:10.226102+0000","last_peered":"2023-09-23T22:31:10.226102+0000","last_clean":"2023-09-23T22:31:10.226102+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:31:10.226102+0000","last_undegraded":"2023-09-23T22:31:10.226102+0000","last_fullsized":"2023-09-23T22:31:10.226102+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":58,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.820663+0000","last_change":"2023-09-23T22:29:01.593820+0000","last_active":"2023-09-23T22:31:10.820663+0000","last_peered":"2023-09-23T22:31:10.820663+0000","last_clean":"2023-09-23T22:31:10.820663+0000","last_became_active":"2023-09-23T22:29:01.593176+0000","last_became_peered":"2023-09-23T22:29:01.593176+0000","last_unstale":"2023-09-23T22:31:10.820663+0000","last_undegraded":"2023-09-23T22:31:10.820663+0000","last_fullsized":"2023-09-23T22:31:10.820663+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:33.824095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":378,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463269+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:31:13.463269+0000","last_peered":"2023-09-23T22:31:13.463269+0000","last_clean":"2023-09-23T22:31:13.463269+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:31:13.463269+0000","last_undegraded":"2023-09-23T22:31:13.463269+0000","last_fullsized":"2023-09-23T22:31:13.463269+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":315,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T22:31:13.463572+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:31:13.463572+0000","last_peered":"2023-09-23T22:31:13.463572+0000","last_clean":"2023-09-23T22:31:13.463572+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:31:13.463572+0000","last_undegraded":"2023-09-23T22:31:13.463572+0000","last_fullsized":"2023-09-23T22:31:13.463572+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":435,"reported_epoch":376,"state":"active+clean","last_fresh":"2023-09-23T22:31:10.225810+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:31:10.225810+0000","last_peered":"2023-09-23T22:31:10.225810+0000","last_clean":"2023-09-23T22:31:10.225810+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:31:10.225810+0000","last_undegraded":"2023-09-23T22:31:10.225810+0000","last_fullsized":"2023-09-23T22:31:10.225810+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":26454,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16546,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":292814848,"num_objects":4468,"num_object_clones":0,"num_object_copies":13404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4468,"num_write_kb":285952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":595197952,"data_stored":595197952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4468,"ondisk_log_size":4468,"up":48,"acting":48,"num_store_stats":6},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":423698,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":132,"num_read_kb":249,"num_write":66,"num_write_kb":660,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2254133,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":187,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234322,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":357836,"kb_used_data":36292,"kb_used_omap":0,"kb_used_meta":321536,"kb_avail":93387316,"statfs":{"total":95995035648,"available":95628611584,"internally_reserved":0,"allocated":37163008,"data_stored":35745533,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":329252864},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:13 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":6.1479999999999997,"15min":4.1870000000000003},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":4.0519999999999996,"5min":220.66,"15min":220.66},"last":0.878},{"interface":"front","average":{"1min":1.5049999999999999,"5min":6.3280000000000003,"15min":4.5540000000000003},"min":{"1min":0.29099999999999998,"5min":0.246,"15min":0.217},"max":{"1min":5.6799999999999997,"5min":220.59899999999999,"15min":220.59899999999999},"last":0.94499999999999995}]},{"osd":1,"last update":"Sat Sep 23 22:30:29 2023","interfaces":[{"interface":"back","average":{"1min":4.9500000000000002,"5min":4.9409999999999998,"15min":4.5590000000000002},"min":{"1min":0.27200000000000002,"5min":0.25,"15min":0.23799999999999999},"max":{"1min":76.984999999999999,"5min":144.77500000000001,"15min":144.77500000000001},"last":0.86299999999999999},{"interface":"front","average":{"1min":5.2080000000000002,"5min":5.2359999999999998,"15min":4.8239999999999998},"min":{"1min":0.28299999999999997,"5min":0.28100000000000003,"15min":0.24299999999999999},"max":{"1min":79.069999999999993,"5min":144.751,"15min":144.751},"last":1.018}]},{"osd":2,"last update":"Sat Sep 23 22:30:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":1.2210000000000001,"15min":1.2210000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7279999999999998,"5min":5.7279999999999998,"15min":5.7279999999999998},"last":0.83399999999999996},{"interface":"front","average":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":3.0529999999999999,"5min":3.0529999999999999,"15min":3.0529999999999999},"last":0.93200000000000005}]},{"osd":3,"last update":"Sat Sep 23 22:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":4.5060000000000002,"15min":3.6850000000000001},"min":{"1min":0.29399999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":3.4260000000000002,"5min":219.90700000000001,"15min":219.90700000000001},"last":0.90700000000000003},{"interface":"front","average":{"1min":2.1059999999999999,"5min":5.1550000000000002,"15min":3.944},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.234,"5min":219.976,"15min":219.976},"last":1.05}]},{"osd":4,"last update":"Sat Sep 23 22:31:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":0.69699999999999995,"15min":1.4319999999999999},"min":{"1min":0.27900000000000003,"5min":0.24299999999999999,"15min":0.23499999999999999},"max":{"1min":8.0180000000000007,"5min":8.0180000000000007,"15min":88.543000000000006},"last":0.88800000000000001},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.64500000000000002,"15min":1.349},"min":{"1min":0.27700000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.7649999999999997,"5min":5.7649999999999997,"15min":88.495999999999995},"last":0.73399999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.57199999999999995,"15min":1.3080000000000001},"min":{"1min":0.31,"5min":0.249,"15min":0.24299999999999999},"max":{"1min":2.4750000000000001,"5min":2.4750000000000001,"15min":93.816000000000003},"last":0.748},{"interface":"front","average":{"1min":1.5840000000000001,"5min":0.82699999999999996,"15min":1.6220000000000001},"min":{"1min":0.39100000000000001,"5min":0.309,"15min":0.27300000000000002},"max":{"1min":8.0459999999999994,"5min":8.0459999999999994,"15min":95.579999999999998},"last":0.79900000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:31:02 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.65600000000000003,"15min":0.78400000000000003},"min":{"1min":0.33400000000000002,"5min":0.29499999999999998,"15min":0.28999999999999998},"max":{"1min":6.0880000000000001,"5min":6.0880000000000001,"15min":9.0139999999999993},"last":0.78900000000000003},{"interface":"front","average":{"1min":1.224,"5min":0.74199999999999999,"15min":1.028},"min":{"1min":0.36399999999999999,"5min":0.33700000000000002,"15min":0.32800000000000001},"max":{"1min":5.109,"5min":5.109,"15min":13.362},"last":0.98599999999999999}]}]},{"osd":6,"up_from":46,"seq":197568495958,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":130452,"kb_used_data":8844,"kb_used_omap":0,"kb_used_meta":121600,"kb_avail":93614700,"statfs":{"total":95995035648,"available":95861452800,"internally_reserved":0,"allocated":9056256,"data_stored":7631917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":124518400},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:17 2023","interfaces":[{"interface":"back","average":{"1min":5.298,"5min":2.2309999999999999,"15min":2.1160000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":55.206000000000003,"5min":55.206000000000003,"15min":108.773},"last":0.64300000000000002},{"interface":"front","average":{"1min":6.0060000000000002,"5min":2.6219999999999999,"15min":3.1429999999999998},"min":{"1min":0.251,"5min":0.22600000000000001,"15min":0.21099999999999999},"max":{"1min":55.375999999999998,"5min":55.375999999999998,"15min":135.45599999999999},"last":0.58099999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":2.1030000000000002,"15min":3.6320000000000001},"min":{"1min":0.27800000000000002,"5min":0.26000000000000001,"15min":0.22800000000000001},"max":{"1min":6.8380000000000001,"5min":80.430000000000007,"15min":129.964},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.129,"5min":2.6640000000000001,"15min":3.9449999999999998},"min":{"1min":0.35699999999999998,"5min":0.254,"15min":0.254},"max":{"1min":6.9249999999999998,"5min":84.192999999999998,"15min":131.18600000000001},"last":0.60899999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:30:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":1.9039999999999999,"15min":1.9039999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.6559999999999997,"5min":6.6559999999999997,"15min":6.6559999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.1499999999999999,"5min":2.1499999999999999,"15min":2.1499999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":9.7319999999999993,"5min":9.7319999999999993,"15min":9.7319999999999993},"last":0.53300000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:30:37 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.77,"15min":3.4159999999999999},"min":{"1min":0.36799999999999999,"5min":0.316,"15min":0.311},"max":{"1min":5.7640000000000002,"5min":42.523000000000003,"15min":125.125},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.631,"5min":1.9359999999999999,"15min":3.2410000000000001},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":7.4589999999999996,"5min":39.926000000000002,"15min":76.406999999999996},"last":0.58899999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:30:48 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":0.65900000000000003,"15min":2.5129999999999999},"min":{"1min":0.309,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.8650000000000002,"5min":6.8650000000000002,"15min":305.30200000000002},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.1319999999999999,"5min":0.62,"15min":2.4630000000000001},"min":{"1min":0.30399999999999999,"5min":0.255,"15min":0.255},"max":{"1min":5.5979999999999999,"5min":5.5979999999999999,"15min":305.40300000000002},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:30:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":0.68799999999999994,"15min":1.056},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.6660000000000004,"5min":5.6660000000000004,"15min":67.045000000000002},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.63800000000000001,"15min":1.107},"min":{"1min":0.28499999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":4.0170000000000003,"5min":5.5910000000000002,"15min":66.802000000000007},"last":0.624}]},{"osd":7,"last update":"Sat Sep 23 22:30:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.60199999999999998,"15min":1.024},"min":{"1min":0.33800000000000002,"5min":0.314,"15min":0.314},"max":{"1min":3.1850000000000001,"5min":3.7509999999999999,"15min":82.801000000000002},"last":0.499},{"interface":"front","average":{"1min":0.96899999999999997,"5min":0.64300000000000002,"15min":1.0069999999999999},"min":{"1min":0.33300000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.6989999999999998,"5min":5.6989999999999998,"15min":83.052999999999997},"last":0.44800000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795231,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":775232,"kb_used_data":231288,"kb_used_omap":7,"kb_used_meta":543928,"kb_avail":92969920,"statfs":{"total":95995035648,"available":95201198080,"internally_reserved":0,"allocated":236838912,"data_stored":234791135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":556982644},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5219999999999998,"5min":4.9199999999999999,"15min":3.9049999999999998},"min":{"1min":0.24399999999999999,"5min":0.216,"15min":0.189},"max":{"1min":18.045000000000002,"5min":90.923000000000002,"15min":196.095},"last":0.21299999999999999},{"interface":"front","average":{"1min":4.5499999999999998,"5min":5.4550000000000001,"15min":4.5899999999999999},"min":{"1min":0.35899999999999999,"5min":0.25600000000000001,"15min":0.20799999999999999},"max":{"1min":14.1,"5min":92.093000000000004,"15min":196.85300000000001},"last":0.19800000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:30:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":5.8250000000000002,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":0.314},{"interface":"front","average":{"1min":2.371,"5min":2.371,"15min":2.371},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.2349999999999994,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":0.41599999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":4.2590000000000003,"15min":4.452},"min":{"1min":0.33900000000000002,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":9.3089999999999993,"5min":88.340000000000003,"15min":160.499},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.5179999999999998,"5min":3.8109999999999999,"15min":4.2389999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.223},"max":{"1min":8.1590000000000007,"5min":91.781000000000006,"15min":160.465},"last":0.68700000000000006}]},{"osd":4,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.151,"15min":2.2389999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.0470000000000002,"5min":9.6539999999999999,"15min":234.79900000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.694,"5min":1.7370000000000001,"15min":2.8210000000000002},"min":{"1min":0.39900000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":9.093,"5min":9.093,"15min":234.81700000000001},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":3.5110000000000001,"5min":2.0659999999999998,"15min":2.3940000000000001},"min":{"1min":0.40300000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.2759999999999998,"5min":10.973000000000001,"15min":196.79900000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.389,"5min":1.6499999999999999,"15min":2.2869999999999999},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1200000000000001,"5min":7.8120000000000003,"15min":196.68299999999999},"last":0.65900000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.8089999999999999,"15min":1.7949999999999999},"min":{"1min":0.41399999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.7779999999999996,"5min":8.5540000000000003,"15min":16.314},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.377,"15min":1.3080000000000001},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":3.4729999999999999,"5min":9.6910000000000007,"15min":11.385999999999999},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.4910000000000001,"5min":1.9099999999999999,"15min":1.9350000000000001},"min":{"1min":0.44900000000000001,"5min":0.39500000000000002,"15min":0.32900000000000001},"max":{"1min":8.0660000000000007,"5min":8.6059999999999999,"15min":16.265000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.3700000000000001,"15min":1.3220000000000001},"min":{"1min":0.439,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.165,"5min":9.7129999999999992,"15min":12.045999999999999},"last":0.52300000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958760,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":935112,"kb_used_data":355968,"kb_used_omap":8,"kb_used_meta":579127,"kb_avail":92810040,"statfs":{"total":95995035648,"available":95037480960,"internally_reserved":0,"allocated":364511232,"data_stored":362397147,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":593026922},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:30:32 2023","interfaces":[{"interface":"back","average":{"1min":6.282,"5min":4.0720000000000001,"15min":3.851},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":89.439999999999998,"5min":89.439999999999998,"15min":169.77199999999999},"last":0.23899999999999999},{"interface":"front","average":{"1min":8.2460000000000004,"5min":4.0750000000000002,"15min":4.0979999999999999},"min":{"1min":0.26400000000000001,"5min":0.218,"15min":0.20300000000000001},"max":{"1min":89.287999999999997,"5min":89.287999999999997,"15min":169.52099999999999},"last":0.59799999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:30:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1960000000000002,"5min":3.1960000000000002,"15min":3.1960000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":22.263000000000002,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.8340000000000001,"5min":1.8340000000000001,"15min":1.8340000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":7.1920000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.32600000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:30:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.149,"15min":3.6070000000000002},"min":{"1min":0.23200000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.234,"5min":15.404,"15min":146.87299999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.9460000000000002,"5min":2.335,"15min":3.7709999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":22.303000000000001,"5min":22.303000000000001,"15min":145.48099999999999},"last":0.371}]},{"osd":4,"last update":"Sat Sep 23 22:31:12 2023","interfaces":[{"interface":"back","average":{"1min":3.556,"5min":2.1240000000000001,"15min":2.536},"min":{"1min":0.309,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":11.474,"5min":11.474,"15min":82.825000000000003},"last":0.623},{"interface":"front","average":{"1min":2.125,"5min":1.2370000000000001,"15min":1.702},"min":{"1min":0.33000000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.2279999999999998,"5min":7.2279999999999998,"15min":75.930999999999997},"last":0.78100000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:31:12 2023","interfaces":[{"interface":"back","average":{"1min":3.0619999999999998,"5min":2.258,"15min":2.9590000000000001},"min":{"1min":0.34499999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.0009999999999994,"5min":13.564,"15min":93.685000000000002},"last":0.98199999999999998},{"interface":"front","average":{"1min":2.4329999999999998,"5min":1.4410000000000001,"15min":2.206},"min":{"1min":0.309,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.8190000000000008,"5min":13.486000000000001,"15min":93.546000000000006},"last":0.73399999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:31:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7189999999999999,"5min":2.2469999999999999,"15min":2.2789999999999999},"min":{"1min":0.40000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":12.955,"5min":12.955,"15min":16.366},"last":0.44},{"interface":"front","average":{"1min":3.008,"5min":1.4870000000000001,"15min":1.637},"min":{"1min":0.38,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":14.208,"5min":14.208,"15min":22.263000000000002},"last":0.49199999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:31:12 2023","interfaces":[{"interface":"back","average":{"1min":2.9089999999999998,"5min":1.8560000000000001,"15min":2.0779999999999998},"min":{"1min":0.41799999999999998,"5min":0.40200000000000002,"15min":0.29999999999999999},"max":{"1min":22.338999999999999,"5min":22.338999999999999,"15min":32.773000000000003},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.391,"5min":2.121,"15min":2.306},"min":{"1min":0.40300000000000002,"5min":0.307,"15min":0.307},"max":{"1min":10.666,"5min":10.901,"15min":16.381},"last":0.47899999999999998}]}]},{"osd":2,"up_from":0,"seq":1498943586323,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468177,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019227,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":285776,"kb_used_data":8584,"kb_used_omap":0,"kb_used_meta":277184,"kb_avail":93459376,"statfs":{"total":95995035648,"available":95702401024,"internally_reserved":0,"allocated":8790016,"data_stored":7369242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":283836416},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7430000000000001,"5min":4.2300000000000004,"15min":3.5699999999999998},"min":{"1min":0.20699999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.9559999999999995,"5min":104.252,"15min":150.63499999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.3399999999999999,"5min":4.5119999999999996,"15min":3.7080000000000002},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.9740000000000002,"5min":104.294,"15min":150.672},"last":0.59899999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":6.2060000000000004,"5min":3.9550000000000001,"15min":3.1869999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.21299999999999999},"max":{"1min":81.122,"5min":95.114000000000004,"15min":107.346},"last":0.57299999999999995},{"interface":"front","average":{"1min":6.1959999999999997,"5min":3.9399999999999999,"15min":3.528},"min":{"1min":0.26700000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":81.090999999999994,"5min":95.222999999999999,"15min":107.33499999999999},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:30:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1699999999999999,"5min":2.1699999999999999,"15min":2.1699999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6790000000000003,"5min":7.6790000000000003,"15min":7.6790000000000003},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.54,"5min":1.54,"15min":1.54},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.6379999999999999,"5min":4.6379999999999999,"15min":4.6379999999999999},"last":0.47199999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.597,"5min":4.0629999999999997,"15min":2.4630000000000001},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.0940000000000003,"5min":102.896,"15min":102.896},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.2040000000000002,"5min":4.2220000000000004,"15min":2.5739999999999998},"min":{"1min":0.32700000000000001,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":14.318,"5min":112.667,"15min":112.667},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:30:39 2023","interfaces":[{"interface":"back","average":{"1min":2.286,"5min":0.76800000000000002,"15min":1.0029999999999999},"min":{"1min":0.26000000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.407,"5min":13.407,"15min":28.399000000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.7709999999999999,"5min":0.79200000000000004,"15min":1.036},"min":{"1min":0.29799999999999999,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":14.212,"5min":14.212,"15min":28.350999999999999},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:30:55 2023","interfaces":[{"interface":"back","average":{"1min":1.464,"5min":0.66000000000000003,"15min":0.82099999999999995},"min":{"1min":0.32300000000000001,"5min":0.26400000000000001,"15min":0.251},"max":{"1min":6.0110000000000001,"5min":6.0110000000000001,"15min":7.8550000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.3440000000000001,"5min":0.65900000000000003,"15min":0.80500000000000005},"min":{"1min":0.35599999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.0149999999999997,"5min":5.0149999999999997,"15min":8.5210000000000008},"last":0.39600000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.59499999999999997,"15min":0.88600000000000001},"min":{"1min":0.33100000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3179999999999996,"5min":4.3179999999999996,"15min":8.577},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.4119999999999999,"5min":0.69099999999999995,"15min":0.88700000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9319999999999999,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":0.40699999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757595,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":342724,"kb_used_data":8828,"kb_used_omap":0,"kb_used_meta":333888,"kb_avail":93402428,"statfs":{"total":95995035648,"available":95644086272,"internally_reserved":0,"allocated":9039872,"data_stored":7627149,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":341901312},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:30:14 2023","interfaces":[{"interface":"back","average":{"1min":3.4399999999999999,"5min":7.1689999999999996,"15min":4.3899999999999997},"min":{"1min":0.23899999999999999,"5min":0.22,"15min":0.20499999999999999},"max":{"1min":16.109999999999999,"5min":163.54499999999999,"15min":163.54499999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.7709999999999999,"5min":6.5010000000000003,"15min":4.2089999999999996},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":5.7949999999999999,"5min":162.81800000000001,"15min":162.81800000000001},"last":0.35199999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":3.9329999999999998,"15min":3.8740000000000001},"min":{"1min":0.38100000000000001,"5min":0.27900000000000003,"15min":0.23799999999999999},"max":{"1min":7.7779999999999996,"5min":113.514,"15min":113.514},"last":0.313},{"interface":"front","average":{"1min":3.1240000000000001,"5min":3.9990000000000001,"15min":3.8860000000000001},"min":{"1min":0.33100000000000002,"5min":0.255,"15min":0.252},"max":{"1min":7.6319999999999997,"5min":113.42700000000001,"15min":113.42700000000001},"last":0.48299999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:30:50 2023","interfaces":[{"interface":"back","average":{"1min":2.411,"5min":2.411,"15min":2.411},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":10.451000000000001,"5min":10.451000000000001,"15min":10.451000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.5289999999999999,"5min":2.5289999999999999,"15min":2.5289999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.9569999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.372}]},{"osd":3,"last update":"Sat Sep 23 22:30:33 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":5.883,"15min":4.1040000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.246},"max":{"1min":9.3369999999999997,"5min":132.59,"15min":132.59},"last":0.40300000000000002},{"interface":"front","average":{"1min":3.024,"5min":6.0949999999999998,"15min":4.1040000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":7.6479999999999997,"5min":129.381,"15min":129.381},"last":0.41199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:30:27 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.9610000000000001,"15min":1.528},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.24099999999999999},"max":{"1min":3.7240000000000002,"5min":126.679,"15min":126.679},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.879,"5min":1.98,"15min":1.5589999999999999},"min":{"1min":0.309,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":2.7610000000000001,"5min":126.78100000000001,"15min":126.78100000000001},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 22:30:46 2023","interfaces":[{"interface":"back","average":{"1min":4.9059999999999997,"5min":1.4039999999999999,"15min":1.462},"min":{"1min":0.36499999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":70.581999999999994,"5min":70.581999999999994,"15min":114.83499999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":5.6050000000000004,"5min":1.6599999999999999,"15min":1.4430000000000001},"min":{"1min":0.33600000000000002,"5min":0.248,"15min":0.248},"max":{"1min":71.093000000000004,"5min":71.093000000000004,"15min":110.905},"last":0.38600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.624,"15min":0.91600000000000004},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":4.1120000000000001,"5min":4.827,"15min":8.8030000000000008},"last":0.495},{"interface":"front","average":{"1min":0.92100000000000004,"5min":0.58999999999999997,"15min":0.77900000000000003},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.308},"max":{"1min":3.7570000000000001,"5min":3.7570000000000001,"15min":7.6609999999999996},"last":0.501}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":312082432,"data_stored":312082432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":273022976,"data_stored":273022976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":10092544,"data_stored":10092544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:31:21.914 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T22:31:21.914 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:31:21.914 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [4, 5, 7, 0, 1, 6] 2023-09-23T22:31:21.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T22:31:22.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:22.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:22.033 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:22.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 15935 15919 142.11 127.875 0.00777844 0.0070287 2023-09-23T22:31:22.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:22.200 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:22.276 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:22.292 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:22.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:22.402 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:22.475 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:22.609 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:22.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:22.719 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:22.764 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:22.773 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:22 smithi070 ceph-mon[192652]: pgmap v973: 99 pgs: 99 active+clean; 707 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T22:31:22.775 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:22.876 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:22.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:22.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:22.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:22 smithi026 ceph-mon[215775]: pgmap v973: 99 pgs: 99 active+clean; 707 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T22:31:22.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:22 smithi026 ceph-mon[223014]: pgmap v973: 99 pgs: 99 active+clean; 707 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T22:31:22.985 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:23.043 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:23.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 18260 18244 142.508 145.312 0.00939781 0.00701095 2023-09-23T22:31:23.141 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:23.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:23.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:23.249 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:23.275 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:23.315 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:23.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:23.418 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:23.449 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:23.490 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:23.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:23.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:23.593 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:23.654 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:23.755 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:23.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:23.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:23.862 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:23.917 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:23.999 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:24.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:24.006 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:24.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 15 19693 19678 136.632 89.625 0.00656713 0.00731307 2023-09-23T22:31:24.107 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:24.118 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:31:24.154 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:24.432 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:24.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:24.722 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:31:24.723 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:31:25.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 21340 21324 133.256 102.875 0.00749406 0.00749826 2023-09-23T22:31:25.111 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:25 smithi070 ceph-mon[192652]: pgmap v974: 99 pgs: 99 active+clean; 707 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T22:31:25.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:25 smithi026 ceph-mon[223014]: pgmap v974: 99 pgs: 99 active+clean; 707 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T22:31:25.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:25 smithi026 ceph-mon[215775]: pgmap v974: 99 pgs: 99 active+clean; 707 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T22:31:25.266 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:25.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:25.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:25.371 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:25.433 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:25.440 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:25.531 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:25.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:25.540 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:25.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:25.701 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:25.748 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:25.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:25.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:25.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:25.907 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:25.994 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:26.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:26.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:26.091 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 23334 23318 132.47 124.625 0.00880296 0.00754296 2023-09-23T22:31:26.102 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:26.120 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3155235988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:26.135 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:31:26.160 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:26.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3155235988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:26.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3155235988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:26.336 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:26.340 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:31:26.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:26.349 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:26.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:26.510 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:26.629 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:26.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:26.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:26.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:26 smithi026 conmon[215751]: 2023-09-23T22:31:26.297+0000 7f4580f04700 -1 mon.a@0(leader).osd e379 definitely_dead 0 2023-09-23T22:31:26.738 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:26.778 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:26.853 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:26.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:26.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:26.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:26.995 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:27.064 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:27.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:27.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:27.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 25331 25315 131.831 124.812 0.00177864 0.00755795 2023-09-23T22:31:27.129 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.7. 2023-09-23T22:31:27.169 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:27.195 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:27.235 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:27.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:27.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: pgmap v975: 99 pgs: 99 active+clean; 1.3 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: failed to encode map e379 with expected crc 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: failed to encode map e379 with expected crc 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3155235988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: osdmap e379: 8 total, 8 up, 6 in 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2638763341' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:31:27.336 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[215775]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: pgmap v975: 99 pgs: 99 active+clean; 1.3 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: failed to encode map e379 with expected crc 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: failed to encode map e379 with expected crc 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3155235988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: osdmap e379: 8 total, 8 up, 6 in 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2638763341' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:31:27.337 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:27 smithi026 ceph-mon[223014]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:31:27.340 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:27.363 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: pgmap v975: 99 pgs: 99 active+clean; 1.3 GiB data, 5.9 GiB used, 530 GiB / 536 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T22:31:27.363 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:31:27.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:31:27.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: failed to encode map e379 with expected crc 2023-09-23T22:31:27.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: failed to encode map e379 with expected crc 2023-09-23T22:31:27.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3155235988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:31:27.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: osdmap e379: 8 total, 8 up, 6 in 2023-09-23T22:31:27.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2638763341' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:31:27.364 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:31:27.365 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:27 smithi070 ceph-mon[192652]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:31:27.365 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:27.407 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:27.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:27.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:27.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:27.540 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:28.042 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:28.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:28.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:28.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 25352 25336 121.792 1.3125 0.00190311 0.00760193 2023-09-23T22:31:28.146 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:28.181 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:28.227 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:28.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[223014]: failed to encode map e380 with expected crc 2023-09-23T22:31:28.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[223014]: failed to encode map e380 with expected crc 2023-09-23T22:31:28.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:31:28.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[223014]: osdmap e380: 8 total, 8 up, 5 in 2023-09-23T22:31:28.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[223014]: osdmap e381: 8 total, 8 up, 5 in 2023-09-23T22:31:28.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[215775]: failed to encode map e380 with expected crc 2023-09-23T22:31:28.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[215775]: failed to encode map e380 with expected crc 2023-09-23T22:31:28.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:31:28.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[215775]: osdmap e380: 8 total, 8 up, 5 in 2023-09-23T22:31:28.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:28 smithi026 ceph-mon[215775]: osdmap e381: 8 total, 8 up, 5 in 2023-09-23T22:31:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:28.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:28.341 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:28.366 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:28.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:28.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:28 smithi070 ceph-mon[192652]: failed to encode map e380 with expected crc 2023-09-23T22:31:28.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:28 smithi070 ceph-mon[192652]: failed to encode map e380 with expected crc 2023-09-23T22:31:28.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:28 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:31:28.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:28 smithi070 ceph-mon[192652]: osdmap e380: 8 total, 8 up, 5 in 2023-09-23T22:31:28.413 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:28 smithi070 ceph-mon[192652]: osdmap e381: 8 total, 8 up, 5 in 2023-09-23T22:31:28.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:28.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:28.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:28.541 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:28.581 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:28.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:28.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:28.688 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:28.718 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:28.727 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:28.772 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:28.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:28.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:28.805 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:31:28.825 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:31:28.844 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:31:28.864 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:31:28.878 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:28.883 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:31:28.903 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:31:28.905 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:28.922 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:31:28.943 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:31:28.944 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:31:28.944 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:31:28.944 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:31:28.944 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:31:28.944 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:31:28.945 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:31:28.945 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:31:28.945 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:31:28.947 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:28.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:28.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:29.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:29.083 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:29.090 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 25352 25336 113.093 0 - 0.00760193 2023-09-23T22:31:29.227 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:29.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: pgmap v978: 99 pgs: 99 active+clean; 1.3 GiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 127 MiB/s wr, 2.02k op/s 2023-09-23T22:31:29.233 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.233 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.233 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.233 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.233 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1955495340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:29.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[223014]: osdmap e382: 8 total, 8 up, 5 in 2023-09-23T22:31:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:29.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:29.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: pgmap v978: 99 pgs: 99 active+clean; 1.3 GiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 127 MiB/s wr, 2.02k op/s 2023-09-23T22:31:29.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1955495340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:29.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:29 smithi026 ceph-mon[215775]: osdmap e382: 8 total, 8 up, 5 in 2023-09-23T22:31:29.338 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:29.390 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: pgmap v978: 99 pgs: 99 active+clean; 1.3 GiB data, 5.0 GiB used, 442 GiB / 447 GiB avail; 127 MiB/s wr, 2.02k op/s 2023-09-23T22:31:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: failed to encode map e381 with expected crc 2023-09-23T22:31:29.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1955495340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:29.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:29 smithi070 ceph-mon[192652]: osdmap e382: 8 total, 8 up, 5 in 2023-09-23T22:31:29.962 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:31:29.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:31:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:31:29] "GET /metrics HTTP/1.1" 200 36353 "" "Prometheus/2.43.0" 2023-09-23T22:31:30.091 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 26009 25993 108.291 20.5312 0.00770361 0.00922693 2023-09-23T22:31:30.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[215775]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[215775]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[215775]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[215775]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[215775]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[215775]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[215775]: osdmap e383: 8 total, 8 up, 5 in 2023-09-23T22:31:30.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[223014]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[223014]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[223014]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[223014]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[223014]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[223014]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:30 smithi026 ceph-mon[223014]: osdmap e383: 8 total, 8 up, 5 in 2023-09-23T22:31:30.529 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:30.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:30.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:30 smithi070 ceph-mon[192652]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:30 smithi070 ceph-mon[192652]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:30 smithi070 ceph-mon[192652]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:30 smithi070 ceph-mon[192652]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:30 smithi070 ceph-mon[192652]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:30 smithi070 ceph-mon[192652]: failed to encode map e382 with expected crc 2023-09-23T22:31:30.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:30 smithi070 ceph-mon[192652]: osdmap e383: 8 total, 8 up, 5 in 2023-09-23T22:31:30.647 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:30.719 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:30.777 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:30.920 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:30.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:30.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:31.033 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:31.091 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 27531 27515 107.465 95.125 0.0207438 0.00929907 2023-09-23T22:31:31.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:31.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[215775]: pgmap v981: 99 pgs: 99 active+clean; 1.4 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 23 MiB/s wr, 360 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T22:31:31.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[215775]: failed to encode map e383 with expected crc 2023-09-23T22:31:31.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[215775]: failed to encode map e383 with expected crc 2023-09-23T22:31:31.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[215775]: failed to encode map e383 with expected crc 2023-09-23T22:31:31.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[215775]: osdmap e384: 8 total, 8 up, 5 in 2023-09-23T22:31:31.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[223014]: pgmap v981: 99 pgs: 99 active+clean; 1.4 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 23 MiB/s wr, 360 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T22:31:31.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[223014]: failed to encode map e383 with expected crc 2023-09-23T22:31:31.179 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[223014]: failed to encode map e383 with expected crc 2023-09-23T22:31:31.179 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[223014]: failed to encode map e383 with expected crc 2023-09-23T22:31:31.179 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:31 smithi026 ceph-mon[223014]: osdmap e384: 8 total, 8 up, 5 in 2023-09-23T22:31:31.244 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:31.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:31.284 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:31:31.284 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:31:31.285 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:31:31.285 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:31:31.351 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:31.420 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:31.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:31 smithi070 ceph-mon[192652]: pgmap v981: 99 pgs: 99 active+clean; 1.4 GiB data, 5.2 GiB used, 442 GiB / 447 GiB avail; 23 MiB/s wr, 360 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T22:31:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:31 smithi070 ceph-mon[192652]: failed to encode map e383 with expected crc 2023-09-23T22:31:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:31 smithi070 ceph-mon[192652]: failed to encode map e383 with expected crc 2023-09-23T22:31:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:31 smithi070 ceph-mon[192652]: failed to encode map e383 with expected crc 2023-09-23T22:31:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:31 smithi070 ceph-mon[192652]: osdmap e384: 8 total, 8 up, 5 in 2023-09-23T22:31:31.656 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:31.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:31.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:31.762 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:31.808 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:32.091 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 29071 29055 106.805 96.25 0.0118413 0.00935602 2023-09-23T22:31:32.184 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:32.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:32.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:32.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:32.347 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:32.454 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:32.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:32 smithi070 ceph-mon[192652]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:32 smithi070 ceph-mon[192652]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:32 smithi070 ceph-mon[192652]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:32 smithi070 ceph-mon[192652]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:32 smithi070 ceph-mon[192652]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:32.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:32.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[223014]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[223014]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[223014]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[223014]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[223014]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[215775]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[215775]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[215775]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[215775]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:32 smithi026 ceph-mon[215775]: failed to encode map e384 with expected crc 2023-09-23T22:31:32.556 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:31:32.568 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:32.625 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:32.718 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:32.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:32.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:32.828 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:32.887 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:33.091 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 30340 30324 105.277 79.3125 0.00173042 0.00944248 2023-09-23T22:31:33.453 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6] out_osds: [3, 2, 7] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:31:33.453 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T22:31:33.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:33 smithi026 ceph-mon[223014]: pgmap v984: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 85 active+clean; 1.6 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s; 7929/77355 objects degraded (10.250%); 3023/77355 objects misplaced (3.908%); 1.1 MiB/s, 2 keys/s, 31 objects/s recovering 2023-09-23T22:31:33.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:33 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 7929/77355 objects degraded (10.250%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:33.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:33 smithi026 ceph-mon[215775]: pgmap v984: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 85 active+clean; 1.6 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s; 7929/77355 objects degraded (10.250%); 3023/77355 objects misplaced (3.908%); 1.1 MiB/s, 2 keys/s, 31 objects/s recovering 2023-09-23T22:31:33.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:33 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 7929/77355 objects degraded (10.250%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:33.594 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:33.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:33.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:33.609 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:31:33.630 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:33 smithi070 ceph-mon[192652]: pgmap v984: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 85 active+clean; 1.6 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 79 MiB/s wr, 1.26k op/s; 7929/77355 objects degraded (10.250%); 3023/77355 objects misplaced (3.908%); 1.1 MiB/s, 2 keys/s, 31 objects/s recovering 2023-09-23T22:31:33.630 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:33 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 7929/77355 objects degraded (10.250%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:33.703 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:33.790 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:33.965 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:33.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:33.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:34.080 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:34.091 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 31534 31518 103.664 74.625 0.0168915 0.00963954 2023-09-23T22:31:34.142 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:34.231 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:34.241 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:34.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:34.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:34.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:34.415 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:34.492 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:34.804 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:34.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:34.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:34.909 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:34.946 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:35.020 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:35.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:35.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:35.091 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:31:35.092142+0000 min lat: 0.00111004 max lat: 2.14499 avg lat: 0.00975845 2023-09-23T22:31:35.092 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:31:35.092 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 32791 32775 102.408 78.5625 0.0159676 0.00975845 2023-09-23T22:31:35.133 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:35.184 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:35.233 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:31:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:35 smithi026 ceph-mon[223014]: pgmap v985: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 85 active+clean; 1.6 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 58 MiB/s wr, 926 op/s; 7929/77355 objects degraded (10.250%); 3023/77355 objects misplaced (3.908%); 822 KiB/s, 1 keys/s, 23 objects/s recovering 2023-09-23T22:31:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1635532568' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:35.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:35 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:35 smithi026 ceph-mon[215775]: pgmap v985: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 85 active+clean; 1.6 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 58 MiB/s wr, 926 op/s; 7929/77355 objects degraded (10.250%); 3023/77355 objects misplaced (3.908%); 822 KiB/s, 1 keys/s, 23 objects/s recovering 2023-09-23T22:31:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1635532568' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:35 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:35.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:35 smithi070 ceph-mon[192652]: pgmap v985: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 85 active+clean; 1.6 GiB data, 5.9 GiB used, 441 GiB / 447 GiB avail; 58 MiB/s wr, 926 op/s; 7929/77355 objects degraded (10.250%); 3023/77355 objects misplaced (3.908%); 822 KiB/s, 1 keys/s, 23 objects/s recovering 2023-09-23T22:31:35.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1635532568' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:35.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:35 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:31:35.661 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:35.665 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:35.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:35.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:35.770 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:35.829 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:35.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:35.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:36.041 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:36.091 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 34087 34071 101.389 81 0.0142541 0.00985778 2023-09-23T22:31:36.112 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:36.228 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:36.228 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:31:36.228 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[215775]: osdmap e385: 8 total, 8 up, 5 in 2023-09-23T22:31:36.228 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[215775]: failed to encode map e385 with expected crc 2023-09-23T22:31:36.228 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[215775]: pgmap v987: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.0 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 107 MiB/s wr, 1.71k op/s; 7929/97236 objects degraded (8.154%); 2170/97236 objects misplaced (2.232%); 9.6 MiB/s, 1 keys/s, 162 objects/s recovering 2023-09-23T22:31:36.228 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[215775]: failed to encode map e385 with expected crc 2023-09-23T22:31:36.229 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:36.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:36.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:36.344 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:36.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:36 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:36.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:31:36.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:36 smithi070 ceph-mon[192652]: osdmap e385: 8 total, 8 up, 5 in 2023-09-23T22:31:36.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:36 smithi070 ceph-mon[192652]: failed to encode map e385 with expected crc 2023-09-23T22:31:36.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:36 smithi070 ceph-mon[192652]: pgmap v987: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.0 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 107 MiB/s wr, 1.71k op/s; 7929/97236 objects degraded (8.154%); 2170/97236 objects misplaced (2.232%); 9.6 MiB/s, 1 keys/s, 162 objects/s recovering 2023-09-23T22:31:36.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:36 smithi070 ceph-mon[192652]: failed to encode map e385 with expected crc 2023-09-23T22:31:36.369 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:36.410 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:36.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:36.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:36.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:36.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:31:36.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[223014]: osdmap e385: 8 total, 8 up, 5 in 2023-09-23T22:31:36.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[223014]: failed to encode map e385 with expected crc 2023-09-23T22:31:36.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[223014]: pgmap v987: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.0 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 107 MiB/s wr, 1.71k op/s; 7929/97236 objects degraded (8.154%); 2170/97236 objects misplaced (2.232%); 9.6 MiB/s, 1 keys/s, 162 objects/s recovering 2023-09-23T22:31:36.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:36 smithi026 ceph-mon[223014]: failed to encode map e385 with expected crc 2023-09-23T22:31:36.515 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:36.575 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:37.093 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 35329 35313 100.308 77.625 0.00278602 0.00994766 2023-09-23T22:31:37.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:37 smithi026 ceph-mon[223014]: failed to encode map e385 with expected crc 2023-09-23T22:31:37.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:37 smithi026 ceph-mon[223014]: failed to encode map e385 with expected crc 2023-09-23T22:31:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:37 smithi026 ceph-mon[215775]: failed to encode map e385 with expected crc 2023-09-23T22:31:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:37 smithi026 ceph-mon[215775]: failed to encode map e385 with expected crc 2023-09-23T22:31:37.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:37 smithi070 ceph-mon[192652]: failed to encode map e385 with expected crc 2023-09-23T22:31:37.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:37 smithi070 ceph-mon[192652]: failed to encode map e385 with expected crc 2023-09-23T22:31:38.093 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 36350 36334 98.7145 63.8125 0.00486621 0.0101213 2023-09-23T22:31:38.404 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:38.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:38.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:38 smithi026 ceph-mon[223014]: pgmap v988: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.0 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 56 MiB/s wr, 890 op/s; 7929/97236 objects degraded (8.154%); 2170/97236 objects misplaced (2.232%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T22:31:38.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7929/97236 objects degraded (8.154%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:38.414 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:38 smithi026 ceph-mon[215775]: pgmap v988: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.0 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 56 MiB/s wr, 890 op/s; 7929/97236 objects degraded (8.154%); 2170/97236 objects misplaced (2.232%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T22:31:38.414 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7929/97236 objects degraded (8.154%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:38.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:38.563 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:38.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:38 smithi070 ceph-mon[192652]: pgmap v988: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.0 GiB data, 7.1 GiB used, 440 GiB / 447 GiB avail; 56 MiB/s wr, 890 op/s; 7929/97236 objects degraded (8.154%); 2170/97236 objects misplaced (2.232%); 7.2 MiB/s, 114 objects/s recovering 2023-09-23T22:31:38.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7929/97236 objects degraded (8.154%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:38.740 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:38.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:38.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:38.851 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:38.911 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:39.093 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 37774 37758 98.3094 89 0.0123064 0.0101665 2023-09-23T22:31:39.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:31:39.657 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:31:39.695 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:31:39.728 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:31:39.763 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:31:39.798 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:31:39.837 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:31:39.842 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:39.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:39.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:39.869 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:31:39.871 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:31:39.872 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:31:39.872 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:31:39.872 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:31:39.872 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:31:39.872 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:31:39.872 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:31:39.872 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:31:39.904 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:31:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:31:39] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2023-09-23T22:31:39.950 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:39.996 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:40.096 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 39071 39055 97.6192 81.0625 0.00710348 0.010238 2023-09-23T22:31:40.096 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:40.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:40.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:40.208 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3982605718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3982605718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:40.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:40.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:40.254 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3982605718' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:40.259 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:40.353 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:40.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:40.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:40.464 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:40.495 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:40.605 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:31:40.931 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:40.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:40.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:41.039 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:41.094 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 40235 40219 96.6617 72.75 0.00557719 0.0103391 2023-09-23T22:31:41.122 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T22:31:41.145 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:41.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:41 smithi026 ceph-mon[223014]: pgmap v989: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.1 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 61 MiB/s wr, 975 op/s; 7929/102051 objects degraded (7.770%); 2170/102051 objects misplaced (2.126%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:31:41.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/918722520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:31:41.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:41 smithi026 ceph-mon[215775]: pgmap v989: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.1 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 61 MiB/s wr, 975 op/s; 7929/102051 objects degraded (7.770%); 2170/102051 objects misplaced (2.126%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:31:41.194 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/918722520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:31:41.364 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:41.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:41.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:41.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:41 smithi070 ceph-mon[192652]: pgmap v989: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.1 GiB data, 7.4 GiB used, 440 GiB / 447 GiB avail; 61 MiB/s wr, 975 op/s; 7929/102051 objects degraded (7.770%); 2170/102051 objects misplaced (2.126%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:31:41.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/918722520' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:31:41.472 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:41.536 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:41.649 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:41.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:41.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:41.756 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:41.784 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:41.853 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:41.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:41.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:41.966 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:42.020 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:42.073 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:31:42.074 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:31:42.094 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 41351 41335 95.6648 69.75 0.0107141 0.010447 2023-09-23T22:31:42.105 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/918722520' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:31:42.105 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:42 smithi070 ceph-mon[192652]: osdmap e386: 8 total, 8 up, 5 in 2023-09-23T22:31:42.105 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:42 smithi070 ceph-mon[192652]: failed to encode map e386 with expected crc 2023-09-23T22:31:42.105 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:42 smithi070 ceph-mon[192652]: failed to encode map e386 with expected crc 2023-09-23T22:31:42.106 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:42 smithi070 ceph-mon[192652]: failed to encode map e386 with expected crc 2023-09-23T22:31:42.115 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:42.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:42.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:42.235 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:42.294 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:42.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/918722520' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:31:42.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[223014]: osdmap e386: 8 total, 8 up, 5 in 2023-09-23T22:31:42.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[223014]: failed to encode map e386 with expected crc 2023-09-23T22:31:42.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[223014]: failed to encode map e386 with expected crc 2023-09-23T22:31:42.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[223014]: failed to encode map e386 with expected crc 2023-09-23T22:31:42.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/918722520' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:31:42.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[215775]: osdmap e386: 8 total, 8 up, 5 in 2023-09-23T22:31:42.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[215775]: failed to encode map e386 with expected crc 2023-09-23T22:31:42.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[215775]: failed to encode map e386 with expected crc 2023-09-23T22:31:42.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:42 smithi026 ceph-mon[215775]: failed to encode map e386 with expected crc 2023-09-23T22:31:42.435 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:42.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:42.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:42.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:42.613 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:42.683 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:42.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:42.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:42.792 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:42.856 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:42.928 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:31:42.977 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:42.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:42.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:43.069 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:31:43.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:43.097 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 42619 42603 95.0685 79.25 0.00614744 0.0105119 2023-09-23T22:31:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:43 smithi070 ceph-mon[192652]: pgmap v991: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.4 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 101 MiB/s wr, 1.61k op/s; 7929/116043 objects degraded (6.833%); 1073/116043 objects misplaced (0.925%); 15 MiB/s, 243 objects/s recovering 2023-09-23T22:31:43.119 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7929/116043 objects degraded (6.833%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:43.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:43 smithi026 ceph-mon[215775]: pgmap v991: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.4 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 101 MiB/s wr, 1.61k op/s; 7929/116043 objects degraded (6.833%); 1073/116043 objects misplaced (0.925%); 15 MiB/s, 243 objects/s recovering 2023-09-23T22:31:43.203 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7929/116043 objects degraded (6.833%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:43.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:43 smithi026 ceph-mon[223014]: pgmap v991: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.4 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 101 MiB/s wr, 1.61k op/s; 7929/116043 objects degraded (6.833%); 1073/116043 objects misplaced (0.925%); 15 MiB/s, 243 objects/s recovering 2023-09-23T22:31:43.206 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7929/116043 objects degraded (6.833%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:43.227 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:43.229 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:31:43.326 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:43.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:43.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:43.383 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:43.441 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:43.479 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:43.577 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:43.584 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:43.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:43.688 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:43.724 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:43.890 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:44.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:44.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:44.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:44.097 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 42845 42829 92.2778 14.125 0.00414359 0.0105127 2023-09-23T22:31:44.133 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:44.179 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:44.207 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:44.319 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:44.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:44.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:44 smithi070 ceph-mon[192652]: osdmap e387: 8 total, 8 up, 5 in 2023-09-23T22:31:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:44 smithi070 ceph-mon[192652]: failed to encode map e387 with expected crc 2023-09-23T22:31:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:44 smithi070 ceph-mon[192652]: failed to encode map e387 with expected crc 2023-09-23T22:31:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:44 smithi070 ceph-mon[192652]: failed to encode map e387 with expected crc 2023-09-23T22:31:44.423 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:44.449 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:44.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:44 smithi026 ceph-mon[223014]: osdmap e387: 8 total, 8 up, 5 in 2023-09-23T22:31:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:44 smithi026 ceph-mon[223014]: failed to encode map e387 with expected crc 2023-09-23T22:31:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:44 smithi026 ceph-mon[223014]: failed to encode map e387 with expected crc 2023-09-23T22:31:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:44 smithi026 ceph-mon[223014]: failed to encode map e387 with expected crc 2023-09-23T22:31:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:44 smithi026 ceph-mon[215775]: osdmap e387: 8 total, 8 up, 5 in 2023-09-23T22:31:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:44 smithi026 ceph-mon[215775]: failed to encode map e387 with expected crc 2023-09-23T22:31:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:44 smithi026 ceph-mon[215775]: failed to encode map e387 with expected crc 2023-09-23T22:31:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:44 smithi026 ceph-mon[215775]: failed to encode map e387 with expected crc 2023-09-23T22:31:44.493 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:44.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:44.602 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:44.628 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:44.671 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:44.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:44.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:44.777 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:44.807 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:45.097 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 43011 42995 89.5482 10.375 0.00269088 0.0108471 2023-09-23T22:31:45.280 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:45.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:45.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:45.384 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:45.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[223014]: pgmap v993: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.4 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 49 MiB/s wr, 783 op/s; 7929/116043 objects degraded (6.833%); 1073/116043 objects misplaced (0.925%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T22:31:45.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[223014]: failed to encode map e388 with expected crc 2023-09-23T22:31:45.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[223014]: osdmap e388: 8 total, 8 up, 5 in 2023-09-23T22:31:45.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[223014]: failed to encode map e388 with expected crc 2023-09-23T22:31:45.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[223014]: failed to encode map e388 with expected crc 2023-09-23T22:31:45.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:45.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[215775]: pgmap v993: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.4 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 49 MiB/s wr, 783 op/s; 7929/116043 objects degraded (6.833%); 1073/116043 objects misplaced (0.925%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T22:31:45.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[215775]: failed to encode map e388 with expected crc 2023-09-23T22:31:45.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[215775]: osdmap e388: 8 total, 8 up, 5 in 2023-09-23T22:31:45.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[215775]: failed to encode map e388 with expected crc 2023-09-23T22:31:45.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[215775]: failed to encode map e388 with expected crc 2023-09-23T22:31:45.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:45 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:45.434 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:45 smithi070 ceph-mon[192652]: pgmap v993: 99 pgs: 2 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 2.4 GiB data, 8.4 GiB used, 439 GiB / 447 GiB avail; 49 MiB/s wr, 783 op/s; 7929/116043 objects degraded (6.833%); 1073/116043 objects misplaced (0.925%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T22:31:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:45 smithi070 ceph-mon[192652]: failed to encode map e388 with expected crc 2023-09-23T22:31:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:45 smithi070 ceph-mon[192652]: osdmap e388: 8 total, 8 up, 5 in 2023-09-23T22:31:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:45 smithi070 ceph-mon[192652]: failed to encode map e388 with expected crc 2023-09-23T22:31:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:45 smithi070 ceph-mon[192652]: failed to encode map e388 with expected crc 2023-09-23T22:31:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:45 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:46.097 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 43977 43961 88.6071 60.375 0.00295099 0.0112287 2023-09-23T22:31:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[215775]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.132 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[215775]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[215775]: osdmap e389: 8 total, 8 up, 5 in 2023-09-23T22:31:46.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[215775]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.133 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[215775]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[223014]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[223014]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[223014]: osdmap e389: 8 total, 8 up, 5 in 2023-09-23T22:31:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[223014]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:46 smithi026 ceph-mon[223014]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.609 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:46.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:46.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:46.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:46 smithi070 ceph-mon[192652]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:46 smithi070 ceph-mon[192652]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:46 smithi070 ceph-mon[192652]: osdmap e389: 8 total, 8 up, 5 in 2023-09-23T22:31:46.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:46 smithi070 ceph-mon[192652]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:46 smithi070 ceph-mon[192652]: failed to encode map e389 with expected crc 2023-09-23T22:31:46.719 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:46.798 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:47.040 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:47.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:47.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:47.097 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 45262 45246 88.3477 80.3125 0.00822303 0.0113126 2023-09-23T22:31:47.146 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:47.209 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:47 smithi026 ceph-mon[223014]: pgmap v996: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 60 MiB/s wr, 966 op/s; 6337/128979 objects degraded (4.913%); 1275/128979 objects misplaced (0.989%); 20 MiB/s, 311 objects/s recovering 2023-09-23T22:31:47.209 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:47 smithi026 ceph-mon[215775]: pgmap v996: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 60 MiB/s wr, 966 op/s; 6337/128979 objects degraded (4.913%); 1275/128979 objects misplaced (0.989%); 20 MiB/s, 311 objects/s recovering 2023-09-23T22:31:47.217 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:47.319 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:47.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:47.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:47.427 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:31:47.494 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:47 smithi070 ceph-mon[192652]: pgmap v996: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 60 MiB/s wr, 966 op/s; 6337/128979 objects degraded (4.913%); 1275/128979 objects misplaced (0.989%); 20 MiB/s, 311 objects/s recovering 2023-09-23T22:31:47.504 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:47.598 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:31:47.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:47.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:31:47.711 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:47.768 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:48.097 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 46460 46444 87.9393 74.875 0.00586205 0.0113667 2023-09-23T22:31:48.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4089028998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:48.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:48 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:48.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/75059135' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:31:48.145 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6337/128979 objects degraded (4.913%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:48.157 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:31:48.158 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:31:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4089028998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:48 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/75059135' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:31:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6337/128979 objects degraded (4.913%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4089028998' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:48 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:31:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/75059135' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:31:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6337/128979 objects degraded (4.913%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:49.098 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 47687 47671 87.6082 76.6875 0.00231823 0.0113924 2023-09-23T22:31:49.426 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:49.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[215775]: pgmap v997: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 45 MiB/s wr, 718 op/s; 6337/128979 objects degraded (4.913%); 1275/128979 objects misplaced (0.989%); 15 MiB/s, 231 objects/s recovering 2023-09-23T22:31:49.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:49.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[215775]: failed to encode map e390 with expected crc 2023-09-23T22:31:49.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:31:49.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/75059135' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:31:49.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[215775]: osdmap e390: 8 total, 8 up, 5 in 2023-09-23T22:31:49.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[223014]: pgmap v997: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 45 MiB/s wr, 718 op/s; 6337/128979 objects degraded (4.913%); 1275/128979 objects misplaced (0.989%); 15 MiB/s, 231 objects/s recovering 2023-09-23T22:31:49.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:49.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[223014]: failed to encode map e390 with expected crc 2023-09-23T22:31:49.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:31:49.434 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/75059135' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:31:49.434 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:49 smithi026 ceph-mon[223014]: osdmap e390: 8 total, 8 up, 5 in 2023-09-23T22:31:49.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:49.535 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:49.565 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:49.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:49 smithi070 ceph-mon[192652]: pgmap v997: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.6 GiB data, 9.3 GiB used, 438 GiB / 447 GiB avail; 45 MiB/s wr, 718 op/s; 6337/128979 objects degraded (4.913%); 1275/128979 objects misplaced (0.989%); 15 MiB/s, 231 objects/s recovering 2023-09-23T22:31:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:49 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:31:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:49 smithi070 ceph-mon[192652]: failed to encode map e390 with expected crc 2023-09-23T22:31:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:49 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:31:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:49 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/75059135' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:31:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:49 smithi070 ceph-mon[192652]: osdmap e390: 8 total, 8 up, 5 in 2023-09-23T22:31:49.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:31:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:31:49] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2023-09-23T22:31:49.909 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:31:49.941 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:31:49.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:31:50.010 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:31:50.046 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:31:50.081 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:31:50.097 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 48725 48709 86.9586 64.875 0.0102262 0.0114957 2023-09-23T22:31:50.118 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:31:50.158 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:31:50.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:31:50.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:31:50.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:31:50.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:31:50.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:31:50.165 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:31:50.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:31:50.166 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:31:50.197 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:50.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:50.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:50.240 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T22:31:50.301 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:50.395 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:50.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1747996870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:50.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1747996870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:50.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1747996870' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:31:51.098 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 49872 49856 86.5343 71.6875 0.00823272 0.0115503 2023-09-23T22:31:51.347 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:51.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:51.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:51.417 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:31:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:31:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:31:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:31:51.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e364/351 cost 1049576)", 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:41.461728+0000", 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 69.951087051000002, 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042939699999999998, 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.461728+0000", 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.461728+0000", 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.461726+0000", 2023-09-23T22:31:51.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.461741+0000", 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4423999999999999e-05 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.461772+0000", 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1168999999999999e-05 2023-09-23T22:31:51.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.461782+0000", 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7999999999999993e-06 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.461853+0000", 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0769e-05 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.462157+0000", 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030456599999999998 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e364/351 cost 1049576)", 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:41.593967+0000", 2023-09-23T22:31:51.422 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 69.818847899000005, 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001277095, 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.593967+0000", 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.593967+0000", 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.593966+0000", 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.593978+0000", 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1593000000000001e-05 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.593993+0000", 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.487e-05 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.594000+0000", 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0280000000000003e-06 2023-09-23T22:31:51.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.594986+0000", 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098627299999999997 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.595244+0000", 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000258108 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.426 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e364/351 cost 1049576)", 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:41.701543+0000", 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 69.711271369000002, 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049661400000000004, 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.701543+0000", 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.701543+0000", 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.701542+0000", 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.701559+0000", 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6909999999999999e-05 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.701582+0000", 2023-09-23T22:31:51.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3419000000000001e-05 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.701603+0000", 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0899000000000001e-05 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.701668+0000", 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4333999999999998e-05 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:41.702040+0000", 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037235700000000003 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e364/351 cost 1049576)", 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:42.277240+0000", 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 69.135574332999994, 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057519199999999998, 2023-09-23T22:31:51.431 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.277240+0000", 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.277240+0000", 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.277239+0000", 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.277255+0000", 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5716e-05 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.277276+0000", 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1509999999999999e-05 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.277285+0000", 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0410000000000003e-06 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.277342+0000", 2023-09-23T22:31:51.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6447999999999999e-05 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.277816+0000", 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047384400000000001 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e364/351 cost 1049576)", 2023-09-23T22:31:51.435 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:42.525765+0000", 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 68.887050008000003, 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025160030000000002, 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.525765+0000", 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.525765+0000", 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.525763+0000", 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.525782+0000", 2023-09-23T22:31:51.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8981e-05 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.525805+0000", 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2645000000000001e-05 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.525829+0000", 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4303000000000001e-05 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.528024+0000", 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021951090000000002 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.528281+0000", 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025633200000000001 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e364/351 cost 1049576)", 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:42.766976+0000", 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 68.645838964000006, 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001744174, 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.766976+0000", 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.766976+0000", 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.766974+0000", 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.766990+0000", 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5579000000000001e-05 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.767012+0000", 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2473e-05 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.767021+0000", 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7539999999999995e-06 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.768469+0000", 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001448235 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:42.768720+0000", 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000250778 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.443 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e365/351 cost 1049576)", 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:43.654059+0000", 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 67.758756075999997, 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025948630000000002, 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:43.654059+0000", 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:43.654059+0000", 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:43.654057+0000", 2023-09-23T22:31:51.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:43.654076+0000", 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9000000000000001e-05 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:43.654099+0000", 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3102e-05 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:43.654110+0000", 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0841e-05 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:43.654183+0000", 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2360999999999994e-05 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:43.656654+0000", 2023-09-23T22:31:51.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024710029999999998 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e365/351 cost 1049576)", 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:44.378191+0000", 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 67.034623523999997, 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069919040000000002, 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.448 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.378191+0000", 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.378191+0000", 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.378190+0000", 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.378206+0000", 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5911000000000001e-05 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.378228+0000", 2023-09-23T22:31:51.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1725e-05 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.378237+0000", 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1209999999999996e-06 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.384851+0000", 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066142589999999999 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.385183+0000", 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000331956 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e365/351 cost 1049576)", 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:44.687150+0000", 2023-09-23T22:31:51.452 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 66.725665155000002, 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027411380000000002, 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.687150+0000", 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.687150+0000", 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.687148+0000", 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.687169+0000", 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.067e-05 2023-09-23T22:31:51.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.687195+0000", 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6299e-05 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.687206+0000", 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1012999999999999e-05 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.687297+0000", 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1460999999999997e-05 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.689891+0000", 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025933729999999999 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e365/351 cost 1049576)", 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:44.719505+0000", 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 66.693309580000005, 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002798011, 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.719505+0000", 2023-09-23T22:31:51.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.719505+0000", 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.719504+0000", 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.719520+0000", 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6033999999999999e-05 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.719541+0000", 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0704999999999999e-05 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.719551+0000", 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0241000000000001e-05 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.719598+0000", 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6823999999999998e-05 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:44.722303+0000", 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027054459999999998 2023-09-23T22:31:51.460 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e365/351 cost 1049576)", 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:45.153443+0000", 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 66.259372240999994, 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048905900000000002, 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153443+0000", 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153443+0000", 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153441+0000", 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153460+0000", 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8867000000000001e-05 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153483+0000", 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2532999999999999e-05 2023-09-23T22:31:51.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153493+0000", 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9839999999999996e-06 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153591+0000", 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8250000000000003e-05 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153932+0000", 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034074300000000001 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e365/351 cost 1049576)", 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:45.153510+0000", 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 66.259304682000007, 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00066929299999999995, 2023-09-23T22:31:51.465 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153510+0000", 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153510+0000", 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153510+0000", 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153516+0000", 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5910000000000001e-06 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153522+0000", 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6699999999999999e-06 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153528+0000", 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.331e-06 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.153954+0000", 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000425401 2023-09-23T22:31:51.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.154179+0000", 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022558999999999999 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e366/351 cost 1049576)", 2023-09-23T22:31:51.469 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:45.462452+0000", 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 65.950362338000005, 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026152829999999999, 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.462452+0000", 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.462452+0000", 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.462451+0000", 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.462466+0000", 2023-09-23T22:31:51.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4688999999999999e-05 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.462488+0000", 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1824999999999999e-05 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.462497+0000", 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6509999999999996e-06 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.462568+0000", 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0321999999999995e-05 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.465068+0000", 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024999660000000002 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e366/351 cost 1049576)", 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:45.538508+0000", 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 65.874307267999995, 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049600399999999998, 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.538508+0000", 2023-09-23T22:31:51.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.538508+0000", 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.538506+0000", 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.538524+0000", 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8624999999999999e-05 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.538546+0000", 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1994999999999999e-05 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.538555+0000", 2023-09-23T22:31:51.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9420000000000001e-06 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.538622+0000", 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6457000000000004e-05 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.539004+0000", 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038176900000000001 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.477 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e366/351 cost 1049576)", 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:45.930545+0000", 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 65.482269428999999, 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058732460000000004, 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.930545+0000", 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.930545+0000", 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.930544+0000", 2023-09-23T22:31:51.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.930559+0000", 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5078e-05 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.930581+0000", 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2586999999999999e-05 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.930591+0000", 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5950000000000005e-06 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.930666+0000", 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5346999999999995e-05 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:45.936419+0000", 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057521930000000001 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e366/351 cost 1049576)", 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:46.312862+0000", 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 65.099952364000004, 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026961020000000001, 2023-09-23T22:31:51.482 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.312862+0000", 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.312862+0000", 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.312861+0000", 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.312878+0000", 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.666e-05 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.312900+0000", 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2065999999999998e-05 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.312910+0000", 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6250000000000002e-06 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.312967+0000", 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7422e-05 2023-09-23T22:31:51.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.315559+0000", 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025914789999999998 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e366/351 cost 1049576)", 2023-09-23T22:31:51.486 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:46.806093+0000", 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 64.606721633000006, 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00102301, 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806093+0000", 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806093+0000", 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806092+0000", 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806108+0000", 2023-09-23T22:31:51.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6024000000000001e-05 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806129+0000", 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0673e-05 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806138+0000", 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9190000000000005e-06 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806189+0000", 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1437000000000002e-05 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.807116+0000", 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00092705800000000003 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e366/351 cost 1049576)", 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:46.806155+0000", 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 64.606660046000002, 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005356263, 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806155+0000", 2023-09-23T22:31:51.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806155+0000", 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806154+0000", 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806161+0000", 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2720000000000003e-06 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806167+0000", 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6780000000000003e-06 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.806174+0000", 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.511e-06 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.807142+0000", 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00096766899999999999 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:46.811511+0000", 2023-09-23T22:31:51.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043694959999999996 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e366/351 cost 1049576)", 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:47.723315+0000", 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.689500201000001, 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040293949999999999, 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.495 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723315+0000", 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723315+0000", 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723313+0000", 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723329+0000", 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5432000000000001e-05 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723350+0000", 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1263e-05 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723359+0000", 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5000000000000005e-06 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723448+0000", 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8967000000000004e-05 2023-09-23T22:31:51.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.727344+0000", 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003895683 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPull(7.c e366/351 cost 1049576)", 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:30:47.723373+0000", 2023-09-23T22:31:51.499 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.689442145000001, 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066794369999999999, 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723373+0000", 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:51.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723373+0000", 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723372+0000", 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723379+0000", 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7360000000000004e-06 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723385+0000", 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7579999999999998e-06 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.723391+0000", 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.02e-06 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.728843+0000", 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054520740000000003 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:30:47.730052+0000", 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012091700000000001 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:51.503 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:51.504 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:31:51.504 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:51.505 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:51 smithi026 ceph-mon[215775]: pgmap v999: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 57 MiB/s wr, 918 op/s; 6337/132576 objects degraded (4.780%); 1275/132576 objects misplaced (0.962%); 15 MiB/s, 231 objects/s recovering 2023-09-23T22:31:51.505 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:51 smithi026 ceph-mon[223014]: pgmap v999: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 57 MiB/s wr, 918 op/s; 6337/132576 objects degraded (4.780%); 1275/132576 objects misplaced (0.962%); 15 MiB/s, 231 objects/s recovering 2023-09-23T22:31:51.557 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:51 smithi070 ceph-mon[192652]: pgmap v999: 99 pgs: 2 peering, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 92 active+clean; 2.7 GiB data, 9.5 GiB used, 438 GiB / 447 GiB avail; 57 MiB/s wr, 918 op/s; 6337/132576 objects degraded (4.780%); 1275/132576 objects misplaced (0.962%); 15 MiB/s, 231 objects/s recovering 2023-09-23T22:31:51.559 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:51.561 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:51.713 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:51.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:51.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:51.827 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:51.900 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:52.013 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:52.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:52.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:52.098 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 51131 51115 86.3221 78.6875 0.0167389 0.0115784 2023-09-23T22:31:52.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:52.160 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:52.243 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:52.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:52.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:52.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:31:52.350 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:52.409 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:52.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:52.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:52.518 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:52.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:52.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:52.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:31:52.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:52.684 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:52.827 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:31:52.828 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:52.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:52.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:52.932 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:52.991 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:53.098 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 52059 52043 85.5759 58 0.00758943 0.0116795 2023-09-23T22:31:53.196 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:53.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:53.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:53.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:53 smithi026 ceph-mon[223014]: pgmap v1000: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s; 6337/145161 objects degraded (4.365%); 877/145161 objects misplaced (0.604%); 15 MiB/s, 240 objects/s recovering 2023-09-23T22:31:53.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6337/145161 objects degraded (4.365%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:53.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:53 smithi026 ceph-mon[215775]: pgmap v1000: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s; 6337/145161 objects degraded (4.365%); 877/145161 objects misplaced (0.604%); 15 MiB/s, 240 objects/s recovering 2023-09-23T22:31:53.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6337/145161 objects degraded (4.365%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:53.304 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:31:53.341 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:53.358 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:31:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:53 smithi070 ceph-mon[192652]: pgmap v1000: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 82 MiB/s wr, 1.31k op/s; 6337/145161 objects degraded (4.365%); 877/145161 objects misplaced (0.604%); 15 MiB/s, 240 objects/s recovering 2023-09-23T22:31:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6337/145161 objects degraded (4.365%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:53.856 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:31:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:53.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:31:53.927 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:53.966 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:31:54.020 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:54.098 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 53010 52994 84.905 59.4375 0.0166182 0.0117708 2023-09-23T22:31:54.125 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:31:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:54.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:31:54.235 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:54.270 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:54.328 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:54.329 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:31:54.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:54.444 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:54.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:54.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:54.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:54.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:31:54.636 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:54.694 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:54.832 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:54.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:54.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:54.937 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:31:55.014 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:55.099 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:31:55.099397+0000 min lat: 0.00111004 max lat: 2.14499 avg lat: 0.0118241 2023-09-23T22:31:55.099 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:31:55.099 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 54127 54111 84.5275 69.8125 0.0176424 0.0118241 2023-09-23T22:31:55.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:31:55.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:55.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:31:55.180 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:55 smithi026 ceph-mon[215775]: pgmap v1001: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 40 MiB/s wr, 639 op/s; 6337/145161 objects degraded (4.365%); 877/145161 objects misplaced (0.604%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T22:31:55.180 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:55.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:55.271 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:55 smithi070 ceph-mon[192652]: pgmap v1001: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 40 MiB/s wr, 639 op/s; 6337/145161 objects degraded (4.365%); 877/145161 objects misplaced (0.604%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T22:31:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:55.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:55 smithi026 ceph-mon[223014]: pgmap v1001: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.0 GiB data, 10 GiB used, 437 GiB / 447 GiB avail; 40 MiB/s wr, 639 op/s; 6337/145161 objects degraded (4.365%); 877/145161 objects misplaced (0.604%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T22:31:55.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:56.099 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 55208 55192 84.1136 67.5625 0.0196953 0.0118831 2023-09-23T22:31:56.212 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T22:31:56.214 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:56.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2091868031' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:31:56.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:56 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:31:56.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2091868031' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:31:56.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:56 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:31:56.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:56.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:56.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:56.365 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2091868031' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:31:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:56 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:31:57.099 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 56314 56298 83.7567 69.125 0.0211001 0.0119338 2023-09-23T22:31:57.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[215775]: pgmap v1002: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 68 MiB/s wr, 1.08k op/s; 6337/161415 objects degraded (3.926%); 350/161415 objects misplaced (0.217%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:31:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[215775]: failed to encode map e391 with expected crc 2023-09-23T22:31:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:31:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[215775]: osdmap e391: 8 total, 8 up, 5 in 2023-09-23T22:31:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[215775]: failed to encode map e391 with expected crc 2023-09-23T22:31:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[223014]: pgmap v1002: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 68 MiB/s wr, 1.08k op/s; 6337/161415 objects degraded (3.926%); 350/161415 objects misplaced (0.217%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:31:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[223014]: failed to encode map e391 with expected crc 2023-09-23T22:31:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:31:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[223014]: osdmap e391: 8 total, 8 up, 5 in 2023-09-23T22:31:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:57 smithi026 ceph-mon[223014]: failed to encode map e391 with expected crc 2023-09-23T22:31:57.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:57 smithi070 ceph-mon[192652]: pgmap v1002: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 68 MiB/s wr, 1.08k op/s; 6337/161415 objects degraded (3.926%); 350/161415 objects misplaced (0.217%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:31:57.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:57 smithi070 ceph-mon[192652]: failed to encode map e391 with expected crc 2023-09-23T22:31:57.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:31:57.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:57 smithi070 ceph-mon[192652]: osdmap e391: 8 total, 8 up, 5 in 2023-09-23T22:31:57.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:57 smithi070 ceph-mon[192652]: failed to encode map e391 with expected crc 2023-09-23T22:31:58.099 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 57381 57365 83.3596 66.6875 0.00253662 0.0119758 2023-09-23T22:31:58.365 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:31:58.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6337/161415 objects degraded (3.926%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:58.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6337/161415 objects degraded (3.926%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:58.534 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:31:58.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:58.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:31:58.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6337/161415 objects degraded (3.926%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:31:58.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:31:58.692 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:58.793 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:31:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:58.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:31:58.900 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:31:58.987 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:59.099 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 58353 58337 82.8458 60.75 0.00694834 0.0120649 2023-09-23T22:31:59.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:59 smithi026 ceph-mon[215775]: pgmap v1004: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 72 MiB/s wr, 1.15k op/s; 6337/161415 objects degraded (3.926%); 350/161415 objects misplaced (0.217%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T22:31:59.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:59 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:59.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:59 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:59.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:31:59 smithi026 ceph-mon[215775]: osdmap e392: 8 total, 8 up, 5 in 2023-09-23T22:31:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:59 smithi026 ceph-mon[223014]: pgmap v1004: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 72 MiB/s wr, 1.15k op/s; 6337/161415 objects degraded (3.926%); 350/161415 objects misplaced (0.217%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T22:31:59.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:59 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:59.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:59 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:59.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:31:59 smithi026 ceph-mon[223014]: osdmap e392: 8 total, 8 up, 5 in 2023-09-23T22:31:59.496 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:31:59.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:59.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:31:59.571 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:31:59.571 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:31:59.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:59.571 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPushReply(8.e 392/381 [PushReplyOp(8:797dec05:::benchmark_data_smithi026_497658_object1718:head)])", 2023-09-23T22:31:59.571 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:31:59.570002+0000", 2023-09-23T22:31:59.571 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00089318600000000002, 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00091474299999999998, 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:59.570002+0000", 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:59.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:59.570002+0000", 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:59.570001+0000", 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:31:59.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:59.570015+0000", 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3499999999999999e-05 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:59.570033+0000", 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8413000000000001e-05 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:59.570044+0000", 2023-09-23T22:31:59.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1486e-05 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:59.570107+0000", 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2422999999999995e-05 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 1 2023-09-23T22:31:59.575 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:31:59.589 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:31:59.600 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:59.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:59 smithi070 ceph-mon[192652]: pgmap v1004: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 72 MiB/s wr, 1.15k op/s; 6337/161415 objects degraded (3.926%); 350/161415 objects misplaced (0.217%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T22:31:59.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:59 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:59.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:59 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:31:59.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:31:59 smithi070 ceph-mon[192652]: osdmap e392: 8 total, 8 up, 5 in 2023-09-23T22:31:59.627 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:59.629 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:31:59.658 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:31:59.669 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:59.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:59.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:59.698 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:31:59.721 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:31:59.745 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:31:59.767 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:31:59.773 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:31:59.791 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:31:59.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:31:59.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:31:59.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:31:59.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:31:59.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:31:59.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:31:59.793 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:31:59.794 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:31:59.800 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:31:59.828 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:31:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:31:59] "GET /metrics HTTP/1.1" 200 36368 "" "Prometheus/2.43.0" 2023-09-23T22:31:59.841 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:31:59.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:31:59.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:31:59.947 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:31:59.980 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:00.099 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 58522 58506 81.2397 10.5625 0.00265292 0.0120591 2023-09-23T22:32:00.120 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:32:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[215775]: failed to encode map e392 with expected crc 2023-09-23T22:32:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[215775]: failed to encode map e392 with expected crc 2023-09-23T22:32:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2903697976' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:00.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/790583544' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:32:00.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:00.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/790583544' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:32:00.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[215775]: osdmap e393: 8 total, 8 up, 5 in 2023-09-23T22:32:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[223014]: failed to encode map e392 with expected crc 2023-09-23T22:32:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[223014]: failed to encode map e392 with expected crc 2023-09-23T22:32:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2903697976' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/790583544' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:32:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:00.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/790583544' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:32:00.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:00 smithi026 ceph-mon[223014]: osdmap e393: 8 total, 8 up, 5 in 2023-09-23T22:32:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:00 smithi070 ceph-mon[192652]: failed to encode map e392 with expected crc 2023-09-23T22:32:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:00 smithi070 ceph-mon[192652]: failed to encode map e392 with expected crc 2023-09-23T22:32:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2903697976' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/790583544' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:32:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:00 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/790583544' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:32:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:00 smithi070 ceph-mon[192652]: osdmap e393: 8 total, 8 up, 5 in 2023-09-23T22:32:01.099 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 59396 59380 80.6611 54.625 0.00268519 0.0123896 2023-09-23T22:32:01.363 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:01.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:01.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:01.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:01 smithi026 ceph-mon[215775]: pgmap v1006: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 53 MiB/s wr, 852 op/s; 6337/165618 objects degraded (3.826%); 350/165618 objects misplaced (0.211%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:32:01.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:01 smithi026 ceph-mon[215775]: failed to encode map e393 with expected crc 2023-09-23T22:32:01.371 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:01 smithi026 ceph-mon[215775]: failed to encode map e393 with expected crc 2023-09-23T22:32:01.371 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:01 smithi026 ceph-mon[223014]: pgmap v1006: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 53 MiB/s wr, 852 op/s; 6337/165618 objects degraded (3.826%); 350/165618 objects misplaced (0.211%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:32:01.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:01 smithi026 ceph-mon[223014]: failed to encode map e393 with expected crc 2023-09-23T22:32:01.372 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:01 smithi026 ceph-mon[223014]: failed to encode map e393 with expected crc 2023-09-23T22:32:01.472 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:01.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:01 smithi070 ceph-mon[192652]: pgmap v1006: 99 pgs: 1 active+recovering+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 53 MiB/s wr, 852 op/s; 6337/165618 objects degraded (3.826%); 350/165618 objects misplaced (0.211%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:32:01.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:01 smithi070 ceph-mon[192652]: failed to encode map e393 with expected crc 2023-09-23T22:32:01.518 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:01 smithi070 ceph-mon[192652]: failed to encode map e393 with expected crc 2023-09-23T22:32:01.523 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:01.617 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:01.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:01.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:01.729 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:01.798 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:01.872 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:32:01.905 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:01.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:01.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:02.011 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:02.071 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:02.100 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 60411 60395 80.2933 63.4375 0.00715314 0.0124483 2023-09-23T22:32:02.449 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:02.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:02.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:02.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:02 smithi026 ceph-mon[215775]: pgmap v1008: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 49 MiB/s wr, 785 op/s; 4800/175563 objects degraded (2.734%); 1379/175563 objects misplaced (0.785%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T22:32:02.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:02 smithi026 ceph-mon[223014]: pgmap v1008: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 49 MiB/s wr, 785 op/s; 4800/175563 objects degraded (2.734%); 1379/175563 objects misplaced (0.785%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T22:32:02.556 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:02.587 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:02.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:02 smithi070 ceph-mon[192652]: pgmap v1008: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 49 MiB/s wr, 785 op/s; 4800/175563 objects degraded (2.734%); 1379/175563 objects misplaced (0.785%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T22:32:02.651 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:02.772 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:02.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:02.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:02.786 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:02.877 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:02.946 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:03.092 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:03.100 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 61560 61544 80.1163 71.8125 0.0137306 0.0124759 2023-09-23T22:32:03.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:03.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:03.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:03.262 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:03.348 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:03.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:03.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:03.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4800/175563 objects degraded (2.734%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:03.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4800/175563 objects degraded (2.734%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:03.453 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:03.512 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4800/175563 objects degraded (2.734%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:03.641 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:03.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:03.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:03.747 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:03.809 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:32:03.814 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:03.908 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:03.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:03.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:04.027 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:04.086 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:04.100 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 62701 62685 79.9364 71.3125 0.013555 0.0125047 2023-09-23T22:32:04.192 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:04.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:04.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:04.234 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:04 smithi026 ceph-mon[223014]: pgmap v1009: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 40 MiB/s wr, 640 op/s; 4800/175563 objects degraded (2.734%); 1379/175563 objects misplaced (0.785%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:32:04.234 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:04 smithi026 ceph-mon[215775]: pgmap v1009: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 40 MiB/s wr, 640 op/s; 4800/175563 objects degraded (2.734%); 1379/175563 objects misplaced (0.785%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:32:04.299 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:04.338 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:04 smithi070 ceph-mon[192652]: pgmap v1009: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 3 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 40 MiB/s wr, 640 op/s; 4800/175563 objects degraded (2.734%); 1379/175563 objects misplaced (0.785%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T22:32:04.343 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:04.454 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:04.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:04.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:04.566 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:04.626 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:32:04.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:05.100 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 63601 63585 79.4626 56.25 0.00522921 0.0125475 2023-09-23T22:32:05.647 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:05.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:05.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:05.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:05.830 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:05.988 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:05.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:05.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:06.096 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:06.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 64537 64521 79.0515 58.5 0.0177424 0.0126444 2023-09-23T22:32:06.162 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:06.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:06.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:06.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:06.448 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:06.484 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:06.554 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:06.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:06 smithi026 ceph-mon[215775]: pgmap v1010: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 75 MiB/s wr, 1.20k op/s; 4800/190176 objects degraded (2.524%); 903/190176 objects misplaced (0.475%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:32:06.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:06 smithi026 ceph-mon[223014]: pgmap v1010: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 75 MiB/s wr, 1.20k op/s; 4800/190176 objects degraded (2.524%); 903/190176 objects misplaced (0.475%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:32:06.698 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:06.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:06.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:06.803 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:06.820 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:06.902 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:06.990 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:32:06.990 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:32:07.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 65551 65535 78.75 63.375 0.0151062 0.0126915 2023-09-23T22:32:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:06 smithi070 ceph-mon[192652]: pgmap v1010: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 75 MiB/s wr, 1.20k op/s; 4800/190176 objects degraded (2.524%); 903/190176 objects misplaced (0.475%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T22:32:07.211 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:07.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:07.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:07.319 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:07.414 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:07.668 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:32:07.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3374644208' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:32:07.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:07 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:32:07.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3374644208' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:32:07.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:07 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:32:08.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 66703 66687 78.6225 72 0.00978485 0.0127072 2023-09-23T22:32:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3374644208' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:32:08.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:07 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:32:08.977 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[215775]: pgmap v1011: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 71 MiB/s wr, 1.13k op/s; 4800/190176 objects degraded (2.524%); 903/190176 objects misplaced (0.475%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T22:32:08.977 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:32:08.977 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[215775]: osdmap e394: 8 total, 8 up, 5 in 2023-09-23T22:32:08.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[215775]: failed to encode map e394 with expected crc 2023-09-23T22:32:08.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[215775]: failed to encode map e394 with expected crc 2023-09-23T22:32:08.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4800/190176 objects degraded (2.524%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:08.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[223014]: pgmap v1011: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 71 MiB/s wr, 1.13k op/s; 4800/190176 objects degraded (2.524%); 903/190176 objects misplaced (0.475%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T22:32:08.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:32:08.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[223014]: osdmap e394: 8 total, 8 up, 5 in 2023-09-23T22:32:08.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[223014]: failed to encode map e394 with expected crc 2023-09-23T22:32:08.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[223014]: failed to encode map e394 with expected crc 2023-09-23T22:32:08.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4800/190176 objects degraded (2.524%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:09.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:32:09.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 67702 67686 78.3227 62.4375 0.00872728 0.0127616 2023-09-23T22:32:09.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:08 smithi070 ceph-mon[192652]: pgmap v1011: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 71 MiB/s wr, 1.13k op/s; 4800/190176 objects degraded (2.524%); 903/190176 objects misplaced (0.475%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T22:32:09.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:08 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:32:09.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:08 smithi070 ceph-mon[192652]: osdmap e394: 8 total, 8 up, 5 in 2023-09-23T22:32:09.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:08 smithi070 ceph-mon[192652]: failed to encode map e394 with expected crc 2023-09-23T22:32:09.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:08 smithi070 ceph-mon[192652]: failed to encode map e394 with expected crc 2023-09-23T22:32:09.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4800/190176 objects degraded (2.524%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:09.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:32:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:32:09] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.43.0" 2023-09-23T22:32:09.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:09.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:09.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:09.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:09.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:09.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:09.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:09.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:09.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:09.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:09.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:10.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 68584 68568 77.9009 55.125 0.00895902 0.0128315 2023-09-23T22:32:10.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:10.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:10.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:10.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:32:10.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:32:10.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:10.823 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:10.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:10.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:10.930 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:10.984 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:11.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 69756 69740 77.8177 73.25 0.00986356 0.0128453 2023-09-23T22:32:11.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:10 smithi070 ceph-mon[192652]: pgmap v1013: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 40 MiB/s wr, 644 op/s; 4800/193845 objects degraded (2.476%); 903/193845 objects misplaced (0.466%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:32:11.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:10 smithi026 ceph-mon[215775]: pgmap v1013: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 40 MiB/s wr, 644 op/s; 4800/193845 objects degraded (2.476%); 903/193845 objects misplaced (0.466%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:32:11.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:10 smithi026 ceph-mon[223014]: pgmap v1013: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 40 MiB/s wr, 644 op/s; 4800/193845 objects degraded (2.476%); 903/193845 objects misplaced (0.466%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:32:11.606 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T22:32:11.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:11.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:11.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:11.875 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:11.955 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:12.001 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:12.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 70676 70660 77.4612 57.5 0.00986258 0.0129048 2023-09-23T22:32:12.544 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:12.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:12.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:12.655 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:12.705 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:12.812 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:12.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:12.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:12.829 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:32:12.874 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:32:12.917 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:32:12.923 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:12.930 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:12.955 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:32:12.998 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:32:13.013 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:13.045 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:32:13.085 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:32:13.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 71705 71689 77.2344 64.3125 0.0127747 0.0129416 2023-09-23T22:32:13.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:12 smithi070 ceph-mon[192652]: pgmap v1014: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 61 MiB/s wr, 982 op/s; 4800/205050 objects degraded (2.341%); 498/205050 objects misplaced (0.243%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:32:13.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3736471724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:13.136 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:32:13.155 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:32:13.155 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:32:13.155 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:32:13.155 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:32:13.155 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:32:13.156 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:32:13.156 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:32:13.156 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:32:13.171 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:13.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:13.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:13.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:12 smithi026 ceph-mon[215775]: pgmap v1014: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 61 MiB/s wr, 982 op/s; 4800/205050 objects degraded (2.341%); 498/205050 objects misplaced (0.243%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:32:13.178 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3736471724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:13.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:12 smithi026 ceph-mon[223014]: pgmap v1014: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 61 MiB/s wr, 982 op/s; 4800/205050 objects degraded (2.341%); 498/205050 objects misplaced (0.243%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:32:13.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3736471724' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:13.279 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:13.344 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:13.455 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:13.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:13.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:13.565 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:13.621 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:13.662 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:13.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:13.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:13.769 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:13.821 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:14.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 72722 72706 77.0027 63.5625 0.0199011 0.0129815 2023-09-23T22:32:14.137 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:32:14.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4800/205050 objects degraded (2.341%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:14.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2496159463' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:14.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:13 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:14.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4800/205050 objects degraded (2.341%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:14.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2496159463' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:14.164 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:13 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:14.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4800/205050 objects degraded (2.341%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:14.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2496159463' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:14.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:13 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:14.493 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:14.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:14.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:14.597 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:14.623 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:14.667 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:14.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:14.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:14.772 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:14.799 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:14.844 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:14.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:14.951 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:14.982 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:15.101 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:32:15.102180+0000 min lat: 0.00111004 max lat: 2.14499 avg lat: 0.0129729 2023-09-23T22:32:15.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:32:15.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 72991 72975 75.9996 16.8125 0.0029957 0.0129729 2023-09-23T22:32:15.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[223014]: pgmap v1015: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 61 MiB/s wr, 982 op/s; 4800/205050 objects degraded (2.341%); 498/205050 objects misplaced (0.243%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:32:15.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:32:15.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:32:15.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[223014]: osdmap e395: 8 total, 8 up, 5 in 2023-09-23T22:32:15.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[223014]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[223014]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[223014]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[223014]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[215775]: pgmap v1015: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 61 MiB/s wr, 982 op/s; 4800/205050 objects degraded (2.341%); 498/205050 objects misplaced (0.243%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[215775]: osdmap e395: 8 total, 8 up, 5 in 2023-09-23T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[215775]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[215775]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[215775]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:14 smithi026 ceph-mon[215775]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.363 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:15.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:15.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:14 smithi070 ceph-mon[192652]: pgmap v1015: 99 pgs: 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 61 MiB/s wr, 982 op/s; 4800/205050 objects degraded (2.341%); 498/205050 objects misplaced (0.243%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:32:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:14 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:32:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:14 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:32:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:14 smithi070 ceph-mon[192652]: osdmap e395: 8 total, 8 up, 5 in 2023-09-23T22:32:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:14 smithi070 ceph-mon[192652]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:14 smithi070 ceph-mon[192652]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:14 smithi070 ceph-mon[192652]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:14 smithi070 ceph-mon[192652]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.471 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:15.551 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:15.670 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:15.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:15.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:15.780 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:15.843 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:15.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[215775]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.889 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[215775]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[215775]: failed to encode map e396 with expected crc 2023-09-23T22:32:15.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[215775]: failed to encode map e396 with expected crc 2023-09-23T22:32:15.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[215775]: osdmap e396: 8 total, 8 up, 5 in 2023-09-23T22:32:15.890 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[215775]: failed to encode map e396 with expected crc 2023-09-23T22:32:15.890 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[223014]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[223014]: failed to encode map e395 with expected crc 2023-09-23T22:32:15.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[223014]: failed to encode map e396 with expected crc 2023-09-23T22:32:15.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[223014]: failed to encode map e396 with expected crc 2023-09-23T22:32:15.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[223014]: osdmap e396: 8 total, 8 up, 5 in 2023-09-23T22:32:15.891 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:15 smithi026 ceph-mon[223014]: failed to encode map e396 with expected crc 2023-09-23T22:32:16.040 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:16.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:16.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:16.056 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:32:16.056 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:32:16.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:32:16.056 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83452 7.e 7:7212fb35:::benchmark_data_smithi026_410749_object83451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.976183+0000", 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 368.07598181600002, 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.748405781, 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.057 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83452 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976183+0000", 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976183+0000", 2023-09-23T22:32:16.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976181+0000", 2023-09-23T22:32:16.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:16.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976213+0000", 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2251e-05 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976230+0000", 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6982000000000001e-05 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976242+0000", 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.219e-05 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976670+0000", 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042763099999999999 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976680+0000", 2023-09-23T22:32:16.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9739999999999997e-06 2023-09-23T22:32:16.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703132+0000", 2023-09-23T22:32:16.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7264517640000001 2023-09-23T22:32:16.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705750+0000", 2023-09-23T22:32:16.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026186209999999998 2023-09-23T22:32:16.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:32:16.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705939+0000", 2023-09-23T22:32:16.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018901399999999999 2023-09-23T22:32:16.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.712672+0000", 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067324810000000002 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.716114+0000", 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034423779999999998 2023-09-23T22:32:16.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720906+0000", 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047918270000000002 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720954+0000", 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8099000000000001e-05 2023-09-23T22:32:16.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.724589+0000", 2023-09-23T22:32:16.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036344390000000002 2023-09-23T22:32:16.066 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.066 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.066 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83457 7.5 7:ae7c1643:::benchmark_data_smithi026_410749_object83456:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.976262+0000", 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 368.07590286700002, 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.709729018, 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.067 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83457 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976262+0000", 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976262+0000", 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976261+0000", 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976329+0000", 2023-09-23T22:32:16.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7922999999999993e-05 2023-09-23T22:32:16.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976339+0000", 2023-09-23T22:32:16.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0276e-05 2023-09-23T22:32:16.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976347+0000", 2023-09-23T22:32:16.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5920000000000003e-06 2023-09-23T22:32:16.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976407+0000", 2023-09-23T22:32:16.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0486000000000003e-05 2023-09-23T22:32:16.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976419+0000", 2023-09-23T22:32:16.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1871e-05 2023-09-23T22:32:16.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.662197+0000", 2023-09-23T22:32:16.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6857776609999999 2023-09-23T22:32:16.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.662351+0000", 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153938 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.662471+0000", 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012044099999999999 2023-09-23T22:32:16.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.669804+0000", 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073327050000000001 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.670014+0000", 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000209817 2023-09-23T22:32:16.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.681794+0000", 2023-09-23T22:32:16.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011780163 2023-09-23T22:32:16.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.681867+0000", 2023-09-23T22:32:16.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3206000000000003e-05 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.685991+0000", 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041235569999999999 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115950 7.c 7:34d3d495:::benchmark_data_smithi026_410749_object25613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:27.660282+0000", 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 228.39188280499999, 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0168458550000001, 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.077 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115950 2023-09-23T22:32:16.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.078 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660282+0000", 2023-09-23T22:32:16.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660282+0000", 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660280+0000", 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660288+0000", 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9459999999999998e-06 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660301+0000", 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2959000000000001e-05 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660309+0000", 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3289999999999994e-06 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.510116+0000", 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.849806879 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.525207+0000", 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015090694999999999 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.592500+0000", 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067292850000000001 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.597705+0000", 2023-09-23T22:32:16.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052054579999999996 2023-09-23T22:32:16.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:32:16.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.608095+0000", 2023-09-23T22:32:16.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010389557000000001 2023-09-23T22:32:16.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.640854+0000", 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032759282000000001 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.642265+0000", 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014111219999999999 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.645324+0000", 2023-09-23T22:32:16.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030586519999999998 2023-09-23T22:32:16.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.647991+0000", 2023-09-23T22:32:16.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026672029999999999 2023-09-23T22:32:16.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.652086+0000", 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040946309999999996 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.677128+0000", 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025042180000000001 2023-09-23T22:32:16.086 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115953 7.4 7:2a290ef6:::benchmark_data_smithi026_410749_object25616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:27.887950+0000", 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 228.164214939, 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7040569749999999, 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.087 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115953 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887950+0000", 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887950+0000", 2023-09-23T22:32:16.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887949+0000", 2023-09-23T22:32:16.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:16.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887955+0000", 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6150000000000002e-06 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887969+0000", 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3961000000000001e-05 2023-09-23T22:32:16.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887977+0000", 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.571e-06 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.560215+0000", 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6722379759999999 2023-09-23T22:32:16.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.560336+0000", 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121458 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.560445+0000", 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108448 2023-09-23T22:32:16.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.565997+0000", 2023-09-23T22:32:16.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055525820000000004 2023-09-23T22:32:16.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.566005+0000", 2023-09-23T22:32:16.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5989999999999996e-06 2023-09-23T22:32:16.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.591899+0000", 2023-09-23T22:32:16.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025894090000000002 2023-09-23T22:32:16.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.591964+0000", 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4912000000000005e-05 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.592007+0000", 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3003999999999997e-05 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117517 7.c 7:31f07fac:::benchmark_data_smithi026_410749_object27180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.499427+0000", 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.55273772199999, 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.410524804, 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.096 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117517 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.499427+0000", 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.499427+0000", 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.499426+0000", 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.499431+0000", 2023-09-23T22:32:16.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2950000000000001e-06 2023-09-23T22:32:16.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.499441+0000", 2023-09-23T22:32:16.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0104e-05 2023-09-23T22:32:16.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.499447+0000", 2023-09-23T22:32:16.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3639999999999998e-06 2023-09-23T22:32:16.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.582074+0000", 2023-09-23T22:32:16.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0826273070000001 2023-09-23T22:32:16.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.604389+0000", 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022314931999999999 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.713041+0000", 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.10865211800000001 2023-09-23T22:32:16.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.719792+0000", 2023-09-23T22:32:16.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067511000000000003 2023-09-23T22:32:16.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:32:16.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.733646+0000", 2023-09-23T22:32:16.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013853493999999999 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.825904+0000", 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.092258223 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.873481+0000", 2023-09-23T22:32:16.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.047577282999999998 2023-09-23T22:32:16.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.877112+0000", 2023-09-23T22:32:16.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036308460000000001 2023-09-23T22:32:16.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.881642+0000", 2023-09-23T22:32:16.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045295229999999997 2023-09-23T22:32:16.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.890159+0000", 2023-09-23T22:32:16.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0085173720000000005 2023-09-23T22:32:16.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.909952+0000", 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019792858 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117521 7.c 7:3ff628c6:::benchmark_data_smithi026_410749_object27184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.527841+0000", 2023-09-23T22:32:16.106 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.52432322999999, 2023-09-23T22:32:16.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.418152536, 2023-09-23T22:32:16.107 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.107 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.107 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.107 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.107 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.107 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117521 2023-09-23T22:32:16.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.107 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.527841+0000", 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.527841+0000", 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.527840+0000", 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.527846+0000", 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.429e-06 2023-09-23T22:32:16.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.527854+0000", 2023-09-23T22:32:16.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3680000000000002e-06 2023-09-23T22:32:16.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.527859+0000", 2023-09-23T22:32:16.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4890000000000003e-06 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.605670+0000", 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.077810347 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.625881+0000", 2023-09-23T22:32:16.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020211650000000001 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.782667+0000", 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.15678520700000001 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.795399+0000", 2023-09-23T22:32:16.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012731948999999999 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.805751+0000", 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010352971000000001 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.870989+0000", 2023-09-23T22:32:16.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.065237358999999995 2023-09-23T22:32:16.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.912074+0000", 2023-09-23T22:32:16.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.04108473 2023-09-23T22:32:16.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.914980+0000", 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002906914 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.917638+0000", 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026570790000000001 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.923014+0000", 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053767980000000003 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.945994+0000", 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022979626 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117528 7.c 7:3d671bb3:::benchmark_data_smithi026_410749_object27191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.558906+0000", 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.49325883399999, 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.678631703, 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.117 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.118 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117528 2023-09-23T22:32:16.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.118 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558906+0000", 2023-09-23T22:32:16.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558906+0000", 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558905+0000", 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:16.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558910+0000", 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6879999999999996e-06 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558922+0000", 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1355000000000001e-05 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558928+0000", 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0589999999999999e-06 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.947133+0000", 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.388205039 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:32:16.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.970820+0000", 2023-09-23T22:32:16.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023686631999999999 2023-09-23T22:32:16.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.055724+0000", 2023-09-23T22:32:16.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.084904269000000004 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.063411+0000", 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076874899999999999 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.076919+0000", 2023-09-23T22:32:16.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013507277 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.143918+0000", 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.066999283000000007 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.215628+0000", 2023-09-23T22:32:16.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.071710228000000001 2023-09-23T22:32:16.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.217535+0000", 2023-09-23T22:32:16.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00190687 2023-09-23T22:32:16.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.219406+0000", 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018707789999999999 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.222250+0000", 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028438980000000001 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.237538+0000", 2023-09-23T22:32:16.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015287945000000001 2023-09-23T22:32:16.127 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.127 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.127 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.127 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117532 7.c 7:34bdbea4:::benchmark_data_smithi026_410749_object27195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:32:16.127 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.692044+0000", 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.36012025700001, 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.585611748, 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117532 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.128 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692044+0000", 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692044+0000", 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692043+0000", 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692052+0000", 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8149999999999995e-06 2023-09-23T22:32:16.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692064+0000", 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.287e-05 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692073+0000", 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5059999999999995e-06 2023-09-23T22:32:16.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.972002+0000", 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2799290349999999 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.988907+0000", 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016904634000000002 2023-09-23T22:32:16.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.117641+0000", 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12873475000000001 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.122617+0000", 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049758129999999999 2023-09-23T22:32:16.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.134973+0000", 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012355414 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.212825+0000", 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.077852370000000004 2023-09-23T22:32:16.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.240596+0000", 2023-09-23T22:32:16.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027771139 2023-09-23T22:32:16.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.242949+0000", 2023-09-23T22:32:16.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023527230000000001 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.245243+0000", 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022938670000000002 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.249823+0000", 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045802660000000004 2023-09-23T22:32:16.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.277656+0000", 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027833238 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117533 7.c 7:3256a181:::benchmark_data_smithi026_410749_object27196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:32:16.137 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.722390+0000", 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.32977426400001, 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5963978089999999, 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117533 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.138 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722390+0000", 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722390+0000", 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722389+0000", 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722399+0000", 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0902e-05 2023-09-23T22:32:16.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722415+0000", 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5068999999999999e-05 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722425+0000", 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0562e-05 2023-09-23T22:32:16.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.989730+0000", 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.267304719 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.013955+0000", 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024225126999999999 2023-09-23T22:32:16.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.176096+0000", 2023-09-23T22:32:16.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16214060799999999 2023-09-23T22:32:16.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.186383+0000", 2023-09-23T22:32:16.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010287566999999999 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.198704+0000", 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01232137 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.238136+0000", 2023-09-23T22:32:16.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.039431352000000003 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.282727+0000", 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.044590888000000002 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.287402+0000", 2023-09-23T22:32:16.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046750940000000003 2023-09-23T22:32:16.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.292487+0000", 2023-09-23T22:32:16.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050851259999999997 2023-09-23T22:32:16.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.297353+0000", 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004866006 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.318788+0000", 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021435279000000002 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117546 7.4 7:2ecd81bb:::benchmark_data_smithi026_410749_object27209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.890332+0000", 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.16183221700001, 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4286364540000001, 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.148 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117546 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.890332+0000", 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.890332+0000", 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.890331+0000", 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.890337+0000", 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.716e-06 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.890348+0000", 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1375999999999999e-05 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.890354+0000", 2023-09-23T22:32:16.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.604e-06 2023-09-23T22:32:16.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.287455+0000", 2023-09-23T22:32:16.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3971011930000001 2023-09-23T22:32:16.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.287568+0000", 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112346 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.287661+0000", 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3307999999999995e-05 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.292592+0000", 2023-09-23T22:32:16.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049310309999999998 2023-09-23T22:32:16.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.292600+0000", 2023-09-23T22:32:16.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9680000000000006e-06 2023-09-23T22:32:16.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.318876+0000", 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026276574 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.318929+0000", 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2800000000000003e-05 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.318969+0000", 2023-09-23T22:32:16.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9634e-05 2023-09-23T22:32:16.156 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.156 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.156 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.156 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117547 7.4 7:25f29dcd:::benchmark_data_smithi026_410749_object27210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.903744+0000", 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.148421181, 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.450844671, 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117547 2023-09-23T22:32:16.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.903744+0000", 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.903744+0000", 2023-09-23T22:32:16.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.903742+0000", 2023-09-23T22:32:16.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:16.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.903750+0000", 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0029999999999994e-06 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.903765+0000", 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5017e-05 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.903773+0000", 2023-09-23T22:32:16.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9640000000000003e-06 2023-09-23T22:32:16.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.289891+0000", 2023-09-23T22:32:16.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3861185650000001 2023-09-23T22:32:16.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.290012+0000", 2023-09-23T22:32:16.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120122 2023-09-23T22:32:16.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:32:16.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.290101+0000", 2023-09-23T22:32:16.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9225e-05 2023-09-23T22:32:16.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.292627+0000", 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025263170000000001 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.319002+0000", 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026374578999999999 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.354497+0000", 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.035495280999999997 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.354549+0000", 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1892999999999999e-05 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.354588+0000", 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9350000000000001e-05 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117550 7.c 7:387185b4:::benchmark_data_smithi026_410749_object27213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:44.045345+0000", 2023-09-23T22:32:16.165 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.006820049, 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.502615335, 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117550 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045345+0000", 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045345+0000", 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045342+0000", 2023-09-23T22:32:16.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:32:16.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045353+0000", 2023-09-23T22:32:16.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1263e-05 2023-09-23T22:32:16.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045371+0000", 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7277000000000001e-05 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045382+0000", 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1154e-05 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.319779+0000", 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.274396912 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.351357+0000", 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031578531999999999 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.409179+0000", 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.057821600000000001 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.414769+0000", 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055900719999999997 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.425662+0000", 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010892808 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.525001+0000", 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.099338953999999993 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.525613+0000", 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00061288300000000005 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.527861+0000", 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022476280000000002 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.529884+0000", 2023-09-23T22:32:16.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002022739 2023-09-23T22:32:16.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.533315+0000", 2023-09-23T22:32:16.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034314139999999998 2023-09-23T22:32:16.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.547960+0000", 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014644714 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117565 7.c 7:3381838d:::benchmark_data_smithi026_410749_object27228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:32:16.175 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:44.108957+0000", 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 211.94320725200001, 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4684593859999999, 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117565 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.108957+0000", 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.108957+0000", 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.108956+0000", 2023-09-23T22:32:16.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:16.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.108963+0000", 2023-09-23T22:32:16.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7780000000000002e-06 2023-09-23T22:32:16.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.108977+0000", 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4329e-05 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.108985+0000", 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7039999999999994e-06 2023-09-23T22:32:16.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.353043+0000", 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.244057798 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.375090+0000", 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022046936999999999 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.469199+0000", 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.094109039000000005 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.480334+0000", 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011135135000000001 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:32:16.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.506109+0000", 2023-09-23T22:32:16.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025775078999999999 2023-09-23T22:32:16.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.548901+0000", 2023-09-23T22:32:16.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.042792017000000002 2023-09-23T22:32:16.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.551325+0000", 2023-09-23T22:32:16.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024240799999999999 2023-09-23T22:32:16.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.553738+0000", 2023-09-23T22:32:16.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024127369999999999 2023-09-23T22:32:16.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.556081+0000", 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023435589999999998 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.560183+0000", 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041013730000000002 2023-09-23T22:32:16.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.577417+0000", 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017234151999999999 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.185 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119469 7.c 7:37fe5ea5:::benchmark_data_smithi026_410749_object29132:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:32:16.186 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.322397+0000", 2023-09-23T22:32:16.186 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 182.72976799099999, 2023-09-23T22:32:16.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9204321049999997, 2023-09-23T22:32:16.186 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.186 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.186 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.186 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.186 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.186 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119469 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322397+0000", 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322397+0000", 2023-09-23T22:32:16.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322396+0000", 2023-09-23T22:32:16.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:32:16.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322399+0000", 2023-09-23T22:32:16.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.548e-06 2023-09-23T22:32:16.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322408+0000", 2023-09-23T22:32:16.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2410000000000001e-06 2023-09-23T22:32:16.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322413+0000", 2023-09-23T22:32:16.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.135e-06 2023-09-23T22:32:16.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322457+0000", 2023-09-23T22:32:16.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3455999999999999e-05 2023-09-23T22:32:16.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322472+0000", 2023-09-23T22:32:16.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5319999999999999e-05 2023-09-23T22:32:16.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053967+0000", 2023-09-23T22:32:16.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.73149523100000002 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053971+0000", 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.737e-06 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.223011+0000", 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1690396930000002 2023-09-23T22:32:16.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.224476+0000", 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001465273 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.226493+0000", 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020173280000000001 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.238434+0000", 2023-09-23T22:32:16.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011941267 2023-09-23T22:32:16.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.238582+0000", 2023-09-23T22:32:16.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147921 2023-09-23T22:32:16.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.238990+0000", 2023-09-23T22:32:16.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040729999999999998 2023-09-23T22:32:16.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.239462+0000", 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047265700000000003 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.240135+0000", 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00067237199999999997 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.242829+0000", 2023-09-23T22:32:16.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026940850000000001 2023-09-23T22:32:16.197 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.197 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.197 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.197 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119490 7.c 7:3eaefa36:::benchmark_data_smithi026_410749_object29153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:32:16.197 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.356832+0000", 2023-09-23T22:32:16.197 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 182.69533272199999, 2023-09-23T22:32:16.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8905167999999999, 2023-09-23T22:32:16.198 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.198 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.198 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119490 2023-09-23T22:32:16.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.198 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356832+0000", 2023-09-23T22:32:16.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356832+0000", 2023-09-23T22:32:16.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356831+0000", 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356836+0000", 2023-09-23T22:32:16.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1410000000000004e-06 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356845+0000", 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8470000000000008e-06 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356852+0000", 2023-09-23T22:32:16.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8990000000000004e-06 2023-09-23T22:32:16.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356902+0000", 2023-09-23T22:32:16.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0081999999999999e-05 2023-09-23T22:32:16.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356914+0000", 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1416e-05 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053982+0000", 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.69706834200000001 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053986+0000", 2023-09-23T22:32:16.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3809999999999999e-06 2023-09-23T22:32:16.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.230378+0000", 2023-09-23T22:32:16.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1763928799999999 2023-09-23T22:32:16.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.231993+0000", 2023-09-23T22:32:16.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001614376 2023-09-23T22:32:16.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:32:16.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.234257+0000", 2023-09-23T22:32:16.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022642140000000001 2023-09-23T22:32:16.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.242966+0000", 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087088479999999999 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.243295+0000", 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032858300000000002 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.243696+0000", 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040189499999999999 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.244112+0000", 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041517399999999998 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.244703+0000", 2023-09-23T22:32:16.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059169699999999997 2023-09-23T22:32:16.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.247349+0000", 2023-09-23T22:32:16.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002645484 2023-09-23T22:32:16.208 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.208 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119497 7.c 7:362bd5ae:::benchmark_data_smithi026_410749_object29160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.362138+0000", 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 182.690026751, 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9043871029999999, 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.209 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.210 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.210 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119497 2023-09-23T22:32:16.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.210 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362138+0000", 2023-09-23T22:32:16.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362138+0000", 2023-09-23T22:32:16.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362136+0000", 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362143+0000", 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1210000000000001e-06 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362156+0000", 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2310000000000001e-05 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362165+0000", 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7140000000000005e-06 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362228+0000", 2023-09-23T22:32:16.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2415999999999993e-05 2023-09-23T22:32:16.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362236+0000", 2023-09-23T22:32:16.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8429999999999999e-06 2023-09-23T22:32:16.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053996+0000", 2023-09-23T22:32:16.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.69175968499999996 2023-09-23T22:32:16.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053999+0000", 2023-09-23T22:32:16.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3210000000000001e-06 2023-09-23T22:32:16.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.247536+0000", 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1935366379999999 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.248998+0000", 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001461681 2023-09-23T22:32:16.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:32:16.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.251332+0000", 2023-09-23T22:32:16.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023344030000000001 2023-09-23T22:32:16.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.262650+0000", 2023-09-23T22:32:16.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011318004 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.262974+0000", 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032428099999999999 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.263074+0000", 2023-09-23T22:32:16.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9837e-05 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.263467+0000", 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039298099999999998 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.264022+0000", 2023-09-23T22:32:16.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055437699999999999 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.266525+0000", 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025033540000000002 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119502 7.c 7:39b9d062:::benchmark_data_smithi026_410749_object29165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.368183+0000", 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 182.68398209099999, 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9164363140000003, 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.221 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119502 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368183+0000", 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368183+0000", 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368182+0000", 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368187+0000", 2023-09-23T22:32:16.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6529999999999999e-06 2023-09-23T22:32:16.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368199+0000", 2023-09-23T22:32:16.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1875000000000001e-05 2023-09-23T22:32:16.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368207+0000", 2023-09-23T22:32:16.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0499999999999992e-06 2023-09-23T22:32:16.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368265+0000", 2023-09-23T22:32:16.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8074e-05 2023-09-23T22:32:16.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368277+0000", 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1640999999999999e-05 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054010+0000", 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68573267699999996 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054013+0000", 2023-09-23T22:32:16.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3349999999999998e-06 2023-09-23T22:32:16.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.254782+0000", 2023-09-23T22:32:16.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2007688540000001 2023-09-23T22:32:16.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.256184+0000", 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014024669999999999 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.258340+0000", 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021561169999999999 2023-09-23T22:32:16.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.266945+0000", 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086043339999999999 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.267354+0000", 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040936299999999999 2023-09-23T22:32:16.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.267803+0000", 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044911400000000002 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.282079+0000", 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014275512000000001 2023-09-23T22:32:16.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.282581+0000", 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050250700000000002 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.284619+0000", 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002037681 2023-09-23T22:32:16.231 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119523 7.c 7:3440ea60:::benchmark_data_smithi026_410749_object29186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.498288+0000", 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 182.55387711700001, 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7907858110000001, 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.232 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119523 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498288+0000", 2023-09-23T22:32:16.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498288+0000", 2023-09-23T22:32:16.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498287+0000", 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498292+0000", 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4890000000000003e-06 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498301+0000", 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9789999999999999e-06 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498315+0000", 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3635000000000001e-05 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498356+0000", 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1279999999999998e-05 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498362+0000", 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1199999999999999e-06 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054024+0000", 2023-09-23T22:32:16.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.555662137 2023-09-23T22:32:16.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054028+0000", 2023-09-23T22:32:16.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.343e-06 2023-09-23T22:32:16.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.267979+0000", 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2139512649999999 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.269550+0000", 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001571645 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.271850+0000", 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022990599999999999 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285119+0000", 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013269311000000001 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285481+0000", 2023-09-23T22:32:16.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036202899999999999 2023-09-23T22:32:16.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285593+0000", 2023-09-23T22:32:16.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111935 2023-09-23T22:32:16.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.286135+0000", 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054262699999999995 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.286669+0000", 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00053313299999999998 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.289073+0000", 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002404822 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119530 7.c 7:36830f67:::benchmark_data_smithi026_410749_object29193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:32:16.243 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.567375+0000", 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 182.48478925800001, 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7261192759999999, 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119530 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.244 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567375+0000", 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567375+0000", 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567374+0000", 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567380+0000", 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1850000000000003e-06 2023-09-23T22:32:16.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567389+0000", 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5410000000000006e-06 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567394+0000", 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1220000000000003e-06 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567454+0000", 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9857e-05 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567465+0000", 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1219e-05 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054039+0000", 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.48657352799999998 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054042+0000", 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5599999999999998e-06 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.274794+0000", 2023-09-23T22:32:16.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2207516300000001 2023-09-23T22:32:16.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.276186+0000", 2023-09-23T22:32:16.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001392162 2023-09-23T22:32:16.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.278086+0000", 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001899502 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285725+0000", 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076398430000000003 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.289503+0000", 2023-09-23T22:32:16.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037773630000000002 2023-09-23T22:32:16.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.289936+0000", 2023-09-23T22:32:16.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043365299999999999 2023-09-23T22:32:16.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.290271+0000", 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033435799999999999 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.290826+0000", 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000554742 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.293495+0000", 2023-09-23T22:32:16.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002669121 2023-09-23T22:32:16.254 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.254 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.254 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.254 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119565 7.c 7:366e6103:::benchmark_data_smithi026_410749_object29228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.701580+0000", 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 182.350585102, 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6102326480000002, 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119565 2023-09-23T22:32:16.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701580+0000", 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701580+0000", 2023-09-23T22:32:16.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701579+0000", 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701583+0000", 2023-09-23T22:32:16.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4100000000000001e-06 2023-09-23T22:32:16.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:16.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701596+0000", 2023-09-23T22:32:16.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3318e-05 2023-09-23T22:32:16.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:16.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701602+0000", 2023-09-23T22:32:16.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2179999999999998e-06 2023-09-23T22:32:16.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701649+0000", 2023-09-23T22:32:16.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7822999999999999e-05 2023-09-23T22:32:16.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701661+0000", 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1739000000000001e-05 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054055+0000", 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.35239350400000002 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:16.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054059+0000", 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.605e-06 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.293665+0000", 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2396055229999998 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:16.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.295090+0000", 2023-09-23T22:32:16.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001425078 2023-09-23T22:32:16.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:32:16.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.297158+0000", 2023-09-23T22:32:16.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002068318 2023-09-23T22:32:16.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:16.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.307433+0000", 2023-09-23T22:32:16.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010274817 2023-09-23T22:32:16.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.307802+0000", 2023-09-23T22:32:16.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036851599999999999 2023-09-23T22:32:16.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.308198+0000", 2023-09-23T22:32:16.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039608500000000001 2023-09-23T22:32:16.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:16.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.308594+0000", 2023-09-23T22:32:16.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039657100000000002 2023-09-23T22:32:16.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.309219+0000", 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00062512600000000005 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.311812+0000", 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025928660000000001 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.265 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.266 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.266 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:16.266 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:16.266 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:32:16.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:16.267 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 73987 73971 75.7741 62.25 0.0146799 0.0131919 2023-09-23T22:32:16.319 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:16.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:15 smithi070 ceph-mon[192652]: failed to encode map e395 with expected crc 2023-09-23T22:32:16.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:15 smithi070 ceph-mon[192652]: failed to encode map e395 with expected crc 2023-09-23T22:32:16.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:15 smithi070 ceph-mon[192652]: failed to encode map e396 with expected crc 2023-09-23T22:32:16.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:15 smithi070 ceph-mon[192652]: failed to encode map e396 with expected crc 2023-09-23T22:32:16.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:15 smithi070 ceph-mon[192652]: osdmap e396: 8 total, 8 up, 5 in 2023-09-23T22:32:16.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:15 smithi070 ceph-mon[192652]: failed to encode map e396 with expected crc 2023-09-23T22:32:17.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 74935 74919 75.5075 59.25 0.00956747 0.0132383 2023-09-23T22:32:17.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:17 smithi026 ceph-mon[223014]: pgmap v1018: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s; 3198/218883 objects degraded (1.461%); 1433/218883 objects misplaced (0.655%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T22:32:17.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:17 smithi026 ceph-mon[215775]: pgmap v1018: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s; 3198/218883 objects degraded (1.461%); 1433/218883 objects misplaced (0.655%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T22:32:17.555 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:17.556 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:32:17.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:17.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:17.565 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:32:17.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:32:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:17 smithi070 ceph-mon[192652]: pgmap v1018: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 76 MiB/s wr, 1.21k op/s; 3198/218883 objects degraded (1.461%); 1433/218883 objects misplaced (0.655%); 8.5 MiB/s, 135 objects/s recovering 2023-09-23T22:32:17.665 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:17.857 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:17.997 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:18.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:18.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:18.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 75886 75870 75.2524 59.4375 0.010697 0.0132834 2023-09-23T22:32:18.105 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:18.155 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:18.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3198/218883 objects degraded (1.461%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:18.261 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:18.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:18.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:18.346 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:18.368 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:18.372 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:18.412 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:18.425 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3198/218883 objects degraded (1.461%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:18.430 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:18.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3198/218883 objects degraded (1.461%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:18.540 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:18.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:18.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:18.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:18.705 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:18.841 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:18.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:18.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:18.989 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:19.064 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:19.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:19.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 76839 76823 75.0072 59.5625 0.0145841 0.013327 2023-09-23T22:32:19.171 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:19.180 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T22:32:19.230 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:19.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:19 smithi026 ceph-mon[215775]: pgmap v1019: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s; 3198/218883 objects degraded (1.461%); 1433/218883 objects misplaced (0.655%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:32:19.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2412244652' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:32:19.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:32:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:19 smithi026 ceph-mon[223014]: pgmap v1019: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s; 3198/218883 objects degraded (1.461%); 1433/218883 objects misplaced (0.655%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:32:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2412244652' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:32:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:32:19.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:19 smithi070 ceph-mon[192652]: pgmap v1019: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 65 MiB/s wr, 1.04k op/s; 3198/218883 objects degraded (1.461%); 1433/218883 objects misplaced (0.655%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:32:19.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2412244652' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:32:19.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:32:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:32:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:32:19] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.43.0" 2023-09-23T22:32:20.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 77699 77683 74.6801 53.75 0.0219005 0.0133802 2023-09-23T22:32:20.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:20 smithi026 ceph-mon[223014]: failed to encode map e397 with expected crc 2023-09-23T22:32:20.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:20 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:32:20.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:20 smithi026 ceph-mon[223014]: osdmap e397: 8 total, 8 up, 5 in 2023-09-23T22:32:20.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:20 smithi026 ceph-mon[223014]: failed to encode map e397 with expected crc 2023-09-23T22:32:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:20 smithi026 ceph-mon[215775]: failed to encode map e397 with expected crc 2023-09-23T22:32:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:20 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:32:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:20 smithi026 ceph-mon[215775]: osdmap e397: 8 total, 8 up, 5 in 2023-09-23T22:32:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:20 smithi026 ceph-mon[215775]: failed to encode map e397 with expected crc 2023-09-23T22:32:20.628 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:20 smithi070 ceph-mon[192652]: failed to encode map e397 with expected crc 2023-09-23T22:32:20.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:20 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:32:20.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:20 smithi070 ceph-mon[192652]: osdmap e397: 8 total, 8 up, 5 in 2023-09-23T22:32:20.629 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:20 smithi070 ceph-mon[192652]: failed to encode map e397 with expected crc 2023-09-23T22:32:21.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 78405 78389 74.2171 44.125 0.0175389 0.0134416 2023-09-23T22:32:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:21 smithi026 ceph-mon[215775]: pgmap v1021: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 59 MiB/s wr, 947 op/s; 3198/222102 objects degraded (1.440%); 1433/222102 objects misplaced (0.645%); 6.9 MiB/s, 111 objects/s recovering 2023-09-23T22:32:21.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:21 smithi026 ceph-mon[223014]: pgmap v1021: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 59 MiB/s wr, 947 op/s; 3198/222102 objects degraded (1.440%); 1433/222102 objects misplaced (0.645%); 6.9 MiB/s, 111 objects/s recovering 2023-09-23T22:32:21.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:21 smithi070 ceph-mon[192652]: pgmap v1021: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 59 MiB/s wr, 947 op/s; 3198/222102 objects degraded (1.440%); 1433/222102 objects misplaced (0.645%); 6.9 MiB/s, 111 objects/s recovering 2023-09-23T22:32:21.722 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:21.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:21.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:21.826 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:21.874 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:22.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 79253 79237 73.9003 53 0.0210909 0.0135266 2023-09-23T22:32:22.354 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:32:22.354 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:32:22.556 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:22.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:22.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:22.669 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:22.737 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:22.884 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:32:22.884 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:32:22.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.884 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:80006 8.1 8:865d38f1:::benchmark_data_smithi026_497658_object80005:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.878617+0000", 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.003017353, 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030587050000000001, 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 80006 2023-09-23T22:32:22.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.886 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878617+0000", 2023-09-23T22:32:22.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878617+0000", 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878616+0000", 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878640+0000", 2023-09-23T22:32:22.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4362000000000002e-05 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878648+0000", 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2449999999999998e-06 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878655+0000", 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7939999999999997e-06 2023-09-23T22:32:22.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878933+0000", 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027826700000000001 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.879111+0000", 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017793799999999999 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.879235+0000", 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012400700000000001 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:80007 8.f 8:ff7678cc:::benchmark_data_smithi026_497658_object80006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T22:32:22.890 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.878664+0000", 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0029709620000000002, 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031210589999999998, 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 80007 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878664+0000", 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878664+0000", 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878663+0000", 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878685+0000", 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1668e-05 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878691+0000", 2023-09-23T22:32:22.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5649999999999998e-06 2023-09-23T22:32:22.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878697+0000", 2023-09-23T22:32:22.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9349999999999999e-06 2023-09-23T22:32:22.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881377+0000", 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026793110000000002 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881554+0000", 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000177376 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881702+0000", 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147618 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45842.0:80011 8.0 e397/373 8:0209c5fa:::benchmark_data_smithi026_497658_object80010:head v 397'4873, mlcod=397'4871)", 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.878770+0000", 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.002864639, 2023-09-23T22:32:22.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003138039, 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878770+0000", 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878770+0000", 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878769+0000", 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878813+0000", 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.32e-05 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878826+0000", 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.292e-05 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878834+0000", 2023-09-23T22:32:22.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8400000000000001e-06 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878886+0000", 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1483999999999997e-05 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878906+0000", 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9927000000000001e-05 2023-09-23T22:32:22.900 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.45842.0:80004 8.f e397/373)", 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.881375+0000", 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00025967299999999998, 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00056700999999999995, 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.901 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:32:22.902 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881375+0000", 2023-09-23T22:32:22.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881375+0000", 2023-09-23T22:32:22.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881374+0000", 2023-09-23T22:32:22.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:22.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881385+0000", 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0832999999999999e-05 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881398+0000", 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2733e-05 2023-09-23T22:32:22.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881406+0000", 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4120000000000001e-06 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45842.0:80008 8.d e397/381 8:b05c88b0:::benchmark_data_smithi026_497658_object80007:head v 397'4983, mlcod=0'0)", 2023-09-23T22:32:22.905 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.881354+0000", 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00028097799999999998, 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00061900200000000001, 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881354+0000", 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881354+0000", 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881353+0000", 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881409+0000", 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6109000000000001e-05 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881422+0000", 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3076e-05 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881430+0000", 2023-09-23T22:32:22.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.456e-06 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881494+0000", 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3634999999999996e-05 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881522+0000", 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8251000000000001e-05 2023-09-23T22:32:22.909 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.910 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.910 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.910 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:79989 8.f 8:fa9014c2:::benchmark_data_smithi026_497658_object79988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T22:32:22.910 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.865413+0000", 2023-09-23T22:32:22.910 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.016221395999999999, 2023-09-23T22:32:22.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016599262, 2023-09-23T22:32:22.910 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79989 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.865413+0000", 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.865413+0000", 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.865412+0000", 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:22.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.865461+0000", 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9208e-05 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.865477+0000", 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6362999999999999e-05 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.865484+0000", 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7599999999999997e-06 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.874378+0000", 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088934819999999994 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.874556+0000", 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000178352 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.874718+0000", 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016229300000000001 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:22.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881306+0000", 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065875040000000001 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881346+0000", 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0654000000000001e-05 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45842.0:79997 8.7 e397/373 8:e3fe5a4f:::benchmark_data_smithi026_497658_object79996:head v 397'5054, mlcod=397'5052)", 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.868110+0000", 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.013524665999999999, 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013952799, 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.868110+0000", 2023-09-23T22:32:22.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.868110+0000", 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.868109+0000", 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:32:22.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.868184+0000", 2023-09-23T22:32:22.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.504e-05 2023-09-23T22:32:22.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.868203+0000", 2023-09-23T22:32:22.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9045000000000001e-05 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.868213+0000", 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8330000000000006e-06 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871428+0000", 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032156979999999999 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871452+0000", 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3790000000000001e-05 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:79996 8.3 8:c643176c:::benchmark_data_smithi026_497658_object79995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.871425+0000", 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.01020945, 2023-09-23T22:32:22.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010676096, 2023-09-23T22:32:22.922 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.922 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:32:22.922 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:22.922 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:32:22.922 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:32:22.922 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 79996 2023-09-23T22:32:22.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.922 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871425+0000", 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871425+0000", 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871425+0000", 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871449+0000", 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3994000000000001e-05 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871457+0000", 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.96e-06 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871462+0000", 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5509999999999999e-06 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.875538+0000", 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040760120000000004 2023-09-23T22:32:22.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.875675+0000", 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137298 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,6", 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.875821+0000", 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145988 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:22.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.880953+0000", 2023-09-23T22:32:22.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051317840000000003 2023-09-23T22:32:22.927 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.927 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.927 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.927 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:80000 8.1 8:8390311c:::benchmark_data_smithi026_497658_object79999:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T22:32:22.927 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.871468+0000", 2023-09-23T22:32:22.928 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.010166407000000001, 2023-09-23T22:32:22.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010680340999999999, 2023-09-23T22:32:22.928 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.928 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:32:22.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:22.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:32:22.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 80000 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871468+0000", 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871468+0000", 2023-09-23T22:32:22.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871468+0000", 2023-09-23T22:32:22.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:32:22.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871488+0000", 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0741e-05 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871495+0000", 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6710000000000002e-06 2023-09-23T22:32:22.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871500+0000", 2023-09-23T22:32:22.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0549999999999999e-06 2023-09-23T22:32:22.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:22.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.874485+0000", 2023-09-23T22:32:22.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029850269999999999 2023-09-23T22:32:22.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:22.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.874597+0000", 2023-09-23T22:32:22.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112175 2023-09-23T22:32:22.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:32:22.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.874690+0000", 2023-09-23T22:32:22.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2668000000000001e-05 2023-09-23T22:32:22.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:22.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878907+0000", 2023-09-23T22:32:22.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042170890000000003 2023-09-23T22:32:22.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:22.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.881670+0000", 2023-09-23T22:32:22.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027630860000000001 2023-09-23T22:32:22.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.935 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.935 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:80004 8.f 8:f767994f:::benchmark_data_smithi026_497658_object80003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T22:32:22.935 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:32:22.871514+0000", 2023-09-23T22:32:22.936 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.010120657, 2023-09-23T22:32:22.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010684312, 2023-09-23T22:32:22.936 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:22.936 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:32:22.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:22.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:32:22.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:32:22.936 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 80004 2023-09-23T22:32:22.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.937 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:22.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:22.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871514+0000", 2023-09-23T22:32:22.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871514+0000", 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871514+0000", 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:22.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871539+0000", 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5329000000000001e-05 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871546+0000", 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.951e-06 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.871551+0000", 2023-09-23T22:32:22.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9359999999999998e-06 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.877558+0000", 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060073660000000001 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878042+0000", 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00048417499999999999 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:22.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:32:22.878277+0000", 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000234806 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 10 2023-09-23T22:32:22.941 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:32:22.942 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:22.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:22.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:23.050 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:23.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 80229 80213 73.7106 61 0.0107992 0.0135615 2023-09-23T22:32:23.119 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:32:23.145 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:23.147 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:32:23.178 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:32:23.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:23 smithi026 ceph-mon[223014]: pgmap v1022: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s; 3198/232488 objects degraded (1.376%); 1063/232488 objects misplaced (0.457%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:32:23.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1969629239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:23.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3198/232488 objects degraded (1.376%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:23.185 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:23 smithi026 ceph-mon[215775]: pgmap v1022: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s; 3198/232488 objects degraded (1.376%); 1063/232488 objects misplaced (0.457%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:32:23.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1969629239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:23.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3198/232488 objects degraded (1.376%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:23.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:32:23.243 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:32:23.249 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:23.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:23.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:23.275 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:32:23.314 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:32:23.353 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:32:23.357 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:32:23.357 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:32:23.357 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:32:23.358 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:32:23.358 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:32:23.358 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:32:23.358 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:32:23.358 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:32:23.362 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:23.418 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:23 smithi070 ceph-mon[192652]: pgmap v1022: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 77 MiB/s wr, 1.23k op/s; 3198/232488 objects degraded (1.376%); 1063/232488 objects misplaced (0.457%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:32:23.418 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1969629239' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:23.418 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3198/232488 objects degraded (1.376%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:23.421 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:23.468 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:23.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:23.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:23.574 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:23.600 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:23.707 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:23.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:23.824 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:23.931 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:24.078 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:24.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:24.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 81031 81015 73.3687 50.125 0.0109856 0.0136245 2023-09-23T22:32:24.189 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:24.219 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:24.670 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:32:24.815 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:24.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:24.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:24.932 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:25.021 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:25.028 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:25.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 81848 81832 73.05 51.0625 0.0272407 0.0136839 2023-09-23T22:32:25.132 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:25.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:25.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:25.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:25 smithi026 ceph-mon[215775]: pgmap v1023: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 34 MiB/s wr, 537 op/s; 3198/232488 objects degraded (1.376%); 1063/232488 objects misplaced (0.457%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:32:25.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:25.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:25.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:25 smithi026 ceph-mon[223014]: pgmap v1023: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 34 MiB/s wr, 537 op/s; 3198/232488 objects degraded (1.376%); 1063/232488 objects misplaced (0.457%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:32:25.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:25.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:25.243 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:25.305 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:25 smithi070 ceph-mon[192652]: pgmap v1023: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 34 MiB/s wr, 537 op/s; 3198/232488 objects degraded (1.376%); 1063/232488 objects misplaced (0.457%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:32:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:25.432 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:25.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:25.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:25.546 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:25.606 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:25.733 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:25.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:25.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:25.842 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:25.888 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:25.932 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:25.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:26.038 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:26.098 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:26.102 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 82645 82629 72.7227 49.8125 0.0167172 0.0137454 2023-09-23T22:32:26.114 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:32:26.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:26.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:26.209 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3517376505' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:32:26.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:26.312 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:26.364 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3517376505' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:32:26.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3517376505' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:32:26.368 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:26.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:32:27.001 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:27.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:27.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:27.096 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:27.104 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 83497 83481 72.4509 53.25 0.00941211 0.0137971 2023-09-23T22:32:27.106 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:27.163 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:27.281 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:27.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:27.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:27.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[223014]: pgmap v1024: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 865 op/s; 3198/244854 objects degraded (1.306%); 525/244854 objects misplaced (0.214%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T22:32:27.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:27.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3517376505' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:32:27.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[223014]: osdmap e398: 8 total, 8 up, 5 in 2023-09-23T22:32:27.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[223014]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[223014]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[223014]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[223014]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[215775]: pgmap v1024: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 865 op/s; 3198/244854 objects degraded (1.306%); 525/244854 objects misplaced (0.214%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T22:32:27.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:27.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3517376505' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:32:27.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[215775]: osdmap e398: 8 total, 8 up, 5 in 2023-09-23T22:32:27.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[215775]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[215775]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[215775]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:27 smithi026 ceph-mon[215775]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:27 smithi070 ceph-mon[192652]: pgmap v1024: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 865 op/s; 3198/244854 objects degraded (1.306%); 525/244854 objects misplaced (0.214%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T22:32:27.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:27 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:27.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3517376505' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:32:27.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:27 smithi070 ceph-mon[192652]: osdmap e398: 8 total, 8 up, 5 in 2023-09-23T22:32:27.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:27 smithi070 ceph-mon[192652]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:27 smithi070 ceph-mon[192652]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:27 smithi070 ceph-mon[192652]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:27 smithi070 ceph-mon[192652]: failed to encode map e398 with expected crc 2023-09-23T22:32:27.396 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:27.454 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:27.554 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:27.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:27.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:27.719 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:27.900 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:27.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:27.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:27.962 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:32:27.964 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:32:28.005 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:28.013 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:28.061 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:28.104 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 84458 84442 72.2812 60.0625 0.0133409 0.0138296 2023-09-23T22:32:28.195 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:28.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:28.305 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:28.329 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:28 smithi070 ceph-mon[192652]: failed to encode map e398 with expected crc 2023-09-23T22:32:28.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:28.420 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:28.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:28.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:28.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:28 smithi026 ceph-mon[223014]: failed to encode map e398 with expected crc 2023-09-23T22:32:28.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:28 smithi026 ceph-mon[215775]: failed to encode map e398 with expected crc 2023-09-23T22:32:28.532 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:28.580 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:29.104 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 85345 85329 72.0535 55.4375 0.0182614 0.0138737 2023-09-23T22:32:29.132 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:32:29.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:29 smithi070 ceph-mon[192652]: pgmap v1026: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s; 3198/244854 objects degraded (1.306%); 525/244854 objects misplaced (0.214%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T22:32:29.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:29 smithi070 ceph-mon[192652]: from='client.55723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:32:29.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:29 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3198/244854 objects degraded (1.306%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:29.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:29 smithi026 ceph-mon[215775]: pgmap v1026: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s; 3198/244854 objects degraded (1.306%); 525/244854 objects misplaced (0.214%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T22:32:29.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:29 smithi026 ceph-mon[215775]: from='client.55723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:32:29.431 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:29 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3198/244854 objects degraded (1.306%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:29.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:29 smithi026 ceph-mon[223014]: pgmap v1026: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 64 MiB/s wr, 1.03k op/s; 3198/244854 objects degraded (1.306%); 525/244854 objects misplaced (0.214%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T22:32:29.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:29 smithi026 ceph-mon[223014]: from='client.55723 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:32:29.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:29 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3198/244854 objects degraded (1.306%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:32:29.463 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1026,"stamp":"2023-09-23T22:32:27.570489+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5351872435,"num_objects":81618,"num_object_clones":0,"num_object_copies":244854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3198,"num_objects_misplaced":525,"num_objects_unfound":0,"num_objects_dirty":81618,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":82497,"num_write_kb":5234485,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7636,"num_bytes_recovered":500400491,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82492,"ondisk_log_size":82492,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":17666872,"kb_used_data":15402768,"kb_used_omap":15,"kb_used_meta":2264048,"kb_avail":451058888,"statfs":{"total":479975178240,"available":461884301312,"internally_reserved":0,"allocated":15772434432,"data_stored":15763770505,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2318385374},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":567345152,"num_objects":8657,"num_object_clones":0,"num_object_copies":25971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-908,"num_objects_unfound":0,"num_objects_dirty":8657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8657,"num_write_kb":554048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":908,"num_bytes_recovered":59506688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.410138"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":424,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131866+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:32:15.131866+0000","last_peered":"2023-09-23T22:32:15.131866+0000","last_clean":"2023-09-23T22:32:15.131866+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:32:15.131866+0000","last_undegraded":"2023-09-23T22:32:15.131866+0000","last_fullsized":"2023-09-23T22:32:15.131866+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":113,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.134317+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:32:15.134317+0000","last_peered":"2023-09-23T22:32:15.134317+0000","last_clean":"2023-09-23T22:32:15.134317+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:32:15.134317+0000","last_undegraded":"2023-09-23T22:32:15.134317+0000","last_fullsized":"2023-09-23T22:32:15.134317+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":113,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.134287+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:32:15.134287+0000","last_peered":"2023-09-23T22:32:15.134287+0000","last_clean":"2023-09-23T22:32:15.134287+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:32:15.134287+0000","last_undegraded":"2023-09-23T22:32:15.134287+0000","last_fullsized":"2023-09-23T22:32:15.134287+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":285,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132766+0000","last_change":"2023-09-23T22:31:28.120861+0000","last_active":"2023-09-23T22:32:15.132766+0000","last_peered":"2023-09-23T22:32:15.132766+0000","last_clean":"2023-09-23T22:32:15.132766+0000","last_became_active":"2023-09-23T22:31:28.120394+0000","last_became_peered":"2023-09-23T22:31:28.120394+0000","last_unstale":"2023-09-23T22:32:15.132766+0000","last_undegraded":"2023-09-23T22:32:15.132766+0000","last_fullsized":"2023-09-23T22:32:15.132766+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:19.470258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":452,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:19.170791+0000","last_change":"2023-09-23T22:31:28.114528+0000","last_active":"2023-09-23T22:32:19.170791+0000","last_peered":"2023-09-23T22:32:19.170791+0000","last_clean":"2023-09-23T22:32:19.170791+0000","last_became_active":"2023-09-23T22:31:28.114177+0000","last_became_peered":"2023-09-23T22:31:28.114177+0000","last_unstale":"2023-09-23T22:32:19.170791+0000","last_undegraded":"2023-09-23T22:32:19.170791+0000","last_fullsized":"2023-09-23T22:32:19.170791+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:46.680433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":78,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.144062+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:32:15.144062+0000","last_peered":"2023-09-23T22:32:15.144062+0000","last_clean":"2023-09-23T22:32:15.144062+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:32:15.144062+0000","last_undegraded":"2023-09-23T22:32:15.144062+0000","last_fullsized":"2023-09-23T22:32:15.144062+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":313,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132902+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:32:15.132902+0000","last_peered":"2023-09-23T22:32:15.132902+0000","last_clean":"2023-09-23T22:32:15.132902+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:32:15.132902+0000","last_undegraded":"2023-09-23T22:32:15.132902+0000","last_fullsized":"2023-09-23T22:32:15.132902+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":78,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143339+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:32:15.143339+0000","last_peered":"2023-09-23T22:32:15.143339+0000","last_clean":"2023-09-23T22:32:15.143339+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:32:15.143339+0000","last_undegraded":"2023-09-23T22:32:15.143339+0000","last_fullsized":"2023-09-23T22:32:15.143339+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":397,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138209+0000","last_change":"2023-09-23T22:31:28.116916+0000","last_active":"2023-09-23T22:32:15.138209+0000","last_peered":"2023-09-23T22:32:15.138209+0000","last_clean":"2023-09-23T22:32:15.138209+0000","last_became_active":"2023-09-23T22:31:28.116577+0000","last_became_peered":"2023-09-23T22:31:28.116577+0000","last_unstale":"2023-09-23T22:32:15.138209+0000","last_undegraded":"2023-09-23T22:32:15.138209+0000","last_fullsized":"2023-09-23T22:32:15.138209+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:33.092789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":88,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143586+0000","last_change":"2023-09-23T22:31:28.122684+0000","last_active":"2023-09-23T22:32:15.143586+0000","last_peered":"2023-09-23T22:32:15.143586+0000","last_clean":"2023-09-23T22:32:15.143586+0000","last_became_active":"2023-09-23T22:31:28.122427+0000","last_became_peered":"2023-09-23T22:31:28.122427+0000","last_unstale":"2023-09-23T22:32:15.143586+0000","last_undegraded":"2023-09-23T22:32:15.143586+0000","last_fullsized":"2023-09-23T22:32:15.143586+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:15.559367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":144,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132176+0000","last_change":"2023-09-23T22:31:28.153199+0000","last_active":"2023-09-23T22:32:15.132176+0000","last_peered":"2023-09-23T22:32:15.132176+0000","last_clean":"2023-09-23T22:32:15.132176+0000","last_became_active":"2023-09-23T22:31:28.129257+0000","last_became_peered":"2023-09-23T22:31:28.129257+0000","last_unstale":"2023-09-23T22:32:15.132176+0000","last_undegraded":"2023-09-23T22:32:15.132176+0000","last_fullsized":"2023-09-23T22:32:15.132176+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:03.743836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":45,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.134715+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:32:15.134715+0000","last_peered":"2023-09-23T22:32:15.134715+0000","last_clean":"2023-09-23T22:32:15.134715+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:32:15.134715+0000","last_undegraded":"2023-09-23T22:32:15.134715+0000","last_fullsized":"2023-09-23T22:32:15.134715+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":481,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132234+0000","last_change":"2023-09-23T22:31:28.145916+0000","last_active":"2023-09-23T22:32:15.132234+0000","last_peered":"2023-09-23T22:32:15.132234+0000","last_clean":"2023-09-23T22:32:15.132234+0000","last_became_active":"2023-09-23T22:31:28.120703+0000","last_became_peered":"2023-09-23T22:31:28.120703+0000","last_unstale":"2023-09-23T22:32:15.132234+0000","last_undegraded":"2023-09-23T22:32:15.132234+0000","last_fullsized":"2023-09-23T22:32:15.132234+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:07.819247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":395,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132076+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:32:15.132076+0000","last_peered":"2023-09-23T22:32:15.132076+0000","last_clean":"2023-09-23T22:32:15.132076+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:32:15.132076+0000","last_undegraded":"2023-09-23T22:32:15.132076+0000","last_fullsized":"2023-09-23T22:32:15.132076+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":388,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131795+0000","last_change":"2023-09-23T22:31:28.149149+0000","last_active":"2023-09-23T22:32:15.131795+0000","last_peered":"2023-09-23T22:32:15.131795+0000","last_clean":"2023-09-23T22:32:15.131795+0000","last_became_active":"2023-09-23T22:31:28.121120+0000","last_became_peered":"2023-09-23T22:31:28.121120+0000","last_unstale":"2023-09-23T22:32:15.131795+0000","last_undegraded":"2023-09-23T22:32:15.131795+0000","last_fullsized":"2023-09-23T22:32:15.131795+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.434137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":359,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138292+0000","last_change":"2023-09-23T22:31:28.119520+0000","last_active":"2023-09-23T22:32:15.138292+0000","last_peered":"2023-09-23T22:32:15.138292+0000","last_clean":"2023-09-23T22:32:15.138292+0000","last_became_active":"2023-09-23T22:31:28.118810+0000","last_became_peered":"2023-09-23T22:31:28.118810+0000","last_unstale":"2023-09-23T22:32:15.138292+0000","last_undegraded":"2023-09-23T22:32:15.138292+0000","last_fullsized":"2023-09-23T22:32:15.138292+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:36.007062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":302,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131915+0000","last_change":"2023-09-23T22:31:28.114537+0000","last_active":"2023-09-23T22:32:15.131915+0000","last_peered":"2023-09-23T22:32:15.131915+0000","last_clean":"2023-09-23T22:32:15.131915+0000","last_became_active":"2023-09-23T22:31:28.114176+0000","last_became_peered":"2023-09-23T22:31:28.114176+0000","last_unstale":"2023-09-23T22:32:15.131915+0000","last_undegraded":"2023-09-23T22:32:15.131915+0000","last_fullsized":"2023-09-23T22:32:15.131915+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.324173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":406,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138782+0000","last_change":"2023-09-23T22:31:28.132494+0000","last_active":"2023-09-23T22:32:15.138782+0000","last_peered":"2023-09-23T22:32:15.138782+0000","last_clean":"2023-09-23T22:32:15.138782+0000","last_became_active":"2023-09-23T22:31:28.132197+0000","last_became_peered":"2023-09-23T22:31:28.132197+0000","last_unstale":"2023-09-23T22:32:15.138782+0000","last_undegraded":"2023-09-23T22:32:15.138782+0000","last_fullsized":"2023-09-23T22:32:15.138782+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:36.181016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":340,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131744+0000","last_change":"2023-09-23T22:31:28.119156+0000","last_active":"2023-09-23T22:32:15.131744+0000","last_peered":"2023-09-23T22:32:15.131744+0000","last_clean":"2023-09-23T22:32:15.131744+0000","last_became_active":"2023-09-23T22:31:28.118382+0000","last_became_peered":"2023-09-23T22:31:28.118382+0000","last_unstale":"2023-09-23T22:32:15.131744+0000","last_undegraded":"2023-09-23T22:32:15.131744+0000","last_fullsized":"2023-09-23T22:32:15.131744+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:10.140079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":293,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132902+0000","last_change":"2023-09-23T22:31:28.121397+0000","last_active":"2023-09-23T22:32:15.132902+0000","last_peered":"2023-09-23T22:32:15.132902+0000","last_clean":"2023-09-23T22:32:15.132902+0000","last_became_active":"2023-09-23T22:31:28.120847+0000","last_became_peered":"2023-09-23T22:31:28.120847+0000","last_unstale":"2023-09-23T22:32:15.132902+0000","last_undegraded":"2023-09-23T22:32:15.132902+0000","last_fullsized":"2023-09-23T22:32:15.132902+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:42.407658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":88,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143258+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:32:15.143258+0000","last_peered":"2023-09-23T22:32:15.143258+0000","last_clean":"2023-09-23T22:32:15.143258+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:32:15.143258+0000","last_undegraded":"2023-09-23T22:32:15.143258+0000","last_fullsized":"2023-09-23T22:32:15.143258+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":88,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143886+0000","last_change":"2023-09-23T22:31:28.128965+0000","last_active":"2023-09-23T22:32:15.143886+0000","last_peered":"2023-09-23T22:32:15.143886+0000","last_clean":"2023-09-23T22:32:15.143886+0000","last_became_active":"2023-09-23T22:31:28.128224+0000","last_became_peered":"2023-09-23T22:31:28.128224+0000","last_unstale":"2023-09-23T22:32:15.143886+0000","last_undegraded":"2023-09-23T22:32:15.143886+0000","last_fullsized":"2023-09-23T22:32:15.143886+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:11.301143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"397'5057","reported_seq":5089,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:24.997183+0000","last_change":"2023-09-23T22:31:08.226781+0000","last_active":"2023-09-23T22:32:24.997183+0000","last_peered":"2023-09-23T22:32:24.997183+0000","last_clean":"2023-09-23T22:32:24.997183+0000","last_became_active":"2023-09-23T22:31:08.226511+0000","last_became_peered":"2023-09-23T22:31:08.226511+0000","last_unstale":"2023-09-23T22:32:24.997183+0000","last_undegraded":"2023-09-23T22:32:24.997183+0000","last_fullsized":"2023-09-23T22:32:24.997183+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5057,"log_dups_size":0,"ondisk_log_size":5057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:03.735360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331415552,"num_objects":5057,"num_object_clones":0,"num_object_copies":15171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5057,"num_write_kb":323648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":406,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132284+0000","last_change":"2023-09-23T22:31:28.139498+0000","last_active":"2023-09-23T22:32:15.132284+0000","last_peered":"2023-09-23T22:32:15.132284+0000","last_clean":"2023-09-23T22:32:15.132284+0000","last_became_active":"2023-09-23T22:31:28.113206+0000","last_became_peered":"2023-09-23T22:31:28.113206+0000","last_unstale":"2023-09-23T22:32:15.132284+0000","last_undegraded":"2023-09-23T22:32:15.132284+0000","last_fullsized":"2023-09-23T22:32:15.132284+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:11.905312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":399,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.137849+0000","last_change":"2023-09-23T22:31:28.131730+0000","last_active":"2023-09-23T22:32:15.137849+0000","last_peered":"2023-09-23T22:32:15.137849+0000","last_clean":"2023-09-23T22:32:15.137849+0000","last_became_active":"2023-09-23T22:31:28.131383+0000","last_became_peered":"2023-09-23T22:31:28.131383+0000","last_unstale":"2023-09-23T22:32:15.137849+0000","last_undegraded":"2023-09-23T22:32:15.137849+0000","last_fullsized":"2023-09-23T22:32:15.137849+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:56.993827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"397'5106","reported_seq":8341,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:25.020510+0000","last_change":"2023-09-23T22:32:00.147475+0000","last_active":"2023-09-23T22:32:25.020510+0000","last_peered":"2023-09-23T22:32:25.020510+0000","last_clean":"2023-09-23T22:32:25.020510+0000","last_became_active":"2023-09-23T22:32:00.121320+0000","last_became_peered":"2023-09-23T22:32:00.121320+0000","last_unstale":"2023-09-23T22:32:25.020510+0000","last_undegraded":"2023-09-23T22:32:25.020510+0000","last_fullsized":"2023-09-23T22:32:25.020510+0000","mapping_epoch":392,"log_start":"377'100","ondisk_log_start":"377'100","created":373,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5006,"log_dups_size":0,"ondisk_log_size":5006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:17:59.934876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334626816,"num_objects":5106,"num_object_clones":0,"num_object_copies":15318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5106,"num_write_kb":326784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1591,"num_bytes_recovered":104267776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":348,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132329+0000","last_change":"2023-09-23T22:31:28.125280+0000","last_active":"2023-09-23T22:32:15.132329+0000","last_peered":"2023-09-23T22:32:15.132329+0000","last_clean":"2023-09-23T22:32:15.132329+0000","last_became_active":"2023-09-23T22:31:28.110856+0000","last_became_peered":"2023-09-23T22:31:28.110856+0000","last_unstale":"2023-09-23T22:32:15.132329+0000","last_undegraded":"2023-09-23T22:32:15.132329+0000","last_fullsized":"2023-09-23T22:32:15.132329+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:32.197056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":424,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131631+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:32:15.131631+0000","last_peered":"2023-09-23T22:32:15.131631+0000","last_clean":"2023-09-23T22:32:15.131631+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:32:15.131631+0000","last_undegraded":"2023-09-23T22:32:15.131631+0000","last_fullsized":"2023-09-23T22:32:15.131631+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"397'5141","reported_seq":5182,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:32:24.824644+0000","last_change":"2023-09-23T22:31:29.183097+0000","last_active":"2023-09-23T22:32:24.824644+0000","last_peered":"2023-09-23T22:32:24.824644+0000","last_clean":"2023-09-23T22:31:27.059725+0000","last_became_active":"2023-09-23T22:31:29.178921+0000","last_became_peered":"2023-09-23T22:31:29.178921+0000","last_unstale":"2023-09-23T22:32:24.824644+0000","last_undegraded":"2023-09-23T22:31:29.114095+0000","last_fullsized":"2023-09-23T22:31:29.105229+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5141,"log_dups_size":0,"ondisk_log_size":5141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:28.838973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336920576,"num_objects":5141,"num_object_clones":0,"num_object_copies":15423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5141,"num_write_kb":329024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":3527},{"shards":"0,6,7","objects":1614}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":364,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138828+0000","last_change":"2023-09-23T22:31:28.136321+0000","last_active":"2023-09-23T22:32:15.138828+0000","last_peered":"2023-09-23T22:32:15.138828+0000","last_clean":"2023-09-23T22:32:15.138828+0000","last_became_active":"2023-09-23T22:31:28.131102+0000","last_became_peered":"2023-09-23T22:31:28.131102+0000","last_unstale":"2023-09-23T22:32:15.138828+0000","last_undegraded":"2023-09-23T22:32:15.138828+0000","last_fullsized":"2023-09-23T22:32:15.138828+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:24.768099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":88,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143655+0000","last_change":"2023-09-23T22:31:28.125138+0000","last_active":"2023-09-23T22:32:15.143655+0000","last_peered":"2023-09-23T22:32:15.143655+0000","last_clean":"2023-09-23T22:32:15.143655+0000","last_became_active":"2023-09-23T22:31:28.124716+0000","last_became_peered":"2023-09-23T22:31:28.124716+0000","last_unstale":"2023-09-23T22:32:15.143655+0000","last_undegraded":"2023-09-23T22:32:15.143655+0000","last_fullsized":"2023-09-23T22:32:15.143655+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.718931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"397'5178","reported_seq":5210,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:25.087588+0000","last_change":"2023-09-23T22:31:08.225591+0000","last_active":"2023-09-23T22:32:25.087588+0000","last_peered":"2023-09-23T22:32:25.087588+0000","last_clean":"2023-09-23T22:32:25.087588+0000","last_became_active":"2023-09-23T22:31:08.225286+0000","last_became_peered":"2023-09-23T22:31:08.225286+0000","last_unstale":"2023-09-23T22:32:25.087588+0000","last_undegraded":"2023-09-23T22:32:25.087588+0000","last_fullsized":"2023-09-23T22:32:25.087588+0000","mapping_epoch":373,"log_start":"377'100","ondisk_log_start":"377'100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5078,"log_dups_size":0,"ondisk_log_size":5078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:14.895634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339345408,"num_objects":5178,"num_object_clones":0,"num_object_copies":15534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5178,"num_write_kb":331392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":427,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138348+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:32:15.138348+0000","last_peered":"2023-09-23T22:32:15.138348+0000","last_clean":"2023-09-23T22:32:15.138348+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:32:15.138348+0000","last_undegraded":"2023-09-23T22:32:15.138348+0000","last_fullsized":"2023-09-23T22:32:15.138348+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":282,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132184+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:32:15.132184+0000","last_peered":"2023-09-23T22:32:15.132184+0000","last_clean":"2023-09-23T22:32:15.132184+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:32:15.132184+0000","last_undegraded":"2023-09-23T22:32:15.132184+0000","last_fullsized":"2023-09-23T22:32:15.132184+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"397'4992","reported_seq":5024,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:25.066839+0000","last_change":"2023-09-23T22:31:08.226199+0000","last_active":"2023-09-23T22:32:25.066839+0000","last_peered":"2023-09-23T22:32:25.066839+0000","last_clean":"2023-09-23T22:32:25.066839+0000","last_became_active":"2023-09-23T22:31:08.225930+0000","last_became_peered":"2023-09-23T22:31:08.225930+0000","last_unstale":"2023-09-23T22:32:25.066839+0000","last_undegraded":"2023-09-23T22:32:25.066839+0000","last_fullsized":"2023-09-23T22:32:25.066839+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4992,"log_dups_size":0,"ondisk_log_size":4992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:16.294443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327155712,"num_objects":4992,"num_object_clones":0,"num_object_copies":14976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4992,"num_write_kb":319488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":742,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.137841+0000","last_change":"2023-09-23T22:31:31.125903+0000","last_active":"2023-09-23T22:32:15.137841+0000","last_peered":"2023-09-23T22:32:15.137841+0000","last_clean":"2023-09-23T22:32:15.137841+0000","last_became_active":"2023-09-23T22:31:31.125630+0000","last_became_peered":"2023-09-23T22:31:31.125630+0000","last_unstale":"2023-09-23T22:32:15.137841+0000","last_undegraded":"2023-09-23T22:32:15.137841+0000","last_fullsized":"2023-09-23T22:32:15.137841+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:24.405922+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":229,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132293+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:32:15.132293+0000","last_peered":"2023-09-23T22:32:15.132293+0000","last_clean":"2023-09-23T22:32:15.132293+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:32:15.132293+0000","last_undegraded":"2023-09-23T22:32:15.132293+0000","last_fullsized":"2023-09-23T22:32:15.132293+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":405,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131901+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:32:15.131901+0000","last_peered":"2023-09-23T22:32:15.131901+0000","last_clean":"2023-09-23T22:32:15.131901+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:32:15.131901+0000","last_undegraded":"2023-09-23T22:32:15.131901+0000","last_fullsized":"2023-09-23T22:32:15.131901+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"397'5105","reported_seq":5137,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:24.144262+0000","last_change":"2023-09-23T22:31:08.227045+0000","last_active":"2023-09-23T22:32:24.144262+0000","last_peered":"2023-09-23T22:32:24.144262+0000","last_clean":"2023-09-23T22:32:24.144262+0000","last_became_active":"2023-09-23T22:31:08.226758+0000","last_became_peered":"2023-09-23T22:31:08.226758+0000","last_unstale":"2023-09-23T22:32:24.144262+0000","last_undegraded":"2023-09-23T22:32:24.144262+0000","last_fullsized":"2023-09-23T22:32:24.144262+0000","mapping_epoch":373,"log_start":"377'300","ondisk_log_start":"377'300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4805,"log_dups_size":0,"ondisk_log_size":4805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:55.118636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334561280,"num_objects":5105,"num_object_clones":0,"num_object_copies":15315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5105,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":778,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138067+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:32:15.138067+0000","last_peered":"2023-09-23T22:32:15.138067+0000","last_clean":"2023-09-23T22:32:15.138067+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:32:15.138067+0000","last_undegraded":"2023-09-23T22:32:15.138067+0000","last_fullsized":"2023-09-23T22:32:15.138067+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":78,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143778+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:32:15.143778+0000","last_peered":"2023-09-23T22:32:15.143778+0000","last_clean":"2023-09-23T22:32:15.143778+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:32:15.143778+0000","last_undegraded":"2023-09-23T22:32:15.143778+0000","last_fullsized":"2023-09-23T22:32:15.143778+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":432,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132054+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:32:15.132054+0000","last_peered":"2023-09-23T22:32:15.132054+0000","last_clean":"2023-09-23T22:32:15.132054+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:32:15.132054+0000","last_undegraded":"2023-09-23T22:32:15.132054+0000","last_fullsized":"2023-09-23T22:32:15.132054+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"397'5105","reported_seq":5137,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:24.991125+0000","last_change":"2023-09-23T22:31:08.225049+0000","last_active":"2023-09-23T22:32:24.991125+0000","last_peered":"2023-09-23T22:32:24.991125+0000","last_clean":"2023-09-23T22:32:24.991125+0000","last_became_active":"2023-09-23T22:31:08.224765+0000","last_became_peered":"2023-09-23T22:31:08.224765+0000","last_unstale":"2023-09-23T22:32:24.991125+0000","last_undegraded":"2023-09-23T22:32:24.991125+0000","last_fullsized":"2023-09-23T22:32:24.991125+0000","mapping_epoch":373,"log_start":"377'300","ondisk_log_start":"377'300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4805,"log_dups_size":0,"ondisk_log_size":4805,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.842171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334561280,"num_objects":5105,"num_object_clones":0,"num_object_copies":15315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5105,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":388,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131534+0000","last_change":"2023-09-23T22:31:28.119358+0000","last_active":"2023-09-23T22:32:15.131534+0000","last_peered":"2023-09-23T22:32:15.131534+0000","last_clean":"2023-09-23T22:32:15.131534+0000","last_became_active":"2023-09-23T22:31:28.118823+0000","last_became_peered":"2023-09-23T22:31:28.118823+0000","last_unstale":"2023-09-23T22:32:15.131534+0000","last_undegraded":"2023-09-23T22:32:15.131534+0000","last_fullsized":"2023-09-23T22:32:15.131534+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:25.483176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":413,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132241+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:32:15.132241+0000","last_peered":"2023-09-23T22:32:15.132241+0000","last_clean":"2023-09-23T22:32:15.132241+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:32:15.132241+0000","last_undegraded":"2023-09-23T22:32:15.132241+0000","last_fullsized":"2023-09-23T22:32:15.132241+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"397'5130","reported_seq":5163,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:24.121798+0000","last_change":"2023-09-23T22:31:08.226959+0000","last_active":"2023-09-23T22:32:24.121798+0000","last_peered":"2023-09-23T22:32:24.121798+0000","last_clean":"2023-09-23T22:32:24.121798+0000","last_became_active":"2023-09-23T22:31:08.226652+0000","last_became_peered":"2023-09-23T22:31:08.226652+0000","last_unstale":"2023-09-23T22:32:24.121798+0000","last_undegraded":"2023-09-23T22:32:24.121798+0000","last_fullsized":"2023-09-23T22:32:24.121798+0000","mapping_epoch":373,"log_start":"377'300","ondisk_log_start":"377'300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4830,"log_dups_size":0,"ondisk_log_size":4830,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:22.758103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336199680,"num_objects":5130,"num_object_clones":0,"num_object_copies":15390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5130,"num_write_kb":328320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":751,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138091+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:32:15.138091+0000","last_peered":"2023-09-23T22:32:15.138091+0000","last_clean":"2023-09-23T22:32:15.138091+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:32:15.138091+0000","last_undegraded":"2023-09-23T22:32:15.138091+0000","last_fullsized":"2023-09-23T22:32:15.138091+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":114,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.133866+0000","last_change":"2023-09-23T22:31:28.120600+0000","last_active":"2023-09-23T22:32:15.133866+0000","last_peered":"2023-09-23T22:32:15.133866+0000","last_clean":"2023-09-23T22:32:15.133866+0000","last_became_active":"2023-09-23T22:31:28.120281+0000","last_became_peered":"2023-09-23T22:31:28.120281+0000","last_unstale":"2023-09-23T22:32:15.133866+0000","last_undegraded":"2023-09-23T22:32:15.133866+0000","last_fullsized":"2023-09-23T22:32:15.133866+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:10.594022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":114,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:19.179963+0000","last_change":"2023-09-23T22:31:28.111270+0000","last_active":"2023-09-23T22:32:19.179963+0000","last_peered":"2023-09-23T22:32:19.179963+0000","last_clean":"2023-09-23T22:32:19.179963+0000","last_became_active":"2023-09-23T22:31:28.110921+0000","last_became_peered":"2023-09-23T22:31:28.110921+0000","last_unstale":"2023-09-23T22:32:19.179963+0000","last_undegraded":"2023-09-23T22:32:19.179963+0000","last_fullsized":"2023-09-23T22:32:19.179963+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:05.417671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"397'5077","reported_seq":8202,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:24.780925+0000","last_change":"2023-09-23T22:32:15.139562+0000","last_active":"2023-09-23T22:32:24.780925+0000","last_peered":"2023-09-23T22:32:24.780925+0000","last_clean":"2023-09-23T22:32:24.780925+0000","last_became_active":"2023-09-23T22:32:15.135437+0000","last_became_peered":"2023-09-23T22:32:15.135437+0000","last_unstale":"2023-09-23T22:32:24.780925+0000","last_undegraded":"2023-09-23T22:32:24.780925+0000","last_fullsized":"2023-09-23T22:32:24.780925+0000","mapping_epoch":395,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5077,"log_dups_size":0,"ondisk_log_size":5077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:36.457421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332726272,"num_objects":5077,"num_object_clones":0,"num_object_copies":15231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5077,"num_write_kb":324928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1537,"num_bytes_recovered":100728832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":672,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143981+0000","last_change":"2023-09-23T22:31:31.129491+0000","last_active":"2023-09-23T22:32:15.143981+0000","last_peered":"2023-09-23T22:32:15.143981+0000","last_clean":"2023-09-23T22:32:15.143981+0000","last_became_active":"2023-09-23T22:31:31.129163+0000","last_became_peered":"2023-09-23T22:31:31.129163+0000","last_unstale":"2023-09-23T22:32:15.143981+0000","last_undegraded":"2023-09-23T22:32:15.143981+0000","last_fullsized":"2023-09-23T22:32:15.143981+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.141229+0000","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":773,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132369+0000","last_change":"2023-09-23T22:31:31.129492+0000","last_active":"2023-09-23T22:32:15.132369+0000","last_peered":"2023-09-23T22:32:15.132369+0000","last_clean":"2023-09-23T22:32:15.132369+0000","last_became_active":"2023-09-23T22:31:31.128706+0000","last_became_peered":"2023-09-23T22:31:31.128706+0000","last_unstale":"2023-09-23T22:32:15.132369+0000","last_undegraded":"2023-09-23T22:32:15.132369+0000","last_fullsized":"2023-09-23T22:32:15.132369+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:16.180152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":464,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138687+0000","last_change":"2023-09-23T22:31:28.130412+0000","last_active":"2023-09-23T22:32:15.138687+0000","last_peered":"2023-09-23T22:32:15.138687+0000","last_clean":"2023-09-23T22:32:15.138687+0000","last_became_active":"2023-09-23T22:31:28.130127+0000","last_became_peered":"2023-09-23T22:31:28.130127+0000","last_unstale":"2023-09-23T22:32:15.138687+0000","last_undegraded":"2023-09-23T22:32:15.138687+0000","last_fullsized":"2023-09-23T22:32:15.138687+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.088651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":105,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.134588+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:32:15.134588+0000","last_peered":"2023-09-23T22:32:15.134588+0000","last_clean":"2023-09-23T22:32:15.134588+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:32:15.134588+0000","last_undegraded":"2023-09-23T22:32:15.134588+0000","last_fullsized":"2023-09-23T22:32:15.134588+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"397'5135","reported_seq":7332,"reported_epoch":397,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:32:25.063419+0000","last_change":"2023-09-23T22:32:14.078072+0000","last_active":"2023-09-23T22:32:25.063419+0000","last_peered":"2023-09-23T22:32:25.063419+0000","last_clean":"2023-09-23T22:31:27.057464+0000","last_became_active":"2023-09-23T22:31:29.175049+0000","last_became_peered":"2023-09-23T22:31:29.175049+0000","last_unstale":"2023-09-23T22:32:25.063419+0000","last_undegraded":"2023-09-23T22:32:25.063419+0000","last_fullsized":"2023-09-23T22:31:29.103539+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5135,"log_dups_size":0,"ondisk_log_size":5135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:10:49.194906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336527360,"num_objects":5135,"num_object_clones":0,"num_object_copies":15405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":525,"num_objects_unfound":0,"num_objects_dirty":5135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5135,"num_write_kb":328640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1077,"num_bytes_recovered":70582272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0],"avail_no_missing":["5","0","7"],"object_location_counts":[{"shards":"0,5","objects":4610},{"shards":"0,5,7","objects":525}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":741,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138069+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:32:15.138069+0000","last_peered":"2023-09-23T22:32:15.138069+0000","last_clean":"2023-09-23T22:32:15.138069+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:32:15.138069+0000","last_undegraded":"2023-09-23T22:32:15.138069+0000","last_fullsized":"2023-09-23T22:32:15.138069+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1275,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138325+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:32:15.138325+0000","last_peered":"2023-09-23T22:32:15.138325+0000","last_clean":"2023-09-23T22:32:15.138325+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:32:15.138325+0000","last_undegraded":"2023-09-23T22:32:15.138325+0000","last_fullsized":"2023-09-23T22:32:15.138325+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":436,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131688+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:32:15.131688+0000","last_peered":"2023-09-23T22:32:15.131688+0000","last_clean":"2023-09-23T22:32:15.131688+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:32:15.131688+0000","last_undegraded":"2023-09-23T22:32:15.131688+0000","last_fullsized":"2023-09-23T22:32:15.131688+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":88,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.144037+0000","last_change":"2023-09-23T22:31:28.123596+0000","last_active":"2023-09-23T22:32:15.144037+0000","last_peered":"2023-09-23T22:32:15.144037+0000","last_clean":"2023-09-23T22:32:15.144037+0000","last_became_active":"2023-09-23T22:31:28.123179+0000","last_became_peered":"2023-09-23T22:31:28.123179+0000","last_unstale":"2023-09-23T22:32:15.144037+0000","last_undegraded":"2023-09-23T22:32:15.144037+0000","last_fullsized":"2023-09-23T22:32:15.144037+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:53.106243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.b","version":"397'5052","reported_seq":8086,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:24.997717+0000","last_change":"2023-09-23T22:31:44.145615+0000","last_active":"2023-09-23T22:32:24.997717+0000","last_peered":"2023-09-23T22:32:24.997717+0000","last_clean":"2023-09-23T22:32:24.997717+0000","last_became_active":"2023-09-23T22:31:44.142103+0000","last_became_peered":"2023-09-23T22:31:44.142103+0000","last_unstale":"2023-09-23T22:32:24.997717+0000","last_undegraded":"2023-09-23T22:32:24.997717+0000","last_fullsized":"2023-09-23T22:32:24.997717+0000","mapping_epoch":387,"log_start":"377'200","ondisk_log_start":"377'200","created":373,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4852,"log_dups_size":0,"ondisk_log_size":4852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:15.488007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331087872,"num_objects":5052,"num_object_clones":0,"num_object_copies":15156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5052,"num_write_kb":323328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1492,"num_bytes_recovered":97779712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":747,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131910+0000","last_change":"2023-09-23T22:31:31.130599+0000","last_active":"2023-09-23T22:32:15.131910+0000","last_peered":"2023-09-23T22:32:15.131910+0000","last_clean":"2023-09-23T22:32:15.131910+0000","last_became_active":"2023-09-23T22:31:31.127784+0000","last_became_peered":"2023-09-23T22:31:31.127784+0000","last_unstale":"2023-09-23T22:32:15.131910+0000","last_undegraded":"2023-09-23T22:32:15.131910+0000","last_fullsized":"2023-09-23T22:32:15.131910+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:28.788741+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":613,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131814+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:32:15.131814+0000","last_peered":"2023-09-23T22:32:15.131814+0000","last_clean":"2023-09-23T22:32:15.131814+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:32:15.131814+0000","last_undegraded":"2023-09-23T22:32:15.131814+0000","last_fullsized":"2023-09-23T22:32:15.131814+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":78,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143376+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:32:15.143376+0000","last_peered":"2023-09-23T22:32:15.143376+0000","last_clean":"2023-09-23T22:32:15.143376+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:32:15.143376+0000","last_undegraded":"2023-09-23T22:32:15.143376+0000","last_fullsized":"2023-09-23T22:32:15.143376+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":45,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.134952+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:32:15.134952+0000","last_peered":"2023-09-23T22:32:15.134952+0000","last_clean":"2023-09-23T22:32:15.134952+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:32:15.134952+0000","last_undegraded":"2023-09-23T22:32:15.134952+0000","last_fullsized":"2023-09-23T22:32:15.134952+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"397'5090","reported_seq":5122,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:25.049568+0000","last_change":"2023-09-23T22:31:08.227185+0000","last_active":"2023-09-23T22:32:25.049568+0000","last_peered":"2023-09-23T22:32:25.049568+0000","last_clean":"2023-09-23T22:32:25.049568+0000","last_became_active":"2023-09-23T22:31:08.226918+0000","last_became_peered":"2023-09-23T22:31:08.226918+0000","last_unstale":"2023-09-23T22:32:25.049568+0000","last_undegraded":"2023-09-23T22:32:25.049568+0000","last_fullsized":"2023-09-23T22:32:25.049568+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5090,"log_dups_size":0,"ondisk_log_size":5090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:54.257936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333578240,"num_objects":5090,"num_object_clones":0,"num_object_copies":15270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5090,"num_write_kb":325760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1159,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138161+0000","last_change":"2023-09-23T22:31:31.127033+0000","last_active":"2023-09-23T22:32:15.138161+0000","last_peered":"2023-09-23T22:32:15.138161+0000","last_clean":"2023-09-23T22:32:15.138161+0000","last_became_active":"2023-09-23T22:31:31.126601+0000","last_became_peered":"2023-09-23T22:31:31.126601+0000","last_unstale":"2023-09-23T22:32:15.138161+0000","last_undegraded":"2023-09-23T22:32:15.138161+0000","last_fullsized":"2023-09-23T22:32:15.138161+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:00.619278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":781,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132088+0000","last_change":"2023-09-23T22:31:31.126756+0000","last_active":"2023-09-23T22:32:15.132088+0000","last_peered":"2023-09-23T22:32:15.132088+0000","last_clean":"2023-09-23T22:32:15.132088+0000","last_became_active":"2023-09-23T22:31:31.126409+0000","last_became_peered":"2023-09-23T22:31:31.126409+0000","last_unstale":"2023-09-23T22:32:15.132088+0000","last_undegraded":"2023-09-23T22:32:15.132088+0000","last_fullsized":"2023-09-23T22:32:15.132088+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.605240+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":278,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138179+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:32:15.138179+0000","last_peered":"2023-09-23T22:32:15.138179+0000","last_clean":"2023-09-23T22:32:15.138179+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:32:15.138179+0000","last_undegraded":"2023-09-23T22:32:15.138179+0000","last_fullsized":"2023-09-23T22:32:15.138179+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":88,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.144472+0000","last_change":"2023-09-23T22:31:28.124597+0000","last_active":"2023-09-23T22:32:15.144472+0000","last_peered":"2023-09-23T22:32:15.144472+0000","last_clean":"2023-09-23T22:32:15.144472+0000","last_became_active":"2023-09-23T22:31:28.124245+0000","last_became_peered":"2023-09-23T22:31:28.124245+0000","last_unstale":"2023-09-23T22:32:15.144472+0000","last_undegraded":"2023-09-23T22:32:15.144472+0000","last_fullsized":"2023-09-23T22:32:15.144472+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.558632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"397'5054","reported_seq":5087,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:24.153496+0000","last_change":"2023-09-23T22:31:08.223664+0000","last_active":"2023-09-23T22:32:24.153496+0000","last_peered":"2023-09-23T22:32:24.153496+0000","last_clean":"2023-09-23T22:32:24.153496+0000","last_became_active":"2023-09-23T22:31:08.223323+0000","last_became_peered":"2023-09-23T22:31:08.223323+0000","last_unstale":"2023-09-23T22:32:24.153496+0000","last_undegraded":"2023-09-23T22:32:24.153496+0000","last_fullsized":"2023-09-23T22:32:24.153496+0000","mapping_epoch":373,"log_start":"377'200","ondisk_log_start":"377'200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4854,"log_dups_size":0,"ondisk_log_size":4854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:38.536861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331218944,"num_objects":5054,"num_object_clones":0,"num_object_copies":15162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5054,"num_write_kb":323456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":629,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143682+0000","last_change":"2023-09-23T22:31:21.362707+0000","last_active":"2023-09-23T22:32:15.143682+0000","last_peered":"2023-09-23T22:32:15.143682+0000","last_clean":"2023-09-23T22:32:15.143682+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:32:15.143682+0000","last_undegraded":"2023-09-23T22:32:15.143682+0000","last_fullsized":"2023-09-23T22:32:15.143682+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:58.622607+0000","scrub_duration":0.0034076229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":776,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.133907+0000","last_change":"2023-09-23T22:31:28.147724+0000","last_active":"2023-09-23T22:32:15.133907+0000","last_peered":"2023-09-23T22:32:15.133907+0000","last_clean":"2023-09-23T22:32:15.133907+0000","last_became_active":"2023-09-23T22:31:28.120238+0000","last_became_peered":"2023-09-23T22:31:28.120238+0000","last_unstale":"2023-09-23T22:32:15.133907+0000","last_undegraded":"2023-09-23T22:32:15.133907+0000","last_fullsized":"2023-09-23T22:32:15.133907+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:53.259496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":292,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131858+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:32:15.131858+0000","last_peered":"2023-09-23T22:32:15.131858+0000","last_clean":"2023-09-23T22:32:15.131858+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:32:15.131858+0000","last_undegraded":"2023-09-23T22:32:15.131858+0000","last_fullsized":"2023-09-23T22:32:15.131858+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":285,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132815+0000","last_change":"2023-09-23T22:31:28.121081+0000","last_active":"2023-09-23T22:32:15.132815+0000","last_peered":"2023-09-23T22:32:15.132815+0000","last_clean":"2023-09-23T22:32:15.132815+0000","last_became_active":"2023-09-23T22:31:28.120399+0000","last_became_peered":"2023-09-23T22:31:28.120399+0000","last_unstale":"2023-09-23T22:32:15.132815+0000","last_undegraded":"2023-09-23T22:32:15.132815+0000","last_fullsized":"2023-09-23T22:32:15.132815+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:11.267296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"397'5182","reported_seq":5215,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:24.129679+0000","last_change":"2023-09-23T22:31:08.224259+0000","last_active":"2023-09-23T22:32:24.129679+0000","last_peered":"2023-09-23T22:32:24.129679+0000","last_clean":"2023-09-23T22:32:24.129679+0000","last_became_active":"2023-09-23T22:31:08.224062+0000","last_became_peered":"2023-09-23T22:31:08.224062+0000","last_unstale":"2023-09-23T22:32:24.129679+0000","last_undegraded":"2023-09-23T22:32:24.129679+0000","last_fullsized":"2023-09-23T22:32:24.129679+0000","mapping_epoch":373,"log_start":"377'400","ondisk_log_start":"377'400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:18.330753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339607552,"num_objects":5182,"num_object_clones":0,"num_object_copies":15546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5182,"num_write_kb":331648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":747,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.144203+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:32:15.144203+0000","last_peered":"2023-09-23T22:32:15.144203+0000","last_clean":"2023-09-23T22:32:15.144203+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:32:15.144203+0000","last_undegraded":"2023-09-23T22:32:15.144203+0000","last_fullsized":"2023-09-23T22:32:15.144203+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":324,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132545+0000","last_change":"2023-09-23T22:31:28.152186+0000","last_active":"2023-09-23T22:32:15.132545+0000","last_peered":"2023-09-23T22:32:15.132545+0000","last_clean":"2023-09-23T22:32:15.132545+0000","last_became_active":"2023-09-23T22:31:28.111292+0000","last_became_peered":"2023-09-23T22:31:28.111292+0000","last_unstale":"2023-09-23T22:32:15.132545+0000","last_undegraded":"2023-09-23T22:32:15.132545+0000","last_fullsized":"2023-09-23T22:32:15.132545+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:06.097398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":323,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138392+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:32:15.138392+0000","last_peered":"2023-09-23T22:32:15.138392+0000","last_clean":"2023-09-23T22:32:15.138392+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:32:15.138392+0000","last_undegraded":"2023-09-23T22:32:15.138392+0000","last_fullsized":"2023-09-23T22:32:15.138392+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":265,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132151+0000","last_change":"2023-09-23T22:31:28.106985+0000","last_active":"2023-09-23T22:32:15.132151+0000","last_peered":"2023-09-23T22:32:15.132151+0000","last_clean":"2023-09-23T22:32:15.132151+0000","last_became_active":"2023-09-23T22:31:28.105451+0000","last_became_peered":"2023-09-23T22:31:28.105451+0000","last_unstale":"2023-09-23T22:32:15.132151+0000","last_undegraded":"2023-09-23T22:32:15.132151+0000","last_fullsized":"2023-09-23T22:32:15.132151+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:43.614928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"397'5111","reported_seq":8365,"reported_epoch":397,"state":"active+clean","last_fresh":"2023-09-23T22:32:24.837285+0000","last_change":"2023-09-23T22:31:45.135267+0000","last_active":"2023-09-23T22:32:24.837285+0000","last_peered":"2023-09-23T22:32:24.837285+0000","last_clean":"2023-09-23T22:32:24.837285+0000","last_became_active":"2023-09-23T22:31:45.131633+0000","last_became_peered":"2023-09-23T22:31:45.131633+0000","last_unstale":"2023-09-23T22:32:24.837285+0000","last_undegraded":"2023-09-23T22:32:24.837285+0000","last_fullsized":"2023-09-23T22:32:24.837285+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5111,"log_dups_size":0,"ondisk_log_size":5111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:00.859364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334954496,"num_objects":5111,"num_object_clones":0,"num_object_copies":15333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5111,"num_write_kb":327104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1601,"num_bytes_recovered":104923136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":751,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131927+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:32:15.131927+0000","last_peered":"2023-09-23T22:32:15.131927+0000","last_clean":"2023-09-23T22:32:15.131927+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:32:15.131927+0000","last_undegraded":"2023-09-23T22:32:15.131927+0000","last_fullsized":"2023-09-23T22:32:15.131927+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":229,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132801+0000","last_change":"2023-09-23T22:31:28.131347+0000","last_active":"2023-09-23T22:32:15.132801+0000","last_peered":"2023-09-23T22:32:15.132801+0000","last_clean":"2023-09-23T22:32:15.132801+0000","last_became_active":"2023-09-23T22:31:28.114169+0000","last_became_peered":"2023-09-23T22:31:28.114169+0000","last_unstale":"2023-09-23T22:32:15.132801+0000","last_undegraded":"2023-09-23T22:32:15.132801+0000","last_fullsized":"2023-09-23T22:32:15.132801+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:30.007254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":893,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.134868+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:32:15.134868+0000","last_peered":"2023-09-23T22:32:15.134868+0000","last_clean":"2023-09-23T22:32:15.134868+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:32:15.134868+0000","last_undegraded":"2023-09-23T22:32:15.134868+0000","last_fullsized":"2023-09-23T22:32:15.134868+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":337,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131545+0000","last_change":"2023-09-23T22:31:28.114550+0000","last_active":"2023-09-23T22:32:15.131545+0000","last_peered":"2023-09-23T22:32:15.131545+0000","last_clean":"2023-09-23T22:32:15.131545+0000","last_became_active":"2023-09-23T22:31:28.114217+0000","last_became_peered":"2023-09-23T22:31:28.114217+0000","last_unstale":"2023-09-23T22:32:15.131545+0000","last_undegraded":"2023-09-23T22:32:15.131545+0000","last_fullsized":"2023-09-23T22:32:15.131545+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:13.616772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.d","version":"397'5075","reported_seq":5117,"reported_epoch":397,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:32:24.967475+0000","last_change":"2023-09-23T22:31:29.186960+0000","last_active":"2023-09-23T22:32:24.967475+0000","last_peered":"2023-09-23T22:32:24.967475+0000","last_clean":"2023-09-23T22:31:27.062039+0000","last_became_active":"2023-09-23T22:31:29.181566+0000","last_became_peered":"2023-09-23T22:31:29.181566+0000","last_unstale":"2023-09-23T22:32:24.967475+0000","last_undegraded":"2023-09-23T22:31:29.116070+0000","last_fullsized":"2023-09-23T22:31:29.103794+0000","mapping_epoch":381,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5075,"log_dups_size":0,"ondisk_log_size":5075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:13.831015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332595200,"num_objects":5075,"num_object_clones":0,"num_object_copies":15225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1584,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5075,"num_write_kb":324800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":3491},{"shards":"5,6,7","objects":1584}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":677,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138437+0000","last_change":"2023-09-23T22:31:45.133529+0000","last_active":"2023-09-23T22:32:15.138437+0000","last_peered":"2023-09-23T22:32:15.138437+0000","last_clean":"2023-09-23T22:32:15.138437+0000","last_became_active":"2023-09-23T22:31:45.133117+0000","last_became_peered":"2023-09-23T22:31:45.133117+0000","last_unstale":"2023-09-23T22:32:15.138437+0000","last_undegraded":"2023-09-23T22:32:15.138437+0000","last_fullsized":"2023-09-23T22:32:15.138437+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.591643+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.591643+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:28.957050+0000","scrub_duration":0.0073638979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":108,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.143800+0000","last_change":"2023-09-23T22:31:28.156716+0000","last_active":"2023-09-23T22:32:15.143800+0000","last_peered":"2023-09-23T22:32:15.143800+0000","last_clean":"2023-09-23T22:32:15.143800+0000","last_became_active":"2023-09-23T22:31:28.128617+0000","last_became_peered":"2023-09-23T22:31:28.128617+0000","last_unstale":"2023-09-23T22:32:15.143800+0000","last_undegraded":"2023-09-23T22:32:15.143800+0000","last_fullsized":"2023-09-23T22:32:15.143800+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:34.465027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":261,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132702+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:32:15.132702+0000","last_peered":"2023-09-23T22:32:15.132702+0000","last_clean":"2023-09-23T22:32:15.132702+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:32:15.132702+0000","last_undegraded":"2023-09-23T22:32:15.132702+0000","last_fullsized":"2023-09-23T22:32:15.132702+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":370,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131543+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:32:15.131543+0000","last_peered":"2023-09-23T22:32:15.131543+0000","last_clean":"2023-09-23T22:32:15.131543+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:32:15.131543+0000","last_undegraded":"2023-09-23T22:32:15.131543+0000","last_fullsized":"2023-09-23T22:32:15.131543+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":56,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132100+0000","last_change":"2023-09-23T22:31:28.111728+0000","last_active":"2023-09-23T22:32:15.132100+0000","last_peered":"2023-09-23T22:32:15.132100+0000","last_clean":"2023-09-23T22:32:15.132100+0000","last_became_active":"2023-09-23T22:31:28.111369+0000","last_became_peered":"2023-09-23T22:31:28.111369+0000","last_unstale":"2023-09-23T22:32:15.132100+0000","last_undegraded":"2023-09-23T22:32:15.132100+0000","last_fullsized":"2023-09-23T22:32:15.132100+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:42.837575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.134066+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:32:15.134066+0000","last_peered":"2023-09-23T22:32:15.134066+0000","last_clean":"2023-09-23T22:32:15.134066+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:32:15.134066+0000","last_undegraded":"2023-09-23T22:32:15.134066+0000","last_fullsized":"2023-09-23T22:32:15.134066+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":387,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138369+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:32:15.138369+0000","last_peered":"2023-09-23T22:32:15.138369+0000","last_clean":"2023-09-23T22:32:15.138369+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:32:15.138369+0000","last_undegraded":"2023-09-23T22:32:15.138369+0000","last_fullsized":"2023-09-23T22:32:15.138369+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":422,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131880+0000","last_change":"2023-09-23T22:31:28.111351+0000","last_active":"2023-09-23T22:32:15.131880+0000","last_peered":"2023-09-23T22:32:15.131880+0000","last_clean":"2023-09-23T22:32:15.131880+0000","last_became_active":"2023-09-23T22:31:28.111006+0000","last_became_peered":"2023-09-23T22:31:28.111006+0000","last_unstale":"2023-09-23T22:32:15.131880+0000","last_undegraded":"2023-09-23T22:32:15.131880+0000","last_fullsized":"2023-09-23T22:32:15.131880+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:51.435513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":412,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.138739+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:32:15.138739+0000","last_peered":"2023-09-23T22:32:15.138739+0000","last_clean":"2023-09-23T22:32:15.138739+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:32:15.138739+0000","last_undegraded":"2023-09-23T22:32:15.138739+0000","last_fullsized":"2023-09-23T22:32:15.138739+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":261,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131780+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:32:15.131780+0000","last_peered":"2023-09-23T22:32:15.131780+0000","last_clean":"2023-09-23T22:32:15.131780+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:32:15.131780+0000","last_undegraded":"2023-09-23T22:32:15.131780+0000","last_fullsized":"2023-09-23T22:32:15.131780+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":88,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.144038+0000","last_change":"2023-09-23T22:31:28.126203+0000","last_active":"2023-09-23T22:32:15.144038+0000","last_peered":"2023-09-23T22:32:15.144038+0000","last_clean":"2023-09-23T22:32:15.144038+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:31:28.125529+0000","last_unstale":"2023-09-23T22:32:15.144038+0000","last_undegraded":"2023-09-23T22:32:15.144038+0000","last_fullsized":"2023-09-23T22:32:15.144038+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:47.359771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":397,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131911+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:32:15.131911+0000","last_peered":"2023-09-23T22:32:15.131911+0000","last_clean":"2023-09-23T22:32:15.131911+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:32:15.131911+0000","last_undegraded":"2023-09-23T22:32:15.131911+0000","last_fullsized":"2023-09-23T22:32:15.131911+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":335,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.132802+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:32:15.132802+0000","last_peered":"2023-09-23T22:32:15.132802+0000","last_clean":"2023-09-23T22:32:15.132802+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:32:15.132802+0000","last_undegraded":"2023-09-23T22:32:15.132802+0000","last_fullsized":"2023-09-23T22:32:15.132802+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":455,"reported_epoch":396,"state":"active+clean","last_fresh":"2023-09-23T22:32:15.131670+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:32:15.131670+0000","last_peered":"2023-09-23T22:32:15.131670+0000","last_clean":"2023-09-23T22:32:15.131670+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:32:15.131670+0000","last_undegraded":"2023-09-23T22:32:15.131670+0000","last_fullsized":"2023-09-23T22:32:15.131670+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":31404,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5347082240,"num_objects":81590,"num_object_clones":0,"num_object_copies":244770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3198,"num_objects_misplaced":525,"num_objects_unfound":0,"num_objects_dirty":81590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":81590,"num_write_kb":5221760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7298,"num_bytes_recovered":478281728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16297820160,"data_stored":16297820160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79690,"ondisk_log_size":79690,"up":48,"acting":45,"num_store_stats":6},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1115744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234336,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495972,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2794772,"kb_used_data":2619468,"kb_used_omap":0,"kb_used_meta":175296,"kb_avail":90950380,"statfs":{"total":95995035648,"available":93133189120,"internally_reserved":0,"allocated":2682335232,"data_stored":2680877807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":179503104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":7.2089999999999996,"5min":5.6589999999999998,"15min":2.9900000000000002},"min":{"1min":0.22900000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":57.936999999999998,"5min":213.23699999999999,"15min":213.23699999999999},"last":1.444},{"interface":"front","average":{"1min":7.1070000000000002,"5min":5.992,"15min":4.0529999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":58.526000000000003,"5min":213.304,"15min":213.304},"last":0.501}]},{"osd":1,"last update":"Sat Sep 23 22:31:41 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.9790000000000001,"15min":3.4510000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":6.5839999999999996,"5min":80.430000000000007,"15min":129.964},"last":1.4099999999999999},{"interface":"front","average":{"1min":0.97199999999999998,"5min":2.4169999999999998,"15min":3.7730000000000001},"min":{"1min":0.27100000000000002,"5min":0.27000000000000002,"15min":0.254},"max":{"1min":4.2850000000000001,"5min":84.192999999999998,"15min":131.18600000000001},"last":0.47299999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.8680000000000001,"15min":1.8919999999999999},"min":{"1min":0.317,"5min":0.311,"15min":0.311},"max":{"1min":5.5839999999999996,"5min":6.6559999999999997,"15min":6.6559999999999997},"last":0.628},{"interface":"front","average":{"1min":0.96599999999999997,"5min":1.913,"15min":2.0710000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.21,"5min":9.7319999999999993,"15min":9.7319999999999993},"last":0.64400000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:31:41 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":1.9039999999999999,"15min":3.2570000000000001},"min":{"1min":0.378,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":5.5519999999999996,"5min":42.523000000000003,"15min":125.125},"last":3.8239999999999998},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.8120000000000001,"15min":2.9910000000000001},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":2.782,"5min":39.926000000000002,"15min":76.406999999999996},"last":3.7810000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:31:49 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":0.85599999999999998,"15min":2.5600000000000001},"min":{"1min":0.27600000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.2460000000000004,"5min":6.8650000000000002,"15min":305.30200000000002},"last":0.439},{"interface":"front","average":{"1min":2.6240000000000001,"5min":1.0620000000000001,"15min":2.5960000000000001},"min":{"1min":0.29299999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":8.1850000000000005,"5min":8.1850000000000005,"15min":305.40300000000002},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:31:49 2023","interfaces":[{"interface":"back","average":{"1min":7.5199999999999996,"5min":2.1099999999999999,"15min":1.5169999999999999},"min":{"1min":0.317,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":88.194999999999993,"5min":88.194999999999993,"15min":88.194999999999993},"last":0.95799999999999996},{"interface":"front","average":{"1min":5.9800000000000004,"5min":1.736,"15min":1.427},"min":{"1min":0.30599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":88.025000000000006,"5min":88.025000000000006,"15min":88.025000000000006},"last":2.1019999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":0.95399999999999996,"15min":1.0980000000000001},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.2270000000000003,"5min":9.2270000000000003,"15min":82.801000000000002},"last":2.0179999999999998},{"interface":"front","average":{"1min":2.4630000000000001,"5min":1.105,"15min":1.1459999999999999},"min":{"1min":0.41999999999999998,"5min":0.33300000000000002,"15min":0.29699999999999999},"max":{"1min":8.7690000000000001,"5min":8.7690000000000001,"15min":83.052999999999997},"last":0.59699999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795244,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3441436,"kb_used_data":2805652,"kb_used_omap":7,"kb_used_meta":635768,"kb_avail":90303716,"statfs":{"total":95995035648,"available":92471005184,"internally_reserved":0,"allocated":2872987648,"data_stored":2870872997,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":651026804},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:31:30 2023","interfaces":[{"interface":"back","average":{"1min":17.125,"5min":6.5919999999999996,"15min":4.9900000000000002},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.189},"max":{"1min":202.73500000000001,"5min":202.73500000000001,"15min":202.73500000000001},"last":2.6720000000000002},{"interface":"front","average":{"1min":18.056999999999999,"5min":6.899,"15min":5.7370000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":202.80799999999999,"5min":202.80799999999999,"15min":202.80799999999999},"last":0.72999999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:31:50 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.94299999999999995,"15min":1.0189999999999999},"min":{"1min":0.26900000000000002,"5min":0.216,"15min":0.216},"max":{"1min":1.0660000000000001,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":1.7250000000000001},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.1259999999999999,"15min":2.2890000000000001},"min":{"1min":0.24299999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.5030000000000001,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":0.81799999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":2.1139999999999999,"15min":4.4800000000000004},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":4.8540000000000001,"5min":13.939,"15min":160.499},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.978,"15min":4.2389999999999999},"min":{"1min":0.27100000000000002,"5min":0.255,"15min":0.223},"max":{"1min":4.8769999999999998,"5min":13.515000000000001,"15min":160.465},"last":0.29499999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:31:30 2023","interfaces":[{"interface":"back","average":{"1min":0.85999999999999999,"5min":1.109,"15min":2.2410000000000001},"min":{"1min":0.28399999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.7880000000000003,"5min":9.6539999999999999,"15min":234.79900000000001},"last":3.4489999999999998},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.4299999999999999,"15min":2.851},"min":{"1min":0.33700000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.3699999999999992,"5min":9.093,"15min":234.81700000000001},"last":4.4539999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.7390000000000001,"15min":2.4239999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":4.681,"5min":10.973000000000001,"15min":196.79900000000001},"last":2.0979999999999999},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.339,"15min":2.3039999999999998},"min":{"1min":0.313,"5min":0.313,"15min":0.29099999999999998},"max":{"1min":4.7729999999999997,"5min":6.8049999999999997,"15min":196.68299999999999},"last":1.222}]},{"osd":6,"last update":"Sat Sep 23 22:31:30 2023","interfaces":[{"interface":"back","average":{"1min":5.556,"5min":2.2639999999999998,"15min":2.1000000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.30399999999999999},"max":{"1min":77.033000000000001,"5min":77.033000000000001,"15min":77.033000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":4.9329999999999998,"5min":2.0619999999999998,"15min":1.5629999999999999},"min":{"1min":0.34699999999999998,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":76.965999999999994,"5min":76.965999999999994,"15min":76.965999999999994},"last":3.0049999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":1.599,"15min":1.9670000000000001},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.32900000000000001},"max":{"1min":5.3140000000000001,"5min":8.0660000000000007,"15min":16.265000000000001},"last":2.4870000000000001},{"interface":"front","average":{"1min":1.4059999999999999,"5min":1.4299999999999999,"15min":1.3300000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":5.8760000000000003,"5min":9.7129999999999992,"15min":12.045999999999999},"last":2.2559999999999998}]}]},{"osd":0,"up_from":93,"seq":399431958773,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3627704,"kb_used_data":2924528,"kb_used_omap":8,"kb_used_meta":703159,"kb_avail":90117448,"statfs":{"total":95995035648,"available":92280266752,"internally_reserved":0,"allocated":2994716672,"data_stored":2992520504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":720035690},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:31:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":3.8029999999999999,"15min":3.8780000000000001},"min":{"1min":0.20499999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":7.407,"5min":89.439999999999998,"15min":169.77199999999999},"last":1.054},{"interface":"front","average":{"1min":1.3180000000000001,"5min":3.7080000000000002,"15min":4.1399999999999997},"min":{"1min":0.219,"5min":0.218,"15min":0.20300000000000001},"max":{"1min":6.6630000000000003,"5min":89.287999999999997,"15min":169.52099999999999},"last":0.97299999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:31:52 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":2.7029999999999998,"15min":3.0310000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.202,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":1.6180000000000001,"15min":1.762},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.3290000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.40100000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:31:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.7549999999999999,"15min":3.6339999999999999},"min":{"1min":0.28999999999999998,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":10.292999999999999,"5min":15.404,"15min":146.87299999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.319,"5min":2.1120000000000001,"15min":3.7850000000000001},"min":{"1min":0.27500000000000002,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":6.5789999999999997,"5min":22.303000000000001,"15min":145.48099999999999},"last":0.80500000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":2.109,"15min":2.5339999999999998},"min":{"1min":0.30199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.2060000000000004,"5min":11.474,"15min":82.825000000000003},"last":3.8130000000000002},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.4119999999999999,"15min":1.7529999999999999},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.444,"5min":7.2279999999999998,"15min":75.930999999999997},"last":0.72099999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":2.21,"15min":2.9689999999999999},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2510000000000003,"5min":13.564,"15min":93.685000000000002},"last":3.8929999999999998},{"interface":"front","average":{"1min":1.841,"5min":1.633,"15min":2.246},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4139999999999997,"5min":13.486000000000001,"15min":93.546000000000006},"last":0.68500000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3290000000000002,"15min":2.3140000000000001},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.308},"max":{"1min":10.183999999999999,"5min":12.955,"15min":16.366},"last":6.4109999999999996},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.6830000000000001,"15min":1.6799999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":6.5060000000000002,"5min":14.208,"15min":22.263000000000002},"last":0.89200000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.8500000000000001,"15min":2},"min":{"1min":0.42599999999999999,"5min":0.40200000000000002,"15min":0.29999999999999999},"max":{"1min":3.0110000000000001,"5min":22.338999999999999,"15min":32.773000000000003},"last":0.88100000000000001},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.1389999999999998,"15min":2.2890000000000001},"min":{"1min":0.47899999999999998,"5min":0.307,"15min":0.307},"max":{"1min":8.7550000000000008,"5min":10.901,"15min":16.381},"last":3.863}]}]},{"osd":2,"up_from":0,"seq":1498943586337,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468191,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019241,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4143120,"kb_used_data":3790344,"kb_used_omap":0,"kb_used_meta":352768,"kb_avail":89602032,"statfs":{"total":95995035648,"available":91752480768,"internally_reserved":0,"allocated":3881312256,"data_stored":3879858376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":361234432},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:31:25 2023","interfaces":[{"interface":"back","average":{"1min":1.974,"5min":3.8130000000000002,"15min":3.6560000000000001},"min":{"1min":0.29199999999999998,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.638999999999999,"5min":104.252,"15min":150.63499999999999},"last":100.06999999999999},{"interface":"front","average":{"1min":2.2429999999999999,"5min":4.1269999999999998,"15min":3.7810000000000001},"min":{"1min":0.26900000000000002,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":15.701000000000001,"5min":104.294,"15min":150.672},"last":100.048}]},{"osd":1,"last update":"Sat Sep 23 22:31:25 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":3.9929999999999999,"15min":3.2389999999999999},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.21299999999999999},"max":{"1min":9.0150000000000006,"5min":95.114000000000004,"15min":107.346},"last":94.197999999999993},{"interface":"front","average":{"1min":1.472,"5min":4.069,"15min":3.556},"min":{"1min":0.33400000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.7039999999999997,"5min":95.222999999999999,"15min":107.33499999999999},"last":94.460999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:31:53 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.911,"15min":2.0830000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.7270000000000003,"5min":7.6790000000000003,"15min":7.6790000000000003},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.4890000000000001,"15min":1.5229999999999999},"min":{"1min":0.376,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":8.1159999999999997,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":3.3679999999999999,"15min":2.46},"min":{"1min":0.36299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7119999999999997,"5min":102.896,"15min":102.896},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.5049999999999999,"5min":3.6539999999999999,"15min":2.593},"min":{"1min":0.37,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":6.6980000000000004,"5min":112.667,"15min":112.667},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.93300000000000005,"15min":0.97899999999999998},"min":{"1min":0.253,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.4050000000000002,"5min":13.407,"15min":28.399000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.3,"5min":0.95099999999999996,"15min":1.0469999999999999},"min":{"1min":0.313,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":4.2240000000000002,"5min":14.212,"15min":28.350999999999999},"last":0.33800000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.80600000000000005,"15min":0.83399999999999996},"min":{"1min":0.30299999999999999,"5min":0.26400000000000001,"15min":0.251},"max":{"1min":3.6859999999999999,"5min":6.0110000000000001,"15min":7.8550000000000004},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.756,"5min":0.90700000000000003,"15min":0.85099999999999998},"min":{"1min":0.32800000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.6950000000000003,"5min":6.6950000000000003,"15min":8.5210000000000008},"last":0.35799999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:31:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":0.80900000000000005,"15min":0.92200000000000004},"min":{"1min":0.39500000000000002,"5min":0.30399999999999999,"15min":0.27500000000000002},"max":{"1min":5.6799999999999997,"5min":5.6799999999999997,"15min":8.577},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.85399999999999998,"5min":0.76000000000000001,"15min":0.874},"min":{"1min":0.37,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.1699999999999999,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":0.32600000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757609,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3659840,"kb_used_data":3262776,"kb_used_omap":0,"kb_used_meta":397056,"kb_avail":90085312,"statfs":{"total":95995035648,"available":92247359488,"internally_reserved":0,"allocated":3341082624,"data_stored":3339640821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":406585344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":13.648999999999999,"5min":3.9780000000000002,"15min":4.5839999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":124.892,"5min":124.892,"15min":163.54499999999999},"last":3.6840000000000002},{"interface":"front","average":{"1min":13.523,"5min":3.528,"15min":4.3230000000000004},"min":{"1min":0.34999999999999998,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":123.807,"5min":123.807,"15min":162.81800000000001},"last":4.5170000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:31:34 2023","interfaces":[{"interface":"back","average":{"1min":4.2480000000000002,"5min":3.7919999999999998,"15min":4.0970000000000004},"min":{"1min":0.313,"5min":0.29099999999999998,"15min":0.23799999999999999},"max":{"1min":61.621000000000002,"5min":113.514,"15min":113.514},"last":0.29599999999999999},{"interface":"front","average":{"1min":4.6079999999999997,"5min":3.9159999999999999,"15min":4.1360000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.252},"max":{"1min":61.645000000000003,"5min":113.42700000000001,"15min":113.42700000000001},"last":0.48499999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":2.133,"15min":2.3180000000000001},"min":{"1min":0.34200000000000003,"5min":0.316,"15min":0.316},"max":{"1min":7.2480000000000002,"5min":10.451000000000001,"15min":10.451000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.83899999999999997,"5min":2.1909999999999998,"15min":2.4159999999999999},"min":{"1min":0.33000000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.4249999999999998,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":3.6349999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:31:34 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":3.1499999999999999,"15min":3.8090000000000002},"min":{"1min":0.39400000000000002,"5min":0.30199999999999999,"15min":0.246},"max":{"1min":4.8369999999999997,"5min":132.59,"15min":132.59},"last":3.706},{"interface":"front","average":{"1min":0.745,"5min":3.3380000000000001,"15min":3.758},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":2.9889999999999999,"5min":129.381,"15min":129.381},"last":0.505}]},{"osd":4,"last update":"Sat Sep 23 22:31:29 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":2.0499999999999998,"15min":1.544},"min":{"1min":0.27700000000000002,"5min":0.254,"15min":0.24099999999999999},"max":{"1min":4.5339999999999998,"5min":126.679,"15min":126.679},"last":1.2470000000000001},{"interface":"front","average":{"1min":0.95399999999999996,"5min":2.0800000000000001,"15min":1.5649999999999999},"min":{"1min":0.28799999999999998,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":3.7029999999999998,"5min":126.78100000000001,"15min":126.78100000000001},"last":3.7320000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":1.504,"15min":1.456},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.5489999999999999,"5min":70.581999999999994,"15min":114.83499999999999},"last":4},{"interface":"front","average":{"1min":1.206,"5min":1.8109999999999999,"15min":1.4670000000000001},"min":{"1min":0.32600000000000001,"5min":0.248,"15min":0.248},"max":{"1min":6.2649999999999997,"5min":71.093000000000004,"15min":110.905},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":3.7650000000000001,"5min":1.24,"15min":1.107},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":40.548000000000002,"5min":40.548000000000002,"15min":40.548000000000002},"last":3.2000000000000002},{"interface":"front","average":{"1min":3.6970000000000001,"5min":1.3440000000000001,"15min":0.98899999999999999},"min":{"1min":0.38,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":41.073,"5min":41.073,"15min":41.073},"last":0.80400000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3008692224,"data_stored":3008692224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2886205440,"data_stored":2886205440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3877699584,"data_stored":3877699584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3343122432,"data_stored":3343122432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2691563520,"data_stored":2691563520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":490536960,"data_stored":490536960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:32:29.470 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:32:29.551 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:29.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:29.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:29.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:29.712 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:29.737 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:32:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:32:29] "GET /metrics HTTP/1.1" 200 36370 "" "Prometheus/2.43.0" 2023-09-23T22:32:29.809 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:29.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:29.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:29.926 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:29.994 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:30.106 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 86302 86286 71.8886 59.8125 0.0131022 0.0139053 2023-09-23T22:32:30.175 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:30.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:30.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:30.243 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:32:30.243 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:32:30.283 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:30.330 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:30.406 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:30.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:30.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:30.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:30.543 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:31.106 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 86611 86595 71.1969 19.3125 0.00347119 0.0138908 2023-09-23T22:32:31.302 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:31.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:31.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:31.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:31 smithi026 ceph-mon[215775]: pgmap v1027: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 868 op/s; 3198/248181 objects degraded (1.289%); 525/248181 objects misplaced (0.212%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T22:32:31.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:31 smithi026 ceph-mon[215775]: osdmap e399: 8 total, 8 up, 5 in 2023-09-23T22:32:31.387 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:31 smithi026 ceph-mon[215775]: failed to encode map e399 with expected crc 2023-09-23T22:32:31.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:31 smithi026 ceph-mon[223014]: pgmap v1027: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 868 op/s; 3198/248181 objects degraded (1.289%); 525/248181 objects misplaced (0.212%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T22:32:31.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:31 smithi026 ceph-mon[223014]: osdmap e399: 8 total, 8 up, 5 in 2023-09-23T22:32:31.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:31 smithi026 ceph-mon[223014]: failed to encode map e399 with expected crc 2023-09-23T22:32:31.403 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:32:31.410 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:31.431 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:31.521 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:31.595 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:31 smithi070 ceph-mon[192652]: pgmap v1027: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 54 MiB/s wr, 868 op/s; 3198/248181 objects degraded (1.289%); 525/248181 objects misplaced (0.212%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T22:32:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:31 smithi070 ceph-mon[192652]: osdmap e399: 8 total, 8 up, 5 in 2023-09-23T22:32:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:31 smithi070 ceph-mon[192652]: failed to encode map e399 with expected crc 2023-09-23T22:32:31.689 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:31.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:31.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:31.803 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:31.882 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:31.977 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:31.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:31.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:32.056 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:32:32.084 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:32.094 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:32:32.106 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 87418 87402 70.9273 50.4375 0.0214932 0.0140939 2023-09-23T22:32:32.136 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:32:32.160 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:32.180 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:32:32.216 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:32:32.250 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[223014]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.251 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[223014]: osdmap e400: 8 total, 8 up, 5 in 2023-09-23T22:32:32.251 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[223014]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.251 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[223014]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.251 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[223014]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.251 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/624835531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:32.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[215775]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[215775]: osdmap e400: 8 total, 8 up, 5 in 2023-09-23T22:32:32.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[215775]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[215775]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[215775]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.255 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:32 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/624835531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:32.265 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:32:32.308 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:32:32.347 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:32:32.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:32:32.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:32:32.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:32:32.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:32:32.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:32:32.351 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:32:32.352 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:32:32.352 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:32:32.359 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:32.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:32.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:32.466 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:32.532 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:32 smithi070 ceph-mon[192652]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.533 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:32 smithi070 ceph-mon[192652]: osdmap e400: 8 total, 8 up, 5 in 2023-09-23T22:32:32.533 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:32 smithi070 ceph-mon[192652]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.533 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:32 smithi070 ceph-mon[192652]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.533 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:32 smithi070 ceph-mon[192652]: failed to encode map e400 with expected crc 2023-09-23T22:32:32.533 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:32 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/624835531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:32.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:32.624 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:32.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:32.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:32.741 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:32.786 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:33.106 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 88234 88218 70.6717 51 0.00322071 0.014117 2023-09-23T22:32:33.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:33 smithi026 ceph-mon[223014]: pgmap v1030: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 740 op/s; 1614/258195 objects degraded (0.625%); 1584/258195 objects misplaced (0.613%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T22:32:33.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 1614/258195 objects degraded (0.625%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:33.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:33 smithi026 ceph-mon[215775]: pgmap v1030: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 740 op/s; 1614/258195 objects degraded (0.625%); 1584/258195 objects misplaced (0.613%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T22:32:33.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 1614/258195 objects degraded (0.625%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:33 smithi070 ceph-mon[192652]: pgmap v1030: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 46 MiB/s wr, 740 op/s; 1614/258195 objects degraded (0.625%); 1584/258195 objects misplaced (0.613%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T22:32:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 1614/258195 objects degraded (0.625%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:33.886 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:33.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:33.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:33.992 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:34.045 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:34.106 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 89004 88988 70.3863 48.125 0.0187063 0.0142031 2023-09-23T22:32:34.562 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:34.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:34.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:34.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:34.677 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:34.745 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:34.884 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:34.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:34.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:35.004 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:35.070 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:35.106 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:32:35.107102+0000 min lat: 0.00111004 max lat: 2.14499 avg lat: 0.0142311 2023-09-23T22:32:35.106 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:32:35.107 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 89946 89930 70.2424 58.875 0.0156557 0.0142311 2023-09-23T22:32:35.184 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:35.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:35.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:35.295 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:35.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:35 smithi026 ceph-mon[215775]: pgmap v1031: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 37 MiB/s wr, 595 op/s; 1614/258195 objects degraded (0.625%); 1584/258195 objects misplaced (0.613%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:32:35.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:35 smithi026 ceph-mon[223014]: pgmap v1031: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 37 MiB/s wr, 595 op/s; 1614/258195 objects degraded (0.625%); 1584/258195 objects misplaced (0.613%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:32:35.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:35 smithi070 ceph-mon[192652]: pgmap v1031: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 37 MiB/s wr, 595 op/s; 1614/258195 objects degraded (0.625%); 1584/258195 objects misplaced (0.613%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T22:32:35.387 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:35.657 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:35.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:35.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:35.767 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:35.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:32:35.846 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:35.955 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:35.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:35.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:32:36.013 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:32:36.066 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:36.106 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 90899 90883 70.1104 59.5625 0.0168222 0.0142588 2023-09-23T22:32:36.127 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:36.182 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:32:36.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:36 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/326225906' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:36.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:36.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:36 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/326225906' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:36.479 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:36.486 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:36.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:36.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:36.592 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:36.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:36 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/326225906' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:36.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:36.622 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:36.750 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:36.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:36.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:36.865 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:37.098 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:37.106 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 91793 91777 69.9368 55.875 0.017443 0.0142933 2023-09-23T22:32:37.221 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:37.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:37.331 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:37.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:37 smithi070 ceph-mon[192652]: pgmap v1032: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s; 1614/269187 objects degraded (0.600%); 1139/269187 objects misplaced (0.423%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:32:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:37 smithi070 ceph-mon[192652]: from='client.36009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:32:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:37 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:37 smithi070 ceph-mon[192652]: failed to encode map e401 with expected crc 2023-09-23T22:32:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:37 smithi070 ceph-mon[192652]: failed to encode map e401 with expected crc 2023-09-23T22:32:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:37 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:32:37.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:37 smithi070 ceph-mon[192652]: osdmap e401: 8 total, 8 up, 5 in 2023-09-23T22:32:37.378 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:37.418 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:37.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:37.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:37.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[223014]: pgmap v1032: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s; 1614/269187 objects degraded (0.600%); 1139/269187 objects misplaced (0.423%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:32:37.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[223014]: from='client.36009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:32:37.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:37.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[223014]: failed to encode map e401 with expected crc 2023-09-23T22:32:37.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[223014]: failed to encode map e401 with expected crc 2023-09-23T22:32:37.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:32:37.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[223014]: osdmap e401: 8 total, 8 up, 5 in 2023-09-23T22:32:37.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[215775]: pgmap v1032: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 63 MiB/s wr, 1.01k op/s; 1614/269187 objects degraded (0.600%); 1139/269187 objects misplaced (0.423%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:32:37.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[215775]: from='client.36009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:32:37.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:37.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[215775]: failed to encode map e401 with expected crc 2023-09-23T22:32:37.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[215775]: failed to encode map e401 with expected crc 2023-09-23T22:32:37.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:32:37.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:37 smithi026 ceph-mon[215775]: osdmap e401: 8 total, 8 up, 5 in 2023-09-23T22:32:37.522 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:37.568 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:37.941 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1032,"stamp":"2023-09-23T22:32:35.576069+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5883434931,"num_objects":89729,"num_object_clones":0,"num_object_copies":269187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1614,"num_objects_misplaced":1139,"num_objects_unfound":0,"num_objects_dirty":89729,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":90608,"num_write_kb":5753589,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8607,"num_bytes_recovered":564035947,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84903,"ondisk_log_size":84903,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":19346944,"kb_used_data":17012312,"kb_used_omap":15,"kb_used_meta":2334576,"kb_avail":449378816,"statfs":{"total":479975178240,"available":460163907584,"internally_reserved":0,"allocated":17420607488,"data_stored":17411926550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2390606046},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,8,5,5,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":531562496,"num_objects":8111,"num_object_clones":0,"num_object_copies":24333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1584,"num_objects_misplaced":614,"num_objects_unfound":0,"num_objects_dirty":8111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8111,"num_write_kb":519104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":971,"num_bytes_recovered":63635456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005580"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":427,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157238+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:32:30.157238+0000","last_peered":"2023-09-23T22:32:30.157238+0000","last_clean":"2023-09-23T22:32:30.157238+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:32:30.157238+0000","last_undegraded":"2023-09-23T22:32:30.157238+0000","last_fullsized":"2023-09-23T22:32:30.157238+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":116,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.153493+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:32:30.153493+0000","last_peered":"2023-09-23T22:32:30.153493+0000","last_clean":"2023-09-23T22:32:30.153493+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:32:30.153493+0000","last_undegraded":"2023-09-23T22:32:30.153493+0000","last_fullsized":"2023-09-23T22:32:30.153493+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":116,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.146988+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:32:30.146988+0000","last_peered":"2023-09-23T22:32:30.146988+0000","last_clean":"2023-09-23T22:32:30.146988+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:32:30.146988+0000","last_undegraded":"2023-09-23T22:32:30.146988+0000","last_fullsized":"2023-09-23T22:32:30.146988+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":288,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143045+0000","last_change":"2023-09-23T22:31:28.120861+0000","last_active":"2023-09-23T22:32:30.143045+0000","last_peered":"2023-09-23T22:32:30.143045+0000","last_clean":"2023-09-23T22:32:30.143045+0000","last_became_active":"2023-09-23T22:31:28.120394+0000","last_became_peered":"2023-09-23T22:31:28.120394+0000","last_unstale":"2023-09-23T22:32:30.143045+0000","last_undegraded":"2023-09-23T22:32:30.143045+0000","last_fullsized":"2023-09-23T22:32:30.143045+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:19.470258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":455,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142004+0000","last_change":"2023-09-23T22:31:28.114528+0000","last_active":"2023-09-23T22:32:30.142004+0000","last_peered":"2023-09-23T22:32:30.142004+0000","last_clean":"2023-09-23T22:32:30.142004+0000","last_became_active":"2023-09-23T22:31:28.114177+0000","last_became_peered":"2023-09-23T22:31:28.114177+0000","last_unstale":"2023-09-23T22:32:30.142004+0000","last_undegraded":"2023-09-23T22:32:30.142004+0000","last_fullsized":"2023-09-23T22:32:30.142004+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:46.680433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":81,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143859+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:32:30.143859+0000","last_peered":"2023-09-23T22:32:30.143859+0000","last_clean":"2023-09-23T22:32:30.143859+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:32:30.143859+0000","last_undegraded":"2023-09-23T22:32:30.143859+0000","last_fullsized":"2023-09-23T22:32:30.143859+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":316,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.138543+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:32:30.138543+0000","last_peered":"2023-09-23T22:32:30.138543+0000","last_clean":"2023-09-23T22:32:30.138543+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:32:30.138543+0000","last_undegraded":"2023-09-23T22:32:30.138543+0000","last_fullsized":"2023-09-23T22:32:30.138543+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":81,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142286+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:32:30.142286+0000","last_peered":"2023-09-23T22:32:30.142286+0000","last_clean":"2023-09-23T22:32:30.142286+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:32:30.142286+0000","last_undegraded":"2023-09-23T22:32:30.142286+0000","last_fullsized":"2023-09-23T22:32:30.142286+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":400,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140309+0000","last_change":"2023-09-23T22:31:28.116916+0000","last_active":"2023-09-23T22:32:30.140309+0000","last_peered":"2023-09-23T22:32:30.140309+0000","last_clean":"2023-09-23T22:32:30.140309+0000","last_became_active":"2023-09-23T22:31:28.116577+0000","last_became_peered":"2023-09-23T22:31:28.116577+0000","last_unstale":"2023-09-23T22:32:30.140309+0000","last_undegraded":"2023-09-23T22:32:30.140309+0000","last_fullsized":"2023-09-23T22:32:30.140309+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:33.092789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":91,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143229+0000","last_change":"2023-09-23T22:31:28.122684+0000","last_active":"2023-09-23T22:32:30.143229+0000","last_peered":"2023-09-23T22:32:30.143229+0000","last_clean":"2023-09-23T22:32:30.143229+0000","last_became_active":"2023-09-23T22:31:28.122427+0000","last_became_peered":"2023-09-23T22:31:28.122427+0000","last_unstale":"2023-09-23T22:32:30.143229+0000","last_undegraded":"2023-09-23T22:32:30.143229+0000","last_fullsized":"2023-09-23T22:32:30.143229+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:15.559367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":147,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157889+0000","last_change":"2023-09-23T22:31:28.153199+0000","last_active":"2023-09-23T22:32:30.157889+0000","last_peered":"2023-09-23T22:32:30.157889+0000","last_clean":"2023-09-23T22:32:30.157889+0000","last_became_active":"2023-09-23T22:31:28.129257+0000","last_became_peered":"2023-09-23T22:31:28.129257+0000","last_unstale":"2023-09-23T22:32:30.157889+0000","last_undegraded":"2023-09-23T22:32:30.157889+0000","last_fullsized":"2023-09-23T22:32:30.157889+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:03.743836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.154811+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:32:30.154811+0000","last_peered":"2023-09-23T22:32:30.154811+0000","last_clean":"2023-09-23T22:32:30.154811+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:32:30.154811+0000","last_undegraded":"2023-09-23T22:32:30.154811+0000","last_fullsized":"2023-09-23T22:32:30.154811+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":484,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157604+0000","last_change":"2023-09-23T22:31:28.145916+0000","last_active":"2023-09-23T22:32:30.157604+0000","last_peered":"2023-09-23T22:32:30.157604+0000","last_clean":"2023-09-23T22:32:30.157604+0000","last_became_active":"2023-09-23T22:31:28.120703+0000","last_became_peered":"2023-09-23T22:31:28.120703+0000","last_unstale":"2023-09-23T22:32:30.157604+0000","last_undegraded":"2023-09-23T22:32:30.157604+0000","last_fullsized":"2023-09-23T22:32:30.157604+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:07.819247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":398,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157581+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:32:30.157581+0000","last_peered":"2023-09-23T22:32:30.157581+0000","last_clean":"2023-09-23T22:32:30.157581+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:32:30.157581+0000","last_undegraded":"2023-09-23T22:32:30.157581+0000","last_fullsized":"2023-09-23T22:32:30.157581+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":391,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157675+0000","last_change":"2023-09-23T22:31:28.149149+0000","last_active":"2023-09-23T22:32:30.157675+0000","last_peered":"2023-09-23T22:32:30.157675+0000","last_clean":"2023-09-23T22:32:30.157675+0000","last_became_active":"2023-09-23T22:31:28.121120+0000","last_became_peered":"2023-09-23T22:31:28.121120+0000","last_unstale":"2023-09-23T22:32:30.157675+0000","last_undegraded":"2023-09-23T22:32:30.157675+0000","last_fullsized":"2023-09-23T22:32:30.157675+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.434137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":362,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140452+0000","last_change":"2023-09-23T22:31:28.119520+0000","last_active":"2023-09-23T22:32:30.140452+0000","last_peered":"2023-09-23T22:32:30.140452+0000","last_clean":"2023-09-23T22:32:30.140452+0000","last_became_active":"2023-09-23T22:31:28.118810+0000","last_became_peered":"2023-09-23T22:31:28.118810+0000","last_unstale":"2023-09-23T22:32:30.140452+0000","last_undegraded":"2023-09-23T22:32:30.140452+0000","last_fullsized":"2023-09-23T22:32:30.140452+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:36.007062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":305,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.139122+0000","last_change":"2023-09-23T22:31:28.114537+0000","last_active":"2023-09-23T22:32:30.139122+0000","last_peered":"2023-09-23T22:32:30.139122+0000","last_clean":"2023-09-23T22:32:30.139122+0000","last_became_active":"2023-09-23T22:31:28.114176+0000","last_became_peered":"2023-09-23T22:31:28.114176+0000","last_unstale":"2023-09-23T22:32:30.139122+0000","last_undegraded":"2023-09-23T22:32:30.139122+0000","last_fullsized":"2023-09-23T22:32:30.139122+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.324173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":409,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143709+0000","last_change":"2023-09-23T22:31:28.132494+0000","last_active":"2023-09-23T22:32:30.143709+0000","last_peered":"2023-09-23T22:32:30.143709+0000","last_clean":"2023-09-23T22:32:30.143709+0000","last_became_active":"2023-09-23T22:31:28.132197+0000","last_became_peered":"2023-09-23T22:31:28.132197+0000","last_unstale":"2023-09-23T22:32:30.143709+0000","last_undegraded":"2023-09-23T22:32:30.143709+0000","last_fullsized":"2023-09-23T22:32:30.143709+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:36.181016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":343,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.156925+0000","last_change":"2023-09-23T22:31:28.119156+0000","last_active":"2023-09-23T22:32:30.156925+0000","last_peered":"2023-09-23T22:32:30.156925+0000","last_clean":"2023-09-23T22:32:30.156925+0000","last_became_active":"2023-09-23T22:31:28.118382+0000","last_became_peered":"2023-09-23T22:31:28.118382+0000","last_unstale":"2023-09-23T22:32:30.156925+0000","last_undegraded":"2023-09-23T22:32:30.156925+0000","last_fullsized":"2023-09-23T22:32:30.156925+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:10.140079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":296,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143214+0000","last_change":"2023-09-23T22:31:28.121397+0000","last_active":"2023-09-23T22:32:30.143214+0000","last_peered":"2023-09-23T22:32:30.143214+0000","last_clean":"2023-09-23T22:32:30.143214+0000","last_became_active":"2023-09-23T22:31:28.120847+0000","last_became_peered":"2023-09-23T22:31:28.120847+0000","last_unstale":"2023-09-23T22:32:30.143214+0000","last_undegraded":"2023-09-23T22:32:30.143214+0000","last_fullsized":"2023-09-23T22:32:30.143214+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:42.407658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":91,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142167+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:32:30.142167+0000","last_peered":"2023-09-23T22:32:30.142167+0000","last_clean":"2023-09-23T22:32:30.142167+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:32:30.142167+0000","last_undegraded":"2023-09-23T22:32:30.142167+0000","last_fullsized":"2023-09-23T22:32:30.142167+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143665+0000","last_change":"2023-09-23T22:31:28.128965+0000","last_active":"2023-09-23T22:32:30.143665+0000","last_peered":"2023-09-23T22:32:30.143665+0000","last_clean":"2023-09-23T22:32:30.143665+0000","last_became_active":"2023-09-23T22:31:28.128224+0000","last_became_peered":"2023-09-23T22:31:28.128224+0000","last_unstale":"2023-09-23T22:32:30.143665+0000","last_undegraded":"2023-09-23T22:32:30.143665+0000","last_fullsized":"2023-09-23T22:32:30.143665+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:11.301143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"400'5586","reported_seq":5621,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:35.095298+0000","last_change":"2023-09-23T22:31:08.226781+0000","last_active":"2023-09-23T22:32:35.095298+0000","last_peered":"2023-09-23T22:32:35.095298+0000","last_clean":"2023-09-23T22:32:35.095298+0000","last_became_active":"2023-09-23T22:31:08.226511+0000","last_became_peered":"2023-09-23T22:31:08.226511+0000","last_unstale":"2023-09-23T22:32:35.095298+0000","last_undegraded":"2023-09-23T22:32:35.095298+0000","last_fullsized":"2023-09-23T22:32:35.095298+0000","mapping_epoch":373,"log_start":"377'100","ondisk_log_start":"377'100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:03.735360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366084096,"num_objects":5586,"num_object_clones":0,"num_object_copies":16758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5586,"num_write_kb":357504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":409,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.158327+0000","last_change":"2023-09-23T22:31:28.139498+0000","last_active":"2023-09-23T22:32:30.158327+0000","last_peered":"2023-09-23T22:32:30.158327+0000","last_clean":"2023-09-23T22:32:30.158327+0000","last_became_active":"2023-09-23T22:31:28.113206+0000","last_became_peered":"2023-09-23T22:31:28.113206+0000","last_unstale":"2023-09-23T22:32:30.158327+0000","last_undegraded":"2023-09-23T22:32:30.158327+0000","last_fullsized":"2023-09-23T22:32:30.158327+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:11.905312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":402,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.144294+0000","last_change":"2023-09-23T22:31:28.131730+0000","last_active":"2023-09-23T22:32:30.144294+0000","last_peered":"2023-09-23T22:32:30.144294+0000","last_clean":"2023-09-23T22:32:30.144294+0000","last_became_active":"2023-09-23T22:31:28.131383+0000","last_became_peered":"2023-09-23T22:31:28.131383+0000","last_unstale":"2023-09-23T22:32:30.144294+0000","last_undegraded":"2023-09-23T22:32:30.144294+0000","last_fullsized":"2023-09-23T22:32:30.144294+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:56.993827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"400'5655","reported_seq":8892,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:35.079488+0000","last_change":"2023-09-23T22:32:00.147475+0000","last_active":"2023-09-23T22:32:35.079488+0000","last_peered":"2023-09-23T22:32:35.079488+0000","last_clean":"2023-09-23T22:32:35.079488+0000","last_became_active":"2023-09-23T22:32:00.121320+0000","last_became_peered":"2023-09-23T22:32:00.121320+0000","last_unstale":"2023-09-23T22:32:35.079488+0000","last_undegraded":"2023-09-23T22:32:35.079488+0000","last_fullsized":"2023-09-23T22:32:35.079488+0000","mapping_epoch":392,"log_start":"378'600","ondisk_log_start":"378'600","created":373,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5055,"log_dups_size":0,"ondisk_log_size":5055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:17:59.934876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370606080,"num_objects":5655,"num_object_clones":0,"num_object_copies":16965,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5655,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5655,"num_write_kb":361920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1591,"num_bytes_recovered":104267776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":351,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.158186+0000","last_change":"2023-09-23T22:31:28.125280+0000","last_active":"2023-09-23T22:32:30.158186+0000","last_peered":"2023-09-23T22:32:30.158186+0000","last_clean":"2023-09-23T22:32:30.158186+0000","last_became_active":"2023-09-23T22:31:28.110856+0000","last_became_peered":"2023-09-23T22:31:28.110856+0000","last_unstale":"2023-09-23T22:32:30.158186+0000","last_undegraded":"2023-09-23T22:32:30.158186+0000","last_fullsized":"2023-09-23T22:32:30.158186+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:32.197056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":427,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.156847+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:32:30.156847+0000","last_peered":"2023-09-23T22:32:30.156847+0000","last_clean":"2023-09-23T22:32:30.156847+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:32:30.156847+0000","last_undegraded":"2023-09-23T22:32:30.156847+0000","last_fullsized":"2023-09-23T22:32:30.156847+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"400'5638","reported_seq":5682,"reported_epoch":400,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:32:34.869010+0000","last_change":"2023-09-23T22:31:29.183097+0000","last_active":"2023-09-23T22:32:34.869010+0000","last_peered":"2023-09-23T22:32:34.869010+0000","last_clean":"2023-09-23T22:31:27.059725+0000","last_became_active":"2023-09-23T22:31:29.178921+0000","last_became_peered":"2023-09-23T22:31:29.178921+0000","last_unstale":"2023-09-23T22:32:34.869010+0000","last_undegraded":"2023-09-23T22:31:29.114095+0000","last_fullsized":"2023-09-23T22:31:29.105229+0000","mapping_epoch":381,"log_start":"377'200","ondisk_log_start":"377'200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5438,"log_dups_size":0,"ondisk_log_size":5438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:28.838973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369491968,"num_objects":5638,"num_object_clones":0,"num_object_copies":16914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5638,"num_write_kb":360832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":4024},{"shards":"0,6,7","objects":1614}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":367,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143786+0000","last_change":"2023-09-23T22:31:28.136321+0000","last_active":"2023-09-23T22:32:30.143786+0000","last_peered":"2023-09-23T22:32:30.143786+0000","last_clean":"2023-09-23T22:32:30.143786+0000","last_became_active":"2023-09-23T22:31:28.131102+0000","last_became_peered":"2023-09-23T22:31:28.131102+0000","last_unstale":"2023-09-23T22:32:30.143786+0000","last_undegraded":"2023-09-23T22:32:30.143786+0000","last_fullsized":"2023-09-23T22:32:30.143786+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:24.768099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":91,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142464+0000","last_change":"2023-09-23T22:31:28.125138+0000","last_active":"2023-09-23T22:32:30.142464+0000","last_peered":"2023-09-23T22:32:30.142464+0000","last_clean":"2023-09-23T22:32:30.142464+0000","last_became_active":"2023-09-23T22:31:28.124716+0000","last_became_peered":"2023-09-23T22:31:28.124716+0000","last_unstale":"2023-09-23T22:32:30.142464+0000","last_undegraded":"2023-09-23T22:32:30.142464+0000","last_fullsized":"2023-09-23T22:32:30.142464+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.718931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"400'5718","reported_seq":5753,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:35.074277+0000","last_change":"2023-09-23T22:31:08.225591+0000","last_active":"2023-09-23T22:32:35.074277+0000","last_peered":"2023-09-23T22:32:35.074277+0000","last_clean":"2023-09-23T22:32:35.074277+0000","last_became_active":"2023-09-23T22:31:08.225286+0000","last_became_peered":"2023-09-23T22:31:08.225286+0000","last_unstale":"2023-09-23T22:32:35.074277+0000","last_undegraded":"2023-09-23T22:32:35.074277+0000","last_fullsized":"2023-09-23T22:32:35.074277+0000","mapping_epoch":373,"log_start":"378'700","ondisk_log_start":"378'700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5018,"log_dups_size":0,"ondisk_log_size":5018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:14.895634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374734848,"num_objects":5718,"num_object_clones":0,"num_object_copies":17154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5718,"num_write_kb":365952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":430,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140857+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:32:30.140857+0000","last_peered":"2023-09-23T22:32:30.140857+0000","last_clean":"2023-09-23T22:32:30.140857+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:32:30.140857+0000","last_undegraded":"2023-09-23T22:32:30.140857+0000","last_fullsized":"2023-09-23T22:32:30.140857+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":285,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140143+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:32:30.140143+0000","last_peered":"2023-09-23T22:32:30.140143+0000","last_clean":"2023-09-23T22:32:30.140143+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:32:30.140143+0000","last_undegraded":"2023-09-23T22:32:30.140143+0000","last_fullsized":"2023-09-23T22:32:30.140143+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"400'5454","reported_seq":5489,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:35.056184+0000","last_change":"2023-09-23T22:31:08.226199+0000","last_active":"2023-09-23T22:32:35.056184+0000","last_peered":"2023-09-23T22:32:35.056184+0000","last_clean":"2023-09-23T22:32:35.056184+0000","last_became_active":"2023-09-23T22:31:08.225930+0000","last_became_peered":"2023-09-23T22:31:08.225930+0000","last_unstale":"2023-09-23T22:32:35.056184+0000","last_undegraded":"2023-09-23T22:32:35.056184+0000","last_fullsized":"2023-09-23T22:32:35.056184+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5454,"log_dups_size":0,"ondisk_log_size":5454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:16.294443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357433344,"num_objects":5454,"num_object_clones":0,"num_object_copies":16362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5454,"num_write_kb":349056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":745,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.139996+0000","last_change":"2023-09-23T22:31:31.125903+0000","last_active":"2023-09-23T22:32:30.139996+0000","last_peered":"2023-09-23T22:32:30.139996+0000","last_clean":"2023-09-23T22:32:30.139996+0000","last_became_active":"2023-09-23T22:31:31.125630+0000","last_became_peered":"2023-09-23T22:31:31.125630+0000","last_unstale":"2023-09-23T22:32:30.139996+0000","last_undegraded":"2023-09-23T22:32:30.139996+0000","last_fullsized":"2023-09-23T22:32:30.139996+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:24.405922+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":232,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140223+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:32:30.140223+0000","last_peered":"2023-09-23T22:32:30.140223+0000","last_clean":"2023-09-23T22:32:30.140223+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:32:30.140223+0000","last_undegraded":"2023-09-23T22:32:30.140223+0000","last_fullsized":"2023-09-23T22:32:30.140223+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":408,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157309+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:32:30.157309+0000","last_peered":"2023-09-23T22:32:30.157309+0000","last_clean":"2023-09-23T22:32:30.157309+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:32:30.157309+0000","last_undegraded":"2023-09-23T22:32:30.157309+0000","last_fullsized":"2023-09-23T22:32:30.157309+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"400'5610","reported_seq":5646,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:34.153452+0000","last_change":"2023-09-23T22:31:08.227045+0000","last_active":"2023-09-23T22:32:34.153452+0000","last_peered":"2023-09-23T22:32:34.153452+0000","last_clean":"2023-09-23T22:32:34.153452+0000","last_became_active":"2023-09-23T22:31:08.226758+0000","last_became_peered":"2023-09-23T22:31:08.226758+0000","last_unstale":"2023-09-23T22:32:34.153452+0000","last_undegraded":"2023-09-23T22:32:34.153452+0000","last_fullsized":"2023-09-23T22:32:34.153452+0000","mapping_epoch":373,"log_start":"378'800","ondisk_log_start":"378'800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4810,"log_dups_size":0,"ondisk_log_size":4810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:55.118636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367656960,"num_objects":5610,"num_object_clones":0,"num_object_copies":16830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5610,"num_write_kb":359040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":781,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.144488+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:32:30.144488+0000","last_peered":"2023-09-23T22:32:30.144488+0000","last_clean":"2023-09-23T22:32:30.144488+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:32:30.144488+0000","last_undegraded":"2023-09-23T22:32:30.144488+0000","last_fullsized":"2023-09-23T22:32:30.144488+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":81,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142582+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:32:30.142582+0000","last_peered":"2023-09-23T22:32:30.142582+0000","last_clean":"2023-09-23T22:32:30.142582+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:32:30.142582+0000","last_undegraded":"2023-09-23T22:32:30.142582+0000","last_fullsized":"2023-09-23T22:32:30.142582+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":435,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157788+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:32:30.157788+0000","last_peered":"2023-09-23T22:32:30.157788+0000","last_clean":"2023-09-23T22:32:30.157788+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:32:30.157788+0000","last_undegraded":"2023-09-23T22:32:30.157788+0000","last_fullsized":"2023-09-23T22:32:30.157788+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"400'5612","reported_seq":5647,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:35.040522+0000","last_change":"2023-09-23T22:31:08.225049+0000","last_active":"2023-09-23T22:32:35.040522+0000","last_peered":"2023-09-23T22:32:35.040522+0000","last_clean":"2023-09-23T22:32:35.040522+0000","last_became_active":"2023-09-23T22:31:08.224765+0000","last_became_peered":"2023-09-23T22:31:08.224765+0000","last_unstale":"2023-09-23T22:32:35.040522+0000","last_undegraded":"2023-09-23T22:32:35.040522+0000","last_fullsized":"2023-09-23T22:32:35.040522+0000","mapping_epoch":373,"log_start":"378'800","ondisk_log_start":"378'800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4812,"log_dups_size":0,"ondisk_log_size":4812,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.842171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367788032,"num_objects":5612,"num_object_clones":0,"num_object_copies":16836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5612,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":391,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.156657+0000","last_change":"2023-09-23T22:31:28.119358+0000","last_active":"2023-09-23T22:32:30.156657+0000","last_peered":"2023-09-23T22:32:30.156657+0000","last_clean":"2023-09-23T22:32:30.156657+0000","last_became_active":"2023-09-23T22:31:28.118823+0000","last_became_peered":"2023-09-23T22:31:28.118823+0000","last_unstale":"2023-09-23T22:32:30.156657+0000","last_undegraded":"2023-09-23T22:32:30.156657+0000","last_fullsized":"2023-09-23T22:32:30.156657+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:25.483176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":416,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.158033+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:32:30.158033+0000","last_peered":"2023-09-23T22:32:30.158033+0000","last_clean":"2023-09-23T22:32:30.158033+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:32:30.158033+0000","last_undegraded":"2023-09-23T22:32:30.158033+0000","last_fullsized":"2023-09-23T22:32:30.158033+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"400'5632","reported_seq":5668,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:34.217618+0000","last_change":"2023-09-23T22:31:08.226959+0000","last_active":"2023-09-23T22:32:34.217618+0000","last_peered":"2023-09-23T22:32:34.217618+0000","last_clean":"2023-09-23T22:32:34.217618+0000","last_became_active":"2023-09-23T22:31:08.226652+0000","last_became_peered":"2023-09-23T22:31:08.226652+0000","last_unstale":"2023-09-23T22:32:34.217618+0000","last_undegraded":"2023-09-23T22:32:34.217618+0000","last_fullsized":"2023-09-23T22:32:34.217618+0000","mapping_epoch":373,"log_start":"378'800","ondisk_log_start":"378'800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4832,"log_dups_size":0,"ondisk_log_size":4832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:22.758103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":369098752,"num_objects":5632,"num_object_clones":0,"num_object_copies":16896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5632,"num_write_kb":360448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":754,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140378+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:32:30.140378+0000","last_peered":"2023-09-23T22:32:30.140378+0000","last_clean":"2023-09-23T22:32:30.140378+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:32:30.140378+0000","last_undegraded":"2023-09-23T22:32:30.140378+0000","last_fullsized":"2023-09-23T22:32:30.140378+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":117,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.147338+0000","last_change":"2023-09-23T22:31:28.120600+0000","last_active":"2023-09-23T22:32:30.147338+0000","last_peered":"2023-09-23T22:32:30.147338+0000","last_clean":"2023-09-23T22:32:30.147338+0000","last_became_active":"2023-09-23T22:31:28.120281+0000","last_became_peered":"2023-09-23T22:31:28.120281+0000","last_unstale":"2023-09-23T22:32:30.147338+0000","last_undegraded":"2023-09-23T22:32:30.147338+0000","last_fullsized":"2023-09-23T22:32:30.147338+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:10.594022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":117,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.148121+0000","last_change":"2023-09-23T22:31:28.111270+0000","last_active":"2023-09-23T22:32:30.148121+0000","last_peered":"2023-09-23T22:32:30.148121+0000","last_clean":"2023-09-23T22:32:30.148121+0000","last_became_active":"2023-09-23T22:31:28.110921+0000","last_became_peered":"2023-09-23T22:31:28.110921+0000","last_unstale":"2023-09-23T22:32:30.148121+0000","last_undegraded":"2023-09-23T22:32:30.148121+0000","last_fullsized":"2023-09-23T22:32:30.148121+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:05.417671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"400'5593","reported_seq":8721,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:34.857020+0000","last_change":"2023-09-23T22:32:15.139562+0000","last_active":"2023-09-23T22:32:34.857020+0000","last_peered":"2023-09-23T22:32:34.857020+0000","last_clean":"2023-09-23T22:32:34.857020+0000","last_became_active":"2023-09-23T22:32:15.135437+0000","last_became_peered":"2023-09-23T22:32:15.135437+0000","last_unstale":"2023-09-23T22:32:34.857020+0000","last_undegraded":"2023-09-23T22:32:34.857020+0000","last_fullsized":"2023-09-23T22:32:34.857020+0000","mapping_epoch":395,"log_start":"377'200","ondisk_log_start":"377'200","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5393,"log_dups_size":0,"ondisk_log_size":5393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:36.457421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":366542848,"num_objects":5593,"num_object_clones":0,"num_object_copies":16779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5593,"num_write_kb":357952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1537,"num_bytes_recovered":100728832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":675,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143743+0000","last_change":"2023-09-23T22:31:31.129491+0000","last_active":"2023-09-23T22:32:30.143743+0000","last_peered":"2023-09-23T22:32:30.143743+0000","last_clean":"2023-09-23T22:32:30.143743+0000","last_became_active":"2023-09-23T22:31:31.129163+0000","last_became_peered":"2023-09-23T22:31:31.129163+0000","last_unstale":"2023-09-23T22:32:30.143743+0000","last_undegraded":"2023-09-23T22:32:30.143743+0000","last_fullsized":"2023-09-23T22:32:30.143743+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.141229+0000","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":776,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.138335+0000","last_change":"2023-09-23T22:31:31.129492+0000","last_active":"2023-09-23T22:32:30.138335+0000","last_peered":"2023-09-23T22:32:30.138335+0000","last_clean":"2023-09-23T22:32:30.138335+0000","last_became_active":"2023-09-23T22:31:31.128706+0000","last_became_peered":"2023-09-23T22:31:31.128706+0000","last_unstale":"2023-09-23T22:32:30.138335+0000","last_undegraded":"2023-09-23T22:32:30.138335+0000","last_fullsized":"2023-09-23T22:32:30.138335+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:16.180152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":467,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143554+0000","last_change":"2023-09-23T22:31:28.130412+0000","last_active":"2023-09-23T22:32:30.143554+0000","last_peered":"2023-09-23T22:32:30.143554+0000","last_clean":"2023-09-23T22:32:30.143554+0000","last_became_active":"2023-09-23T22:31:28.130127+0000","last_became_peered":"2023-09-23T22:31:28.130127+0000","last_unstale":"2023-09-23T22:32:30.143554+0000","last_undegraded":"2023-09-23T22:32:30.143554+0000","last_fullsized":"2023-09-23T22:32:30.143554+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.088651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":108,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.154663+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:32:30.154663+0000","last_peered":"2023-09-23T22:32:30.154663+0000","last_clean":"2023-09-23T22:32:30.154663+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:32:30.154663+0000","last_undegraded":"2023-09-23T22:32:30.154663+0000","last_fullsized":"2023-09-23T22:32:30.154663+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"400'5613","reported_seq":8872,"reported_epoch":400,"state":"active+recovery_wait","last_fresh":"2023-09-23T22:32:35.095290+0000","last_change":"2023-09-23T22:32:31.203405+0000","last_active":"2023-09-23T22:32:35.095290+0000","last_peered":"2023-09-23T22:32:35.095290+0000","last_clean":"2023-09-23T22:31:27.057464+0000","last_became_active":"2023-09-23T22:32:31.163888+0000","last_became_peered":"2023-09-23T22:32:31.163888+0000","last_unstale":"2023-09-23T22:32:35.095290+0000","last_undegraded":"2023-09-23T22:32:35.095290+0000","last_fullsized":"2023-09-23T22:32:35.095290+0000","mapping_epoch":399,"log_start":"377'100","ondisk_log_start":"377'100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5513,"log_dups_size":0,"ondisk_log_size":5513,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:18.004864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367853568,"num_objects":5613,"num_object_clones":0,"num_object_copies":16839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5613,"num_write_kb":359232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1603,"num_bytes_recovered":105054208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":["5","0","4","7"],"object_location_counts":[{"shards":"0,4,5","objects":5613}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":744,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140261+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:32:30.140261+0000","last_peered":"2023-09-23T22:32:30.140261+0000","last_clean":"2023-09-23T22:32:30.140261+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:32:30.140261+0000","last_undegraded":"2023-09-23T22:32:30.140261+0000","last_fullsized":"2023-09-23T22:32:30.140261+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1278,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140551+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:32:30.140551+0000","last_peered":"2023-09-23T22:32:30.140551+0000","last_clean":"2023-09-23T22:32:30.140551+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:32:30.140551+0000","last_undegraded":"2023-09-23T22:32:30.140551+0000","last_fullsized":"2023-09-23T22:32:30.140551+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":439,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157513+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:32:30.157513+0000","last_peered":"2023-09-23T22:32:30.157513+0000","last_clean":"2023-09-23T22:32:30.157513+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:32:30.157513+0000","last_undegraded":"2023-09-23T22:32:30.157513+0000","last_fullsized":"2023-09-23T22:32:30.157513+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143989+0000","last_change":"2023-09-23T22:31:28.123596+0000","last_active":"2023-09-23T22:32:30.143989+0000","last_peered":"2023-09-23T22:32:30.143989+0000","last_clean":"2023-09-23T22:32:30.143989+0000","last_became_active":"2023-09-23T22:31:28.123179+0000","last_became_peered":"2023-09-23T22:31:28.123179+0000","last_unstale":"2023-09-23T22:32:30.143989+0000","last_undegraded":"2023-09-23T22:32:30.143989+0000","last_fullsized":"2023-09-23T22:32:30.143989+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:53.106243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.b","version":"400'5563","reported_seq":8600,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:34.975166+0000","last_change":"2023-09-23T22:31:44.145615+0000","last_active":"2023-09-23T22:32:34.975166+0000","last_peered":"2023-09-23T22:32:34.975166+0000","last_clean":"2023-09-23T22:32:34.975166+0000","last_became_active":"2023-09-23T22:31:44.142103+0000","last_became_peered":"2023-09-23T22:31:44.142103+0000","last_unstale":"2023-09-23T22:32:34.975166+0000","last_undegraded":"2023-09-23T22:32:34.975166+0000","last_fullsized":"2023-09-23T22:32:34.975166+0000","mapping_epoch":387,"log_start":"378'800","ondisk_log_start":"378'800","created":373,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4763,"log_dups_size":0,"ondisk_log_size":4763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:15.488007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364576768,"num_objects":5563,"num_object_clones":0,"num_object_copies":16689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5563,"num_write_kb":356032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1492,"num_bytes_recovered":97779712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":750,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.139045+0000","last_change":"2023-09-23T22:31:31.130599+0000","last_active":"2023-09-23T22:32:30.139045+0000","last_peered":"2023-09-23T22:32:30.139045+0000","last_clean":"2023-09-23T22:32:30.139045+0000","last_became_active":"2023-09-23T22:31:31.127784+0000","last_became_peered":"2023-09-23T22:31:31.127784+0000","last_unstale":"2023-09-23T22:32:30.139045+0000","last_undegraded":"2023-09-23T22:32:30.139045+0000","last_fullsized":"2023-09-23T22:32:30.139045+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:28.788741+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":617,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:31.136939+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:32:31.136939+0000","last_peered":"2023-09-23T22:32:31.136939+0000","last_clean":"2023-09-23T22:32:31.136939+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:32:31.136939+0000","last_undegraded":"2023-09-23T22:32:31.136939+0000","last_fullsized":"2023-09-23T22:32:31.136939+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":81,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142355+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:32:30.142355+0000","last_peered":"2023-09-23T22:32:30.142355+0000","last_clean":"2023-09-23T22:32:30.142355+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:32:30.142355+0000","last_undegraded":"2023-09-23T22:32:30.142355+0000","last_fullsized":"2023-09-23T22:32:30.142355+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":48,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.147881+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:32:30.147881+0000","last_peered":"2023-09-23T22:32:30.147881+0000","last_clean":"2023-09-23T22:32:30.147881+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:32:30.147881+0000","last_undegraded":"2023-09-23T22:32:30.147881+0000","last_fullsized":"2023-09-23T22:32:30.147881+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"400'5602","reported_seq":5637,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:35.062531+0000","last_change":"2023-09-23T22:31:08.227185+0000","last_active":"2023-09-23T22:32:35.062531+0000","last_peered":"2023-09-23T22:32:35.062531+0000","last_clean":"2023-09-23T22:32:35.062531+0000","last_became_active":"2023-09-23T22:31:08.226918+0000","last_became_peered":"2023-09-23T22:31:08.226918+0000","last_unstale":"2023-09-23T22:32:35.062531+0000","last_undegraded":"2023-09-23T22:32:35.062531+0000","last_fullsized":"2023-09-23T22:32:35.062531+0000","mapping_epoch":373,"log_start":"378'600","ondisk_log_start":"378'600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5002,"log_dups_size":0,"ondisk_log_size":5002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:54.257936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367132672,"num_objects":5602,"num_object_clones":0,"num_object_copies":16806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5602,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1162,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140126+0000","last_change":"2023-09-23T22:31:31.127033+0000","last_active":"2023-09-23T22:32:30.140126+0000","last_peered":"2023-09-23T22:32:30.140126+0000","last_clean":"2023-09-23T22:32:30.140126+0000","last_became_active":"2023-09-23T22:31:31.126601+0000","last_became_peered":"2023-09-23T22:31:31.126601+0000","last_unstale":"2023-09-23T22:32:30.140126+0000","last_undegraded":"2023-09-23T22:32:30.140126+0000","last_fullsized":"2023-09-23T22:32:30.140126+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:00.619278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":784,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142150+0000","last_change":"2023-09-23T22:31:31.126756+0000","last_active":"2023-09-23T22:32:30.142150+0000","last_peered":"2023-09-23T22:32:30.142150+0000","last_clean":"2023-09-23T22:32:30.142150+0000","last_became_active":"2023-09-23T22:31:31.126409+0000","last_became_peered":"2023-09-23T22:31:31.126409+0000","last_unstale":"2023-09-23T22:32:30.142150+0000","last_undegraded":"2023-09-23T22:32:30.142150+0000","last_fullsized":"2023-09-23T22:32:30.142150+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.605240+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":281,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140429+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:32:30.140429+0000","last_peered":"2023-09-23T22:32:30.140429+0000","last_clean":"2023-09-23T22:32:30.140429+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:32:30.140429+0000","last_undegraded":"2023-09-23T22:32:30.140429+0000","last_fullsized":"2023-09-23T22:32:30.140429+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":91,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.146851+0000","last_change":"2023-09-23T22:31:28.124597+0000","last_active":"2023-09-23T22:32:30.146851+0000","last_peered":"2023-09-23T22:32:30.146851+0000","last_clean":"2023-09-23T22:32:30.146851+0000","last_became_active":"2023-09-23T22:31:28.124245+0000","last_became_peered":"2023-09-23T22:31:28.124245+0000","last_unstale":"2023-09-23T22:32:30.146851+0000","last_undegraded":"2023-09-23T22:32:30.146851+0000","last_fullsized":"2023-09-23T22:32:30.146851+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.558632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"400'5556","reported_seq":5592,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:34.131545+0000","last_change":"2023-09-23T22:31:08.223664+0000","last_active":"2023-09-23T22:32:34.131545+0000","last_peered":"2023-09-23T22:32:34.131545+0000","last_clean":"2023-09-23T22:32:34.131545+0000","last_became_active":"2023-09-23T22:31:08.223323+0000","last_became_peered":"2023-09-23T22:31:08.223323+0000","last_unstale":"2023-09-23T22:32:34.131545+0000","last_undegraded":"2023-09-23T22:32:34.131545+0000","last_fullsized":"2023-09-23T22:32:34.131545+0000","mapping_epoch":373,"log_start":"378'700","ondisk_log_start":"378'700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4856,"log_dups_size":0,"ondisk_log_size":4856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:38.536861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364118016,"num_objects":5556,"num_object_clones":0,"num_object_copies":16668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5556,"num_write_kb":355584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":632,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142691+0000","last_change":"2023-09-23T22:31:21.362707+0000","last_active":"2023-09-23T22:32:30.142691+0000","last_peered":"2023-09-23T22:32:30.142691+0000","last_clean":"2023-09-23T22:32:30.142691+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:32:30.142691+0000","last_undegraded":"2023-09-23T22:32:30.142691+0000","last_fullsized":"2023-09-23T22:32:30.142691+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:58.622607+0000","scrub_duration":0.0034076229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":779,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.147426+0000","last_change":"2023-09-23T22:31:28.147724+0000","last_active":"2023-09-23T22:32:30.147426+0000","last_peered":"2023-09-23T22:32:30.147426+0000","last_clean":"2023-09-23T22:32:30.147426+0000","last_became_active":"2023-09-23T22:31:28.120238+0000","last_became_peered":"2023-09-23T22:31:28.120238+0000","last_unstale":"2023-09-23T22:32:30.147426+0000","last_undegraded":"2023-09-23T22:32:30.147426+0000","last_fullsized":"2023-09-23T22:32:30.147426+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:53.259496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":295,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.141090+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:32:30.141090+0000","last_peered":"2023-09-23T22:32:30.141090+0000","last_clean":"2023-09-23T22:32:30.141090+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:32:30.141090+0000","last_undegraded":"2023-09-23T22:32:30.141090+0000","last_fullsized":"2023-09-23T22:32:30.141090+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":288,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143126+0000","last_change":"2023-09-23T22:31:28.121081+0000","last_active":"2023-09-23T22:32:30.143126+0000","last_peered":"2023-09-23T22:32:30.143126+0000","last_clean":"2023-09-23T22:32:30.143126+0000","last_became_active":"2023-09-23T22:31:28.120399+0000","last_became_peered":"2023-09-23T22:31:28.120399+0000","last_unstale":"2023-09-23T22:32:30.143126+0000","last_undegraded":"2023-09-23T22:32:30.143126+0000","last_fullsized":"2023-09-23T22:32:30.143126+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:11.267296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"400'5683","reported_seq":5719,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:34.203676+0000","last_change":"2023-09-23T22:31:08.224259+0000","last_active":"2023-09-23T22:32:34.203676+0000","last_peered":"2023-09-23T22:32:34.203676+0000","last_clean":"2023-09-23T22:32:34.203676+0000","last_became_active":"2023-09-23T22:31:08.224062+0000","last_became_peered":"2023-09-23T22:31:08.224062+0000","last_unstale":"2023-09-23T22:32:34.203676+0000","last_undegraded":"2023-09-23T22:32:34.203676+0000","last_fullsized":"2023-09-23T22:32:34.203676+0000","mapping_epoch":373,"log_start":"378'900","ondisk_log_start":"378'900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4783,"log_dups_size":0,"ondisk_log_size":4783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:18.330753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372441088,"num_objects":5683,"num_object_clones":0,"num_object_copies":17049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5683,"num_write_kb":363712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":750,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142991+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:32:30.142991+0000","last_peered":"2023-09-23T22:32:30.142991+0000","last_clean":"2023-09-23T22:32:30.142991+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:32:30.142991+0000","last_undegraded":"2023-09-23T22:32:30.142991+0000","last_fullsized":"2023-09-23T22:32:30.142991+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":327,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.138747+0000","last_change":"2023-09-23T22:31:28.152186+0000","last_active":"2023-09-23T22:32:30.138747+0000","last_peered":"2023-09-23T22:32:30.138747+0000","last_clean":"2023-09-23T22:32:30.138747+0000","last_became_active":"2023-09-23T22:31:28.111292+0000","last_became_peered":"2023-09-23T22:31:28.111292+0000","last_unstale":"2023-09-23T22:32:30.138747+0000","last_undegraded":"2023-09-23T22:32:30.138747+0000","last_fullsized":"2023-09-23T22:32:30.138747+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:06.097398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":326,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140930+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:32:30.140930+0000","last_peered":"2023-09-23T22:32:30.140930+0000","last_clean":"2023-09-23T22:32:30.140930+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:32:30.140930+0000","last_undegraded":"2023-09-23T22:32:30.140930+0000","last_fullsized":"2023-09-23T22:32:30.140930+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":268,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140074+0000","last_change":"2023-09-23T22:31:28.106985+0000","last_active":"2023-09-23T22:32:30.140074+0000","last_peered":"2023-09-23T22:32:30.140074+0000","last_clean":"2023-09-23T22:32:30.140074+0000","last_became_active":"2023-09-23T22:31:28.105451+0000","last_became_peered":"2023-09-23T22:31:28.105451+0000","last_unstale":"2023-09-23T22:32:30.140074+0000","last_undegraded":"2023-09-23T22:32:30.140074+0000","last_fullsized":"2023-09-23T22:32:30.140074+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:43.614928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"400'5602","reported_seq":8858,"reported_epoch":400,"state":"active+clean","last_fresh":"2023-09-23T22:32:34.839932+0000","last_change":"2023-09-23T22:31:45.135267+0000","last_active":"2023-09-23T22:32:34.839932+0000","last_peered":"2023-09-23T22:32:34.839932+0000","last_clean":"2023-09-23T22:32:34.839932+0000","last_became_active":"2023-09-23T22:31:45.131633+0000","last_became_peered":"2023-09-23T22:31:45.131633+0000","last_unstale":"2023-09-23T22:32:34.839932+0000","last_undegraded":"2023-09-23T22:32:34.839932+0000","last_fullsized":"2023-09-23T22:32:34.839932+0000","mapping_epoch":388,"log_start":"377'200","ondisk_log_start":"377'200","created":373,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5402,"log_dups_size":0,"ondisk_log_size":5402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:00.859364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":367132672,"num_objects":5602,"num_object_clones":0,"num_object_copies":16806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5602,"num_write_kb":358528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1601,"num_bytes_recovered":104923136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":754,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157713+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:32:30.157713+0000","last_peered":"2023-09-23T22:32:30.157713+0000","last_clean":"2023-09-23T22:32:30.157713+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:32:30.157713+0000","last_undegraded":"2023-09-23T22:32:30.157713+0000","last_fullsized":"2023-09-23T22:32:30.157713+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":232,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.138378+0000","last_change":"2023-09-23T22:31:28.131347+0000","last_active":"2023-09-23T22:32:30.138378+0000","last_peered":"2023-09-23T22:32:30.138378+0000","last_clean":"2023-09-23T22:32:30.138378+0000","last_became_active":"2023-09-23T22:31:28.114169+0000","last_became_peered":"2023-09-23T22:31:28.114169+0000","last_unstale":"2023-09-23T22:32:30.138378+0000","last_undegraded":"2023-09-23T22:32:30.138378+0000","last_fullsized":"2023-09-23T22:32:30.138378+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:30.007254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":896,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.147402+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:32:30.147402+0000","last_peered":"2023-09-23T22:32:30.147402+0000","last_clean":"2023-09-23T22:32:30.147402+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:32:30.147402+0000","last_undegraded":"2023-09-23T22:32:30.147402+0000","last_fullsized":"2023-09-23T22:32:30.147402+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":340,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.141916+0000","last_change":"2023-09-23T22:31:28.114550+0000","last_active":"2023-09-23T22:32:30.141916+0000","last_peered":"2023-09-23T22:32:30.141916+0000","last_clean":"2023-09-23T22:32:30.141916+0000","last_became_active":"2023-09-23T22:31:28.114217+0000","last_became_peered":"2023-09-23T22:31:28.114217+0000","last_unstale":"2023-09-23T22:32:30.141916+0000","last_undegraded":"2023-09-23T22:32:30.141916+0000","last_fullsized":"2023-09-23T22:32:30.141916+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:13.616772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.d","version":"400'5584","reported_seq":6520,"reported_epoch":400,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:32:35.107365+0000","last_change":"2023-09-23T22:32:30.117983+0000","last_active":"2023-09-23T22:32:35.107365+0000","last_peered":"2023-09-23T22:32:35.107365+0000","last_clean":"2023-09-23T22:31:27.062039+0000","last_became_active":"2023-09-23T22:31:29.181566+0000","last_became_peered":"2023-09-23T22:31:29.181566+0000","last_unstale":"2023-09-23T22:32:35.107365+0000","last_undegraded":"2023-09-23T22:32:35.107365+0000","last_fullsized":"2023-09-23T22:31:29.103794+0000","mapping_epoch":381,"log_start":"377'100","ondisk_log_start":"377'100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:13.831015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365953024,"num_objects":5584,"num_object_clones":0,"num_object_copies":16752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1139,"num_objects_unfound":0,"num_objects_dirty":5584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5584,"num_write_kb":357376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":445,"num_bytes_recovered":29163520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":4445},{"shards":"5,6,7","objects":1139}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":680,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.141019+0000","last_change":"2023-09-23T22:31:45.133529+0000","last_active":"2023-09-23T22:32:30.141019+0000","last_peered":"2023-09-23T22:32:30.141019+0000","last_clean":"2023-09-23T22:32:30.141019+0000","last_became_active":"2023-09-23T22:31:45.133117+0000","last_became_peered":"2023-09-23T22:31:45.133117+0000","last_unstale":"2023-09-23T22:32:30.141019+0000","last_undegraded":"2023-09-23T22:32:30.141019+0000","last_fullsized":"2023-09-23T22:32:30.141019+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.591643+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.591643+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:28.957050+0000","scrub_duration":0.0073638979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":111,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143582+0000","last_change":"2023-09-23T22:31:28.156716+0000","last_active":"2023-09-23T22:32:30.143582+0000","last_peered":"2023-09-23T22:32:30.143582+0000","last_clean":"2023-09-23T22:32:30.143582+0000","last_became_active":"2023-09-23T22:31:28.128617+0000","last_became_peered":"2023-09-23T22:31:28.128617+0000","last_unstale":"2023-09-23T22:32:30.143582+0000","last_undegraded":"2023-09-23T22:32:30.143582+0000","last_fullsized":"2023-09-23T22:32:30.143582+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:34.465027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":264,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142944+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:32:30.142944+0000","last_peered":"2023-09-23T22:32:30.142944+0000","last_clean":"2023-09-23T22:32:30.142944+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:32:30.142944+0000","last_undegraded":"2023-09-23T22:32:30.142944+0000","last_fullsized":"2023-09-23T22:32:30.142944+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":373,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.156726+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:32:30.156726+0000","last_peered":"2023-09-23T22:32:30.156726+0000","last_clean":"2023-09-23T22:32:30.156726+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:32:30.156726+0000","last_undegraded":"2023-09-23T22:32:30.156726+0000","last_fullsized":"2023-09-23T22:32:30.156726+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":59,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140004+0000","last_change":"2023-09-23T22:31:28.111728+0000","last_active":"2023-09-23T22:32:30.140004+0000","last_peered":"2023-09-23T22:32:30.140004+0000","last_clean":"2023-09-23T22:32:30.140004+0000","last_became_active":"2023-09-23T22:31:28.111369+0000","last_became_peered":"2023-09-23T22:31:28.111369+0000","last_unstale":"2023-09-23T22:32:30.140004+0000","last_undegraded":"2023-09-23T22:32:30.140004+0000","last_fullsized":"2023-09-23T22:32:30.140004+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:42.837575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":150,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.147175+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:32:30.147175+0000","last_peered":"2023-09-23T22:32:30.147175+0000","last_clean":"2023-09-23T22:32:30.147175+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:32:30.147175+0000","last_undegraded":"2023-09-23T22:32:30.147175+0000","last_fullsized":"2023-09-23T22:32:30.147175+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":390,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.140545+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:32:30.140545+0000","last_peered":"2023-09-23T22:32:30.140545+0000","last_clean":"2023-09-23T22:32:30.140545+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:32:30.140545+0000","last_undegraded":"2023-09-23T22:32:30.140545+0000","last_fullsized":"2023-09-23T22:32:30.140545+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":425,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157764+0000","last_change":"2023-09-23T22:31:28.111351+0000","last_active":"2023-09-23T22:32:30.157764+0000","last_peered":"2023-09-23T22:32:30.157764+0000","last_clean":"2023-09-23T22:32:30.157764+0000","last_became_active":"2023-09-23T22:31:28.111006+0000","last_became_peered":"2023-09-23T22:31:28.111006+0000","last_unstale":"2023-09-23T22:32:30.157764+0000","last_undegraded":"2023-09-23T22:32:30.157764+0000","last_fullsized":"2023-09-23T22:32:30.157764+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:51.435513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":415,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.143639+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:32:30.143639+0000","last_peered":"2023-09-23T22:32:30.143639+0000","last_clean":"2023-09-23T22:32:30.143639+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:32:30.143639+0000","last_undegraded":"2023-09-23T22:32:30.143639+0000","last_fullsized":"2023-09-23T22:32:30.143639+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":264,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.157196+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:32:30.157196+0000","last_peered":"2023-09-23T22:32:30.157196+0000","last_clean":"2023-09-23T22:32:30.157196+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:32:30.157196+0000","last_undegraded":"2023-09-23T22:32:30.157196+0000","last_fullsized":"2023-09-23T22:32:30.157196+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":91,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.142834+0000","last_change":"2023-09-23T22:31:28.126203+0000","last_active":"2023-09-23T22:32:30.142834+0000","last_peered":"2023-09-23T22:32:30.142834+0000","last_clean":"2023-09-23T22:32:30.142834+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:31:28.125529+0000","last_unstale":"2023-09-23T22:32:30.142834+0000","last_undegraded":"2023-09-23T22:32:30.142834+0000","last_fullsized":"2023-09-23T22:32:30.142834+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:47.359771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":400,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.141131+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:32:30.141131+0000","last_peered":"2023-09-23T22:32:30.141131+0000","last_clean":"2023-09-23T22:32:30.141131+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:32:30.141131+0000","last_undegraded":"2023-09-23T22:32:30.141131+0000","last_fullsized":"2023-09-23T22:32:30.141131+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":338,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.138465+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:32:30.138465+0000","last_peered":"2023-09-23T22:32:30.138465+0000","last_clean":"2023-09-23T22:32:30.138465+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:32:30.138465+0000","last_undegraded":"2023-09-23T22:32:30.138465+0000","last_fullsized":"2023-09-23T22:32:30.138465+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":458,"reported_epoch":399,"state":"active+clean","last_fresh":"2023-09-23T22:32:30.156430+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:32:30.156430+0000","last_peered":"2023-09-23T22:32:30.156430+0000","last_clean":"2023-09-23T22:32:30.156430+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:32:30.156430+0000","last_undegraded":"2023-09-23T22:32:30.156430+0000","last_fullsized":"2023-09-23T22:32:30.156430+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":31404,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5878644736,"num_objects":89701,"num_object_clones":0,"num_object_copies":269103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1614,"num_objects_misplaced":1139,"num_objects_unfound":0,"num_objects_dirty":89701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":89701,"num_write_kb":5740864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8269,"num_bytes_recovered":541917184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17875664896,"data_stored":17875664896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82101,"ondisk_log_size":82101,"up":48,"acting":46,"num_store_stats":6},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1115744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234338,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495974,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3055596,"kb_used_data":2879652,"kb_used_omap":0,"kb_used_meta":175936,"kb_avail":90689556,"statfs":{"total":95995035648,"available":92866105344,"internally_reserved":0,"allocated":2948763648,"data_stored":2947305895,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":180158464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":7.2089999999999996,"5min":5.6589999999999998,"15min":2.9900000000000002},"min":{"1min":0.22900000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":57.936999999999998,"5min":213.23699999999999,"15min":213.23699999999999},"last":4.4420000000000002},{"interface":"front","average":{"1min":7.1070000000000002,"5min":5.992,"15min":4.0529999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":58.526000000000003,"5min":213.304,"15min":213.304},"last":2.8530000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:31:41 2023","interfaces":[{"interface":"back","average":{"1min":1.212,"5min":1.9790000000000001,"15min":3.4510000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":6.5839999999999996,"5min":80.430000000000007,"15min":129.964},"last":4.5099999999999998},{"interface":"front","average":{"1min":0.97199999999999998,"5min":2.4169999999999998,"15min":3.7730000000000001},"min":{"1min":0.27100000000000002,"5min":0.27000000000000002,"15min":0.254},"max":{"1min":4.2850000000000001,"5min":84.192999999999998,"15min":131.18600000000001},"last":4.6600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.8680000000000001,"15min":1.8919999999999999},"min":{"1min":0.317,"5min":0.311,"15min":0.311},"max":{"1min":5.5839999999999996,"5min":6.6559999999999997,"15min":6.6559999999999997},"last":6.2789999999999999},{"interface":"front","average":{"1min":0.96599999999999997,"5min":1.913,"15min":2.0710000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.21,"5min":9.7319999999999993,"15min":9.7319999999999993},"last":2.8809999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:31:41 2023","interfaces":[{"interface":"back","average":{"1min":1.837,"5min":1.9039999999999999,"15min":3.2570000000000001},"min":{"1min":0.378,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":5.5519999999999996,"5min":42.523000000000003,"15min":125.125},"last":6.165},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.8120000000000001,"15min":2.9910000000000001},"min":{"1min":0.36199999999999999,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":2.782,"5min":39.926000000000002,"15min":76.406999999999996},"last":4.4800000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:31:49 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":0.85599999999999998,"15min":2.5600000000000001},"min":{"1min":0.27600000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.2460000000000004,"5min":6.8650000000000002,"15min":305.30200000000002},"last":2.8090000000000002},{"interface":"front","average":{"1min":2.6240000000000001,"5min":1.0620000000000001,"15min":2.5960000000000001},"min":{"1min":0.29299999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":8.1850000000000005,"5min":8.1850000000000005,"15min":305.40300000000002},"last":6.093}]},{"osd":5,"last update":"Sat Sep 23 22:31:49 2023","interfaces":[{"interface":"back","average":{"1min":7.5199999999999996,"5min":2.1099999999999999,"15min":1.5169999999999999},"min":{"1min":0.317,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":88.194999999999993,"5min":88.194999999999993,"15min":88.194999999999993},"last":6.1920000000000002},{"interface":"front","average":{"1min":5.9800000000000004,"5min":1.736,"15min":1.427},"min":{"1min":0.30599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":88.025000000000006,"5min":88.025000000000006,"15min":88.025000000000006},"last":9.1560000000000006}]},{"osd":7,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":0.95399999999999996,"15min":1.0980000000000001},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.2270000000000003,"5min":9.2270000000000003,"15min":82.801000000000002},"last":4.4020000000000001},{"interface":"front","average":{"1min":2.4630000000000001,"5min":1.105,"15min":1.1459999999999999},"min":{"1min":0.41999999999999998,"5min":0.33300000000000002,"15min":0.29699999999999999},"max":{"1min":8.7690000000000001,"5min":8.7690000000000001,"15min":83.052999999999997},"last":6.2140000000000004}]}]},{"osd":1,"up_from":98,"seq":420906795246,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3773944,"kb_used_data":3120112,"kb_used_omap":7,"kb_used_meta":653816,"kb_avail":89971208,"statfs":{"total":95995035648,"available":92130516992,"internally_reserved":0,"allocated":3194994688,"data_stored":3192875857,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":669507956},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":6.3769999999999998,"5min":5.8250000000000002,"15min":5.1369999999999996},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":107.202,"5min":202.73500000000001,"15min":202.73500000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":7.0449999999999999,"5min":6.3120000000000003,"15min":5.9349999999999996},"min":{"1min":0.24399999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":107.426,"5min":202.80799999999999,"15min":202.80799999999999},"last":0.46700000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:31:50 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.94299999999999995,"15min":1.0189999999999999},"min":{"1min":0.26900000000000002,"5min":0.216,"15min":0.216},"max":{"1min":1.0660000000000001,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":0.23599999999999999},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.1259999999999999,"15min":2.2890000000000001},"min":{"1min":0.24299999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.5030000000000001,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":3.7549999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2669999999999999,"5min":1.732,"15min":4.4980000000000002},"min":{"1min":0.30599999999999999,"5min":0.246,"15min":0.246},"max":{"1min":10.587999999999999,"5min":10.587999999999999,"15min":160.499},"last":3.7890000000000001},{"interface":"front","average":{"1min":1.653,"5min":1.46,"15min":4.1890000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.223},"max":{"1min":5.1920000000000002,"5min":8.1590000000000007,"15min":160.465},"last":4.0069999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.92200000000000004,"15min":2.2440000000000002},"min":{"1min":0.29499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.3730000000000002,"5min":7.0380000000000003,"15min":234.79900000000001},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.478,"15min":2.919},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.4729999999999999,"5min":9.093,"15min":234.81700000000001},"last":3.6989999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":1.6859999999999999,"15min":2.5},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":7.4450000000000003,"5min":9.2759999999999998,"15min":196.79900000000001},"last":5.2489999999999997},{"interface":"front","average":{"1min":1.611,"5min":1.294,"15min":2.3239999999999998},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.29099999999999998},"max":{"1min":5.2889999999999997,"5min":6.8049999999999997,"15min":196.68299999999999},"last":3.9319999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":2.2559999999999998,"15min":2.1379999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":5.2350000000000003,"5min":77.033000000000001,"15min":77.033000000000001},"last":3.9689999999999999},{"interface":"front","average":{"1min":1.681,"5min":1.9630000000000001,"15min":1.583},"min":{"1min":0.442,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":8.3640000000000008,"5min":76.965999999999994,"15min":76.965999999999994},"last":3.8780000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":1.663,"15min":2.0259999999999998},"min":{"1min":0.41499999999999998,"5min":0.38300000000000001,"15min":0.32900000000000001},"max":{"1min":8.0299999999999994,"5min":8.0660000000000007,"15min":16.265000000000001},"last":4.0609999999999999},{"interface":"front","average":{"1min":1.302,"5min":1.2529999999999999,"15min":1.339},"min":{"1min":0.378,"5min":0.318,"15min":0.318},"max":{"1min":3.8119999999999998,"5min":7.1429999999999998,"15min":12.045999999999999},"last":0.83599999999999997}]}]},{"osd":0,"up_from":93,"seq":399431958775,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3941980,"kb_used_data":3220756,"kb_used_omap":8,"kb_used_meta":721207,"kb_avail":89803172,"statfs":{"total":95995035648,"available":91958448128,"internally_reserved":0,"allocated":3298054144,"data_stored":3295846129,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":738516842},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:31:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6850000000000001,"5min":3.8029999999999999,"15min":3.8780000000000001},"min":{"1min":0.20499999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":7.407,"5min":89.439999999999998,"15min":169.77199999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.3180000000000001,"5min":3.7080000000000002,"15min":4.1399999999999997},"min":{"1min":0.219,"5min":0.218,"15min":0.20300000000000001},"max":{"1min":6.6630000000000003,"5min":89.287999999999997,"15min":169.52099999999999},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:31:52 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":2.7029999999999998,"15min":3.0310000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.202,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.83099999999999996},{"interface":"front","average":{"1min":0.75700000000000001,"5min":1.6180000000000001,"15min":1.762},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.3290000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:31:37 2023","interfaces":[{"interface":"back","average":{"1min":1.6819999999999999,"5min":1.7549999999999999,"15min":3.6339999999999999},"min":{"1min":0.28999999999999998,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":10.292999999999999,"5min":15.404,"15min":146.87299999999999},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.319,"5min":2.1120000000000001,"15min":3.7850000000000001},"min":{"1min":0.27500000000000002,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":6.5789999999999997,"5min":22.303000000000001,"15min":145.48099999999999},"last":0.41399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":2.109,"15min":2.5339999999999998},"min":{"1min":0.30199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.2060000000000004,"5min":11.474,"15min":82.825000000000003},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.4119999999999999,"15min":1.7529999999999999},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.444,"5min":7.2279999999999998,"15min":75.930999999999997},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":2.21,"15min":2.9689999999999999},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2510000000000003,"5min":13.564,"15min":93.685000000000002},"last":7.085},{"interface":"front","average":{"1min":1.841,"5min":1.633,"15min":2.246},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4139999999999997,"5min":13.486000000000001,"15min":93.546000000000006},"last":0.433}]},{"osd":6,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3290000000000002,"15min":2.3140000000000001},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.308},"max":{"1min":10.183999999999999,"5min":12.955,"15min":16.366},"last":2.2090000000000001},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.6830000000000001,"15min":1.6799999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":6.5060000000000002,"5min":14.208,"15min":22.263000000000002},"last":0.63300000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.8500000000000001,"15min":2},"min":{"1min":0.42599999999999999,"5min":0.40200000000000002,"15min":0.29999999999999999},"max":{"1min":3.0110000000000001,"5min":22.338999999999999,"15min":32.773000000000003},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.1389999999999998,"15min":2.2890000000000001},"min":{"1min":0.47899999999999998,"5min":0.307,"15min":0.307},"max":{"1min":8.7550000000000008,"5min":10.901,"15min":16.381},"last":0.54000000000000004}]}]},{"osd":2,"up_from":0,"seq":1498943586338,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468193,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019243,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4593000,"kb_used_data":4212064,"kb_used_omap":0,"kb_used_meta":380928,"kb_avail":89152152,"statfs":{"total":95995035648,"available":91291803648,"internally_reserved":0,"allocated":4313153536,"data_stored":4311699328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":390070272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":11.644,"5min":3.5510000000000002,"15min":4.1360000000000001},"min":{"1min":0.216,"5min":0.20699999999999999,"15min":0.19700000000000001},"max":{"1min":101.44499999999999,"5min":101.44499999999999,"15min":150.63499999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":11.930999999999999,"5min":3.7890000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":101.465,"5min":101.465,"15min":150.672},"last":0.36299999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":5.7779999999999996,"5min":3.2109999999999999,"15min":3.4060000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":94.197999999999993,"5min":94.197999999999993,"15min":107.346},"last":153.37700000000001},{"interface":"front","average":{"1min":6.359,"5min":3.508,"15min":3.6499999999999999},"min":{"1min":0.26100000000000001,"5min":0.253,"15min":0.22900000000000001},"max":{"1min":94.460999999999999,"5min":94.460999999999999,"15min":107.33499999999999},"last":153.78399999999999}]},{"osd":2,"last update":"Sat Sep 23 22:31:53 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.911,"15min":2.0830000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.7270000000000003,"5min":7.6790000000000003,"15min":7.6790000000000003},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.4890000000000001,"15min":1.5229999999999999},"min":{"1min":0.376,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":8.1159999999999997,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":0.48899999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":3.3679999999999999,"15min":2.46},"min":{"1min":0.36299999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.7119999999999997,"5min":102.896,"15min":102.896},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.5049999999999999,"5min":3.6539999999999999,"15min":2.593},"min":{"1min":0.37,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":6.6980000000000004,"5min":112.667,"15min":112.667},"last":0.45500000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":0.93300000000000005,"15min":0.97899999999999998},"min":{"1min":0.253,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.4050000000000002,"5min":13.407,"15min":28.399000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.3,"5min":0.95099999999999996,"15min":1.0469999999999999},"min":{"1min":0.313,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":4.2240000000000002,"5min":14.212,"15min":28.350999999999999},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.80600000000000005,"15min":0.83399999999999996},"min":{"1min":0.30299999999999999,"5min":0.26400000000000001,"15min":0.251},"max":{"1min":3.6859999999999999,"5min":6.0110000000000001,"15min":7.8550000000000004},"last":0.497},{"interface":"front","average":{"1min":1.756,"5min":0.90700000000000003,"15min":0.85099999999999998},"min":{"1min":0.32800000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.6950000000000003,"5min":6.6950000000000003,"15min":8.5210000000000008},"last":0.58699999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":1.03,"15min":0.95399999999999996},"min":{"1min":0.376,"5min":0.307,"15min":0.27500000000000002},"max":{"1min":8.1440000000000001,"5min":8.1440000000000001,"15min":8.577},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.23,"5min":0.89400000000000002,"15min":0.89600000000000002},"min":{"1min":0.32600000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.633,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":0.55600000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757611,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3982424,"kb_used_data":3579728,"kb_used_omap":0,"kb_used_meta":402688,"kb_avail":89762728,"statfs":{"total":95995035648,"available":91917033472,"internally_reserved":0,"allocated":3665641472,"data_stored":3664199341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":412352512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,6,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":13.648999999999999,"5min":3.9780000000000002,"15min":4.5839999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":124.892,"5min":124.892,"15min":163.54499999999999},"last":2.8119999999999998},{"interface":"front","average":{"1min":13.523,"5min":3.528,"15min":4.3230000000000004},"min":{"1min":0.34999999999999998,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":123.807,"5min":123.807,"15min":162.81800000000001},"last":4.7599999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":3.7639999999999998,"15min":3.819},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":7.1440000000000001,"5min":113.514,"15min":113.514},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.323,"5min":3.988,"15min":3.8599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":8.4420000000000002,"5min":113.42700000000001,"15min":113.42700000000001},"last":1.847}]},{"osd":2,"last update":"Sat Sep 23 22:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":2.133,"15min":2.3180000000000001},"min":{"1min":0.34200000000000003,"5min":0.316,"15min":0.316},"max":{"1min":7.2480000000000002,"5min":10.451000000000001,"15min":10.451000000000001},"last":1.889},{"interface":"front","average":{"1min":0.83899999999999997,"5min":2.1909999999999998,"15min":2.4159999999999999},"min":{"1min":0.33000000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.4249999999999998,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":2.8410000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.6539999999999999,"15min":3.7719999999999998},"min":{"1min":0.36099999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":8.5459999999999994,"5min":9.3369999999999997,"15min":132.59},"last":2.871},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.7070000000000001,"15min":3.71},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":7.8390000000000004,"5min":7.8390000000000004,"15min":129.381},"last":4.7190000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9849999999999999,"5min":1.3620000000000001,"15min":1.508},"min":{"1min":0.27700000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":36.343000000000004,"5min":36.343000000000004,"15min":126.679},"last":8.9629999999999992},{"interface":"front","average":{"1min":3.9740000000000002,"5min":1.377,"15min":1.512},"min":{"1min":0.35899999999999999,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":36.421999999999997,"5min":36.421999999999997,"15min":126.78100000000001},"last":3.1040000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":1.504,"15min":1.456},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.5489999999999999,"5min":70.581999999999994,"15min":114.83499999999999},"last":3.0710000000000002},{"interface":"front","average":{"1min":1.206,"5min":1.8109999999999999,"15min":1.4670000000000001},"min":{"1min":0.32600000000000001,"5min":0.248,"15min":0.248},"max":{"1min":6.2649999999999997,"5min":71.093000000000004,"15min":110.905},"last":4.7869999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":3.7650000000000001,"5min":1.24,"15min":1.107},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":40.548000000000002,"5min":40.548000000000002,"15min":40.548000000000002},"last":4.8109999999999999},{"interface":"front","average":{"1min":3.6970000000000001,"5min":1.3440000000000001,"15min":0.98899999999999999},"min":{"1min":0.38,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":41.073,"5min":41.073,"15min":41.073},"last":9.0060000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3306553344,"data_stored":3306553344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3207725056,"data_stored":3207725056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4310433792,"data_stored":4310433792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3671195648,"data_stored":3671195648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2963472384,"data_stored":2963472384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":416284672,"data_stored":416284672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:32:37.948 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:32:37.995 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:37.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:38.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:38.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:38.107 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 15 92674 92659 69.7583 55.125 0.00888191 0.014331 2023-09-23T22:32:38.163 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:38.186 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:38.262 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:38.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:38.369 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:38.384 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:32:38.384 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:32:38.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:32:38.384 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:32:38.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.384 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:85129 7.d 7:b998456c:::benchmark_data_smithi026_410749_object85128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:13.158705+0000", 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 385.22109487, 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26241143700000003, 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 85129 2023-09-23T22:32:38.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.158705+0000", 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.158705+0000", 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.158703+0000", 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.158759+0000", 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5275999999999997e-05 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.158778+0000", 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9245e-05 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.158787+0000", 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7260000000000008e-06 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.158953+0000", 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016618899999999999 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.159091+0000", 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013764199999999999 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.159232+0000", 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000141101 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.164111+0000", 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048797129999999999 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.170270+0000", 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061585370000000004 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.421015+0000", 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25074542 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.421073+0000", 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7333000000000001e-05 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.421117+0000", 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4190999999999998e-05 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.392 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:85135 7.d 7:ba0d31e3:::benchmark_data_smithi026_410749_object85134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T22:32:38.393 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:13.171929+0000", 2023-09-23T22:32:38.393 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 385.207871216, 2023-09-23T22:32:38.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.29596194599999998, 2023-09-23T22:32:38.393 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.393 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.393 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.393 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.394 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.394 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 85135 2023-09-23T22:32:38.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.394 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.394 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.394 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.171929+0000", 2023-09-23T22:32:38.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.171929+0000", 2023-09-23T22:32:38.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.171927+0000", 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.171964+0000", 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6501999999999997e-05 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.396 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.171982+0000", 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8476000000000001e-05 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.171992+0000", 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0451999999999999e-05 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.172051+0000", 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8801000000000002e-05 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.172234+0000", 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000182833 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.172406+0000", 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000171676 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.412040+0000", 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23963410499999999 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.424185+0000", 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012144980999999999 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.467784+0000", 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043599271000000002 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.467845+0000", 2023-09-23T22:32:38.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0721999999999999e-05 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:13.467891+0000", 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6109999999999997e-05 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.401 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96581 7.b 7:da8af227:::benchmark_data_smithi026_410749_object6244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.121529+0000", 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 359.258271708, 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0460254070000001, 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96581 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.402 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121529+0000", 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121529+0000", 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121527+0000", 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121535+0000", 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2330000000000006e-06 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121557+0000", 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2014e-05 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121568+0000", 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0974e-05 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121632+0000", 2023-09-23T22:32:38.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.347e-05 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121645+0000", 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2959000000000001e-05 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061244+0000", 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93959893500000002 2023-09-23T22:32:38.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061260+0000", 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.613e-05 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.159051+0000", 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.097790874 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.159184+0000", 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133869 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.159271+0000", 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6310999999999994e-05 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.161846+0000", 2023-09-23T22:32:38.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025755970000000002 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167032+0000", 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005186052 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167199+0000", 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016674600000000001 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167355+0000", 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000155494 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167418+0000", 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3830000000000004e-05 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167554+0000", 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013549800000000001 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96604 7.b 7:dba0fa8d:::benchmark_data_smithi026_410749_object6267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.174896+0000", 2023-09-23T22:32:38.411 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 359.20490405499999, 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.992895646, 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96604 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174896+0000", 2023-09-23T22:32:38.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174896+0000", 2023-09-23T22:32:38.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174894+0000", 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174904+0000", 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8190000000000003e-06 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174927+0000", 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2821000000000001e-05 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174937+0000", 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0549e-05 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.175009+0000", 2023-09-23T22:32:38.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1465999999999996e-05 2023-09-23T22:32:38.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.175023+0000", 2023-09-23T22:32:38.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4574e-05 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061288+0000", 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.88626441600000005 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061298+0000", 2023-09-23T22:32:38.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0083e-05 2023-09-23T22:32:38.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.161879+0000", 2023-09-23T22:32:38.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1005808749999999 2023-09-23T22:32:38.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.161966+0000", 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6909000000000004e-05 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.162040+0000", 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4089000000000002e-05 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167210+0000", 2023-09-23T22:32:38.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051705910000000004 2023-09-23T22:32:38.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167270+0000", 2023-09-23T22:32:38.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0257000000000003e-05 2023-09-23T22:32:38.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167296+0000", 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5072e-05 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167628+0000", 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033241999999999999 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167683+0000", 2023-09-23T22:32:38.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4676999999999998e-05 2023-09-23T22:32:38.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167792+0000", 2023-09-23T22:32:38.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001092 2023-09-23T22:32:38.422 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.423 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.423 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.423 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96657 7.b 7:d2054ff9:::benchmark_data_smithi026_410749_object6320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.423 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.297353+0000", 2023-09-23T22:32:38.423 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 359.08244760999997, 2023-09-23T22:32:38.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8706818570000001, 2023-09-23T22:32:38.423 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96657 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297353+0000", 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297353+0000", 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297351+0000", 2023-09-23T22:32:38.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297359+0000", 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4349999999999997e-06 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297379+0000", 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9244000000000001e-05 2023-09-23T22:32:38.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297390+0000", 2023-09-23T22:32:38.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1504e-05 2023-09-23T22:32:38.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297505+0000", 2023-09-23T22:32:38.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115328 2023-09-23T22:32:38.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297519+0000", 2023-09-23T22:32:38.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3552e-05 2023-09-23T22:32:38.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061322+0000", 2023-09-23T22:32:38.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.76380318800000002 2023-09-23T22:32:38.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061332+0000", 2023-09-23T22:32:38.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0074999999999999e-05 2023-09-23T22:32:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.164234+0000", 2023-09-23T22:32:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.102901989 2023-09-23T22:32:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.164356+0000", 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121854 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.164444+0000", 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.81e-05 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167572+0000", 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031280320000000002 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167853+0000", 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028055799999999998 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167883+0000", 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0664000000000003e-05 2023-09-23T22:32:38.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167932+0000", 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8953999999999999e-05 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167966+0000", 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4050000000000001e-05 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.168034+0000", 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8077999999999994e-05 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96659 7.b 7:d16d3249:::benchmark_data_smithi026_410749_object6322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.434 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.305815+0000", 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 359.07398518399998, 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8766973179999999, 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96659 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305815+0000", 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305815+0000", 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305813+0000", 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305821+0000", 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1159999999999999e-06 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305834+0000", 2023-09-23T22:32:38.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3122000000000001e-05 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305842+0000", 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7489999999999998e-06 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305913+0000", 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0431000000000004e-05 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305925+0000", 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2017e-05 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061357+0000", 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.75543235500000006 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061367+0000", 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0026e-05 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.168087+0000", 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1067203699999999 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.168241+0000", 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153883 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.168393+0000", 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015202299999999999 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175249+0000", 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068552550000000002 2023-09-23T22:32:38.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175306+0000", 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7188000000000002e-05 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182294+0000", 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069879249999999999 2023-09-23T22:32:38.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182309+0000", 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5254e-05 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182372+0000", 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3219000000000006e-05 2023-09-23T22:32:38.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182512+0000", 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140285 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96680 7.b 7:d07ca5d2:::benchmark_data_smithi026_410749_object6343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.444 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.352046+0000", 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 359.02775456500001, 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8377739719999999, 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96680 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352046+0000", 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352046+0000", 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352044+0000", 2023-09-23T22:32:38.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:38.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352052+0000", 2023-09-23T22:32:38.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1629999999999996e-06 2023-09-23T22:32:38.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352071+0000", 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8278e-05 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352080+0000", 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4709999999999996e-06 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352153+0000", 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3120000000000005e-05 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352166+0000", 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2813000000000001e-05 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061392+0000", 2023-09-23T22:32:38.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.70922612900000004 2023-09-23T22:32:38.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061402+0000", 2023-09-23T22:32:38.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0190999999999999e-05 2023-09-23T22:32:38.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.171558+0000", 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1101559830000001 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.171689+0000", 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130395 2023-09-23T22:32:38.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.171813+0000", 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124401 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182245+0000", 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010431573 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182523+0000", 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027883900000000002 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182578+0000", 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4280000000000002e-05 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189651+0000", 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007072811 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189703+0000", 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2033999999999997e-05 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189820+0000", 2023-09-23T22:32:38.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000117064 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96686 7.b 7:de804cf2:::benchmark_data_smithi026_410749_object6349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.359794+0000", 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 359.02000618300002, 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8301854309999999, 2023-09-23T22:32:38.455 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96686 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359794+0000", 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359794+0000", 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359792+0000", 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359803+0000", 2023-09-23T22:32:38.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0549e-05 2023-09-23T22:32:38.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359828+0000", 2023-09-23T22:32:38.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5647e-05 2023-09-23T22:32:38.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359839+0000", 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0575e-05 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359912+0000", 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3368000000000006e-05 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359926+0000", 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3757000000000001e-05 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061427+0000", 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.70150106899999998 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061437+0000", 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9420000000000007e-06 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175332+0000", 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1138945499999999 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175468+0000", 2023-09-23T22:32:38.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136501 2023-09-23T22:32:38.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175599+0000", 2023-09-23T22:32:38.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130784 2023-09-23T22:32:38.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182530+0000", 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069310630000000003 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182624+0000", 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4302999999999995e-05 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182699+0000", 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5062999999999996e-05 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189860+0000", 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071608649999999998 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189907+0000", 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6369000000000003e-05 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189979+0000", 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2970000000000001e-05 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.465 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96695 7.b 7:dbf81365:::benchmark_data_smithi026_410749_object6358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.382326+0000", 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.99747405300002, 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8078309880000001, 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96695 2023-09-23T22:32:38.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382326+0000", 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382326+0000", 2023-09-23T22:32:38.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382325+0000", 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382333+0000", 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7649999999999993e-06 2023-09-23T22:32:38.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382351+0000", 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8131e-05 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382361+0000", 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9890000000000004e-06 2023-09-23T22:32:38.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382441+0000", 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0789000000000002e-05 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382465+0000", 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3802999999999999e-05 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061462+0000", 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.67899700699999999 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061472+0000", 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0189e-05 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.178943+0000", 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1174706839999999 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.179093+0000", 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149708 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.179222+0000", 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012925999999999999 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182586+0000", 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033642749999999999 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182661+0000", 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4276000000000005e-05 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182742+0000", 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0872999999999996e-05 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190018+0000", 2023-09-23T22:32:38.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072768939999999999 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190065+0000", 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6468000000000003e-05 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190157+0000", 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2299000000000005e-05 2023-09-23T22:32:38.475 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96696 7.b 7:d9fb316f:::benchmark_data_smithi026_410749_object6359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.384266+0000", 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.995534615, 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8062237539999999, 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.476 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96696 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384266+0000", 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384266+0000", 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384264+0000", 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384271+0000", 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0369999999999997e-06 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384290+0000", 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8411e-05 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384298+0000", 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8689999999999993e-06 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.388227+0000", 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039287009999999997 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.388243+0000", 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5744e-05 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061496+0000", 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.67325356400000003 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061505+0000", 2023-09-23T22:32:38.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3189999999999995e-06 2023-09-23T22:32:38.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182772+0000", 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.121266777 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182923+0000", 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015133400000000001 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.183082+0000", 2023-09-23T22:32:38.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159405 2023-09-23T22:32:38.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189616+0000", 2023-09-23T22:32:38.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065341460000000002 2023-09-23T22:32:38.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190195+0000", 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057832200000000004 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190320+0000", 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125647 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190329+0000", 2023-09-23T22:32:38.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4719999999999995e-06 2023-09-23T22:32:38.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190377+0000", 2023-09-23T22:32:38.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8112000000000003e-05 2023-09-23T22:32:38.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190489+0000", 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011237499999999999 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.487 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96701 7.b 7:d8abcfde:::benchmark_data_smithi026_410749_object6364:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.488 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.393752+0000", 2023-09-23T22:32:38.488 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.98604794300002, 2023-09-23T22:32:38.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8036669359999999, 2023-09-23T22:32:38.488 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.488 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.488 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.488 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.488 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.488 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96701 2023-09-23T22:32:38.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.489 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393752+0000", 2023-09-23T22:32:38.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393752+0000", 2023-09-23T22:32:38.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393751+0000", 2023-09-23T22:32:38.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:38.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393759+0000", 2023-09-23T22:32:38.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6139999999999992e-06 2023-09-23T22:32:38.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393776+0000", 2023-09-23T22:32:38.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6566e-05 2023-09-23T22:32:38.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393783+0000", 2023-09-23T22:32:38.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4250000000000001e-06 2023-09-23T22:32:38.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393852+0000", 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8328e-05 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393863+0000", 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1618e-05 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061537+0000", 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.66767378899999996 2023-09-23T22:32:38.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061546+0000", 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3919999999999992e-06 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.186152+0000", 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1246053899999999 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.186265+0000", 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113108 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.186366+0000", 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100787 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190277+0000", 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039108930000000004 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190498+0000", 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022148799999999999 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190539+0000", 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1427000000000002e-05 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197233+0000", 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066931229999999996 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197292+0000", 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8946000000000001e-05 2023-09-23T22:32:38.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197419+0000", 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127704 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96708 7.b 7:dc997113:::benchmark_data_smithi026_410749_object6371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.500 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.411022+0000", 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.96877801300002, 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.793953546, 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96708 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411022+0000", 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411022+0000", 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411020+0000", 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411032+0000", 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2551000000000001e-05 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411053+0000", 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0922000000000001e-05 2023-09-23T22:32:38.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411067+0000", 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3495e-05 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411157+0000", 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0869e-05 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411172+0000", 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4972e-05 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061570+0000", 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.65039764600000005 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061580+0000", 2023-09-23T22:32:38.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5040000000000008e-06 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190561+0000", 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1289812429999999 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190672+0000", 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011156500000000001 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190807+0000", 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134771 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197194+0000", 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063872620000000003 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204752+0000", 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007557355 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204761+0000", 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9029999999999993e-06 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204808+0000", 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7187000000000001e-05 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204863+0000", 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5390999999999999e-05 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204976+0000", 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112508 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96711 7.b 7:d9a0e45e:::benchmark_data_smithi026_410749_object6374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.419858+0000", 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.95994204200002, 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7984610889999999, 2023-09-23T22:32:38.510 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96711 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419858+0000", 2023-09-23T22:32:38.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419858+0000", 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419856+0000", 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419865+0000", 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0950000000000002e-06 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419882+0000", 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6135000000000001e-05 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419891+0000", 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0839999999999998e-06 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419955+0000", 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4425999999999996e-05 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419965+0000", 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0171e-05 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061599+0000", 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.64163331400000001 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061604+0000", 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0599999999999998e-06 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.193594+0000", 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1319901510000001 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.193776+0000", 2023-09-23T22:32:38.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018225899999999999 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.193924+0000", 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014823899999999999 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197465+0000", 2023-09-23T22:32:38.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003541237 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204767+0000", 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073014100000000004 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218008+0000", 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013241404 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218125+0000", 2023-09-23T22:32:38.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011712199999999999 2023-09-23T22:32:38.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218183+0000", 2023-09-23T22:32:38.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7797999999999998e-05 2023-09-23T22:32:38.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218319+0000", 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013606100000000001 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96742 7.b 7:d66e3e8b:::benchmark_data_smithi026_410749_object6405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.501784+0000", 2023-09-23T22:32:38.520 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.87801586099999, 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.716760217, 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96742 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501784+0000", 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501784+0000", 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501782+0000", 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501794+0000", 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1527e-05 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501819+0000", 2023-09-23T22:32:38.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5318000000000001e-05 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501831+0000", 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1579e-05 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501934+0000", 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010391399999999999 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501954+0000", 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9068000000000001e-05 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061616+0000", 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.55966211899999996 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061624+0000", 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4319999999999999e-06 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197491+0000", 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.135866592 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197627+0000", 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013658 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197765+0000", 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013777400000000001 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204695+0000", 2023-09-23T22:32:38.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069303510000000004 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204773+0000", 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7848000000000004e-05 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218048+0000", 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013274555 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218390+0000", 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000342295 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218440+0000", 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9842000000000002e-05 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218545+0000", 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104691 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96747 7.b 7:d5f72434:::benchmark_data_smithi026_410749_object6410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.510418+0000", 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.86938203699998, 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.711354499, 2023-09-23T22:32:38.530 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96747 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510418+0000", 2023-09-23T22:32:38.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510418+0000", 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510417+0000", 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:38.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510424+0000", 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7739999999999999e-06 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510436+0000", 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2357999999999999e-05 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510445+0000", 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9479999999999997e-06 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.514417+0000", 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039724570000000004 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.514438+0000", 2023-09-23T22:32:38.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0241000000000002e-05 2023-09-23T22:32:38.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061645+0000", 2023-09-23T22:32:38.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.54720765500000002 2023-09-23T22:32:38.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061651+0000", 2023-09-23T22:32:38.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3360000000000002e-06 2023-09-23T22:32:38.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.201041+0000", 2023-09-23T22:32:38.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.139390049 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.201189+0000", 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014889699999999999 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.201334+0000", 2023-09-23T22:32:38.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000144089 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212102+0000", 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010767995000000001 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212149+0000", 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7225000000000003e-05 2023-09-23T22:32:38.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218091+0000", 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059417910000000001 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.221580+0000", 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003489771 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.221637+0000", 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6943e-05 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.221773+0000", 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135439 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96750 7.b 7:d0c71cb9:::benchmark_data_smithi026_410749_object6413:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.520260+0000", 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.85954017400002, 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7056295430000001, 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96750 2023-09-23T22:32:38.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520260+0000", 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520260+0000", 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520258+0000", 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520267+0000", 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0909999999999993e-06 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520282+0000", 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5747e-05 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520293+0000", 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0292000000000001e-05 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520369+0000", 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6327000000000003e-05 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520386+0000", 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7005999999999999e-05 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061666+0000", 2023-09-23T22:32:38.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.54127963700000004 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061674+0000", 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5380000000000002e-06 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.205004+0000", 2023-09-23T22:32:38.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.143329743 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.205127+0000", 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012354600000000001 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.205248+0000", 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012061 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212157+0000", 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069090289999999997 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212199+0000", 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2474e-05 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218595+0000", 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063951340000000002 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.225719+0000", 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071245279999999998 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.225774+0000", 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5337000000000002e-05 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.225890+0000", 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115142 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96754 7.b 7:d80856d8:::benchmark_data_smithi026_410749_object6417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.533130+0000", 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.846670711, 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7019917420000001, 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.551 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96754 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533130+0000", 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533130+0000", 2023-09-23T22:32:38.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533127+0000", 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533138+0000", 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.044e-05 2023-09-23T22:32:38.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533172+0000", 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3939000000000002e-05 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533183+0000", 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1053000000000001e-05 2023-09-23T22:32:38.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533258+0000", 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5605000000000005e-05 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533271+0000", 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2901000000000001e-05 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061697+0000", 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.52842535099999999 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061705+0000", 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1850000000000004e-06 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.208491+0000", 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1467860519999999 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.208647+0000", 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015611999999999999 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.208815+0000", 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016839099999999999 2023-09-23T22:32:38.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212236+0000", 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034210099999999999 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218561+0000", 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006324311 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218756+0000", 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000195515 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.234972+0000", 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016215947000000001 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235011+0000", 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8461999999999999e-05 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235121+0000", 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011069399999999999 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96780 7.b 7:da651c72:::benchmark_data_smithi026_410749_object6443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.699047+0000", 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 358.68075343300001, 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5362475739999999, 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96780 2023-09-23T22:32:38.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699047+0000", 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699047+0000", 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699045+0000", 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699054+0000", 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.405e-06 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699072+0000", 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8097999999999999e-05 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699079+0000", 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2919999999999998e-06 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699132+0000", 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3053999999999997e-05 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699143+0000", 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0866000000000001e-05 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061734+0000", 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36259154900000001 2023-09-23T22:32:38.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:32:38.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061742+0000", 2023-09-23T22:32:38.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9130000000000003e-06 2023-09-23T22:32:38.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.214041+0000", 2023-09-23T22:32:38.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1522982420000001 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.214184+0000", 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000143299 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.214321+0000", 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137145 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.219748+0000", 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054267309999999997 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.220699+0000", 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00095117499999999996 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.229541+0000", 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088419870000000008 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235160+0000", 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056194740000000002 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235208+0000", 2023-09-23T22:32:38.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7886999999999997e-05 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235294+0000", 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6030000000000001e-05 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.570 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:7720 8.1 8:83214e53:::benchmark_data_smithi026_497658_object7719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:31:17.757749+0000", 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 80.622051511999999, 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.29828353800000001, 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 7720 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.757749+0000", 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.757749+0000", 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.757748+0000", 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.757800+0000", 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1474000000000002e-05 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.757810+0000", 2023-09-23T22:32:38.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0919e-05 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.757817+0000", 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7309999999999996e-06 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.819663+0000", 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.061846102 2023-09-23T22:32:38.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.819819+0000", 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000155453 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.819979+0000", 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000160202 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.821479+0000", 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015002010000000001 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:17.821553+0000", 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4293000000000002e-05 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:18.055949+0000", 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23439585399999999 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:18.055988+0000", 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8522999999999997e-05 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:18.056032+0000", 2023-09-23T22:32:38.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4437000000000002e-05 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:25734 8.1 8:896563b3:::benchmark_data_smithi026_497658_object25733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:31:29.450695+0000", 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 68.929105471, 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26654650699999999, 2023-09-23T22:32:38.578 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 25734 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.450695+0000", 2023-09-23T22:32:38.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.450695+0000", 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.450694+0000", 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.450735+0000", 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1964e-05 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.450749+0000", 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3854e-05 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.450755+0000", 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3199999999999999e-06 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.450810+0000", 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5754000000000002e-05 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.450930+0000", 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119373 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.451050+0000", 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120319 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.452032+0000", 2023-09-23T22:32:38.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098203299999999999 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.452439+0000", 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040736899999999998 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.717155+0000", 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26471581799999999 2023-09-23T22:32:38.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.717204+0000", 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9048000000000002e-05 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:31:29.717241+0000", 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6910999999999999e-05 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.585 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.586 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.586 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:32:38.586 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:32:38.586 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:32:38.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 1614/269187 objects degraded (0.600%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:38.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 1614/269187 objects degraded (0.600%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:38.587 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:38.591 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:38.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:38.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:38.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 1614/269187 objects degraded (0.600%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:38.703 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:38.758 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:38.879 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:38.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:38.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:38.987 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:39.046 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:39.107 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 93599 93583 69.6153 57.75 0.0229995 0.0143592 2023-09-23T22:32:39.175 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:39.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:39.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:39 smithi026 ceph-mon[215775]: pgmap v1034: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 31 MiB/s wr, 492 op/s; 1614/269187 objects degraded (0.600%); 1139/269187 objects misplaced (0.423%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:32:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:39.220 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:32:39.286 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:39.335 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:39.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:39 smithi070 ceph-mon[192652]: pgmap v1034: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 31 MiB/s wr, 492 op/s; 1614/269187 objects degraded (0.600%); 1139/269187 objects misplaced (0.423%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:32:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:39.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:39.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:39 smithi026 ceph-mon[223014]: pgmap v1034: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 31 MiB/s wr, 492 op/s; 1614/269187 objects degraded (0.600%); 1139/269187 objects misplaced (0.423%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:32:39.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:32:39.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:39.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:32:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:32:39] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.43.0" 2023-09-23T22:32:40.108 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 94233 94217 69.2616 39.625 0.0131708 0.0144338 2023-09-23T22:32:40.337 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:40.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:40.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:40.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:32:40.380 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:40.446 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:40.537 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:40.757 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:40.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:40.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:40.865 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:40.911 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:40.936 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:41.040 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:41.044 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:32:41.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:41.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:41.081 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:32:41.111 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 95158 95142 69.1256 57.8125 0.019046 0.0144614 2023-09-23T22:32:41.113 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:32:41.151 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:41.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:32:41.198 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:32:41.204 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:41 smithi070 ceph-mon[192652]: pgmap v1035: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 36 MiB/s wr, 570 op/s; 1614/272649 objects degraded (0.592%); 1139/272649 objects misplaced (0.418%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T22:32:41.204 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2618506591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:41.213 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:41.231 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:32:41.261 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:32:41.292 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:32:41.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:32:41.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:32:41.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:32:41.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:32:41.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:32:41.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:32:41.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:32:41.294 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:32:41.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:41 smithi026 ceph-mon[223014]: pgmap v1035: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 36 MiB/s wr, 570 op/s; 1614/272649 objects degraded (0.592%); 1139/272649 objects misplaced (0.418%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T22:32:41.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2618506591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:41.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:41 smithi026 ceph-mon[215775]: pgmap v1035: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 36 MiB/s wr, 570 op/s; 1614/272649 objects degraded (0.592%); 1139/272649 objects misplaced (0.418%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T22:32:41.325 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2618506591' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:41.364 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:41.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:41.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:41.471 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:41.541 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:41.665 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:41.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:41.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:41.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:41.843 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:41.960 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:41.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:41.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:42.070 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:42.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 96135 96119 69.0329 61.0625 0.0274898 0.0144805 2023-09-23T22:32:42.135 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:42.277 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:42.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:42.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:42.402 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:42.506 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:42.620 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:42.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:42.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:42.732 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:42.814 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:42.917 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:42.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:42.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:43.034 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:43.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:43.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 97125 97109 68.9515 61.875 0.0124694 0.0144981 2023-09-23T22:32:43.192 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:43.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:43.208 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:43 smithi026 ceph-mon[215775]: pgmap v1036: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 50 MiB/s wr, 799 op/s; 1614/282210 objects degraded (0.572%); 800/282210 objects misplaced (0.283%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:32:43.208 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 1614/282210 objects degraded (0.572%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:43.299 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:43.361 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:43.469 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:43.472 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:43 smithi070 ceph-mon[192652]: pgmap v1036: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 50 MiB/s wr, 799 op/s; 1614/282210 objects degraded (0.572%); 800/282210 objects misplaced (0.283%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:32:43.473 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 1614/282210 objects degraded (0.572%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:43.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:43.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:43.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:43 smithi026 ceph-mon[223014]: pgmap v1036: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 50 MiB/s wr, 799 op/s; 1614/282210 objects degraded (0.572%); 800/282210 objects misplaced (0.283%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:32:43.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 1614/282210 objects degraded (0.572%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:43.575 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:43.636 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:43.666 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:32:43.667 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 1614/282210 objects degraded (0.572%), 1 pg degraded, 2 pgs undersized","count":3},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1347,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":401,"num_osds":8,"num_up_osds":8,"osd_up_since":1695508188,"num_in_osds":5,"osd_in_since":1695508287,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":94070,"data_bytes":6167926707,"bytes_used":20687986688,"bytes_avail":459287191552,"bytes_total":479975178240,"degraded_objects":1614,"degraded_total":282210,"degraded_ratio":0.0057191453173168916,"misplaced_objects":800,"misplaced_total":282210,"misplaced_ratio":0.002834768434853478,"recovering_objects_per_sec":78,"recovering_bytes_per_sec":5140348,"recovering_keys_per_sec":0,"num_objects_recovered":785,"num_bytes_recovered":51445760,"num_keys_recovered":0,"write_bytes_sec":52418457,"read_op_per_sec":0,"write_op_per_sec":799},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":24,"modified":"2023-09-23T22:30:47.531589+0000","services":{}},"progress_events":{"5b5ab3a4-2f21-4d0a-896f-44b2b3c7eda7":{"message":"Global Recovery Event (45s)\n [===========================.] (remaining: 1s)","progress":0.96969699859619141,"add_to_ceph_s":true},"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:32:43.758 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:43.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:43.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:43.869 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:43.949 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:44.062 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:44.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:44.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:44.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 97972 97956 68.7715 52.9375 0.0046548 0.0145342 2023-09-23T22:32:44.174 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:44.229 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:44.329 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:44.334 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:44 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3867403224' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:32:44.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:44 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3867403224' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:32:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:44 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3867403224' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:32:44.517 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:44.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:44.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:44.621 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:32:44.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:32:44.625 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:44.735 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:44.815 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:32:44.973 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:44.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:44.981 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:45.082 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:45.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.0204 2023-09-23T22:32:45.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 98889 2023-09-23T22:32:45.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T22:32:45.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T22:32:45.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 68.6573 2023-09-23T22:32:45.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 32.9899 2023-09-23T22:32:45.112 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 194.5 2023-09-23T22:32:45.113 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:32:45.113 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 1098 2023-09-23T22:32:45.113 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 527.846 2023-09-23T22:32:45.113 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 3112 2023-09-23T22:32:45.113 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T22:32:45.113 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.0145607 2023-09-23T22:32:45.113 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.0399476 2023-09-23T22:32:45.113 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 2.14499 2023-09-23T22:32:45.113 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00111004 2023-09-23T22:32:45.120 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:32:45.138 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:45.305 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:45.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:45.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:45.420 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:45.482 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:45 smithi070 ceph-mon[192652]: pgmap v1037: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 50 MiB/s wr, 800 op/s; 1614/282210 objects degraded (0.572%); 800/282210 objects misplaced (0.283%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:32:45.482 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:45 smithi026 ceph-mon[223014]: pgmap v1037: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 50 MiB/s wr, 800 op/s; 1614/282210 objects degraded (0.572%); 800/282210 objects misplaced (0.283%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:32:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:45 smithi026 ceph-mon[215775]: pgmap v1037: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 50 MiB/s wr, 800 op/s; 1614/282210 objects degraded (0.572%); 800/282210 objects misplaced (0.283%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:32:45.590 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:45.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:45.595 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:45.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:45.721 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:45.761 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:45.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:45.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:45.866 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:45.908 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:46.182 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:46.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:46.192 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:46.276 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:46.285 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:32:46.287 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:46.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:46.386 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:46.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1757601286' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:32:46.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:46 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:32:46.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1757601286' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:32:46.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:46 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:32:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1757601286' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:32:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:46 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:32:47.035 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:47.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:47.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:47.141 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:47.212 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:47.316 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:47.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:47.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:47.425 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[215775]: pgmap v1038: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 102 B/s rd, 56 MiB/s wr, 893 op/s; 1614/295986 objects degraded (0.545%); 409/295986 objects misplaced (0.138%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T22:32:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[215775]: failed to encode map e402 with expected crc 2023-09-23T22:32:47.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:32:47.483 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[215775]: osdmap e402: 8 total, 8 up, 5 in 2023-09-23T22:32:47.483 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[215775]: failed to encode map e402 with expected crc 2023-09-23T22:32:47.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[223014]: pgmap v1038: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 102 B/s rd, 56 MiB/s wr, 893 op/s; 1614/295986 objects degraded (0.545%); 409/295986 objects misplaced (0.138%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T22:32:47.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:47.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[223014]: failed to encode map e402 with expected crc 2023-09-23T22:32:47.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:32:47.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[223014]: osdmap e402: 8 total, 8 up, 5 in 2023-09-23T22:32:47.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:47 smithi026 ceph-mon[223014]: failed to encode map e402 with expected crc 2023-09-23T22:32:47.504 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:47 smithi070 ceph-mon[192652]: pgmap v1038: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 102 B/s rd, 56 MiB/s wr, 893 op/s; 1614/295986 objects degraded (0.545%); 409/295986 objects misplaced (0.138%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T22:32:47.504 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:47 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:32:47.504 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:47 smithi070 ceph-mon[192652]: failed to encode map e402 with expected crc 2023-09-23T22:32:47.504 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:47 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:32:47.505 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:47 smithi070 ceph-mon[192652]: osdmap e402: 8 total, 8 up, 5 in 2023-09-23T22:32:47.505 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:47 smithi070 ceph-mon[192652]: failed to encode map e402 with expected crc 2023-09-23T22:32:47.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:47.618 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:47.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:47.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:47.729 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:47.787 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:47.832 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:32:47.832 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:32:47.944 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:47.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:47.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:48.052 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:32:48.102 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:48.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:48 smithi026 ceph-mon[215775]: pgmap v1040: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 102 B/s rd, 56 MiB/s wr, 893 op/s; 1614/295986 objects degraded (0.545%); 409/295986 objects misplaced (0.138%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T22:32:48.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 1614/295986 objects degraded (0.545%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:48 smithi026 ceph-mon[223014]: pgmap v1040: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 102 B/s rd, 56 MiB/s wr, 893 op/s; 1614/295986 objects degraded (0.545%); 409/295986 objects misplaced (0.138%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T22:32:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 1614/295986 objects degraded (0.545%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:48 smithi070 ceph-mon[192652]: pgmap v1040: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 102 B/s rd, 56 MiB/s wr, 893 op/s; 1614/295986 objects degraded (0.545%); 409/295986 objects misplaced (0.138%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T22:32:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 1614/295986 objects degraded (0.545%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:49.093 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:32:49.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:49.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:32:49.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:32:49.199 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:49.260 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:49.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:32:49.354 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:49.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:49.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:49.471 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:49.522 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:49.738 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:32:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:32:49] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.43.0" 2023-09-23T22:32:49.902 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:32:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:49.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:32:50.008 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:50.056 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:50.146 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:50.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:50.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:50.259 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:50.328 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:50.428 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:50.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:50.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:50.537 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:50.571 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:32:50.608 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:32:50.608 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:50.648 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:32:50.677 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:50.707 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:32:50.725 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:50.740 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:32:50.748 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:50.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:50.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:50.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:50 smithi026 ceph-mon[215775]: pgmap v1041: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 102 B/s rd, 50 MiB/s wr, 885 op/s; 1614/294033 objects degraded (0.549%); 409/294033 objects misplaced (0.139%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T22:32:50.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4233296654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:50.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:50 smithi026 ceph-mon[223014]: pgmap v1041: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 102 B/s rd, 50 MiB/s wr, 885 op/s; 1614/294033 objects degraded (0.549%); 409/294033 objects misplaced (0.139%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T22:32:50.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4233296654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:50.783 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:32:50.816 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:32:50.848 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:32:50.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:32:50.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:32:50.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:32:50.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:32:50.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:32:50.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:32:50.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:32:50.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:32:50.862 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:50.911 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:50 smithi070 ceph-mon[192652]: pgmap v1041: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 6.0 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 102 B/s rd, 50 MiB/s wr, 885 op/s; 1614/294033 objects degraded (0.549%); 409/294033 objects misplaced (0.139%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T22:32:51.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4233296654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:32:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:32:51.142 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:32:51.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:51 smithi026 ceph-mon[215775]: osdmap e403: 8 total, 8 up, 5 in 2023-09-23T22:32:51.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:51 smithi026 ceph-mon[215775]: failed to encode map e403 with expected crc 2023-09-23T22:32:51.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:51 smithi026 ceph-mon[215775]: failed to encode map e403 with expected crc 2023-09-23T22:32:51.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:51 smithi026 ceph-mon[215775]: from='client.55753 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:32:51.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:51 smithi026 ceph-mon[223014]: osdmap e403: 8 total, 8 up, 5 in 2023-09-23T22:32:51.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:51 smithi026 ceph-mon[223014]: failed to encode map e403 with expected crc 2023-09-23T22:32:51.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:51 smithi026 ceph-mon[223014]: failed to encode map e403 with expected crc 2023-09-23T22:32:51.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:51 smithi026 ceph-mon[223014]: from='client.55753 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:32:52.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:51 smithi070 ceph-mon[192652]: osdmap e403: 8 total, 8 up, 5 in 2023-09-23T22:32:52.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:51 smithi070 ceph-mon[192652]: failed to encode map e403 with expected crc 2023-09-23T22:32:52.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:51 smithi070 ceph-mon[192652]: failed to encode map e403 with expected crc 2023-09-23T22:32:52.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:51 smithi070 ceph-mon[192652]: from='client.55753 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:32:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[223014]: pgmap v1043: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 127 B/s rd, 38 MiB/s wr, 1.12k op/s; 1614/284415 objects degraded (0.567%); 11/284415 objects misplaced (0.004%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T22:32:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[223014]: osdmap e404: 8 total, 8 up, 5 in 2023-09-23T22:32:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[223014]: failed to encode map e404 with expected crc 2023-09-23T22:32:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[223014]: failed to encode map e404 with expected crc 2023-09-23T22:32:52.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[223014]: failed to encode map e404 with expected crc 2023-09-23T22:32:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[215775]: pgmap v1043: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 127 B/s rd, 38 MiB/s wr, 1.12k op/s; 1614/284415 objects degraded (0.567%); 11/284415 objects misplaced (0.004%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T22:32:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[215775]: osdmap e404: 8 total, 8 up, 5 in 2023-09-23T22:32:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[215775]: failed to encode map e404 with expected crc 2023-09-23T22:32:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[215775]: failed to encode map e404 with expected crc 2023-09-23T22:32:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:52 smithi026 ceph-mon[215775]: failed to encode map e404 with expected crc 2023-09-23T22:32:53.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:52 smithi070 ceph-mon[192652]: pgmap v1043: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 127 B/s rd, 38 MiB/s wr, 1.12k op/s; 1614/284415 objects degraded (0.567%); 11/284415 objects misplaced (0.004%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T22:32:53.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:52 smithi070 ceph-mon[192652]: osdmap e404: 8 total, 8 up, 5 in 2023-09-23T22:32:53.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:52 smithi070 ceph-mon[192652]: failed to encode map e404 with expected crc 2023-09-23T22:32:53.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:52 smithi070 ceph-mon[192652]: failed to encode map e404 with expected crc 2023-09-23T22:32:53.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:52 smithi070 ceph-mon[192652]: failed to encode map e404 with expected crc 2023-09-23T22:32:53.176 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:53.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:53.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:53.288 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:53.320 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:53.741 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:53.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:53.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:53.850 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:53.935 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:54.027 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:54.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:54.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:54.151 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:54.211 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:54 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 1614/284415 objects degraded (0.567%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:54.211 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:54.214 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:54.283 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:54.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:54.390 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:32:54.417 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:54.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:54 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 1614/284415 objects degraded (0.567%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:54.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:54.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:54 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 1614/284415 objects degraded (0.567%), 1 pg degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:32:54.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:32:54.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:32:54.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:54.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:32:54.570 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1042,"stamp":"2023-09-23T22:32:50.662675+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6216030155,"num_objects":94805,"num_object_clones":0,"num_object_copies":284415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1614,"num_objects_misplaced":11,"num_objects_unfound":0,"num_objects_dirty":94805,"num_whiteouts":0,"num_read":5983,"num_read_kb":12185,"num_write":103910,"num_write_kb":6341622,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9735,"num_bytes_recovered":637960555,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85005,"ondisk_log_size":85005,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":20607268,"kb_used_data":18233084,"kb_used_omap":15,"kb_used_meta":2373744,"kb_avail":448118492,"statfs":{"total":479975178240,"available":458873335808,"internally_reserved":0,"allocated":18670678016,"data_stored":18661975968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2430714078},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":48103448,"num_objects":735,"num_object_clones":0,"num_object_copies":2205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-789,"num_objects_unfound":0,"num_objects_dirty":735,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8961,"num_write_kb":310209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":789,"num_bytes_recovered":51707904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.082871"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":429,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.179892+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:32:36.179892+0000","last_peered":"2023-09-23T22:32:36.179892+0000","last_clean":"2023-09-23T22:32:36.179892+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:32:36.179892+0000","last_undegraded":"2023-09-23T22:32:36.179892+0000","last_fullsized":"2023-09-23T22:32:36.179892+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":118,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.190061+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:32:36.190061+0000","last_peered":"2023-09-23T22:32:36.190061+0000","last_clean":"2023-09-23T22:32:36.190061+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:32:36.190061+0000","last_undegraded":"2023-09-23T22:32:36.190061+0000","last_fullsized":"2023-09-23T22:32:36.190061+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":119,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:44.293295+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:32:44.293295+0000","last_peered":"2023-09-23T22:32:44.293295+0000","last_clean":"2023-09-23T22:32:44.293295+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:32:44.293295+0000","last_undegraded":"2023-09-23T22:32:44.293295+0000","last_fullsized":"2023-09-23T22:32:44.293295+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":290,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.191072+0000","last_change":"2023-09-23T22:31:28.120861+0000","last_active":"2023-09-23T22:32:36.191072+0000","last_peered":"2023-09-23T22:32:36.191072+0000","last_clean":"2023-09-23T22:32:36.191072+0000","last_became_active":"2023-09-23T22:31:28.120394+0000","last_became_peered":"2023-09-23T22:31:28.120394+0000","last_unstale":"2023-09-23T22:32:36.191072+0000","last_undegraded":"2023-09-23T22:32:36.191072+0000","last_fullsized":"2023-09-23T22:32:36.191072+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:19.470258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":457,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.195457+0000","last_change":"2023-09-23T22:31:28.114528+0000","last_active":"2023-09-23T22:32:36.195457+0000","last_peered":"2023-09-23T22:32:36.195457+0000","last_clean":"2023-09-23T22:32:36.195457+0000","last_became_active":"2023-09-23T22:31:28.114177+0000","last_became_peered":"2023-09-23T22:31:28.114177+0000","last_unstale":"2023-09-23T22:32:36.195457+0000","last_undegraded":"2023-09-23T22:32:36.195457+0000","last_fullsized":"2023-09-23T22:32:36.195457+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:46.680433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":83,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.184430+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:32:36.184430+0000","last_peered":"2023-09-23T22:32:36.184430+0000","last_clean":"2023-09-23T22:32:36.184430+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:32:36.184430+0000","last_undegraded":"2023-09-23T22:32:36.184430+0000","last_fullsized":"2023-09-23T22:32:36.184430+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":318,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.188632+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:32:36.188632+0000","last_peered":"2023-09-23T22:32:36.188632+0000","last_clean":"2023-09-23T22:32:36.188632+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:32:36.188632+0000","last_undegraded":"2023-09-23T22:32:36.188632+0000","last_fullsized":"2023-09-23T22:32:36.188632+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":83,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.182989+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:32:36.182989+0000","last_peered":"2023-09-23T22:32:36.182989+0000","last_clean":"2023-09-23T22:32:36.182989+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:32:36.182989+0000","last_undegraded":"2023-09-23T22:32:36.182989+0000","last_fullsized":"2023-09-23T22:32:36.182989+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":402,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.175439+0000","last_change":"2023-09-23T22:31:28.116916+0000","last_active":"2023-09-23T22:32:36.175439+0000","last_peered":"2023-09-23T22:32:36.175439+0000","last_clean":"2023-09-23T22:32:36.175439+0000","last_became_active":"2023-09-23T22:31:28.116577+0000","last_became_peered":"2023-09-23T22:31:28.116577+0000","last_unstale":"2023-09-23T22:32:36.175439+0000","last_undegraded":"2023-09-23T22:32:36.175439+0000","last_fullsized":"2023-09-23T22:32:36.175439+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:33.092789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":93,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.183454+0000","last_change":"2023-09-23T22:31:28.122684+0000","last_active":"2023-09-23T22:32:36.183454+0000","last_peered":"2023-09-23T22:32:36.183454+0000","last_clean":"2023-09-23T22:32:36.183454+0000","last_became_active":"2023-09-23T22:31:28.122427+0000","last_became_peered":"2023-09-23T22:31:28.122427+0000","last_unstale":"2023-09-23T22:32:36.183454+0000","last_undegraded":"2023-09-23T22:32:36.183454+0000","last_fullsized":"2023-09-23T22:32:36.183454+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:15.559367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":149,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.180903+0000","last_change":"2023-09-23T22:31:28.153199+0000","last_active":"2023-09-23T22:32:36.180903+0000","last_peered":"2023-09-23T22:32:36.180903+0000","last_clean":"2023-09-23T22:32:36.180903+0000","last_became_active":"2023-09-23T22:31:28.129257+0000","last_became_peered":"2023-09-23T22:31:28.129257+0000","last_unstale":"2023-09-23T22:32:36.180903+0000","last_undegraded":"2023-09-23T22:32:36.180903+0000","last_fullsized":"2023-09-23T22:32:36.180903+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:03.743836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":50,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.188187+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:32:36.188187+0000","last_peered":"2023-09-23T22:32:36.188187+0000","last_clean":"2023-09-23T22:32:36.188187+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:32:36.188187+0000","last_undegraded":"2023-09-23T22:32:36.188187+0000","last_fullsized":"2023-09-23T22:32:36.188187+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":486,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.181049+0000","last_change":"2023-09-23T22:31:28.145916+0000","last_active":"2023-09-23T22:32:36.181049+0000","last_peered":"2023-09-23T22:32:36.181049+0000","last_clean":"2023-09-23T22:32:36.181049+0000","last_became_active":"2023-09-23T22:31:28.120703+0000","last_became_peered":"2023-09-23T22:31:28.120703+0000","last_unstale":"2023-09-23T22:32:36.181049+0000","last_undegraded":"2023-09-23T22:32:36.181049+0000","last_fullsized":"2023-09-23T22:32:36.181049+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:07.819247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":400,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.179932+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:32:36.179932+0000","last_peered":"2023-09-23T22:32:36.179932+0000","last_clean":"2023-09-23T22:32:36.179932+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:32:36.179932+0000","last_undegraded":"2023-09-23T22:32:36.179932+0000","last_fullsized":"2023-09-23T22:32:36.179932+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":393,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.180022+0000","last_change":"2023-09-23T22:31:28.149149+0000","last_active":"2023-09-23T22:32:36.180022+0000","last_peered":"2023-09-23T22:32:36.180022+0000","last_clean":"2023-09-23T22:32:36.180022+0000","last_became_active":"2023-09-23T22:31:28.121120+0000","last_became_peered":"2023-09-23T22:31:28.121120+0000","last_unstale":"2023-09-23T22:32:36.180022+0000","last_undegraded":"2023-09-23T22:32:36.180022+0000","last_fullsized":"2023-09-23T22:32:36.180022+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.434137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":364,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.178084+0000","last_change":"2023-09-23T22:31:28.119520+0000","last_active":"2023-09-23T22:32:36.178084+0000","last_peered":"2023-09-23T22:32:36.178084+0000","last_clean":"2023-09-23T22:32:36.178084+0000","last_became_active":"2023-09-23T22:31:28.118810+0000","last_became_peered":"2023-09-23T22:31:28.118810+0000","last_unstale":"2023-09-23T22:32:36.178084+0000","last_undegraded":"2023-09-23T22:32:36.178084+0000","last_fullsized":"2023-09-23T22:32:36.178084+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:36.007062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":307,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.190657+0000","last_change":"2023-09-23T22:31:28.114537+0000","last_active":"2023-09-23T22:32:36.190657+0000","last_peered":"2023-09-23T22:32:36.190657+0000","last_clean":"2023-09-23T22:32:36.190657+0000","last_became_active":"2023-09-23T22:31:28.114176+0000","last_became_peered":"2023-09-23T22:31:28.114176+0000","last_unstale":"2023-09-23T22:32:36.190657+0000","last_undegraded":"2023-09-23T22:32:36.190657+0000","last_fullsized":"2023-09-23T22:32:36.190657+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.324173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":411,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.177680+0000","last_change":"2023-09-23T22:31:28.132494+0000","last_active":"2023-09-23T22:32:36.177680+0000","last_peered":"2023-09-23T22:32:36.177680+0000","last_clean":"2023-09-23T22:32:36.177680+0000","last_became_active":"2023-09-23T22:31:28.132197+0000","last_became_peered":"2023-09-23T22:31:28.132197+0000","last_unstale":"2023-09-23T22:32:36.177680+0000","last_undegraded":"2023-09-23T22:32:36.177680+0000","last_fullsized":"2023-09-23T22:32:36.177680+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:36.181016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":345,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.180274+0000","last_change":"2023-09-23T22:31:28.119156+0000","last_active":"2023-09-23T22:32:36.180274+0000","last_peered":"2023-09-23T22:32:36.180274+0000","last_clean":"2023-09-23T22:32:36.180274+0000","last_became_active":"2023-09-23T22:31:28.118382+0000","last_became_peered":"2023-09-23T22:31:28.118382+0000","last_unstale":"2023-09-23T22:32:36.180274+0000","last_undegraded":"2023-09-23T22:32:36.180274+0000","last_fullsized":"2023-09-23T22:32:36.180274+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:10.140079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":298,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.191238+0000","last_change":"2023-09-23T22:31:28.121397+0000","last_active":"2023-09-23T22:32:36.191238+0000","last_peered":"2023-09-23T22:32:36.191238+0000","last_clean":"2023-09-23T22:32:36.191238+0000","last_became_active":"2023-09-23T22:31:28.120847+0000","last_became_peered":"2023-09-23T22:31:28.120847+0000","last_unstale":"2023-09-23T22:32:36.191238+0000","last_undegraded":"2023-09-23T22:32:36.191238+0000","last_fullsized":"2023-09-23T22:32:36.191238+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:42.407658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":93,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.182905+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:32:36.182905+0000","last_peered":"2023-09-23T22:32:36.182905+0000","last_clean":"2023-09-23T22:32:36.182905+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:32:36.182905+0000","last_undegraded":"2023-09-23T22:32:36.182905+0000","last_fullsized":"2023-09-23T22:32:36.182905+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":93,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.184259+0000","last_change":"2023-09-23T22:31:28.128965+0000","last_active":"2023-09-23T22:32:36.184259+0000","last_peered":"2023-09-23T22:32:36.184259+0000","last_clean":"2023-09-23T22:32:36.184259+0000","last_became_active":"2023-09-23T22:31:28.128224+0000","last_became_peered":"2023-09-23T22:31:28.128224+0000","last_unstale":"2023-09-23T22:32:36.184259+0000","last_undegraded":"2023-09-23T22:32:36.184259+0000","last_fullsized":"2023-09-23T22:32:36.184259+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:11.301143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"402'6434","reported_seq":6471,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:50.098143+0000","last_change":"2023-09-23T22:31:08.226781+0000","last_active":"2023-09-23T22:32:50.098143+0000","last_peered":"2023-09-23T22:32:50.098143+0000","last_clean":"2023-09-23T22:32:50.098143+0000","last_became_active":"2023-09-23T22:31:08.226511+0000","last_became_peered":"2023-09-23T22:31:08.226511+0000","last_unstale":"2023-09-23T22:32:50.098143+0000","last_undegraded":"2023-09-23T22:32:50.098143+0000","last_fullsized":"2023-09-23T22:32:50.098143+0000","mapping_epoch":373,"log_start":"378'900","ondisk_log_start":"378'900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5534,"log_dups_size":0,"ondisk_log_size":5534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:03.735360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387448832,"num_objects":5912,"num_object_clones":0,"num_object_copies":17736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6434,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":411,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.180645+0000","last_change":"2023-09-23T22:31:28.139498+0000","last_active":"2023-09-23T22:32:36.180645+0000","last_peered":"2023-09-23T22:32:36.180645+0000","last_clean":"2023-09-23T22:32:36.180645+0000","last_became_active":"2023-09-23T22:31:28.113206+0000","last_became_peered":"2023-09-23T22:31:28.113206+0000","last_unstale":"2023-09-23T22:32:36.180645+0000","last_undegraded":"2023-09-23T22:32:36.180645+0000","last_fullsized":"2023-09-23T22:32:36.180645+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:11.905312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":404,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.175462+0000","last_change":"2023-09-23T22:31:28.131730+0000","last_active":"2023-09-23T22:32:36.175462+0000","last_peered":"2023-09-23T22:32:36.175462+0000","last_clean":"2023-09-23T22:32:36.175462+0000","last_became_active":"2023-09-23T22:31:28.131383+0000","last_became_peered":"2023-09-23T22:31:28.131383+0000","last_unstale":"2023-09-23T22:32:36.175462+0000","last_undegraded":"2023-09-23T22:32:36.175462+0000","last_fullsized":"2023-09-23T22:32:36.175462+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:56.993827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"402'6513","reported_seq":9753,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:50.029831+0000","last_change":"2023-09-23T22:32:00.147475+0000","last_active":"2023-09-23T22:32:50.029831+0000","last_peered":"2023-09-23T22:32:50.029831+0000","last_clean":"2023-09-23T22:32:50.029831+0000","last_became_active":"2023-09-23T22:32:00.121320+0000","last_became_peered":"2023-09-23T22:32:00.121320+0000","last_unstale":"2023-09-23T22:32:50.029831+0000","last_undegraded":"2023-09-23T22:32:50.029831+0000","last_fullsized":"2023-09-23T22:32:50.029831+0000","mapping_epoch":392,"log_start":"379'1500","ondisk_log_start":"379'1500","created":373,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5013,"log_dups_size":0,"ondisk_log_size":5013,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:17:59.934876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388825088,"num_objects":5933,"num_object_clones":0,"num_object_copies":17799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6513,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1591,"num_bytes_recovered":104267776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":353,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.184025+0000","last_change":"2023-09-23T22:31:28.125280+0000","last_active":"2023-09-23T22:32:36.184025+0000","last_peered":"2023-09-23T22:32:36.184025+0000","last_clean":"2023-09-23T22:32:36.184025+0000","last_became_active":"2023-09-23T22:31:28.110856+0000","last_became_peered":"2023-09-23T22:31:28.110856+0000","last_unstale":"2023-09-23T22:32:36.184025+0000","last_undegraded":"2023-09-23T22:32:36.184025+0000","last_fullsized":"2023-09-23T22:32:36.184025+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:32.197056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":429,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.180190+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:32:36.180190+0000","last_peered":"2023-09-23T22:32:36.180190+0000","last_clean":"2023-09-23T22:32:36.180190+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:32:36.180190+0000","last_undegraded":"2023-09-23T22:32:36.180190+0000","last_fullsized":"2023-09-23T22:32:36.180190+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"402'6469","reported_seq":6515,"reported_epoch":402,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:32:49.844848+0000","last_change":"2023-09-23T22:31:29.183097+0000","last_active":"2023-09-23T22:32:49.844848+0000","last_peered":"2023-09-23T22:32:49.844848+0000","last_clean":"2023-09-23T22:31:27.059725+0000","last_became_active":"2023-09-23T22:31:29.178921+0000","last_became_peered":"2023-09-23T22:31:29.178921+0000","last_unstale":"2023-09-23T22:32:49.844848+0000","last_undegraded":"2023-09-23T22:31:29.114095+0000","last_fullsized":"2023-09-23T22:31:29.105229+0000","mapping_epoch":381,"log_start":"378'1100","ondisk_log_start":"378'1100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5369,"log_dups_size":0,"ondisk_log_size":5369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:28.838973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387776512,"num_objects":5917,"num_object_clones":0,"num_object_copies":17751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1614,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6469,"num_write_kb":396352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":4303},{"shards":"0,6,7","objects":1338}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":369,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.177768+0000","last_change":"2023-09-23T22:31:28.136321+0000","last_active":"2023-09-23T22:32:36.177768+0000","last_peered":"2023-09-23T22:32:36.177768+0000","last_clean":"2023-09-23T22:32:36.177768+0000","last_became_active":"2023-09-23T22:31:28.131102+0000","last_became_peered":"2023-09-23T22:31:28.131102+0000","last_unstale":"2023-09-23T22:32:36.177768+0000","last_undegraded":"2023-09-23T22:32:36.177768+0000","last_fullsized":"2023-09-23T22:32:36.177768+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:24.768099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":93,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.183247+0000","last_change":"2023-09-23T22:31:28.125138+0000","last_active":"2023-09-23T22:32:36.183247+0000","last_peered":"2023-09-23T22:32:36.183247+0000","last_clean":"2023-09-23T22:32:36.183247+0000","last_became_active":"2023-09-23T22:31:28.124716+0000","last_became_peered":"2023-09-23T22:31:28.124716+0000","last_unstale":"2023-09-23T22:32:36.183247+0000","last_undegraded":"2023-09-23T22:32:36.183247+0000","last_fullsized":"2023-09-23T22:32:36.183247+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.718931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"402'6558","reported_seq":6595,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:50.089110+0000","last_change":"2023-09-23T22:31:08.225591+0000","last_active":"2023-09-23T22:32:50.089110+0000","last_peered":"2023-09-23T22:32:50.089110+0000","last_clean":"2023-09-23T22:32:50.089110+0000","last_became_active":"2023-09-23T22:31:08.225286+0000","last_became_peered":"2023-09-23T22:31:08.225286+0000","last_unstale":"2023-09-23T22:32:50.089110+0000","last_undegraded":"2023-09-23T22:32:50.089110+0000","last_fullsized":"2023-09-23T22:32:50.089110+0000","mapping_epoch":373,"log_start":"379'1500","ondisk_log_start":"379'1500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5058,"log_dups_size":0,"ondisk_log_size":5058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:14.895634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387842048,"num_objects":5918,"num_object_clones":0,"num_object_copies":17754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6558,"num_write_kb":399232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":432,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.178247+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:32:36.178247+0000","last_peered":"2023-09-23T22:32:36.178247+0000","last_clean":"2023-09-23T22:32:36.178247+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:32:36.178247+0000","last_undegraded":"2023-09-23T22:32:36.178247+0000","last_fullsized":"2023-09-23T22:32:36.178247+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":287,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.193331+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:32:36.193331+0000","last_peered":"2023-09-23T22:32:36.193331+0000","last_clean":"2023-09-23T22:32:36.193331+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:32:36.193331+0000","last_undegraded":"2023-09-23T22:32:36.193331+0000","last_fullsized":"2023-09-23T22:32:36.193331+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"402'6273","reported_seq":6310,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:50.112412+0000","last_change":"2023-09-23T22:31:08.226199+0000","last_active":"2023-09-23T22:32:50.112412+0000","last_peered":"2023-09-23T22:32:50.112412+0000","last_clean":"2023-09-23T22:32:50.112412+0000","last_became_active":"2023-09-23T22:31:08.225930+0000","last_became_peered":"2023-09-23T22:31:08.225930+0000","last_unstale":"2023-09-23T22:32:50.112412+0000","last_undegraded":"2023-09-23T22:32:50.112412+0000","last_fullsized":"2023-09-23T22:32:50.112412+0000","mapping_epoch":373,"log_start":"378'800","ondisk_log_start":"378'800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:16.294443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376897536,"num_objects":5751,"num_object_clones":0,"num_object_copies":17253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6273,"num_write_kb":384768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":747,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.175939+0000","last_change":"2023-09-23T22:31:31.125903+0000","last_active":"2023-09-23T22:32:36.175939+0000","last_peered":"2023-09-23T22:32:36.175939+0000","last_clean":"2023-09-23T22:32:36.175939+0000","last_became_active":"2023-09-23T22:31:31.125630+0000","last_became_peered":"2023-09-23T22:31:31.125630+0000","last_unstale":"2023-09-23T22:32:36.175939+0000","last_undegraded":"2023-09-23T22:32:36.175939+0000","last_fullsized":"2023-09-23T22:32:36.175939+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:24.405922+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":234,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.193412+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:32:36.193412+0000","last_peered":"2023-09-23T22:32:36.193412+0000","last_clean":"2023-09-23T22:32:36.193412+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:32:36.193412+0000","last_undegraded":"2023-09-23T22:32:36.193412+0000","last_fullsized":"2023-09-23T22:32:36.193412+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":410,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.180604+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:32:36.180604+0000","last_peered":"2023-09-23T22:32:36.180604+0000","last_clean":"2023-09-23T22:32:36.180604+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:32:36.180604+0000","last_undegraded":"2023-09-23T22:32:36.180604+0000","last_fullsized":"2023-09-23T22:32:36.180604+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"402'6452","reported_seq":6490,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:49.269760+0000","last_change":"2023-09-23T22:31:08.227045+0000","last_active":"2023-09-23T22:32:49.269760+0000","last_peered":"2023-09-23T22:32:49.269760+0000","last_clean":"2023-09-23T22:32:49.269760+0000","last_became_active":"2023-09-23T22:31:08.226758+0000","last_became_peered":"2023-09-23T22:31:08.226758+0000","last_unstale":"2023-09-23T22:32:49.269760+0000","last_undegraded":"2023-09-23T22:32:49.269760+0000","last_fullsized":"2023-09-23T22:32:49.269760+0000","mapping_epoch":373,"log_start":"382'1600","ondisk_log_start":"382'1600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4852,"log_dups_size":0,"ondisk_log_size":4852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:55.118636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393740288,"num_objects":6008,"num_object_clones":0,"num_object_copies":18024,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6008,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6452,"num_write_kb":398720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":783,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.175647+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:32:36.175647+0000","last_peered":"2023-09-23T22:32:36.175647+0000","last_clean":"2023-09-23T22:32:36.175647+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:32:36.175647+0000","last_undegraded":"2023-09-23T22:32:36.175647+0000","last_fullsized":"2023-09-23T22:32:36.175647+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":83,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.183374+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:32:36.183374+0000","last_peered":"2023-09-23T22:32:36.183374+0000","last_clean":"2023-09-23T22:32:36.183374+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:32:36.183374+0000","last_undegraded":"2023-09-23T22:32:36.183374+0000","last_fullsized":"2023-09-23T22:32:36.183374+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":437,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.179637+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:32:36.179637+0000","last_peered":"2023-09-23T22:32:36.179637+0000","last_clean":"2023-09-23T22:32:36.179637+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:32:36.179637+0000","last_undegraded":"2023-09-23T22:32:36.179637+0000","last_fullsized":"2023-09-23T22:32:36.179637+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"402'6460","reported_seq":6497,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:50.066984+0000","last_change":"2023-09-23T22:31:08.225049+0000","last_active":"2023-09-23T22:32:50.066984+0000","last_peered":"2023-09-23T22:32:50.066984+0000","last_clean":"2023-09-23T22:32:50.066984+0000","last_became_active":"2023-09-23T22:31:08.224765+0000","last_became_peered":"2023-09-23T22:31:08.224765+0000","last_unstale":"2023-09-23T22:32:50.066984+0000","last_undegraded":"2023-09-23T22:32:50.066984+0000","last_fullsized":"2023-09-23T22:32:50.066984+0000","mapping_epoch":373,"log_start":"379'1600","ondisk_log_start":"379'1600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4860,"log_dups_size":0,"ondisk_log_size":4860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.842171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389414912,"num_objects":5942,"num_object_clones":0,"num_object_copies":17826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6460,"num_write_kb":396864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":393,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.179559+0000","last_change":"2023-09-23T22:31:28.119358+0000","last_active":"2023-09-23T22:32:36.179559+0000","last_peered":"2023-09-23T22:32:36.179559+0000","last_clean":"2023-09-23T22:32:36.179559+0000","last_became_active":"2023-09-23T22:31:28.118823+0000","last_became_peered":"2023-09-23T22:31:28.118823+0000","last_unstale":"2023-09-23T22:32:36.179559+0000","last_undegraded":"2023-09-23T22:32:36.179559+0000","last_fullsized":"2023-09-23T22:32:36.179559+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:25.483176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":418,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.184182+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:32:36.184182+0000","last_peered":"2023-09-23T22:32:36.184182+0000","last_clean":"2023-09-23T22:32:36.184182+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:32:36.184182+0000","last_undegraded":"2023-09-23T22:32:36.184182+0000","last_fullsized":"2023-09-23T22:32:36.184182+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"402'6455","reported_seq":6493,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:49.269745+0000","last_change":"2023-09-23T22:31:08.226959+0000","last_active":"2023-09-23T22:32:49.269745+0000","last_peered":"2023-09-23T22:32:49.269745+0000","last_clean":"2023-09-23T22:32:49.269745+0000","last_became_active":"2023-09-23T22:31:08.226652+0000","last_became_peered":"2023-09-23T22:31:08.226652+0000","last_unstale":"2023-09-23T22:32:49.269745+0000","last_undegraded":"2023-09-23T22:32:49.269745+0000","last_fullsized":"2023-09-23T22:32:49.269745+0000","mapping_epoch":373,"log_start":"379'1600","ondisk_log_start":"379'1600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4855,"log_dups_size":0,"ondisk_log_size":4855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:22.758103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":392364032,"num_objects":5987,"num_object_clones":0,"num_object_copies":17961,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5987,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6455,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":756,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.177337+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:32:36.177337+0000","last_peered":"2023-09-23T22:32:36.177337+0000","last_clean":"2023-09-23T22:32:36.177337+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:32:36.177337+0000","last_undegraded":"2023-09-23T22:32:36.177337+0000","last_fullsized":"2023-09-23T22:32:36.177337+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":119,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.185956+0000","last_change":"2023-09-23T22:31:28.120600+0000","last_active":"2023-09-23T22:32:36.185956+0000","last_peered":"2023-09-23T22:32:36.185956+0000","last_clean":"2023-09-23T22:32:36.185956+0000","last_became_active":"2023-09-23T22:31:28.120281+0000","last_became_peered":"2023-09-23T22:31:28.120281+0000","last_unstale":"2023-09-23T22:32:36.185956+0000","last_undegraded":"2023-09-23T22:32:36.185956+0000","last_fullsized":"2023-09-23T22:32:36.185956+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:10.594022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":119,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.191668+0000","last_change":"2023-09-23T22:31:28.111270+0000","last_active":"2023-09-23T22:32:36.191668+0000","last_peered":"2023-09-23T22:32:36.191668+0000","last_clean":"2023-09-23T22:32:36.191668+0000","last_became_active":"2023-09-23T22:31:28.110921+0000","last_became_peered":"2023-09-23T22:31:28.110921+0000","last_unstale":"2023-09-23T22:32:36.191668+0000","last_undegraded":"2023-09-23T22:32:36.191668+0000","last_fullsized":"2023-09-23T22:32:36.191668+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:05.417671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"402'6393","reported_seq":9523,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:49.854531+0000","last_change":"2023-09-23T22:32:15.139562+0000","last_active":"2023-09-23T22:32:49.854531+0000","last_peered":"2023-09-23T22:32:49.854531+0000","last_clean":"2023-09-23T22:32:49.854531+0000","last_became_active":"2023-09-23T22:32:15.135437+0000","last_became_peered":"2023-09-23T22:32:15.135437+0000","last_unstale":"2023-09-23T22:32:49.854531+0000","last_undegraded":"2023-09-23T22:32:49.854531+0000","last_fullsized":"2023-09-23T22:32:49.854531+0000","mapping_epoch":395,"log_start":"378'1000","ondisk_log_start":"378'1000","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5393,"log_dups_size":0,"ondisk_log_size":5393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:36.457421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382271488,"num_objects":5833,"num_object_clones":0,"num_object_copies":17499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6393,"num_write_kb":391232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1537,"num_bytes_recovered":100728832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":677,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.184339+0000","last_change":"2023-09-23T22:31:31.129491+0000","last_active":"2023-09-23T22:32:36.184339+0000","last_peered":"2023-09-23T22:32:36.184339+0000","last_clean":"2023-09-23T22:32:36.184339+0000","last_became_active":"2023-09-23T22:31:31.129163+0000","last_became_peered":"2023-09-23T22:31:31.129163+0000","last_unstale":"2023-09-23T22:32:36.184339+0000","last_undegraded":"2023-09-23T22:32:36.184339+0000","last_fullsized":"2023-09-23T22:32:36.184339+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.141229+0000","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":778,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.195668+0000","last_change":"2023-09-23T22:31:31.129492+0000","last_active":"2023-09-23T22:32:36.195668+0000","last_peered":"2023-09-23T22:32:36.195668+0000","last_clean":"2023-09-23T22:32:36.195668+0000","last_became_active":"2023-09-23T22:31:31.128706+0000","last_became_peered":"2023-09-23T22:31:31.128706+0000","last_unstale":"2023-09-23T22:32:36.195668+0000","last_undegraded":"2023-09-23T22:32:36.195668+0000","last_fullsized":"2023-09-23T22:32:36.195668+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:16.180152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":469,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.177510+0000","last_change":"2023-09-23T22:31:28.130412+0000","last_active":"2023-09-23T22:32:36.177510+0000","last_peered":"2023-09-23T22:32:36.177510+0000","last_clean":"2023-09-23T22:32:36.177510+0000","last_became_active":"2023-09-23T22:31:28.130127+0000","last_became_peered":"2023-09-23T22:31:28.130127+0000","last_unstale":"2023-09-23T22:32:36.177510+0000","last_undegraded":"2023-09-23T22:32:36.177510+0000","last_fullsized":"2023-09-23T22:32:36.177510+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.088651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.188054+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:32:36.188054+0000","last_peered":"2023-09-23T22:32:36.188054+0000","last_clean":"2023-09-23T22:32:36.188054+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:32:36.188054+0000","last_undegraded":"2023-09-23T22:32:36.188054+0000","last_fullsized":"2023-09-23T22:32:36.188054+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"402'6403","reported_seq":9665,"reported_epoch":402,"state":"active+recovery_wait","last_fresh":"2023-09-23T22:32:50.090899+0000","last_change":"2023-09-23T22:32:31.203405+0000","last_active":"2023-09-23T22:32:50.090899+0000","last_peered":"2023-09-23T22:32:50.090899+0000","last_clean":"2023-09-23T22:31:27.057464+0000","last_became_active":"2023-09-23T22:32:31.163888+0000","last_became_peered":"2023-09-23T22:32:31.163888+0000","last_unstale":"2023-09-23T22:32:50.090899+0000","last_undegraded":"2023-09-23T22:32:50.090899+0000","last_fullsized":"2023-09-23T22:32:50.090899+0000","mapping_epoch":399,"log_start":"378'900","ondisk_log_start":"378'900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5503,"log_dups_size":0,"ondisk_log_size":5503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:18.004864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385679360,"num_objects":5885,"num_object_clones":0,"num_object_copies":17655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6403,"num_write_kb":393216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1603,"num_bytes_recovered":105054208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":["5","0","4","7"],"object_location_counts":[{"shards":"0,4,5","objects":5885}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":746,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.176211+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:32:36.176211+0000","last_peered":"2023-09-23T22:32:36.176211+0000","last_clean":"2023-09-23T22:32:36.176211+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:32:36.176211+0000","last_undegraded":"2023-09-23T22:32:36.176211+0000","last_fullsized":"2023-09-23T22:32:36.176211+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1280,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.177543+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:32:36.177543+0000","last_peered":"2023-09-23T22:32:36.177543+0000","last_clean":"2023-09-23T22:32:36.177543+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:32:36.177543+0000","last_undegraded":"2023-09-23T22:32:36.177543+0000","last_fullsized":"2023-09-23T22:32:36.177543+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":441,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.179855+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:32:36.179855+0000","last_peered":"2023-09-23T22:32:36.179855+0000","last_clean":"2023-09-23T22:32:36.179855+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:32:36.179855+0000","last_undegraded":"2023-09-23T22:32:36.179855+0000","last_fullsized":"2023-09-23T22:32:36.179855+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":93,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.183305+0000","last_change":"2023-09-23T22:31:28.123596+0000","last_active":"2023-09-23T22:32:36.183305+0000","last_peered":"2023-09-23T22:32:36.183305+0000","last_clean":"2023-09-23T22:32:36.183305+0000","last_became_active":"2023-09-23T22:31:28.123179+0000","last_became_peered":"2023-09-23T22:31:28.123179+0000","last_unstale":"2023-09-23T22:32:36.183305+0000","last_undegraded":"2023-09-23T22:32:36.183305+0000","last_fullsized":"2023-09-23T22:32:36.183305+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:53.106243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.b","version":"402'6364","reported_seq":9403,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:50.045546+0000","last_change":"2023-09-23T22:31:44.145615+0000","last_active":"2023-09-23T22:32:50.045546+0000","last_peered":"2023-09-23T22:32:50.045546+0000","last_clean":"2023-09-23T22:32:50.045546+0000","last_became_active":"2023-09-23T22:31:44.142103+0000","last_became_peered":"2023-09-23T22:31:44.142103+0000","last_unstale":"2023-09-23T22:32:50.045546+0000","last_undegraded":"2023-09-23T22:32:50.045546+0000","last_fullsized":"2023-09-23T22:32:50.045546+0000","mapping_epoch":387,"log_start":"383'1600","ondisk_log_start":"383'1600","created":373,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4764,"log_dups_size":0,"ondisk_log_size":4764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:15.488007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387842048,"num_objects":5918,"num_object_clones":0,"num_object_copies":17754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6364,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1492,"num_bytes_recovered":97779712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":752,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.190564+0000","last_change":"2023-09-23T22:31:31.130599+0000","last_active":"2023-09-23T22:32:36.190564+0000","last_peered":"2023-09-23T22:32:36.190564+0000","last_clean":"2023-09-23T22:32:36.190564+0000","last_became_active":"2023-09-23T22:31:31.127784+0000","last_became_peered":"2023-09-23T22:31:31.127784+0000","last_unstale":"2023-09-23T22:32:36.190564+0000","last_undegraded":"2023-09-23T22:32:36.190564+0000","last_fullsized":"2023-09-23T22:32:36.190564+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:28.788741+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":619,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.188671+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:32:36.188671+0000","last_peered":"2023-09-23T22:32:36.188671+0000","last_clean":"2023-09-23T22:32:36.188671+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:32:36.188671+0000","last_undegraded":"2023-09-23T22:32:36.188671+0000","last_fullsized":"2023-09-23T22:32:36.188671+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.183026+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:32:36.183026+0000","last_peered":"2023-09-23T22:32:36.183026+0000","last_clean":"2023-09-23T22:32:36.183026+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:32:36.183026+0000","last_undegraded":"2023-09-23T22:32:36.183026+0000","last_fullsized":"2023-09-23T22:32:36.183026+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":51,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:46.254096+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:32:46.254096+0000","last_peered":"2023-09-23T22:32:46.254096+0000","last_clean":"2023-09-23T22:32:46.254096+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:32:46.254096+0000","last_undegraded":"2023-09-23T22:32:46.254096+0000","last_fullsized":"2023-09-23T22:32:46.254096+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"402'6410","reported_seq":6447,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:50.079865+0000","last_change":"2023-09-23T22:31:08.227185+0000","last_active":"2023-09-23T22:32:50.079865+0000","last_peered":"2023-09-23T22:32:50.079865+0000","last_clean":"2023-09-23T22:32:50.079865+0000","last_became_active":"2023-09-23T22:31:08.226918+0000","last_became_peered":"2023-09-23T22:31:08.226918+0000","last_unstale":"2023-09-23T22:32:50.079865+0000","last_undegraded":"2023-09-23T22:32:50.079865+0000","last_fullsized":"2023-09-23T22:32:50.079865+0000","mapping_epoch":373,"log_start":"378'1400","ondisk_log_start":"378'1400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5010,"log_dups_size":0,"ondisk_log_size":5010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:54.257936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385613824,"num_objects":5884,"num_object_clones":0,"num_object_copies":17652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6410,"num_write_kb":393408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1164,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.176019+0000","last_change":"2023-09-23T22:31:31.127033+0000","last_active":"2023-09-23T22:32:36.176019+0000","last_peered":"2023-09-23T22:32:36.176019+0000","last_clean":"2023-09-23T22:32:36.176019+0000","last_became_active":"2023-09-23T22:31:31.126601+0000","last_became_peered":"2023-09-23T22:31:31.126601+0000","last_unstale":"2023-09-23T22:32:36.176019+0000","last_undegraded":"2023-09-23T22:32:36.176019+0000","last_fullsized":"2023-09-23T22:32:36.176019+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:00.619278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":786,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.195895+0000","last_change":"2023-09-23T22:31:31.126756+0000","last_active":"2023-09-23T22:32:36.195895+0000","last_peered":"2023-09-23T22:32:36.195895+0000","last_clean":"2023-09-23T22:32:36.195895+0000","last_became_active":"2023-09-23T22:31:31.126409+0000","last_became_peered":"2023-09-23T22:31:31.126409+0000","last_unstale":"2023-09-23T22:32:36.195895+0000","last_undegraded":"2023-09-23T22:32:36.195895+0000","last_fullsized":"2023-09-23T22:32:36.195895+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.605240+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":283,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.177416+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:32:36.177416+0000","last_peered":"2023-09-23T22:32:36.177416+0000","last_clean":"2023-09-23T22:32:36.177416+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:32:36.177416+0000","last_undegraded":"2023-09-23T22:32:36.177416+0000","last_fullsized":"2023-09-23T22:32:36.177416+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":93,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.184664+0000","last_change":"2023-09-23T22:31:28.124597+0000","last_active":"2023-09-23T22:32:36.184664+0000","last_peered":"2023-09-23T22:32:36.184664+0000","last_clean":"2023-09-23T22:32:36.184664+0000","last_became_active":"2023-09-23T22:31:28.124245+0000","last_became_peered":"2023-09-23T22:31:28.124245+0000","last_unstale":"2023-09-23T22:32:36.184664+0000","last_undegraded":"2023-09-23T22:32:36.184664+0000","last_fullsized":"2023-09-23T22:32:36.184664+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.558632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"402'6401","reported_seq":6439,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:49.251258+0000","last_change":"2023-09-23T22:31:08.223664+0000","last_active":"2023-09-23T22:32:49.251258+0000","last_peered":"2023-09-23T22:32:49.251258+0000","last_clean":"2023-09-23T22:32:49.251258+0000","last_became_active":"2023-09-23T22:31:08.223323+0000","last_became_peered":"2023-09-23T22:31:08.223323+0000","last_unstale":"2023-09-23T22:32:49.251258+0000","last_undegraded":"2023-09-23T22:32:49.251258+0000","last_fullsized":"2023-09-23T22:32:49.251258+0000","mapping_epoch":373,"log_start":"379'1600","ondisk_log_start":"379'1600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4801,"log_dups_size":0,"ondisk_log_size":4801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:38.536861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391839744,"num_objects":5979,"num_object_clones":0,"num_object_copies":17937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6401,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":634,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.183880+0000","last_change":"2023-09-23T22:31:21.362707+0000","last_active":"2023-09-23T22:32:36.183880+0000","last_peered":"2023-09-23T22:32:36.183880+0000","last_clean":"2023-09-23T22:32:36.183880+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:32:36.183880+0000","last_undegraded":"2023-09-23T22:32:36.183880+0000","last_fullsized":"2023-09-23T22:32:36.183880+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:58.622607+0000","scrub_duration":0.0034076229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":781,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.186036+0000","last_change":"2023-09-23T22:31:28.147724+0000","last_active":"2023-09-23T22:32:36.186036+0000","last_peered":"2023-09-23T22:32:36.186036+0000","last_clean":"2023-09-23T22:32:36.186036+0000","last_became_active":"2023-09-23T22:31:28.120238+0000","last_became_peered":"2023-09-23T22:31:28.120238+0000","last_unstale":"2023-09-23T22:32:36.186036+0000","last_undegraded":"2023-09-23T22:32:36.186036+0000","last_fullsized":"2023-09-23T22:32:36.186036+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:53.259496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":297,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.188763+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:32:36.188763+0000","last_peered":"2023-09-23T22:32:36.188763+0000","last_clean":"2023-09-23T22:32:36.188763+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:32:36.188763+0000","last_undegraded":"2023-09-23T22:32:36.188763+0000","last_fullsized":"2023-09-23T22:32:36.188763+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":290,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.191154+0000","last_change":"2023-09-23T22:31:28.121081+0000","last_active":"2023-09-23T22:32:36.191154+0000","last_peered":"2023-09-23T22:32:36.191154+0000","last_clean":"2023-09-23T22:32:36.191154+0000","last_became_active":"2023-09-23T22:31:28.120399+0000","last_became_peered":"2023-09-23T22:31:28.120399+0000","last_unstale":"2023-09-23T22:32:36.191154+0000","last_undegraded":"2023-09-23T22:32:36.191154+0000","last_fullsized":"2023-09-23T22:32:36.191154+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:11.267296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"402'6538","reported_seq":6576,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:49.263441+0000","last_change":"2023-09-23T22:31:08.224259+0000","last_active":"2023-09-23T22:32:49.263441+0000","last_peered":"2023-09-23T22:32:49.263441+0000","last_clean":"2023-09-23T22:32:49.263441+0000","last_became_active":"2023-09-23T22:31:08.224062+0000","last_became_peered":"2023-09-23T22:31:08.224062+0000","last_unstale":"2023-09-23T22:32:49.263441+0000","last_undegraded":"2023-09-23T22:32:49.263441+0000","last_fullsized":"2023-09-23T22:32:49.263441+0000","mapping_epoch":373,"log_start":"383'1700","ondisk_log_start":"383'1700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4838,"log_dups_size":0,"ondisk_log_size":4838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:18.330753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402915328,"num_objects":6148,"num_object_clones":0,"num_object_copies":18444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6538,"num_write_kb":405952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":752,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.183978+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:32:36.183978+0000","last_peered":"2023-09-23T22:32:36.183978+0000","last_clean":"2023-09-23T22:32:36.183978+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:32:36.183978+0000","last_undegraded":"2023-09-23T22:32:36.183978+0000","last_fullsized":"2023-09-23T22:32:36.183978+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":329,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.190607+0000","last_change":"2023-09-23T22:31:28.152186+0000","last_active":"2023-09-23T22:32:36.190607+0000","last_peered":"2023-09-23T22:32:36.190607+0000","last_clean":"2023-09-23T22:32:36.190607+0000","last_became_active":"2023-09-23T22:31:28.111292+0000","last_became_peered":"2023-09-23T22:31:28.111292+0000","last_unstale":"2023-09-23T22:32:36.190607+0000","last_undegraded":"2023-09-23T22:32:36.190607+0000","last_fullsized":"2023-09-23T22:32:36.190607+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:06.097398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":328,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.178316+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:32:36.178316+0000","last_peered":"2023-09-23T22:32:36.178316+0000","last_clean":"2023-09-23T22:32:36.178316+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:32:36.178316+0000","last_undegraded":"2023-09-23T22:32:36.178316+0000","last_fullsized":"2023-09-23T22:32:36.178316+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":270,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.193256+0000","last_change":"2023-09-23T22:31:28.106985+0000","last_active":"2023-09-23T22:32:36.193256+0000","last_peered":"2023-09-23T22:32:36.193256+0000","last_clean":"2023-09-23T22:32:36.193256+0000","last_became_active":"2023-09-23T22:31:28.105451+0000","last_became_peered":"2023-09-23T22:31:28.105451+0000","last_unstale":"2023-09-23T22:32:36.193256+0000","last_undegraded":"2023-09-23T22:32:36.193256+0000","last_fullsized":"2023-09-23T22:32:36.193256+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:43.614928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"402'6459","reported_seq":9718,"reported_epoch":402,"state":"active+clean","last_fresh":"2023-09-23T22:32:49.826363+0000","last_change":"2023-09-23T22:31:45.135267+0000","last_active":"2023-09-23T22:32:49.826363+0000","last_peered":"2023-09-23T22:32:49.826363+0000","last_clean":"2023-09-23T22:32:49.826363+0000","last_became_active":"2023-09-23T22:31:45.131633+0000","last_became_peered":"2023-09-23T22:31:45.131633+0000","last_unstale":"2023-09-23T22:32:49.826363+0000","last_undegraded":"2023-09-23T22:32:49.826363+0000","last_fullsized":"2023-09-23T22:32:49.826363+0000","mapping_epoch":388,"log_start":"378'1100","ondisk_log_start":"378'1100","created":373,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5359,"log_dups_size":0,"ondisk_log_size":5359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:00.859364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388169728,"num_objects":5923,"num_object_clones":0,"num_object_copies":17769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6459,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1601,"num_bytes_recovered":104923136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":756,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.179502+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:32:36.179502+0000","last_peered":"2023-09-23T22:32:36.179502+0000","last_clean":"2023-09-23T22:32:36.179502+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:32:36.179502+0000","last_undegraded":"2023-09-23T22:32:36.179502+0000","last_fullsized":"2023-09-23T22:32:36.179502+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":234,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.188468+0000","last_change":"2023-09-23T22:31:28.131347+0000","last_active":"2023-09-23T22:32:36.188468+0000","last_peered":"2023-09-23T22:32:36.188468+0000","last_clean":"2023-09-23T22:32:36.188468+0000","last_became_active":"2023-09-23T22:31:28.114169+0000","last_became_peered":"2023-09-23T22:31:28.114169+0000","last_unstale":"2023-09-23T22:32:36.188468+0000","last_undegraded":"2023-09-23T22:32:36.188468+0000","last_fullsized":"2023-09-23T22:32:36.188468+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:30.007254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":898,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.187692+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:32:36.187692+0000","last_peered":"2023-09-23T22:32:36.187692+0000","last_clean":"2023-09-23T22:32:36.187692+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:32:36.187692+0000","last_undegraded":"2023-09-23T22:32:36.187692+0000","last_fullsized":"2023-09-23T22:32:36.187692+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":342,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.193494+0000","last_change":"2023-09-23T22:31:28.114550+0000","last_active":"2023-09-23T22:32:36.193494+0000","last_peered":"2023-09-23T22:32:36.193494+0000","last_clean":"2023-09-23T22:32:36.193494+0000","last_became_active":"2023-09-23T22:31:28.114217+0000","last_became_peered":"2023-09-23T22:31:28.114217+0000","last_unstale":"2023-09-23T22:32:36.193494+0000","last_undegraded":"2023-09-23T22:32:36.193494+0000","last_fullsized":"2023-09-23T22:32:36.193494+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:13.616772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.d","version":"402'6421","reported_seq":9616,"reported_epoch":402,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T22:32:50.127553+0000","last_change":"2023-09-23T22:32:30.117983+0000","last_active":"2023-09-23T22:32:50.127553+0000","last_peered":"2023-09-23T22:32:50.127553+0000","last_clean":"2023-09-23T22:31:27.062039+0000","last_became_active":"2023-09-23T22:31:29.181566+0000","last_became_peered":"2023-09-23T22:31:29.181566+0000","last_unstale":"2023-09-23T22:32:50.127553+0000","last_undegraded":"2023-09-23T22:32:50.127553+0000","last_fullsized":"2023-09-23T22:31:29.103794+0000","mapping_epoch":381,"log_start":"378'900","ondisk_log_start":"378'900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:19:13.831015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382599192,"num_objects":5839,"num_object_clones":0,"num_object_copies":17517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":11,"num_objects_unfound":0,"num_objects_dirty":5839,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6421,"num_write_kb":392257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1573,"num_bytes_recovered":103088128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6","objects":5828},{"shards":"5,6,7","objects":11}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":682,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.178390+0000","last_change":"2023-09-23T22:31:45.133529+0000","last_active":"2023-09-23T22:32:36.178390+0000","last_peered":"2023-09-23T22:32:36.178390+0000","last_clean":"2023-09-23T22:32:36.178390+0000","last_became_active":"2023-09-23T22:31:45.133117+0000","last_became_peered":"2023-09-23T22:31:45.133117+0000","last_unstale":"2023-09-23T22:32:36.178390+0000","last_undegraded":"2023-09-23T22:32:36.178390+0000","last_fullsized":"2023-09-23T22:32:36.178390+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.591643+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.591643+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:28.957050+0000","scrub_duration":0.0073638979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":113,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.184194+0000","last_change":"2023-09-23T22:31:28.156716+0000","last_active":"2023-09-23T22:32:36.184194+0000","last_peered":"2023-09-23T22:32:36.184194+0000","last_clean":"2023-09-23T22:32:36.184194+0000","last_became_active":"2023-09-23T22:31:28.128617+0000","last_became_peered":"2023-09-23T22:31:28.128617+0000","last_unstale":"2023-09-23T22:32:36.184194+0000","last_undegraded":"2023-09-23T22:32:36.184194+0000","last_fullsized":"2023-09-23T22:32:36.184194+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:34.465027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":266,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.190971+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:32:36.190971+0000","last_peered":"2023-09-23T22:32:36.190971+0000","last_clean":"2023-09-23T22:32:36.190971+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:32:36.190971+0000","last_undegraded":"2023-09-23T22:32:36.190971+0000","last_fullsized":"2023-09-23T22:32:36.190971+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":375,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.180074+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:32:36.180074+0000","last_peered":"2023-09-23T22:32:36.180074+0000","last_clean":"2023-09-23T22:32:36.180074+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:32:36.180074+0000","last_undegraded":"2023-09-23T22:32:36.180074+0000","last_fullsized":"2023-09-23T22:32:36.180074+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":61,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.193179+0000","last_change":"2023-09-23T22:31:28.111728+0000","last_active":"2023-09-23T22:32:36.193179+0000","last_peered":"2023-09-23T22:32:36.193179+0000","last_clean":"2023-09-23T22:32:36.193179+0000","last_became_active":"2023-09-23T22:31:28.111369+0000","last_became_peered":"2023-09-23T22:31:28.111369+0000","last_unstale":"2023-09-23T22:32:36.193179+0000","last_undegraded":"2023-09-23T22:32:36.193179+0000","last_fullsized":"2023-09-23T22:32:36.193179+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:42.837575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":152,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.188972+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:32:36.188972+0000","last_peered":"2023-09-23T22:32:36.188972+0000","last_clean":"2023-09-23T22:32:36.188972+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:32:36.188972+0000","last_undegraded":"2023-09-23T22:32:36.188972+0000","last_fullsized":"2023-09-23T22:32:36.188972+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":392,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.178176+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:32:36.178176+0000","last_peered":"2023-09-23T22:32:36.178176+0000","last_clean":"2023-09-23T22:32:36.178176+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:32:36.178176+0000","last_undegraded":"2023-09-23T22:32:36.178176+0000","last_fullsized":"2023-09-23T22:32:36.178176+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":427,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.180093+0000","last_change":"2023-09-23T22:31:28.111351+0000","last_active":"2023-09-23T22:32:36.180093+0000","last_peered":"2023-09-23T22:32:36.180093+0000","last_clean":"2023-09-23T22:32:36.180093+0000","last_became_active":"2023-09-23T22:31:28.111006+0000","last_became_peered":"2023-09-23T22:31:28.111006+0000","last_unstale":"2023-09-23T22:32:36.180093+0000","last_undegraded":"2023-09-23T22:32:36.180093+0000","last_fullsized":"2023-09-23T22:32:36.180093+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:51.435513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":417,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.177597+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:32:36.177597+0000","last_peered":"2023-09-23T22:32:36.177597+0000","last_clean":"2023-09-23T22:32:36.177597+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:32:36.177597+0000","last_undegraded":"2023-09-23T22:32:36.177597+0000","last_fullsized":"2023-09-23T22:32:36.177597+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":266,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.179814+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:32:36.179814+0000","last_peered":"2023-09-23T22:32:36.179814+0000","last_clean":"2023-09-23T22:32:36.179814+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:32:36.179814+0000","last_undegraded":"2023-09-23T22:32:36.179814+0000","last_fullsized":"2023-09-23T22:32:36.179814+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":93,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.183897+0000","last_change":"2023-09-23T22:31:28.126203+0000","last_active":"2023-09-23T22:32:36.183897+0000","last_peered":"2023-09-23T22:32:36.183897+0000","last_clean":"2023-09-23T22:32:36.183897+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:31:28.125529+0000","last_unstale":"2023-09-23T22:32:36.183897+0000","last_undegraded":"2023-09-23T22:32:36.183897+0000","last_fullsized":"2023-09-23T22:32:36.183897+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:47.359771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":402,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.188855+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:32:36.188855+0000","last_peered":"2023-09-23T22:32:36.188855+0000","last_clean":"2023-09-23T22:32:36.188855+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:32:36.188855+0000","last_undegraded":"2023-09-23T22:32:36.188855+0000","last_fullsized":"2023-09-23T22:32:36.188855+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":340,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.188554+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:32:36.188554+0000","last_peered":"2023-09-23T22:32:36.188554+0000","last_clean":"2023-09-23T22:32:36.188554+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:32:36.188554+0000","last_undegraded":"2023-09-23T22:32:36.188554+0000","last_fullsized":"2023-09-23T22:32:36.188554+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":460,"reported_epoch":401,"state":"active+clean","last_fresh":"2023-09-23T22:32:36.179167+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:32:36.179167+0000","last_peered":"2023-09-23T22:32:36.179167+0000","last_clean":"2023-09-23T22:32:36.179167+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:32:36.179167+0000","last_undegraded":"2023-09-23T22:32:36.179167+0000","last_fullsized":"2023-09-23T22:32:36.179167+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":31404,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":6211239960,"num_objects":94777,"num_object_clones":0,"num_object_copies":284331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1614,"num_objects_misplaced":11,"num_objects_unfound":0,"num_objects_dirty":94777,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":103003,"num_write_kb":6328897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9397,"num_bytes_recovered":615841792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18880999424,"data_stored":18880987208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82203,"ondisk_log_size":82203,"up":48,"acting":46,"num_store_stats":6},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1115744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234341,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495977,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3251008,"kb_used_data":3063736,"kb_used_omap":0,"kb_used_meta":187200,"kb_avail":90494144,"statfs":{"total":95995035648,"available":92666003456,"internally_reserved":0,"allocated":3137265664,"data_stored":3135803601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":191692800},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":7.2089999999999996,"5min":5.6589999999999998,"15min":2.9900000000000002},"min":{"1min":0.22900000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":57.936999999999998,"5min":213.23699999999999,"15min":213.23699999999999},"last":6.2249999999999996},{"interface":"front","average":{"1min":7.1070000000000002,"5min":5.992,"15min":4.0529999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":58.526000000000003,"5min":213.304,"15min":213.304},"last":4.8360000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":5.3650000000000002,"5min":1.8620000000000001,"15min":3.5110000000000001},"min":{"1min":0.27700000000000002,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":58.143999999999998,"5min":58.143999999999998,"15min":129.964},"last":6.2880000000000003},{"interface":"front","average":{"1min":5.2050000000000001,"5min":2.0750000000000002,"15min":3.8039999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":63.073999999999998,"5min":63.073999999999998,"15min":131.18600000000001},"last":4.9610000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.8680000000000001,"15min":1.8919999999999999},"min":{"1min":0.317,"5min":0.311,"15min":0.311},"max":{"1min":5.5839999999999996,"5min":6.6559999999999997,"15min":6.6559999999999997},"last":6.2990000000000004},{"interface":"front","average":{"1min":0.96599999999999997,"5min":1.913,"15min":2.0710000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.21,"5min":9.7319999999999993,"15min":9.7319999999999993},"last":4.9210000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.5940000000000001,"15min":3.145},"min":{"1min":0.56899999999999995,"5min":0.32500000000000001,"15min":0.316},"max":{"1min":6.165,"5min":6.8609999999999998,"15min":125.125},"last":6.3380000000000001},{"interface":"front","average":{"1min":2.2829999999999999,"5min":1.4119999999999999,"15min":2.8610000000000002},"min":{"1min":0.33900000000000002,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":9.2569999999999997,"5min":9.2569999999999997,"15min":76.406999999999996},"last":6.351}]},{"osd":4,"last update":"Sat Sep 23 22:31:49 2023","interfaces":[{"interface":"back","average":{"1min":1.353,"5min":0.85599999999999998,"15min":2.5600000000000001},"min":{"1min":0.27600000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":4.2460000000000004,"5min":6.8650000000000002,"15min":305.30200000000002},"last":4.9889999999999999},{"interface":"front","average":{"1min":2.6240000000000001,"5min":1.0620000000000001,"15min":2.5960000000000001},"min":{"1min":0.29299999999999998,"5min":0.26000000000000001,"15min":0.255},"max":{"1min":8.1850000000000005,"5min":8.1850000000000005,"15min":305.40300000000002},"last":6.3639999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:31:49 2023","interfaces":[{"interface":"back","average":{"1min":7.5199999999999996,"5min":2.1099999999999999,"15min":1.5169999999999999},"min":{"1min":0.317,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":88.194999999999993,"5min":88.194999999999993,"15min":88.194999999999993},"last":6.2539999999999996},{"interface":"front","average":{"1min":5.9800000000000004,"5min":1.736,"15min":1.427},"min":{"1min":0.30599999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":88.025000000000006,"5min":88.025000000000006,"15min":88.025000000000006},"last":6.3810000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":0.95399999999999996,"15min":1.0980000000000001},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.2270000000000003,"5min":9.2270000000000003,"15min":82.801000000000002},"last":6.444},{"interface":"front","average":{"1min":2.4630000000000001,"5min":1.105,"15min":1.1459999999999999},"min":{"1min":0.41999999999999998,"5min":0.33300000000000002,"15min":0.29699999999999999},"max":{"1min":8.7690000000000001,"5min":8.7690000000000001,"15min":83.052999999999997},"last":6.5289999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795249,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4096016,"kb_used_data":3446280,"kb_used_omap":7,"kb_used_meta":649720,"kb_avail":89649136,"statfs":{"total":95995035648,"available":91800715264,"internally_reserved":0,"allocated":3528990720,"data_stored":3526863727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":665313652},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":6.3769999999999998,"5min":5.8250000000000002,"15min":5.1369999999999996},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":107.202,"5min":202.73500000000001,"15min":202.73500000000001},"last":0.20799999999999999},{"interface":"front","average":{"1min":7.0449999999999999,"5min":6.3120000000000003,"15min":5.9349999999999996},"min":{"1min":0.24399999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":107.426,"5min":202.80799999999999,"15min":202.80799999999999},"last":0.23899999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:31:50 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.94299999999999995,"15min":1.0189999999999999},"min":{"1min":0.26900000000000002,"5min":0.216,"15min":0.216},"max":{"1min":1.0660000000000001,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.1499999999999999,"5min":2.1259999999999999,"15min":2.2890000000000001},"min":{"1min":0.24299999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.5030000000000001,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":0.31}]},{"osd":3,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2669999999999999,"5min":1.732,"15min":4.4980000000000002},"min":{"1min":0.30599999999999999,"5min":0.246,"15min":0.246},"max":{"1min":10.587999999999999,"5min":10.587999999999999,"15min":160.499},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.653,"5min":1.46,"15min":4.1890000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.223},"max":{"1min":5.1920000000000002,"5min":8.1590000000000007,"15min":160.465},"last":2.145}]},{"osd":4,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.92200000000000004,"15min":2.2440000000000002},"min":{"1min":0.29499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.3730000000000002,"5min":7.0380000000000003,"15min":234.79900000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.478,"15min":2.919},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.4729999999999999,"5min":9.093,"15min":234.81700000000001},"last":0.38400000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":1.6859999999999999,"15min":2.5},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":7.4450000000000003,"5min":9.2759999999999998,"15min":196.79900000000001},"last":3.234},{"interface":"front","average":{"1min":1.611,"5min":1.294,"15min":2.3239999999999998},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.29099999999999998},"max":{"1min":5.2889999999999997,"5min":6.8049999999999997,"15min":196.68299999999999},"last":2.2149999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":2.2559999999999998,"15min":2.1379999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":5.2350000000000003,"5min":77.033000000000001,"15min":77.033000000000001},"last":2.2440000000000002},{"interface":"front","average":{"1min":1.681,"5min":1.9630000000000001,"15min":1.583},"min":{"1min":0.442,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":8.3640000000000008,"5min":76.965999999999994,"15min":76.965999999999994},"last":2.9049999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":1.663,"15min":2.0259999999999998},"min":{"1min":0.41499999999999998,"5min":0.38300000000000001,"15min":0.32900000000000001},"max":{"1min":8.0299999999999994,"5min":8.0660000000000007,"15min":16.265000000000001},"last":2.2719999999999998},{"interface":"front","average":{"1min":1.302,"5min":1.2529999999999999,"15min":1.339},"min":{"1min":0.378,"5min":0.318,"15min":0.318},"max":{"1min":3.8119999999999998,"5min":7.1429999999999998,"15min":12.045999999999999},"last":2.8239999999999998}]}]},{"osd":0,"up_from":93,"seq":399431958778,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4172144,"kb_used_data":3424680,"kb_used_omap":8,"kb_used_meta":747447,"kb_avail":89573008,"statfs":{"total":95995035648,"available":91722760192,"internally_reserved":0,"allocated":3506872320,"data_stored":3504660215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":765386602},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:32:37 2023","interfaces":[{"interface":"back","average":{"1min":7.5819999999999999,"5min":3.4860000000000002,"15min":4.1769999999999996},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":121.688,"5min":121.688,"15min":169.77199999999999},"last":1.6539999999999999},{"interface":"front","average":{"1min":6.8739999999999997,"5min":3.5529999999999999,"15min":4.2850000000000001},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":122.93300000000001,"5min":122.93300000000001,"15min":169.52099999999999},"last":4.6040000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:31:52 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":2.7029999999999998,"15min":3.0310000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.202,"5min":22.263000000000002,"15min":22.263000000000002},"last":4.2359999999999998},{"interface":"front","average":{"1min":0.75700000000000001,"5min":1.6180000000000001,"15min":1.762},"min":{"1min":0.27300000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.3290000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:32:37 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.45,"15min":3.5419999999999998},"min":{"1min":0.315,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":8.548,"5min":10.292999999999999,"15min":146.87299999999999},"last":3.2010000000000001},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.6259999999999999,"15min":3.5750000000000002},"min":{"1min":0.29299999999999998,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":3.5379999999999998,"5min":22.303000000000001,"15min":145.48099999999999},"last":10.43}]},{"osd":4,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":2.109,"15min":2.5339999999999998},"min":{"1min":0.30199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.2060000000000004,"5min":11.474,"15min":82.825000000000003},"last":3.5630000000000002},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.4119999999999999,"15min":1.7529999999999999},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.444,"5min":7.2279999999999998,"15min":75.930999999999997},"last":0.54000000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":2.21,"15min":2.9689999999999999},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2510000000000003,"5min":13.564,"15min":93.685000000000002},"last":3.6080000000000001},{"interface":"front","average":{"1min":1.841,"5min":1.633,"15min":2.246},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4139999999999997,"5min":13.486000000000001,"15min":93.546000000000006},"last":2.2909999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3290000000000002,"15min":2.3140000000000001},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.308},"max":{"1min":10.183999999999999,"5min":12.955,"15min":16.366},"last":3.6619999999999999},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.6830000000000001,"15min":1.6799999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":6.5060000000000002,"5min":14.208,"15min":22.263000000000002},"last":2.3690000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.8500000000000001,"15min":2},"min":{"1min":0.42599999999999999,"5min":0.40200000000000002,"15min":0.29999999999999999},"max":{"1min":3.0110000000000001,"5min":22.338999999999999,"15min":32.773000000000003},"last":10.377000000000001},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.1389999999999998,"15min":2.2890000000000001},"min":{"1min":0.47899999999999998,"5min":0.307,"15min":0.307},"max":{"1min":8.7550000000000008,"5min":10.901,"15min":16.381},"last":3.641}]}]},{"osd":2,"up_from":0,"seq":1498943586341,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468196,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019246,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4889720,"kb_used_data":4503088,"kb_used_omap":0,"kb_used_meta":386304,"kb_avail":88855432,"statfs":{"total":95995035648,"available":90987962368,"internally_reserved":0,"allocated":4611162112,"data_stored":4609707666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":395575296},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":11.644,"5min":3.5510000000000002,"15min":4.1360000000000001},"min":{"1min":0.216,"5min":0.20699999999999999,"15min":0.19700000000000001},"max":{"1min":101.44499999999999,"5min":101.44499999999999,"15min":150.63499999999999},"last":5.7430000000000003},{"interface":"front","average":{"1min":11.930999999999999,"5min":3.7890000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":101.465,"5min":101.465,"15min":150.672},"last":1.3600000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":5.7779999999999996,"5min":3.2109999999999999,"15min":3.4060000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":94.197999999999993,"5min":94.197999999999993,"15min":107.346},"last":5.798},{"interface":"front","average":{"1min":6.359,"5min":3.508,"15min":3.6499999999999999},"min":{"1min":0.26100000000000001,"5min":0.253,"15min":0.22900000000000001},"max":{"1min":94.460999999999999,"5min":94.460999999999999,"15min":107.33499999999999},"last":1.4039999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:31:53 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.911,"15min":2.0830000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":5.7270000000000003,"5min":7.6790000000000003,"15min":7.6790000000000003},"last":4.3799999999999999},{"interface":"front","average":{"1min":1.2849999999999999,"5min":1.4890000000000001,"15min":1.5229999999999999},"min":{"1min":0.376,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":8.1159999999999997,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":3.04}]},{"osd":3,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.663,"15min":2.472},"min":{"1min":0.36399999999999999,"5min":0.314,"15min":0.29699999999999999},"max":{"1min":4.202,"5min":48.216999999999999,"15min":102.896},"last":2.9180000000000001},{"interface":"front","average":{"1min":1.423,"5min":1.8069999999999999,"15min":2.589},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":7.875,"5min":48.232999999999997,"15min":112.667},"last":2.9740000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":1.2669999999999999,"15min":1.0860000000000001},"min":{"1min":0.30299999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.0220000000000002,"5min":13.407,"15min":28.399000000000001},"last":4.4089999999999998},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2809999999999999,"15min":1.153},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":11.048999999999999,"5min":14.212,"15min":28.350999999999999},"last":6.149}]},{"osd":6,"last update":"Sat Sep 23 22:31:57 2023","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.80600000000000005,"15min":0.83399999999999996},"min":{"1min":0.30299999999999999,"5min":0.26400000000000001,"15min":0.251},"max":{"1min":3.6859999999999999,"5min":6.0110000000000001,"15min":7.8550000000000004},"last":5.774},{"interface":"front","average":{"1min":1.756,"5min":0.90700000000000003,"15min":0.85099999999999998},"min":{"1min":0.32800000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.6950000000000003,"5min":6.6950000000000003,"15min":8.5210000000000008},"last":4.3140000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":1.03,"15min":0.95399999999999996},"min":{"1min":0.376,"5min":0.307,"15min":0.27500000000000002},"max":{"1min":8.1440000000000001,"5min":8.1440000000000001,"15min":8.577},"last":4.3479999999999999},{"interface":"front","average":{"1min":1.23,"5min":0.89400000000000002,"15min":0.89600000000000002},"min":{"1min":0.32600000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.633,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":2.827}]}]},{"osd":5,"up_from":38,"seq":163208757614,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4198380,"kb_used_data":3795300,"kb_used_omap":0,"kb_used_meta":403072,"kb_avail":89546772,"statfs":{"total":95995035648,"available":91695894528,"internally_reserved":0,"allocated":3886387200,"data_stored":3884940759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":412745728},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":13.648999999999999,"5min":3.9780000000000002,"15min":4.5839999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":124.892,"5min":124.892,"15min":163.54499999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":13.523,"5min":3.528,"15min":4.3230000000000004},"min":{"1min":0.34999999999999998,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":123.807,"5min":123.807,"15min":162.81800000000001},"last":2.3599999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":3.7639999999999998,"15min":3.819},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":7.1440000000000001,"5min":113.514,"15min":113.514},"last":2.6720000000000002},{"interface":"front","average":{"1min":2.323,"5min":3.988,"15min":3.8599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":8.4420000000000002,"5min":113.42700000000001,"15min":113.42700000000001},"last":4.1769999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":2.133,"15min":2.3180000000000001},"min":{"1min":0.34200000000000003,"5min":0.316,"15min":0.316},"max":{"1min":7.2480000000000002,"5min":10.451000000000001,"15min":10.451000000000001},"last":4.1299999999999999},{"interface":"front","average":{"1min":0.83899999999999997,"5min":2.1909999999999998,"15min":2.4159999999999999},"min":{"1min":0.33000000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":3.4249999999999998,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.6539999999999999,"15min":3.7719999999999998},"min":{"1min":0.36099999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":8.5459999999999994,"5min":9.3369999999999997,"15min":132.59},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.7070000000000001,"15min":3.71},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":7.8390000000000004,"5min":7.8390000000000004,"15min":129.381},"last":2.46}]},{"osd":4,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9849999999999999,"5min":1.3620000000000001,"15min":1.508},"min":{"1min":0.27700000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":36.343000000000004,"5min":36.343000000000004,"15min":126.679},"last":1.8660000000000001},{"interface":"front","average":{"1min":3.9740000000000002,"5min":1.377,"15min":1.512},"min":{"1min":0.35899999999999999,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":36.421999999999997,"5min":36.421999999999997,"15min":126.78100000000001},"last":4.4960000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:31:46 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":1.504,"15min":1.456},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.5489999999999999,"5min":70.581999999999994,"15min":114.83499999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.206,"5min":1.8109999999999999,"15min":1.4670000000000001},"min":{"1min":0.32600000000000001,"5min":0.248,"15min":0.248},"max":{"1min":6.2649999999999997,"5min":71.093000000000004,"15min":110.905},"last":2.4020000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":3.7650000000000001,"5min":1.24,"15min":1.107},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":40.548000000000002,"5min":40.548000000000002,"15min":40.548000000000002},"last":2.4340000000000002},{"interface":"front","average":{"1min":3.6970000000000001,"5min":1.3440000000000001,"15min":0.98899999999999999},"min":{"1min":0.38,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":41.073,"5min":41.073,"15min":41.073},"last":1.8979999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3490054144,"data_stored":3490054144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3507163136,"data_stored":3507159064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4562223104,"data_stored":4562223104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3861450752,"data_stored":3861446680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3107524608,"data_stored":3107520536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":352583680,"data_stored":352583680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:32:54.614 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:54.691 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:54.821 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:54.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:54.831 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:54.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:55.001 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:55.078 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:55.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:55.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:55.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:55 smithi026 ceph-mon[215775]: pgmap v1045: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 2.2 MiB/s wr, 595 op/s; 1614/284415 objects degraded (0.567%); 11/284415 objects misplaced (0.004%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:32:55.153 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:55 smithi026 ceph-mon[223014]: pgmap v1045: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 2.2 MiB/s wr, 595 op/s; 1614/284415 objects degraded (0.567%); 11/284415 objects misplaced (0.004%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:32:55.193 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:55.249 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:55.360 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:55.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:55 smithi070 ceph-mon[192652]: pgmap v1045: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.8 GiB data, 20 GiB used, 427 GiB / 447 GiB avail; 2.2 MiB/s wr, 595 op/s; 1614/284415 objects degraded (0.567%); 11/284415 objects misplaced (0.004%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T22:32:55.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:55.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:55.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:55.550 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:55.603 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:32:55.603 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:32:55.604 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:32:55.604 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:32:55.604 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:32:55.640 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:55.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:55.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:55.751 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:32:55.751 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:55.821 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:55.940 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:55.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:55.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:56.053 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:32:56.113 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:56.172 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:32:56.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3747935125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:56.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3747935125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:56.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3747935125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:32:57.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:32:57.118 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:32:57.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:57.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:32:57.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:57.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[215775]: pgmap v1046: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 2.0 MiB/s wr, 1.06k op/s; 308/271968 objects degraded (0.113%); 969/271968 objects misplaced (0.356%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T22:32:57.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:32:57.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[215775]: failed to encode map e405 with expected crc 2023-09-23T22:32:57.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3747935125' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:32:57.252 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[215775]: osdmap e405: 8 total, 8 up, 5 in 2023-09-23T22:32:57.252 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[223014]: pgmap v1046: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 2.0 MiB/s wr, 1.06k op/s; 308/271968 objects degraded (0.113%); 969/271968 objects misplaced (0.356%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T22:32:57.252 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:32:57.252 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[223014]: failed to encode map e405 with expected crc 2023-09-23T22:32:57.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3747935125' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:32:57.253 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:57 smithi026 ceph-mon[223014]: osdmap e405: 8 total, 8 up, 5 in 2023-09-23T22:32:57.254 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:57.304 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:57.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:57.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:57.408 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:32:57.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:57 smithi070 ceph-mon[192652]: pgmap v1046: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 2.0 MiB/s wr, 1.06k op/s; 308/271968 objects degraded (0.113%); 969/271968 objects misplaced (0.356%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T22:32:57.466 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:57 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:32:57.467 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:57 smithi070 ceph-mon[192652]: failed to encode map e405 with expected crc 2023-09-23T22:32:57.467 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3747935125' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:32:57.467 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:57 smithi070 ceph-mon[192652]: osdmap e405: 8 total, 8 up, 5 in 2023-09-23T22:32:57.472 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:57.578 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:32:57.600 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:32:57.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:57.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:32:57.722 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:32:57.780 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:57.888 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:32:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:57.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:32:57.907 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:57.934 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:32:57.999 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:58.065 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:58.298 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:58.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:58.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:58.391 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 308/271968 objects degraded (0.113%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:32:58.393 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 308/271968 objects degraded (0.113%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:32:58.409 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:32:58.466 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 308/271968 objects degraded (0.113%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:32:58.650 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:32:58.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:58.656 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:32:58.756 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:32:58.833 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:58.956 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:32:58.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:32:58.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:32:59.073 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:32:59.118 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:32:59.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:32:59 smithi026 ceph-mon[223014]: pgmap v1048: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 599 op/s; 308/271968 objects degraded (0.113%); 969/271968 objects misplaced (0.356%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:32:59.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:32:59 smithi026 ceph-mon[215775]: pgmap v1048: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 599 op/s; 308/271968 objects degraded (0.113%); 969/271968 objects misplaced (0.356%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:32:59.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:32:59 smithi070 ceph-mon[192652]: pgmap v1048: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 599 op/s; 308/271968 objects degraded (0.113%); 969/271968 objects misplaced (0.356%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T22:32:59.805 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:32:59.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:32:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:32:59] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.43.0" 2023-09-23T22:33:00.471 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:00.482 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:00.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:00.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:00.588 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:00.663 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:00.716 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:00.818 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:00.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:00.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:00.925 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:00.996 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:01.099 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:01.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:01.193 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:01 smithi026 ceph-mon[223014]: pgmap v1049: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 667 op/s; 308/268389 objects degraded (0.115%); 969/268389 objects misplaced (0.361%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:33:01.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:01.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:01 smithi070 ceph-mon[192652]: pgmap v1049: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 667 op/s; 308/268389 objects degraded (0.115%); 969/268389 objects misplaced (0.361%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:33:01.264 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:01.370 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:01.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:01.482 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:01.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:01 smithi026 ceph-mon[215775]: pgmap v1049: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 19 GiB used, 428 GiB / 447 GiB avail; 0 B/s wr, 667 op/s; 308/268389 objects degraded (0.115%); 969/268389 objects misplaced (0.361%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:33:01.538 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:01.635 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:01.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:01.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:01.718 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:33:01.743 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:01.747 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:33:01.747 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:33:01.747 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83436 7.2 7:4436d6c5:::benchmark_data_smithi026_410749_object83435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.874035+0000", 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.87095818799997, 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8534463419999998, 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.748 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83436 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874035+0000", 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874035+0000", 2023-09-23T22:33:01.749 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874034+0000", 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874054+0000", 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0642e-05 2023-09-23T22:33:01.750 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874063+0000", 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9560000000000003e-06 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.874069+0000", 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2400000000000004e-06 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.751 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.109468+0000", 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23539858499999999 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.109634+0000", 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016552 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.109834+0000", 2023-09-23T22:33:01.752 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000200714 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119592+0000", 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0097577359999999995 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119603+0000", 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.13e-05 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.753 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.965993+0000", 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84638983099999998 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966014+0000", 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0774000000000001e-05 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.754 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.688706+0000", 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.722691585 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694732+0000", 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060268170000000003 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.727365+0000", 2023-09-23T22:33:01.755 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032632845000000001 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.727481+0000", 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011591600000000001 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83439 7.2 7:41661071:::benchmark_data_smithi026_410749_object83438:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:33:01.756 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875790+0000", 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.86920308200001, 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8703235449999998, 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83439 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.757 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875790+0000", 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875790+0000", 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875789+0000", 2023-09-23T22:33:01.758 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875812+0000", 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3323999999999999e-05 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875835+0000", 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.317e-05 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.759 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875844+0000", 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4370000000000007e-06 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.113024+0000", 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.237180689 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.760 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.113148+0000", 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012339600000000001 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.113277+0000", 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129271 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119636+0000", 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063592079999999999 2023-09-23T22:33:01.761 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119647+0000", 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0547000000000001e-05 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966064+0000", 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84641742499999995 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.762 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966076+0000", 2023-09-23T22:33:01.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1899000000000001e-05 2023-09-23T22:33:01.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.763 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.763 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.699874+0000", 2023-09-23T22:33:01.763 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.733797451 2023-09-23T22:33:01.763 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.763 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:33:01.765 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.702993+0000", 2023-09-23T22:33:01.765 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031193470000000002 2023-09-23T22:33:01.765 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.765 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.765 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746002+0000", 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.04300913 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746113+0000", 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011135 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.766 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83442 7.c 7:3b003b51:::benchmark_data_smithi026_410749_object83441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.86914254700002, 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8103755399999999, 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83442 2023-09-23T22:33:01.767 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.768 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875850+0000", 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875871+0000", 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0897999999999999e-05 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.769 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875878+0000", 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0140000000000001e-06 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875883+0000", 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4360000000000001e-06 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875932+0000", 2023-09-23T22:33:01.770 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8757000000000001e-05 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.876054+0000", 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122371 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.110153+0000", 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.23409861300000001 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.771 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119005+0000", 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088520830000000002 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119022+0000", 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6586999999999999e-05 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.772 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967848+0000", 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84882608400000004 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967861+0000", 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3498e-05 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.653928+0000", 2023-09-23T22:33:01.773 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6860671979999999 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.654575+0000", 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00064664599999999996 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686123+0000", 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031547598000000003 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.774 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686226+0000", 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000103359 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83443 7.a 7:5676589a:::benchmark_data_smithi026_410749_object83442:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:33:01.775 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.86909564699999, 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8791883110000001, 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83443 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.776 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875897+0000", 2023-09-23T22:33:01.777 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875917+0000", 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0571e-05 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875925+0000", 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4800000000000004e-06 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.778 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875930+0000", 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2549999999999997e-06 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120311+0000", 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.24438064000000001 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.779 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120321+0000", 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0532e-05 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966151+0000", 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84582989600000003 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966161+0000", 2023-09-23T22:33:01.780 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0161e-05 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713553+0000", 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7473917839999999 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713688+0000", 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013500599999999999 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.781 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713846+0000", 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015831800000000001 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720327+0000", 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064802289999999997 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.782 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723835+0000", 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035083179999999999 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.752272+0000", 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028437240999999999 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.752361+0000", 2023-09-23T22:33:01.783 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8768000000000001e-05 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.755086+0000", 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027245949999999998 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83444 7.2 7:4ab81381:::benchmark_data_smithi026_410749_object83443:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:33:01.784 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.86905792900001, 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8732291189999999, 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83444 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.785 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875935+0000", 2023-09-23T22:33:01.786 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875955+0000", 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0437000000000001e-05 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875962+0000", 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2590000000000003e-06 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.787 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:06.875968+0000", 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5539999999999997e-06 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120681+0000", 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.24471289900000001 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.788 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.120690+0000", 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4579999999999999e-06 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966109+0000", 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84541897700000002 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966119+0000", 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4369999999999996e-06 2023-09-23T22:33:01.789 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703041+0000", 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.73692275 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703182+0000", 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014080000000000001 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.790 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703342+0000", 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016031500000000001 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.713519+0000", 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010176648 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.791 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723879+0000", 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010359706 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746208+0000", 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022329188999999999 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.746255+0000", 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6820000000000002e-05 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.792 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.749164+0000", 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029093589999999998 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83449 7.c 7:31b18e39:::benchmark_data_smithi026_410749_object83448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e270)", 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.118334+0000", 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.62665849799998, 2023-09-23T22:33:01.793 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5715964209999997, 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83449 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.794 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118334+0000", 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118334+0000", 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118333+0000", 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:33:01.795 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118364+0000", 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1628999999999998e-05 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118376+0000", 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1888e-05 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.796 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.118385+0000", 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4829999999999999e-06 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119686+0000", 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001300998 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.797 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.119696+0000", 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0208e-05 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967893+0000", 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84819666800000004 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967901+0000", 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8619999999999992e-06 2023-09-23T22:33:01.798 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.657644+0000", 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6897430070000001 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.657855+0000", 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021073899999999999 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.799 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.658074+0000", 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021842699999999999 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664296+0000", 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062224689999999996 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.667607+0000", 2023-09-23T22:33:01.800 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033106160000000002 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686269+0000", 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018662099000000001 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686317+0000", 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8285999999999998e-05 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.801 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.689931+0000", 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036137999999999999 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83454 7.a 7:5caa08ff:::benchmark_data_smithi026_410749_object83453:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.125118+0000", 2023-09-23T22:33:01.802 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.61987449999998, 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.634736432, 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83454 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.803 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125118+0000", 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125118+0000", 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125116+0000", 2023-09-23T22:33:01.804 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125144+0000", 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8147999999999999e-05 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125157+0000", 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2418999999999999e-05 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.805 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125167+0000", 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0088999999999999e-05 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125218+0000", 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1601000000000003e-05 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.806 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.125225+0000", 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0539999999999997e-06 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966187+0000", 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84096177699999997 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.807 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966197+0000", 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3619999999999995e-06 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.716965+0000", 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7507684270000001 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.717105+0000", 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013953199999999999 2023-09-23T22:33:01.808 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.717236+0000", 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000131901 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720350+0000", 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031134539999999999 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.809 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.734562+0000", 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014212158000000001 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.756983+0000", 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022420549000000001 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.810 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.757031+0000", 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8467999999999997e-05 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.759855+0000", 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002823791 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.811 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83455 7.c 7:30f89560:::benchmark_data_smithi026_410749_object83454:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.128196+0000", 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.61679685000001, 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5655597960000001, 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83455 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.812 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128196+0000", 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128196+0000", 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.813 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128195+0000", 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128218+0000", 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3824000000000001e-05 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.814 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128235+0000", 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6864e-05 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128245+0000", 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6709999999999994e-06 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128304+0000", 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9104000000000002e-05 2023-09-23T22:33:01.815 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.128316+0000", 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1858999999999999e-05 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967930+0000", 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83961362900000003 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.816 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.967939+0000", 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0000000000000002e-06 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664326+0000", 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.696387901 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664467+0000", 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140433 2023-09-23T22:33:01.817 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.664596+0000", 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129345 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.667813+0000", 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032163579999999999 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.818 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.671373+0000", 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035605559999999999 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.690178+0000", 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018804521000000001 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.819 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.690239+0000", 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1784000000000003e-05 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.693756+0000", 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035164060000000001 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83459 7.c 7:389583a7:::benchmark_data_smithi026_410749_object83458:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:33:01.820 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.131267+0000", 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.61372542700002, 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5663957699999997, 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83459 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.821 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131267+0000", 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131267+0000", 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.822 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131266+0000", 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131287+0000", 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0669000000000001e-05 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131296+0000", 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3179999999999996e-06 2023-09-23T22:33:01.823 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131303+0000", 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3890000000000002e-06 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131356+0000", 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2883999999999997e-05 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.824 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131367+0000", 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1501000000000001e-05 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.968650+0000", 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83728242900000005 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.968662+0000", 2023-09-23T22:33:01.825 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2218999999999999e-05 2023-09-23T22:33:01.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.667852+0000", 2023-09-23T22:33:01.826 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6991896950000001 2023-09-23T22:33:01.826 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.826 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.826 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.826 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.668041+0000", 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018923599999999999 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.668259+0000", 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000218063 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.671597+0000", 2023-09-23T22:33:01.827 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033378510000000002 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.675784+0000", 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004187362 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.693833+0000", 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018049193000000002 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.828 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.693899+0000", 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5381000000000003e-05 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.697663+0000", 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037646799999999998 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.829 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83460 7.2 7:4c76e5c3:::benchmark_data_smithi026_410749_object83459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 414.61309658800002, 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6264806219999999, 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.830 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83460 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:33:01.831 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131896+0000", 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131915+0000", 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9627999999999999e-05 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.832 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131927+0000", 2023-09-23T22:33:01.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2150999999999999e-05 2023-09-23T22:33:01.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.833 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.833 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.833 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131933+0000", 2023-09-23T22:33:01.833 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2390000000000002e-06 2023-09-23T22:33:01.833 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131970+0000", 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7323000000000002e-05 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.131979+0000", 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1439999999999992e-06 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.834 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966248+0000", 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.83426858500000001 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.966259+0000", 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0813e-05 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.835 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.709897+0000", 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.743638695 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.710090+0000", 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000192552 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.710317+0000", 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022765700000000001 2023-09-23T22:33:01.836 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720308+0000", 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099907369999999995 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723922+0000", 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036138960000000001 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.837 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.755137+0000", 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031214848999999999 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.755202+0000", 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4978e-05 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.838 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.758377+0000", 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031750530000000002 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83432 7.8 7:102b4fc4:::benchmark_data_smithi026_410749_object83431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978262+0000", 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.76673130299997, 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7098934809999999, 2023-09-23T22:33:01.839 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83432 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978262+0000", 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.840 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978262+0000", 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978260+0000", 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.841 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978316+0000", 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6307000000000001e-05 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978331+0000", 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4329e-05 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.842 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978341+0000", 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0179e-05 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978398+0000", 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7342000000000001e-05 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.843 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978405+0000", 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2559999999999996e-06 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.686224+0000", 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7078187279999999 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.844 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.688155+0000", 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019310600000000001 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83434 7.8 7:1cb65715:::benchmark_data_smithi026_410749_object83433:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978364+0000", 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.76662893000002, 2023-09-23T22:33:01.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7126752629999999, 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83434 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978364+0000", 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978364+0000", 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978363+0000", 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978410+0000", 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6882999999999998e-05 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978421+0000", 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0555e-05 2023-09-23T22:33:01.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978428+0000", 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.447e-06 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978481+0000", 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2367000000000002e-05 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978491+0000", 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0348e-05 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.690940+0000", 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.712448669 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691039+0000", 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9671000000000002e-05 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83451 7.8 7:10ddb7e8:::benchmark_data_smithi026_410749_object83450:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978444+0000", 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.766549097, 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7421942690000001, 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.851 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83451 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978444+0000", 2023-09-23T22:33:01.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978444+0000", 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978443+0000", 2023-09-23T22:33:01.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:01.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978490+0000", 2023-09-23T22:33:01.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7082000000000001e-05 2023-09-23T22:33:01.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978500+0000", 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9930000000000007e-06 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978507+0000", 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8789999999999997e-06 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978649+0000", 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014189399999999999 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978658+0000", 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2750000000000001e-06 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691064+0000", 2023-09-23T22:33:01.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.712405483 2023-09-23T22:33:01.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691807+0000", 2023-09-23T22:33:01.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00074337199999999996 2023-09-23T22:33:01.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T22:33:01.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.691974+0000", 2023-09-23T22:33:01.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016714799999999999 2023-09-23T22:33:01.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.698213+0000", 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006238848 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.704903+0000", 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066894340000000002 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705854+0000", 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00095168 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.718111+0000", 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012256306999999999 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.718166+0000", 2023-09-23T22:33:01.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5387999999999999e-05 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720638+0000", 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024721159999999999 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83458 7.8 7:14196f44:::benchmark_data_smithi026_410749_object83457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:33:01.861 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 413.76647079000003, 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.744606302, 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83458 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978522+0000", 2023-09-23T22:33:01.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978566+0000", 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4564000000000002e-05 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978585+0000", 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8978999999999999e-05 2023-09-23T22:33:01.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978592+0000", 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1250000000000004e-06 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978682+0000", 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0093999999999995e-05 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.978690+0000", 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3200000000000002e-06 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694418+0000", 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.715727904 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694629+0000", 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021179900000000001 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,7", 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.694812+0000", 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018232099999999999 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.701814+0000", 2023-09-23T22:33:01.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070020819999999998 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705861+0000", 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040473699999999998 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.711791+0000", 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059297259999999997 2023-09-23T22:33:01.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720680+0000", 2023-09-23T22:33:01.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088894569999999999 2023-09-23T22:33:01.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720735+0000", 2023-09-23T22:33:01.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5049000000000001e-05 2023-09-23T22:33:01.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.723128+0000", 2023-09-23T22:33:01.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002393109 2023-09-23T22:33:01.871 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.871 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.871 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.872 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114666 7.a 7:5b4d1c8f:::benchmark_data_smithi026_410749_object24329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:33:01.872 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.103730+0000", 2023-09-23T22:33:01.872 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.641262864, 2023-09-23T22:33:01.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0325042339999999, 2023-09-23T22:33:01.872 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.872 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:33:01.872 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.872 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.873 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.873 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114666 2023-09-23T22:33:01.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.873 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103730+0000", 2023-09-23T22:33:01.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103730+0000", 2023-09-23T22:33:01.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103729+0000", 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103735+0000", 2023-09-23T22:33:01.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.331e-06 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103747+0000", 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1623999999999999e-05 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.103755+0000", 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2430000000000005e-06 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.130376+0000", 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026621280000000001 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.130574+0000", 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019795899999999999 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.135541+0000", 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049666739999999999 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141802+0000", 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062614180000000004 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.144696+0000", 2023-09-23T22:33:01.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028941489999999999 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.283811+0000", 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.139114184 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.283833+0000", 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1937e-05 2023-09-23T22:33:01.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.136221+0000", 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.85238819899999996 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.136234+0000", 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3553000000000001e-05 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.880 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114679 7.3 7:c9482514:::benchmark_data_smithi026_410749_object24342:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.138279+0000", 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.60671377699998, 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99640990200000001, 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.881 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114679 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138279+0000", 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138279+0000", 2023-09-23T22:33:01.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138278+0000", 2023-09-23T22:33:01.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:01.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138285+0000", 2023-09-23T22:33:01.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4590000000000001e-06 2023-09-23T22:33:01.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138298+0000", 2023-09-23T22:33:01.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3152999999999999e-05 2023-09-23T22:33:01.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138307+0000", 2023-09-23T22:33:01.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7600000000000008e-06 2023-09-23T22:33:01.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138382+0000", 2023-09-23T22:33:01.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4432999999999997e-05 2023-09-23T22:33:01.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138592+0000", 2023-09-23T22:33:01.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020998499999999999 2023-09-23T22:33:01.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:33:01.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138787+0000", 2023-09-23T22:33:01.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019538800000000001 2023-09-23T22:33:01.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.286686+0000", 2023-09-23T22:33:01.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.147899482 2023-09-23T22:33:01.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.286694+0000", 2023-09-23T22:33:01.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7000000000000008e-06 2023-09-23T22:33:01.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.134671+0000", 2023-09-23T22:33:01.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84797734300000005 2023-09-23T22:33:01.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.134689+0000", 2023-09-23T22:33:01.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7621e-05 2023-09-23T22:33:01.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.891 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.892 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114682 7.3 7:c74ef5a2:::benchmark_data_smithi026_410749_object24345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T22:33:01.892 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:26.138323+0000", 2023-09-23T22:33:01.892 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.60666995499997, 2023-09-23T22:33:01.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99642287399999996, 2023-09-23T22:33:01.892 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.893 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:33:01.893 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.893 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.893 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.893 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114682 2023-09-23T22:33:01.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.894 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138323+0000", 2023-09-23T22:33:01.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138323+0000", 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138322+0000", 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138326+0000", 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1779999999999999e-06 2023-09-23T22:33:01.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138333+0000", 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5440000000000001e-06 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.138339+0000", 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8529999999999997e-06 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.141825+0000", 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034864309999999999 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.142018+0000", 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019217 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4", 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.142174+0000", 2023-09-23T22:33:01.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015678799999999999 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.292592+0000", 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.15041737199999999 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:26.292603+0000", 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.128e-05 2023-09-23T22:33:01.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.134733+0000", 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84213020299999997 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:27.134746+0000", 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2549e-05 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.899 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114854 7.a 7:514305bc:::benchmark_data_smithi026_410749_object24517:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e305)", 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:52.042477+0000", 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 309.70251641700003, 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.374932499, 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.900 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114854 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042477+0000", 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042477+0000", 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042475+0000", 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042482+0000", 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6210000000000004e-06 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042496+0000", 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3213e-05 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042508+0000", 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2310000000000001e-05 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043832+0000", 2023-09-23T22:33:01.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013236859999999999 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043845+0000", 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3722e-05 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.392953+0000", 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3491075079999999 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393122+0000", 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016948199999999999 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.417296+0000", 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024174047000000001 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.417409+0000", 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112722 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114867 7.a 7:5bbaaced:::benchmark_data_smithi026_410749_object24530:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e305)", 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:52.042515+0000", 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 309.702477907, 2023-09-23T22:33:01.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3830621110000001, 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114867 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042515+0000", 2023-09-23T22:33:01.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042515+0000", 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042514+0000", 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042518+0000", 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0249999999999998e-06 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042525+0000", 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1919999999999999e-06 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042531+0000", 2023-09-23T22:33:01.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8989999999999998e-06 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043871+0000", 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00134023 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043879+0000", 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7200000000000006e-06 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393153+0000", 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3492744459999999 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393299+0000", 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145838 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393456+0000", 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000156928 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.400332+0000", 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068758229999999997 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:33:01.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.400343+0000", 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0963e-05 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.417452+0000", 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017109017000000001 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421935+0000", 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044831059999999997 2023-09-23T22:33:01.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421995+0000", 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9889000000000002e-05 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.425577+0000", 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035825710000000001 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.914 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.915 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.915 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:33:01.915 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:33:01.915 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:33:01.916 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:33:01.916 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:33:01.916 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:33:01.916 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:33:01.916 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:33:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:01.917 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:01.917 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:01.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:01.953 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:33:01.987 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:33:02.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:33:02.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:33:02.010 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:33:02.011 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:33:02.011 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:33:02.011 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:33:02.011 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:33:02.011 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:33:02.018 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:02.105 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:02.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:02.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:02.309 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:02.367 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2655706777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:02.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2655706777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:02.374 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:02.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2655706777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:03.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:03 smithi026 ceph-mon[223014]: pgmap v1050: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 899 op/s; 308/257631 objects degraded (0.120%); 559/257631 objects misplaced (0.217%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:03.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 308/257631 objects degraded (0.120%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:33:03.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:03 smithi026 ceph-mon[215775]: pgmap v1050: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 899 op/s; 308/257631 objects degraded (0.120%); 559/257631 objects misplaced (0.217%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:03.460 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 308/257631 objects degraded (0.120%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:33:03.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:03 smithi070 ceph-mon[192652]: pgmap v1050: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 899 op/s; 308/257631 objects degraded (0.120%); 559/257631 objects misplaced (0.217%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:03.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 308/257631 objects degraded (0.120%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:33:04.176 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:04.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:04.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:04.281 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:04.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:04.942 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:04.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:04.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:04.998 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:33:05.048 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:05.148 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:05.211 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:05 smithi026 ceph-mon[223014]: pgmap v1051: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 892 op/s; 308/257631 objects degraded (0.120%); 559/257631 objects misplaced (0.217%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:33:05.214 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:05 smithi026 ceph-mon[215775]: pgmap v1051: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 892 op/s; 308/257631 objects degraded (0.120%); 559/257631 objects misplaced (0.217%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:33:05.275 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:05.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:05.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:05 smithi070 ceph-mon[192652]: pgmap v1051: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 5.2 GiB data, 18 GiB used, 429 GiB / 447 GiB avail; 0 B/s wr, 892 op/s; 308/257631 objects degraded (0.120%); 559/257631 objects misplaced (0.217%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:33:05.390 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:05.458 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:05.557 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:05.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:05.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:05.723 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:05.743 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:05.845 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:05.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:05.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:05.880 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:33:05.886 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:33:05.955 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:05.990 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:06.031 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:33:06.254 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:33:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/42120280' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:06.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:06 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/42120280' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:06.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:06 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:06.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/42120280' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:06.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:06 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:06.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:06.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:06.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:06.630 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:06.698 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:07.309 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:07.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:07.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:07.413 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:07.453 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: pgmap v1052: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 308/242970 objects degraded (0.127%); 208/242970 objects misplaced (0.086%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:07.453 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: from='client.45968 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:33:07.453 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:07.453 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.453 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:33:07.453 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: osdmap e406: 8 total, 8 up, 5 in 2023-09-23T22:33:07.454 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.454 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.454 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.454 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[215775]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: pgmap v1052: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 308/242970 objects degraded (0.127%); 208/242970 objects misplaced (0.086%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:07.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: from='client.45968 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:33:07.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:07.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.454 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:33:07.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: osdmap e406: 8 total, 8 up, 5 in 2023-09-23T22:33:07.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:07 smithi026 ceph-mon[223014]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.461 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:07.489 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: pgmap v1052: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 308/242970 objects degraded (0.127%); 208/242970 objects misplaced (0.086%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:07.489 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: from='client.45968 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:33:07.489 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:07.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:33:07.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: osdmap e406: 8 total, 8 up, 5 in 2023-09-23T22:33:07.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:07 smithi070 ceph-mon[192652]: failed to encode map e406 with expected crc 2023-09-23T22:33:07.493 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1052,"stamp":"2023-09-23T22:33:05.588257+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5310650315,"num_objects":80990,"num_object_clones":0,"num_object_copies":242970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":308,"num_objects_misplaced":208,"num_objects_unfound":0,"num_objects_dirty":80990,"num_whiteouts":0,"num_read":5983,"num_read_kb":12185,"num_write":117725,"num_write_kb":6341622,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10844,"num_bytes_recovered":710639979,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85120,"ondisk_log_size":85120,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":18053664,"kb_used_data":15671416,"kb_used_omap":15,"kb_used_meta":2382000,"kb_avail":450672096,"statfs":{"total":479975178240,"available":461488226304,"internally_reserved":0,"allocated":16047529984,"data_stored":16038821872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2439168222},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-633470976,"num_objects":-9666,"num_object_clones":0,"num_object_copies":-28998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-761,"num_objects_unfound":0,"num_objects_dirty":-9666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9666,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":761,"num_bytes_recovered":49872896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002979"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":432,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669186+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:32:51.669186+0000","last_peered":"2023-09-23T22:32:51.669186+0000","last_clean":"2023-09-23T22:32:51.669186+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:32:51.669186+0000","last_undegraded":"2023-09-23T22:32:51.669186+0000","last_fullsized":"2023-09-23T22:32:51.669186+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":122,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:54.419958+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:32:54.419958+0000","last_peered":"2023-09-23T22:32:54.419958+0000","last_clean":"2023-09-23T22:32:54.419958+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:32:54.419958+0000","last_undegraded":"2023-09-23T22:32:54.419958+0000","last_fullsized":"2023-09-23T22:32:54.419958+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":122,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.670211+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:32:51.670211+0000","last_peered":"2023-09-23T22:32:51.670211+0000","last_clean":"2023-09-23T22:32:51.670211+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:32:51.670211+0000","last_undegraded":"2023-09-23T22:32:51.670211+0000","last_fullsized":"2023-09-23T22:32:51.670211+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":293,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.665817+0000","last_change":"2023-09-23T22:31:28.120861+0000","last_active":"2023-09-23T22:32:51.665817+0000","last_peered":"2023-09-23T22:32:51.665817+0000","last_clean":"2023-09-23T22:32:51.665817+0000","last_became_active":"2023-09-23T22:31:28.120394+0000","last_became_peered":"2023-09-23T22:31:28.120394+0000","last_unstale":"2023-09-23T22:32:51.665817+0000","last_undegraded":"2023-09-23T22:32:51.665817+0000","last_fullsized":"2023-09-23T22:32:51.665817+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:19.470258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":460,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.663414+0000","last_change":"2023-09-23T22:31:28.114528+0000","last_active":"2023-09-23T22:32:51.663414+0000","last_peered":"2023-09-23T22:32:51.663414+0000","last_clean":"2023-09-23T22:32:51.663414+0000","last_became_active":"2023-09-23T22:31:28.114177+0000","last_became_peered":"2023-09-23T22:31:28.114177+0000","last_unstale":"2023-09-23T22:32:51.663414+0000","last_undegraded":"2023-09-23T22:32:51.663414+0000","last_fullsized":"2023-09-23T22:32:51.663414+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:46.680433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":86,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.672136+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:32:51.672136+0000","last_peered":"2023-09-23T22:32:51.672136+0000","last_clean":"2023-09-23T22:32:51.672136+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:32:51.672136+0000","last_undegraded":"2023-09-23T22:32:51.672136+0000","last_fullsized":"2023-09-23T22:32:51.672136+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":321,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.664276+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:32:51.664276+0000","last_peered":"2023-09-23T22:32:51.664276+0000","last_clean":"2023-09-23T22:32:51.664276+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:32:51.664276+0000","last_undegraded":"2023-09-23T22:32:51.664276+0000","last_fullsized":"2023-09-23T22:32:51.664276+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671133+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:32:51.671133+0000","last_peered":"2023-09-23T22:32:51.671133+0000","last_clean":"2023-09-23T22:32:51.671133+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:32:51.671133+0000","last_undegraded":"2023-09-23T22:32:51.671133+0000","last_fullsized":"2023-09-23T22:32:51.671133+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":405,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668460+0000","last_change":"2023-09-23T22:31:28.116916+0000","last_active":"2023-09-23T22:32:51.668460+0000","last_peered":"2023-09-23T22:32:51.668460+0000","last_clean":"2023-09-23T22:32:51.668460+0000","last_became_active":"2023-09-23T22:31:28.116577+0000","last_became_peered":"2023-09-23T22:31:28.116577+0000","last_unstale":"2023-09-23T22:32:51.668460+0000","last_undegraded":"2023-09-23T22:32:51.668460+0000","last_fullsized":"2023-09-23T22:32:51.668460+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:33.092789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":96,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671343+0000","last_change":"2023-09-23T22:31:28.122684+0000","last_active":"2023-09-23T22:32:51.671343+0000","last_peered":"2023-09-23T22:32:51.671343+0000","last_clean":"2023-09-23T22:32:51.671343+0000","last_became_active":"2023-09-23T22:31:28.122427+0000","last_became_peered":"2023-09-23T22:31:28.122427+0000","last_unstale":"2023-09-23T22:32:51.671343+0000","last_undegraded":"2023-09-23T22:32:51.671343+0000","last_fullsized":"2023-09-23T22:32:51.671343+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:15.559367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":152,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669608+0000","last_change":"2023-09-23T22:31:28.153199+0000","last_active":"2023-09-23T22:32:51.669608+0000","last_peered":"2023-09-23T22:32:51.669608+0000","last_clean":"2023-09-23T22:32:51.669608+0000","last_became_active":"2023-09-23T22:31:28.129257+0000","last_became_peered":"2023-09-23T22:31:28.129257+0000","last_unstale":"2023-09-23T22:32:51.669608+0000","last_undegraded":"2023-09-23T22:32:51.669608+0000","last_fullsized":"2023-09-23T22:32:51.669608+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:03.743836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":54,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:32:56.508369+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:32:56.508369+0000","last_peered":"2023-09-23T22:32:56.508369+0000","last_clean":"2023-09-23T22:32:56.508369+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:32:56.508369+0000","last_undegraded":"2023-09-23T22:32:56.508369+0000","last_fullsized":"2023-09-23T22:32:56.508369+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":489,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669081+0000","last_change":"2023-09-23T22:31:28.145916+0000","last_active":"2023-09-23T22:32:51.669081+0000","last_peered":"2023-09-23T22:32:51.669081+0000","last_clean":"2023-09-23T22:32:51.669081+0000","last_became_active":"2023-09-23T22:31:28.120703+0000","last_became_peered":"2023-09-23T22:31:28.120703+0000","last_unstale":"2023-09-23T22:32:51.669081+0000","last_undegraded":"2023-09-23T22:32:51.669081+0000","last_fullsized":"2023-09-23T22:32:51.669081+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:07.819247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":403,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669536+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:32:51.669536+0000","last_peered":"2023-09-23T22:32:51.669536+0000","last_clean":"2023-09-23T22:32:51.669536+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:32:51.669536+0000","last_undegraded":"2023-09-23T22:32:51.669536+0000","last_fullsized":"2023-09-23T22:32:51.669536+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":396,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669627+0000","last_change":"2023-09-23T22:31:28.149149+0000","last_active":"2023-09-23T22:32:51.669627+0000","last_peered":"2023-09-23T22:32:51.669627+0000","last_clean":"2023-09-23T22:32:51.669627+0000","last_became_active":"2023-09-23T22:31:28.121120+0000","last_became_peered":"2023-09-23T22:31:28.121120+0000","last_unstale":"2023-09-23T22:32:51.669627+0000","last_undegraded":"2023-09-23T22:32:51.669627+0000","last_fullsized":"2023-09-23T22:32:51.669627+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.434137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":367,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669774+0000","last_change":"2023-09-23T22:31:28.119520+0000","last_active":"2023-09-23T22:32:51.669774+0000","last_peered":"2023-09-23T22:32:51.669774+0000","last_clean":"2023-09-23T22:32:51.669774+0000","last_became_active":"2023-09-23T22:31:28.118810+0000","last_became_peered":"2023-09-23T22:31:28.118810+0000","last_unstale":"2023-09-23T22:32:51.669774+0000","last_undegraded":"2023-09-23T22:32:51.669774+0000","last_fullsized":"2023-09-23T22:32:51.669774+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:36.007062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":311,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:32:56.174550+0000","last_change":"2023-09-23T22:31:28.114537+0000","last_active":"2023-09-23T22:32:56.174550+0000","last_peered":"2023-09-23T22:32:56.174550+0000","last_clean":"2023-09-23T22:32:56.174550+0000","last_became_active":"2023-09-23T22:31:28.114176+0000","last_became_peered":"2023-09-23T22:31:28.114176+0000","last_unstale":"2023-09-23T22:32:56.174550+0000","last_undegraded":"2023-09-23T22:32:56.174550+0000","last_fullsized":"2023-09-23T22:32:56.174550+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.324173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":414,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669051+0000","last_change":"2023-09-23T22:31:28.132494+0000","last_active":"2023-09-23T22:32:51.669051+0000","last_peered":"2023-09-23T22:32:51.669051+0000","last_clean":"2023-09-23T22:32:51.669051+0000","last_became_active":"2023-09-23T22:31:28.132197+0000","last_became_peered":"2023-09-23T22:31:28.132197+0000","last_unstale":"2023-09-23T22:32:51.669051+0000","last_undegraded":"2023-09-23T22:32:51.669051+0000","last_fullsized":"2023-09-23T22:32:51.669051+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:36.181016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":348,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669155+0000","last_change":"2023-09-23T22:31:28.119156+0000","last_active":"2023-09-23T22:32:51.669155+0000","last_peered":"2023-09-23T22:32:51.669155+0000","last_clean":"2023-09-23T22:32:51.669155+0000","last_became_active":"2023-09-23T22:31:28.118382+0000","last_became_peered":"2023-09-23T22:31:28.118382+0000","last_unstale":"2023-09-23T22:32:51.669155+0000","last_undegraded":"2023-09-23T22:32:51.669155+0000","last_fullsized":"2023-09-23T22:32:51.669155+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:10.140079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":301,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.666028+0000","last_change":"2023-09-23T22:31:28.121397+0000","last_active":"2023-09-23T22:32:51.666028+0000","last_peered":"2023-09-23T22:32:51.666028+0000","last_clean":"2023-09-23T22:32:51.666028+0000","last_became_active":"2023-09-23T22:31:28.120847+0000","last_became_peered":"2023-09-23T22:31:28.120847+0000","last_unstale":"2023-09-23T22:32:51.666028+0000","last_undegraded":"2023-09-23T22:32:51.666028+0000","last_fullsized":"2023-09-23T22:32:51.666028+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:42.407658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":96,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671045+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:32:51.671045+0000","last_peered":"2023-09-23T22:32:51.671045+0000","last_clean":"2023-09-23T22:32:51.671045+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:32:51.671045+0000","last_undegraded":"2023-09-23T22:32:51.671045+0000","last_fullsized":"2023-09-23T22:32:51.671045+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":96,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.672052+0000","last_change":"2023-09-23T22:31:28.128965+0000","last_active":"2023-09-23T22:32:51.672052+0000","last_peered":"2023-09-23T22:32:51.672052+0000","last_clean":"2023-09-23T22:32:51.672052+0000","last_became_active":"2023-09-23T22:31:28.128224+0000","last_became_peered":"2023-09-23T22:31:28.128224+0000","last_unstale":"2023-09-23T22:32:51.672052+0000","last_undegraded":"2023-09-23T22:32:51.672052+0000","last_fullsized":"2023-09-23T22:32:51.672052+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:11.301143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"405'7317","reported_seq":7357,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:05.120879+0000","last_change":"2023-09-23T22:31:08.226781+0000","last_active":"2023-09-23T22:33:05.120879+0000","last_peered":"2023-09-23T22:33:05.120879+0000","last_clean":"2023-09-23T22:33:05.120879+0000","last_became_active":"2023-09-23T22:31:08.226511+0000","last_became_peered":"2023-09-23T22:31:08.226511+0000","last_unstale":"2023-09-23T22:33:05.120879+0000","last_undegraded":"2023-09-23T22:33:05.120879+0000","last_fullsized":"2023-09-23T22:33:05.120879+0000","mapping_epoch":373,"log_start":"384'1800","ondisk_log_start":"384'1800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5517,"log_dups_size":0,"ondisk_log_size":5517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:03.735360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329580544,"num_objects":5029,"num_object_clones":0,"num_object_copies":15087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7317,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":414,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.670105+0000","last_change":"2023-09-23T22:31:28.139498+0000","last_active":"2023-09-23T22:32:51.670105+0000","last_peered":"2023-09-23T22:32:51.670105+0000","last_clean":"2023-09-23T22:32:51.670105+0000","last_became_active":"2023-09-23T22:31:28.113206+0000","last_became_peered":"2023-09-23T22:31:28.113206+0000","last_unstale":"2023-09-23T22:32:51.670105+0000","last_undegraded":"2023-09-23T22:32:51.670105+0000","last_fullsized":"2023-09-23T22:32:51.670105+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:11.905312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":407,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668137+0000","last_change":"2023-09-23T22:31:28.131730+0000","last_active":"2023-09-23T22:32:51.668137+0000","last_peered":"2023-09-23T22:32:51.668137+0000","last_clean":"2023-09-23T22:32:51.668137+0000","last_became_active":"2023-09-23T22:31:28.131383+0000","last_became_peered":"2023-09-23T22:31:28.131383+0000","last_unstale":"2023-09-23T22:32:51.668137+0000","last_undegraded":"2023-09-23T22:32:51.668137+0000","last_fullsized":"2023-09-23T22:32:51.668137+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:56.993827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"405'7365","reported_seq":10608,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:05.072202+0000","last_change":"2023-09-23T22:32:00.147475+0000","last_active":"2023-09-23T22:33:05.072202+0000","last_peered":"2023-09-23T22:33:05.072202+0000","last_clean":"2023-09-23T22:33:05.072202+0000","last_became_active":"2023-09-23T22:32:00.121320+0000","last_became_peered":"2023-09-23T22:32:00.121320+0000","last_unstale":"2023-09-23T22:33:05.072202+0000","last_undegraded":"2023-09-23T22:33:05.072202+0000","last_fullsized":"2023-09-23T22:33:05.072202+0000","mapping_epoch":392,"log_start":"385'2300","ondisk_log_start":"385'2300","created":373,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5065,"log_dups_size":0,"ondisk_log_size":5065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:17:59.934876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332988416,"num_objects":5081,"num_object_clones":0,"num_object_copies":15243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7365,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1591,"num_bytes_recovered":104267776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":356,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669558+0000","last_change":"2023-09-23T22:31:28.125280+0000","last_active":"2023-09-23T22:32:51.669558+0000","last_peered":"2023-09-23T22:32:51.669558+0000","last_clean":"2023-09-23T22:32:51.669558+0000","last_became_active":"2023-09-23T22:31:28.110856+0000","last_became_peered":"2023-09-23T22:31:28.110856+0000","last_unstale":"2023-09-23T22:32:51.669558+0000","last_undegraded":"2023-09-23T22:32:51.669558+0000","last_fullsized":"2023-09-23T22:32:51.669558+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:32.197056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":432,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669078+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:32:51.669078+0000","last_peered":"2023-09-23T22:32:51.669078+0000","last_clean":"2023-09-23T22:32:51.669078+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:32:51.669078+0000","last_undegraded":"2023-09-23T22:32:51.669078+0000","last_fullsized":"2023-09-23T22:32:51.669078+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"405'7354","reported_seq":9600,"reported_epoch":405,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:33:04.977664+0000","last_change":"2023-09-23T22:32:50.276838+0000","last_active":"2023-09-23T22:33:04.977664+0000","last_peered":"2023-09-23T22:33:04.977664+0000","last_clean":"2023-09-23T22:31:27.059725+0000","last_became_active":"2023-09-23T22:31:29.178921+0000","last_became_peered":"2023-09-23T22:31:29.178921+0000","last_unstale":"2023-09-23T22:33:04.977664+0000","last_undegraded":"2023-09-23T22:31:29.114095+0000","last_fullsized":"2023-09-23T22:31:29.105229+0000","mapping_epoch":381,"log_start":"384'1900","ondisk_log_start":"384'1900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5454,"log_dups_size":0,"ondisk_log_size":5454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:28.838973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329777152,"num_objects":5032,"num_object_clones":0,"num_object_copies":15096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":308,"num_objects_misplaced":208,"num_objects_unfound":0,"num_objects_dirty":5032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7354,"num_write_kb":396352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1098,"num_bytes_recovered":71958528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":4516},{"shards":"0,6,7","objects":208}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":372,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669136+0000","last_change":"2023-09-23T22:31:28.136321+0000","last_active":"2023-09-23T22:32:51.669136+0000","last_peered":"2023-09-23T22:32:51.669136+0000","last_clean":"2023-09-23T22:32:51.669136+0000","last_became_active":"2023-09-23T22:31:28.131102+0000","last_became_peered":"2023-09-23T22:31:28.131102+0000","last_unstale":"2023-09-23T22:32:51.669136+0000","last_undegraded":"2023-09-23T22:32:51.669136+0000","last_fullsized":"2023-09-23T22:32:51.669136+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:24.768099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":96,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671331+0000","last_change":"2023-09-23T22:31:28.125138+0000","last_active":"2023-09-23T22:32:51.671331+0000","last_peered":"2023-09-23T22:32:51.671331+0000","last_clean":"2023-09-23T22:32:51.671331+0000","last_became_active":"2023-09-23T22:31:28.124716+0000","last_became_peered":"2023-09-23T22:31:28.124716+0000","last_unstale":"2023-09-23T22:32:51.671331+0000","last_undegraded":"2023-09-23T22:32:51.671331+0000","last_fullsized":"2023-09-23T22:32:51.671331+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.718931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"405'7395","reported_seq":7435,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:05.061720+0000","last_change":"2023-09-23T22:31:08.225591+0000","last_active":"2023-09-23T22:33:05.061720+0000","last_peered":"2023-09-23T22:33:05.061720+0000","last_clean":"2023-09-23T22:33:05.061720+0000","last_became_active":"2023-09-23T22:31:08.225286+0000","last_became_peered":"2023-09-23T22:31:08.225286+0000","last_unstale":"2023-09-23T22:33:05.061720+0000","last_undegraded":"2023-09-23T22:33:05.061720+0000","last_fullsized":"2023-09-23T22:33:05.061720+0000","mapping_epoch":373,"log_start":"385'2300","ondisk_log_start":"385'2300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5095,"log_dups_size":0,"ondisk_log_size":5095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:14.895634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332988416,"num_objects":5081,"num_object_clones":0,"num_object_copies":15243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7395,"num_write_kb":399232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":435,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669931+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:32:51.669931+0000","last_peered":"2023-09-23T22:32:51.669931+0000","last_clean":"2023-09-23T22:32:51.669931+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:32:51.669931+0000","last_undegraded":"2023-09-23T22:32:51.669931+0000","last_fullsized":"2023-09-23T22:32:51.669931+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":291,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:32:56.174969+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:32:56.174969+0000","last_peered":"2023-09-23T22:32:56.174969+0000","last_clean":"2023-09-23T22:32:56.174969+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:32:56.174969+0000","last_undegraded":"2023-09-23T22:32:56.174969+0000","last_fullsized":"2023-09-23T22:32:56.174969+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"405'7098","reported_seq":7138,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:05.128668+0000","last_change":"2023-09-23T22:31:08.226199+0000","last_active":"2023-09-23T22:33:05.128668+0000","last_peered":"2023-09-23T22:33:05.128668+0000","last_clean":"2023-09-23T22:33:05.128668+0000","last_became_active":"2023-09-23T22:31:08.225930+0000","last_became_peered":"2023-09-23T22:31:08.225930+0000","last_unstale":"2023-09-23T22:33:05.128668+0000","last_undegraded":"2023-09-23T22:33:05.128668+0000","last_fullsized":"2023-09-23T22:33:05.128668+0000","mapping_epoch":373,"log_start":"383'1600","ondisk_log_start":"383'1600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5498,"log_dups_size":0,"ondisk_log_size":5498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:16.294443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322830336,"num_objects":4926,"num_object_clones":0,"num_object_copies":14778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7098,"num_write_kb":384768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":750,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668115+0000","last_change":"2023-09-23T22:31:31.125903+0000","last_active":"2023-09-23T22:32:51.668115+0000","last_peered":"2023-09-23T22:32:51.668115+0000","last_clean":"2023-09-23T22:32:51.668115+0000","last_became_active":"2023-09-23T22:31:31.125630+0000","last_became_peered":"2023-09-23T22:31:31.125630+0000","last_unstale":"2023-09-23T22:32:51.668115+0000","last_undegraded":"2023-09-23T22:32:51.668115+0000","last_fullsized":"2023-09-23T22:32:51.668115+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:24.405922+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":237,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.664470+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:32:51.664470+0000","last_peered":"2023-09-23T22:32:51.664470+0000","last_clean":"2023-09-23T22:32:51.664470+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:32:51.664470+0000","last_undegraded":"2023-09-23T22:32:51.664470+0000","last_fullsized":"2023-09-23T22:32:51.664470+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":413,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668975+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:32:51.668975+0000","last_peered":"2023-09-23T22:32:51.668975+0000","last_clean":"2023-09-23T22:32:51.668975+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:32:51.668975+0000","last_undegraded":"2023-09-23T22:32:51.668975+0000","last_fullsized":"2023-09-23T22:32:51.668975+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"405'7325","reported_seq":7365,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:04.408171+0000","last_change":"2023-09-23T22:31:08.227045+0000","last_active":"2023-09-23T22:33:04.408171+0000","last_peered":"2023-09-23T22:33:04.408171+0000","last_clean":"2023-09-23T22:33:04.408171+0000","last_became_active":"2023-09-23T22:31:08.226758+0000","last_became_peered":"2023-09-23T22:31:08.226758+0000","last_unstale":"2023-09-23T22:33:04.408171+0000","last_undegraded":"2023-09-23T22:33:04.408171+0000","last_fullsized":"2023-09-23T22:33:04.408171+0000","mapping_epoch":373,"log_start":"385'2500","ondisk_log_start":"385'2500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4825,"log_dups_size":0,"ondisk_log_size":4825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:55.118636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336527360,"num_objects":5135,"num_object_clones":0,"num_object_copies":15405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7325,"num_write_kb":398720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":786,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668531+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:32:51.668531+0000","last_peered":"2023-09-23T22:32:51.668531+0000","last_clean":"2023-09-23T22:32:51.668531+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:32:51.668531+0000","last_undegraded":"2023-09-23T22:32:51.668531+0000","last_fullsized":"2023-09-23T22:32:51.668531+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":86,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671481+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:32:51.671481+0000","last_peered":"2023-09-23T22:32:51.671481+0000","last_clean":"2023-09-23T22:32:51.671481+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:32:51.671481+0000","last_undegraded":"2023-09-23T22:32:51.671481+0000","last_fullsized":"2023-09-23T22:32:51.671481+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":440,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669392+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:32:51.669392+0000","last_peered":"2023-09-23T22:32:51.669392+0000","last_clean":"2023-09-23T22:32:51.669392+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:32:51.669392+0000","last_undegraded":"2023-09-23T22:32:51.669392+0000","last_fullsized":"2023-09-23T22:32:51.669392+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"405'7363","reported_seq":7403,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:05.036284+0000","last_change":"2023-09-23T22:31:08.225049+0000","last_active":"2023-09-23T22:33:05.036284+0000","last_peered":"2023-09-23T22:33:05.036284+0000","last_clean":"2023-09-23T22:33:05.036284+0000","last_became_active":"2023-09-23T22:31:08.224765+0000","last_became_peered":"2023-09-23T22:31:08.224765+0000","last_unstale":"2023-09-23T22:33:05.036284+0000","last_undegraded":"2023-09-23T22:33:05.036284+0000","last_fullsized":"2023-09-23T22:33:05.036284+0000","mapping_epoch":373,"log_start":"386'2600","ondisk_log_start":"386'2600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4763,"log_dups_size":0,"ondisk_log_size":4763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.842171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330235904,"num_objects":5039,"num_object_clones":0,"num_object_copies":15117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7363,"num_write_kb":396864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":396,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668855+0000","last_change":"2023-09-23T22:31:28.119358+0000","last_active":"2023-09-23T22:32:51.668855+0000","last_peered":"2023-09-23T22:32:51.668855+0000","last_clean":"2023-09-23T22:32:51.668855+0000","last_became_active":"2023-09-23T22:31:28.118823+0000","last_became_peered":"2023-09-23T22:31:28.118823+0000","last_unstale":"2023-09-23T22:32:51.668855+0000","last_undegraded":"2023-09-23T22:32:51.668855+0000","last_fullsized":"2023-09-23T22:32:51.668855+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:25.483176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":421,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669374+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:32:51.669374+0000","last_peered":"2023-09-23T22:32:51.669374+0000","last_clean":"2023-09-23T22:32:51.669374+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:32:51.669374+0000","last_undegraded":"2023-09-23T22:32:51.669374+0000","last_fullsized":"2023-09-23T22:32:51.669374+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"405'7326","reported_seq":7367,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:04.404556+0000","last_change":"2023-09-23T22:31:08.226959+0000","last_active":"2023-09-23T22:33:04.404556+0000","last_peered":"2023-09-23T22:33:04.404556+0000","last_clean":"2023-09-23T22:33:04.404556+0000","last_became_active":"2023-09-23T22:31:08.226652+0000","last_became_peered":"2023-09-23T22:31:08.226652+0000","last_unstale":"2023-09-23T22:33:04.404556+0000","last_undegraded":"2023-09-23T22:33:04.404556+0000","last_fullsized":"2023-09-23T22:33:04.404556+0000","mapping_epoch":373,"log_start":"385'2500","ondisk_log_start":"385'2500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4826,"log_dups_size":0,"ondisk_log_size":4826,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:22.758103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335282176,"num_objects":5116,"num_object_clones":0,"num_object_copies":15348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7326,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":759,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668337+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:32:51.668337+0000","last_peered":"2023-09-23T22:32:51.668337+0000","last_clean":"2023-09-23T22:32:51.668337+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:32:51.668337+0000","last_undegraded":"2023-09-23T22:32:51.668337+0000","last_fullsized":"2023-09-23T22:32:51.668337+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":122,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.670085+0000","last_change":"2023-09-23T22:31:28.120600+0000","last_active":"2023-09-23T22:32:51.670085+0000","last_peered":"2023-09-23T22:32:51.670085+0000","last_clean":"2023-09-23T22:32:51.670085+0000","last_became_active":"2023-09-23T22:31:28.120281+0000","last_became_peered":"2023-09-23T22:31:28.120281+0000","last_unstale":"2023-09-23T22:32:51.670085+0000","last_undegraded":"2023-09-23T22:32:51.670085+0000","last_fullsized":"2023-09-23T22:32:51.670085+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:10.594022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":122,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671524+0000","last_change":"2023-09-23T22:31:28.111270+0000","last_active":"2023-09-23T22:32:51.671524+0000","last_peered":"2023-09-23T22:32:51.671524+0000","last_clean":"2023-09-23T22:32:51.671524+0000","last_became_active":"2023-09-23T22:31:28.110921+0000","last_became_peered":"2023-09-23T22:31:28.110921+0000","last_unstale":"2023-09-23T22:32:51.671524+0000","last_undegraded":"2023-09-23T22:32:51.671524+0000","last_fullsized":"2023-09-23T22:32:51.671524+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:05.417671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"405'7213","reported_seq":10346,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:04.951595+0000","last_change":"2023-09-23T22:32:15.139562+0000","last_active":"2023-09-23T22:33:04.951595+0000","last_peered":"2023-09-23T22:33:04.951595+0000","last_clean":"2023-09-23T22:33:04.951595+0000","last_became_active":"2023-09-23T22:32:15.135437+0000","last_became_peered":"2023-09-23T22:32:15.135437+0000","last_unstale":"2023-09-23T22:33:04.951595+0000","last_undegraded":"2023-09-23T22:33:04.951595+0000","last_fullsized":"2023-09-23T22:33:04.951595+0000","mapping_epoch":395,"log_start":"384'1800","ondisk_log_start":"384'1800","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5413,"log_dups_size":0,"ondisk_log_size":5413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:36.457421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328531968,"num_objects":5013,"num_object_clones":0,"num_object_copies":15039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7213,"num_write_kb":391232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1537,"num_bytes_recovered":100728832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":680,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.672062+0000","last_change":"2023-09-23T22:31:31.129491+0000","last_active":"2023-09-23T22:32:51.672062+0000","last_peered":"2023-09-23T22:32:51.672062+0000","last_clean":"2023-09-23T22:32:51.672062+0000","last_became_active":"2023-09-23T22:31:31.129163+0000","last_became_peered":"2023-09-23T22:31:31.129163+0000","last_unstale":"2023-09-23T22:32:51.672062+0000","last_undegraded":"2023-09-23T22:32:51.672062+0000","last_fullsized":"2023-09-23T22:32:51.672062+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.141229+0000","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":781,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.664153+0000","last_change":"2023-09-23T22:31:31.129492+0000","last_active":"2023-09-23T22:32:51.664153+0000","last_peered":"2023-09-23T22:32:51.664153+0000","last_clean":"2023-09-23T22:32:51.664153+0000","last_became_active":"2023-09-23T22:31:31.128706+0000","last_became_peered":"2023-09-23T22:31:31.128706+0000","last_unstale":"2023-09-23T22:32:51.664153+0000","last_undegraded":"2023-09-23T22:32:51.664153+0000","last_fullsized":"2023-09-23T22:32:51.664153+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:16.180152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":472,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668874+0000","last_change":"2023-09-23T22:31:28.130412+0000","last_active":"2023-09-23T22:32:51.668874+0000","last_peered":"2023-09-23T22:32:51.668874+0000","last_clean":"2023-09-23T22:32:51.668874+0000","last_became_active":"2023-09-23T22:31:28.130127+0000","last_became_peered":"2023-09-23T22:31:28.130127+0000","last_unstale":"2023-09-23T22:32:51.668874+0000","last_undegraded":"2023-09-23T22:32:51.668874+0000","last_fullsized":"2023-09-23T22:32:51.668874+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.088651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":113,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.670478+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:32:51.670478+0000","last_peered":"2023-09-23T22:32:51.670478+0000","last_clean":"2023-09-23T22:32:51.670478+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:32:51.670478+0000","last_undegraded":"2023-09-23T22:32:51.670478+0000","last_fullsized":"2023-09-23T22:32:51.670478+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"405'7296","reported_seq":10561,"reported_epoch":405,"state":"active+recovery_wait","last_fresh":"2023-09-23T22:33:05.117350+0000","last_change":"2023-09-23T22:32:31.203405+0000","last_active":"2023-09-23T22:33:05.117350+0000","last_peered":"2023-09-23T22:33:05.117350+0000","last_clean":"2023-09-23T22:31:27.057464+0000","last_became_active":"2023-09-23T22:32:31.163888+0000","last_became_peered":"2023-09-23T22:32:31.163888+0000","last_unstale":"2023-09-23T22:33:05.117350+0000","last_undegraded":"2023-09-23T22:33:05.117350+0000","last_fullsized":"2023-09-23T22:33:05.117350+0000","mapping_epoch":399,"log_start":"384'1800","ondisk_log_start":"384'1800","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5496,"log_dups_size":0,"ondisk_log_size":5496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:18.004864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327155712,"num_objects":4992,"num_object_clones":0,"num_object_copies":14976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7296,"num_write_kb":393216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1603,"num_bytes_recovered":105054208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":["5","0","4","7"],"object_location_counts":[{"shards":"0,4,5","objects":4992}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":749,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669490+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:32:51.669490+0000","last_peered":"2023-09-23T22:32:51.669490+0000","last_clean":"2023-09-23T22:32:51.669490+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:32:51.669490+0000","last_undegraded":"2023-09-23T22:32:51.669490+0000","last_fullsized":"2023-09-23T22:32:51.669490+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1283,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668569+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:32:51.668569+0000","last_peered":"2023-09-23T22:32:51.668569+0000","last_clean":"2023-09-23T22:32:51.668569+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:32:51.668569+0000","last_undegraded":"2023-09-23T22:32:51.668569+0000","last_fullsized":"2023-09-23T22:32:51.668569+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":444,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669455+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:32:51.669455+0000","last_peered":"2023-09-23T22:32:51.669455+0000","last_clean":"2023-09-23T22:32:51.669455+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:32:51.669455+0000","last_undegraded":"2023-09-23T22:32:51.669455+0000","last_fullsized":"2023-09-23T22:32:51.669455+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":96,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671276+0000","last_change":"2023-09-23T22:31:28.123596+0000","last_active":"2023-09-23T22:32:51.671276+0000","last_peered":"2023-09-23T22:32:51.671276+0000","last_clean":"2023-09-23T22:32:51.671276+0000","last_became_active":"2023-09-23T22:31:28.123179+0000","last_became_peered":"2023-09-23T22:31:28.123179+0000","last_unstale":"2023-09-23T22:32:51.671276+0000","last_undegraded":"2023-09-23T22:32:51.671276+0000","last_fullsized":"2023-09-23T22:32:51.671276+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:53.106243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.b","version":"405'7177","reported_seq":10219,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:05.034674+0000","last_change":"2023-09-23T22:31:44.145615+0000","last_active":"2023-09-23T22:33:05.034674+0000","last_peered":"2023-09-23T22:33:05.034674+0000","last_clean":"2023-09-23T22:33:05.034674+0000","last_became_active":"2023-09-23T22:31:44.142103+0000","last_became_peered":"2023-09-23T22:31:44.142103+0000","last_unstale":"2023-09-23T22:33:05.034674+0000","last_undegraded":"2023-09-23T22:33:05.034674+0000","last_fullsized":"2023-09-23T22:33:05.034674+0000","mapping_epoch":387,"log_start":"385'2400","ondisk_log_start":"385'2400","created":373,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4777,"log_dups_size":0,"ondisk_log_size":4777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:15.488007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334561280,"num_objects":5105,"num_object_clones":0,"num_object_copies":15315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7177,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1492,"num_bytes_recovered":97779712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":755,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.663889+0000","last_change":"2023-09-23T22:31:31.130599+0000","last_active":"2023-09-23T22:32:51.663889+0000","last_peered":"2023-09-23T22:32:51.663889+0000","last_clean":"2023-09-23T22:32:51.663889+0000","last_became_active":"2023-09-23T22:31:31.127784+0000","last_became_peered":"2023-09-23T22:31:31.127784+0000","last_unstale":"2023-09-23T22:32:51.663889+0000","last_undegraded":"2023-09-23T22:32:51.663889+0000","last_fullsized":"2023-09-23T22:32:51.663889+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:28.788741+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":622,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.663867+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:32:51.663867+0000","last_peered":"2023-09-23T22:32:51.663867+0000","last_clean":"2023-09-23T22:32:51.663867+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:32:51.663867+0000","last_undegraded":"2023-09-23T22:32:51.663867+0000","last_fullsized":"2023-09-23T22:32:51.663867+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":86,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671221+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:32:51.671221+0000","last_peered":"2023-09-23T22:32:51.671221+0000","last_clean":"2023-09-23T22:32:51.671221+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:32:51.671221+0000","last_undegraded":"2023-09-23T22:32:51.671221+0000","last_fullsized":"2023-09-23T22:32:51.671221+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":54,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671390+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:32:51.671390+0000","last_peered":"2023-09-23T22:32:51.671390+0000","last_clean":"2023-09-23T22:32:51.671390+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:32:51.671390+0000","last_undegraded":"2023-09-23T22:32:51.671390+0000","last_fullsized":"2023-09-23T22:32:51.671390+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"405'7267","reported_seq":7307,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:05.066167+0000","last_change":"2023-09-23T22:31:08.227185+0000","last_active":"2023-09-23T22:33:05.066167+0000","last_peered":"2023-09-23T22:33:05.066167+0000","last_clean":"2023-09-23T22:33:05.066167+0000","last_became_active":"2023-09-23T22:31:08.226918+0000","last_became_peered":"2023-09-23T22:31:08.226918+0000","last_unstale":"2023-09-23T22:33:05.066167+0000","last_undegraded":"2023-09-23T22:33:05.066167+0000","last_fullsized":"2023-09-23T22:33:05.066167+0000","mapping_epoch":373,"log_start":"385'2200","ondisk_log_start":"385'2200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5067,"log_dups_size":0,"ondisk_log_size":5067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:54.257936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329449472,"num_objects":5027,"num_object_clones":0,"num_object_copies":15081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5027,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7267,"num_write_kb":393408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1167,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.670374+0000","last_change":"2023-09-23T22:31:31.127033+0000","last_active":"2023-09-23T22:32:51.670374+0000","last_peered":"2023-09-23T22:32:51.670374+0000","last_clean":"2023-09-23T22:32:51.670374+0000","last_became_active":"2023-09-23T22:31:31.126601+0000","last_became_peered":"2023-09-23T22:31:31.126601+0000","last_unstale":"2023-09-23T22:32:51.670374+0000","last_undegraded":"2023-09-23T22:32:51.670374+0000","last_fullsized":"2023-09-23T22:32:51.670374+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:00.619278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":789,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.663567+0000","last_change":"2023-09-23T22:31:31.126756+0000","last_active":"2023-09-23T22:32:51.663567+0000","last_peered":"2023-09-23T22:32:51.663567+0000","last_clean":"2023-09-23T22:32:51.663567+0000","last_became_active":"2023-09-23T22:31:31.126409+0000","last_became_peered":"2023-09-23T22:31:31.126409+0000","last_unstale":"2023-09-23T22:32:51.663567+0000","last_undegraded":"2023-09-23T22:32:51.663567+0000","last_fullsized":"2023-09-23T22:32:51.663567+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.605240+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":286,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668424+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:32:51.668424+0000","last_peered":"2023-09-23T22:32:51.668424+0000","last_clean":"2023-09-23T22:32:51.668424+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:32:51.668424+0000","last_undegraded":"2023-09-23T22:32:51.668424+0000","last_fullsized":"2023-09-23T22:32:51.668424+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":96,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671853+0000","last_change":"2023-09-23T22:31:28.124597+0000","last_active":"2023-09-23T22:32:51.671853+0000","last_peered":"2023-09-23T22:32:51.671853+0000","last_clean":"2023-09-23T22:32:51.671853+0000","last_became_active":"2023-09-23T22:31:28.124245+0000","last_became_peered":"2023-09-23T22:31:28.124245+0000","last_unstale":"2023-09-23T22:32:51.671853+0000","last_undegraded":"2023-09-23T22:32:51.671853+0000","last_fullsized":"2023-09-23T22:32:51.671853+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.558632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"405'7279","reported_seq":7320,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:04.381835+0000","last_change":"2023-09-23T22:31:08.223664+0000","last_active":"2023-09-23T22:33:04.381835+0000","last_peered":"2023-09-23T22:33:04.381835+0000","last_clean":"2023-09-23T22:33:04.381835+0000","last_became_active":"2023-09-23T22:31:08.223323+0000","last_became_peered":"2023-09-23T22:31:08.223323+0000","last_unstale":"2023-09-23T22:33:04.381835+0000","last_undegraded":"2023-09-23T22:33:04.381835+0000","last_fullsized":"2023-09-23T22:33:04.381835+0000","mapping_epoch":373,"log_start":"385'2500","ondisk_log_start":"385'2500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4779,"log_dups_size":0,"ondisk_log_size":4779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:38.536861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334299136,"num_objects":5101,"num_object_clones":0,"num_object_copies":15303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7279,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":637,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671850+0000","last_change":"2023-09-23T22:31:21.362707+0000","last_active":"2023-09-23T22:32:51.671850+0000","last_peered":"2023-09-23T22:32:51.671850+0000","last_clean":"2023-09-23T22:32:51.671850+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:32:51.671850+0000","last_undegraded":"2023-09-23T22:32:51.671850+0000","last_fullsized":"2023-09-23T22:32:51.671850+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:58.622607+0000","scrub_duration":0.0034076229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":784,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.670125+0000","last_change":"2023-09-23T22:31:28.147724+0000","last_active":"2023-09-23T22:32:51.670125+0000","last_peered":"2023-09-23T22:32:51.670125+0000","last_clean":"2023-09-23T22:32:51.670125+0000","last_became_active":"2023-09-23T22:31:28.120238+0000","last_became_peered":"2023-09-23T22:31:28.120238+0000","last_unstale":"2023-09-23T22:32:51.670125+0000","last_undegraded":"2023-09-23T22:32:51.670125+0000","last_fullsized":"2023-09-23T22:32:51.670125+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:53.259496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":300,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.663952+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:32:51.663952+0000","last_peered":"2023-09-23T22:32:51.663952+0000","last_clean":"2023-09-23T22:32:51.663952+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:32:51.663952+0000","last_undegraded":"2023-09-23T22:32:51.663952+0000","last_fullsized":"2023-09-23T22:32:51.663952+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":293,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.665939+0000","last_change":"2023-09-23T22:31:28.121081+0000","last_active":"2023-09-23T22:32:51.665939+0000","last_peered":"2023-09-23T22:32:51.665939+0000","last_clean":"2023-09-23T22:32:51.665939+0000","last_became_active":"2023-09-23T22:31:28.120399+0000","last_became_peered":"2023-09-23T22:31:28.120399+0000","last_unstale":"2023-09-23T22:32:51.665939+0000","last_undegraded":"2023-09-23T22:32:51.665939+0000","last_fullsized":"2023-09-23T22:32:51.665939+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:11.267296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"405'7391","reported_seq":7432,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:04.394144+0000","last_change":"2023-09-23T22:31:08.224259+0000","last_active":"2023-09-23T22:33:04.394144+0000","last_peered":"2023-09-23T22:33:04.394144+0000","last_clean":"2023-09-23T22:33:04.394144+0000","last_became_active":"2023-09-23T22:31:08.224062+0000","last_became_peered":"2023-09-23T22:31:08.224062+0000","last_unstale":"2023-09-23T22:33:04.394144+0000","last_undegraded":"2023-09-23T22:33:04.394144+0000","last_fullsized":"2023-09-23T22:33:04.394144+0000","mapping_epoch":373,"log_start":"386'2600","ondisk_log_start":"386'2600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4791,"log_dups_size":0,"ondisk_log_size":4791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:18.330753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347013120,"num_objects":5295,"num_object_clones":0,"num_object_copies":15885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7391,"num_write_kb":405952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":755,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.672146+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:32:51.672146+0000","last_peered":"2023-09-23T22:32:51.672146+0000","last_clean":"2023-09-23T22:32:51.672146+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:32:51.672146+0000","last_undegraded":"2023-09-23T22:32:51.672146+0000","last_fullsized":"2023-09-23T22:32:51.672146+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":332,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.664041+0000","last_change":"2023-09-23T22:31:28.152186+0000","last_active":"2023-09-23T22:32:51.664041+0000","last_peered":"2023-09-23T22:32:51.664041+0000","last_clean":"2023-09-23T22:32:51.664041+0000","last_became_active":"2023-09-23T22:31:28.111292+0000","last_became_peered":"2023-09-23T22:31:28.111292+0000","last_unstale":"2023-09-23T22:32:51.664041+0000","last_undegraded":"2023-09-23T22:32:51.664041+0000","last_fullsized":"2023-09-23T22:32:51.664041+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:06.097398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":331,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.670033+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:32:51.670033+0000","last_peered":"2023-09-23T22:32:51.670033+0000","last_clean":"2023-09-23T22:32:51.670033+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:32:51.670033+0000","last_undegraded":"2023-09-23T22:32:51.670033+0000","last_fullsized":"2023-09-23T22:32:51.670033+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":273,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.664380+0000","last_change":"2023-09-23T22:31:28.106985+0000","last_active":"2023-09-23T22:32:51.664380+0000","last_peered":"2023-09-23T22:32:51.664380+0000","last_clean":"2023-09-23T22:32:51.664380+0000","last_became_active":"2023-09-23T22:31:28.105451+0000","last_became_peered":"2023-09-23T22:31:28.105451+0000","last_unstale":"2023-09-23T22:32:51.664380+0000","last_undegraded":"2023-09-23T22:32:51.664380+0000","last_fullsized":"2023-09-23T22:32:51.664380+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:43.614928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"405'7356","reported_seq":10618,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:04.968226+0000","last_change":"2023-09-23T22:31:45.135267+0000","last_active":"2023-09-23T22:33:04.968226+0000","last_peered":"2023-09-23T22:33:04.968226+0000","last_clean":"2023-09-23T22:33:04.968226+0000","last_became_active":"2023-09-23T22:31:45.131633+0000","last_became_peered":"2023-09-23T22:31:45.131633+0000","last_unstale":"2023-09-23T22:33:04.968226+0000","last_undegraded":"2023-09-23T22:33:04.968226+0000","last_fullsized":"2023-09-23T22:33:04.968226+0000","mapping_epoch":388,"log_start":"384'1900","ondisk_log_start":"384'1900","created":373,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:00.859364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329383936,"num_objects":5026,"num_object_clones":0,"num_object_copies":15078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7356,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1601,"num_bytes_recovered":104923136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":759,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669255+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:32:51.669255+0000","last_peered":"2023-09-23T22:32:51.669255+0000","last_clean":"2023-09-23T22:32:51.669255+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:32:51.669255+0000","last_undegraded":"2023-09-23T22:32:51.669255+0000","last_fullsized":"2023-09-23T22:32:51.669255+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":237,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.664193+0000","last_change":"2023-09-23T22:31:28.131347+0000","last_active":"2023-09-23T22:32:51.664193+0000","last_peered":"2023-09-23T22:32:51.664193+0000","last_clean":"2023-09-23T22:32:51.664193+0000","last_became_active":"2023-09-23T22:31:28.114169+0000","last_became_peered":"2023-09-23T22:31:28.114169+0000","last_unstale":"2023-09-23T22:32:51.664193+0000","last_undegraded":"2023-09-23T22:32:51.664193+0000","last_fullsized":"2023-09-23T22:32:51.664193+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:30.007254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":901,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.671277+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:32:51.671277+0000","last_peered":"2023-09-23T22:32:51.671277+0000","last_clean":"2023-09-23T22:32:51.671277+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:32:51.671277+0000","last_undegraded":"2023-09-23T22:32:51.671277+0000","last_fullsized":"2023-09-23T22:32:51.671277+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":345,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.663316+0000","last_change":"2023-09-23T22:31:28.114550+0000","last_active":"2023-09-23T22:32:51.663316+0000","last_peered":"2023-09-23T22:32:51.663316+0000","last_clean":"2023-09-23T22:32:51.663316+0000","last_became_active":"2023-09-23T22:31:28.114217+0000","last_became_peered":"2023-09-23T22:31:28.114217+0000","last_unstale":"2023-09-23T22:32:51.663316+0000","last_undegraded":"2023-09-23T22:32:51.663316+0000","last_fullsized":"2023-09-23T22:32:51.663316+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:13.616772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.d","version":"405'7296","reported_seq":10526,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:33:05.141857+0000","last_change":"2023-09-23T22:32:51.714615+0000","last_active":"2023-09-23T22:33:05.141857+0000","last_peered":"2023-09-23T22:33:05.141857+0000","last_clean":"2023-09-23T22:33:05.141857+0000","last_became_active":"2023-09-23T22:32:51.673119+0000","last_became_peered":"2023-09-23T22:32:51.673119+0000","last_unstale":"2023-09-23T22:33:05.141857+0000","last_undegraded":"2023-09-23T22:33:05.141857+0000","last_fullsized":"2023-09-23T22:33:05.141857+0000","mapping_epoch":403,"log_start":"384'1800","ondisk_log_start":"384'1800","created":373,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5496,"log_dups_size":0,"ondisk_log_size":5496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:58.214128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325255192,"num_objects":4964,"num_object_clones":0,"num_object_copies":14892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4964,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7296,"num_write_kb":392257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":103809024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":685,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.670111+0000","last_change":"2023-09-23T22:31:45.133529+0000","last_active":"2023-09-23T22:32:51.670111+0000","last_peered":"2023-09-23T22:32:51.670111+0000","last_clean":"2023-09-23T22:32:51.670111+0000","last_became_active":"2023-09-23T22:31:45.133117+0000","last_became_peered":"2023-09-23T22:31:45.133117+0000","last_unstale":"2023-09-23T22:32:51.670111+0000","last_undegraded":"2023-09-23T22:32:51.670111+0000","last_fullsized":"2023-09-23T22:32:51.670111+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.591643+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.591643+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:28.957050+0000","scrub_duration":0.0073638979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":116,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.672008+0000","last_change":"2023-09-23T22:31:28.156716+0000","last_active":"2023-09-23T22:32:51.672008+0000","last_peered":"2023-09-23T22:32:51.672008+0000","last_clean":"2023-09-23T22:32:51.672008+0000","last_became_active":"2023-09-23T22:31:28.128617+0000","last_became_peered":"2023-09-23T22:31:28.128617+0000","last_unstale":"2023-09-23T22:32:51.672008+0000","last_undegraded":"2023-09-23T22:32:51.672008+0000","last_fullsized":"2023-09-23T22:32:51.672008+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:34.465027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":269,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.665737+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:32:51.665737+0000","last_peered":"2023-09-23T22:32:51.665737+0000","last_clean":"2023-09-23T22:32:51.665737+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:32:51.665737+0000","last_undegraded":"2023-09-23T22:32:51.665737+0000","last_fullsized":"2023-09-23T22:32:51.665737+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":378,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668936+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:32:51.668936+0000","last_peered":"2023-09-23T22:32:51.668936+0000","last_clean":"2023-09-23T22:32:51.668936+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:32:51.668936+0000","last_undegraded":"2023-09-23T22:32:51.668936+0000","last_fullsized":"2023-09-23T22:32:51.668936+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":65,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T22:32:56.174795+0000","last_change":"2023-09-23T22:31:28.111728+0000","last_active":"2023-09-23T22:32:56.174795+0000","last_peered":"2023-09-23T22:32:56.174795+0000","last_clean":"2023-09-23T22:32:56.174795+0000","last_became_active":"2023-09-23T22:31:28.111369+0000","last_became_peered":"2023-09-23T22:31:28.111369+0000","last_unstale":"2023-09-23T22:32:56.174795+0000","last_undegraded":"2023-09-23T22:32:56.174795+0000","last_fullsized":"2023-09-23T22:32:56.174795+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:42.837575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":155,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.670393+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:32:51.670393+0000","last_peered":"2023-09-23T22:32:51.670393+0000","last_clean":"2023-09-23T22:32:51.670393+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:32:51.670393+0000","last_undegraded":"2023-09-23T22:32:51.670393+0000","last_fullsized":"2023-09-23T22:32:51.670393+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":395,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669856+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:32:51.669856+0000","last_peered":"2023-09-23T22:32:51.669856+0000","last_clean":"2023-09-23T22:32:51.669856+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:32:51.669856+0000","last_undegraded":"2023-09-23T22:32:51.669856+0000","last_fullsized":"2023-09-23T22:32:51.669856+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":430,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669704+0000","last_change":"2023-09-23T22:31:28.111351+0000","last_active":"2023-09-23T22:32:51.669704+0000","last_peered":"2023-09-23T22:32:51.669704+0000","last_clean":"2023-09-23T22:32:51.669704+0000","last_became_active":"2023-09-23T22:31:28.111006+0000","last_became_peered":"2023-09-23T22:31:28.111006+0000","last_unstale":"2023-09-23T22:32:51.669704+0000","last_undegraded":"2023-09-23T22:32:51.669704+0000","last_fullsized":"2023-09-23T22:32:51.669704+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:51.435513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":420,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668971+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:32:51.668971+0000","last_peered":"2023-09-23T22:32:51.668971+0000","last_clean":"2023-09-23T22:32:51.668971+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:32:51.668971+0000","last_undegraded":"2023-09-23T22:32:51.668971+0000","last_fullsized":"2023-09-23T22:32:51.668971+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":269,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.669097+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:32:51.669097+0000","last_peered":"2023-09-23T22:32:51.669097+0000","last_clean":"2023-09-23T22:32:51.669097+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:32:51.669097+0000","last_undegraded":"2023-09-23T22:32:51.669097+0000","last_fullsized":"2023-09-23T22:32:51.669097+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":96,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.672100+0000","last_change":"2023-09-23T22:31:28.126203+0000","last_active":"2023-09-23T22:32:51.672100+0000","last_peered":"2023-09-23T22:32:51.672100+0000","last_clean":"2023-09-23T22:32:51.672100+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:31:28.125529+0000","last_unstale":"2023-09-23T22:32:51.672100+0000","last_undegraded":"2023-09-23T22:32:51.672100+0000","last_fullsized":"2023-09-23T22:32:51.672100+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:47.359771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":405,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.664563+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:32:51.664563+0000","last_peered":"2023-09-23T22:32:51.664563+0000","last_clean":"2023-09-23T22:32:51.664563+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:32:51.664563+0000","last_undegraded":"2023-09-23T22:32:51.664563+0000","last_fullsized":"2023-09-23T22:32:51.664563+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":343,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.664336+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:32:51.664336+0000","last_peered":"2023-09-23T22:32:51.664336+0000","last_clean":"2023-09-23T22:32:51.664336+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:32:51.664336+0000","last_undegraded":"2023-09-23T22:32:51.664336+0000","last_fullsized":"2023-09-23T22:32:51.664336+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":463,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T22:32:51.668595+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:32:51.668595+0000","last_peered":"2023-09-23T22:32:51.668595+0000","last_clean":"2023-09-23T22:32:51.668595+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:32:51.668595+0000","last_undegraded":"2023-09-23T22:32:51.668595+0000","last_fullsized":"2023-09-23T22:32:51.668595+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":31404,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5305860120,"num_objects":80962,"num_object_clones":0,"num_object_copies":242886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":308,"num_objects_misplaced":208,"num_objects_unfound":0,"num_objects_dirty":80962,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":116818,"num_write_kb":6328897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10506,"num_bytes_recovered":688521216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16187273216,"data_stored":16187261000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82318,"ondisk_log_size":82318,"up":48,"acting":47,"num_store_stats":6},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1115744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234344,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495980,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2805656,"kb_used_data":2625808,"kb_used_omap":0,"kb_used_meta":179840,"kb_avail":90939496,"statfs":{"total":95995035648,"available":93122043904,"internally_reserved":0,"allocated":2688827392,"data_stored":2687364885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":184156160},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":7.2089999999999996,"5min":5.6589999999999998,"15min":2.9900000000000002},"min":{"1min":0.22900000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":57.936999999999998,"5min":213.23699999999999,"15min":213.23699999999999},"last":0.23499999999999999},{"interface":"front","average":{"1min":7.1070000000000002,"5min":5.992,"15min":4.0529999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":58.526000000000003,"5min":213.304,"15min":213.304},"last":0.82999999999999996}]},{"osd":1,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":5.3650000000000002,"5min":1.8620000000000001,"15min":3.5110000000000001},"min":{"1min":0.27700000000000002,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":58.143999999999998,"5min":58.143999999999998,"15min":129.964},"last":0.55100000000000005},{"interface":"front","average":{"1min":5.2050000000000001,"5min":2.0750000000000002,"15min":3.8039999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":63.073999999999998,"5min":63.073999999999998,"15min":131.18600000000001},"last":1.157}]},{"osd":2,"last update":"Sat Sep 23 22:33:00 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":1.927,"15min":1.911},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.2990000000000004,"5min":6.6559999999999997,"15min":6.6559999999999997},"last":3.0459999999999998},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.016,"15min":2.105},"min":{"1min":0.51300000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.1109999999999998,"5min":9.7319999999999993,"15min":9.7319999999999993},"last":0.621}]},{"osd":3,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.5940000000000001,"15min":3.145},"min":{"1min":0.56899999999999995,"5min":0.32500000000000001,"15min":0.316},"max":{"1min":6.165,"5min":6.8609999999999998,"15min":125.125},"last":3.0720000000000001},{"interface":"front","average":{"1min":2.2829999999999999,"5min":1.4119999999999999,"15min":2.8610000000000002},"min":{"1min":0.33900000000000002,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":9.2569999999999997,"5min":9.2569999999999997,"15min":76.406999999999996},"last":0.66500000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.127,"15min":2.6299999999999999},"min":{"1min":0.29299999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5010000000000003,"5min":6.8650000000000002,"15min":305.30200000000002},"last":2.4580000000000002},{"interface":"front","average":{"1min":4.1020000000000003,"5min":1.782,"15min":2.8319999999999999},"min":{"1min":0.317,"5min":0.29299999999999998,"15min":0.255},"max":{"1min":11.561,"5min":11.561,"15min":305.40300000000002},"last":3.496}]},{"osd":5,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":2.726,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.148,"5min":88.194999999999993,"15min":88.194999999999993},"last":3.0019999999999998},{"interface":"front","average":{"1min":3.1909999999999998,"5min":2.2839999999999998,"15min":1.5860000000000001},"min":{"1min":0.44600000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.2859999999999996,"5min":88.025000000000006,"15min":88.025000000000006},"last":0.45700000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":0.95399999999999996,"15min":1.0980000000000001},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.2270000000000003,"5min":9.2270000000000003,"15min":82.801000000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.4630000000000001,"5min":1.105,"15min":1.1459999999999999},"min":{"1min":0.41999999999999998,"5min":0.33300000000000002,"15min":0.29699999999999999},"max":{"1min":8.7690000000000001,"5min":8.7690000000000001,"15min":83.052999999999997},"last":3.1469999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795252,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3613288,"kb_used_data":2967648,"kb_used_omap":7,"kb_used_meta":645624,"kb_avail":90131864,"statfs":{"total":95995035648,"available":92295028736,"internally_reserved":0,"allocated":3038871552,"data_stored":3036744115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":661119348},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":6.3769999999999998,"5min":5.8250000000000002,"15min":5.1369999999999996},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":107.202,"5min":202.73500000000001,"15min":202.73500000000001},"last":0.24399999999999999},{"interface":"front","average":{"1min":7.0449999999999999,"5min":6.3120000000000003,"15min":5.9349999999999996},"min":{"1min":0.24399999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":107.426,"5min":202.80799999999999,"15min":202.80799999999999},"last":0.45600000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":0.94999999999999996,"15min":1.0209999999999999},"min":{"1min":0.23599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":3.8820000000000001,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.24,"5min":1.8999999999999999,"15min":2.214},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.9859999999999998,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2669999999999999,"5min":1.732,"15min":4.4980000000000002},"min":{"1min":0.30599999999999999,"5min":0.246,"15min":0.246},"max":{"1min":10.587999999999999,"5min":10.587999999999999,"15min":160.499},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.653,"5min":1.46,"15min":4.1890000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.223},"max":{"1min":5.1920000000000002,"5min":8.1590000000000007,"15min":160.465},"last":0.72499999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.92200000000000004,"15min":2.2440000000000002},"min":{"1min":0.29499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.3730000000000002,"5min":7.0380000000000003,"15min":234.79900000000001},"last":0.503},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.478,"15min":2.919},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.4729999999999999,"5min":9.093,"15min":234.81700000000001},"last":0.56799999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":1.6859999999999999,"15min":2.5},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":7.4450000000000003,"5min":9.2759999999999998,"15min":196.79900000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.611,"5min":1.294,"15min":2.3239999999999998},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.29099999999999998},"max":{"1min":5.2889999999999997,"5min":6.8049999999999997,"15min":196.68299999999999},"last":2.367}]},{"osd":6,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":2.2559999999999998,"15min":2.1379999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":5.2350000000000003,"5min":77.033000000000001,"15min":77.033000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":1.681,"5min":1.9630000000000001,"15min":1.583},"min":{"1min":0.442,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":8.3640000000000008,"5min":76.965999999999994,"15min":76.965999999999994},"last":5.2050000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":1.663,"15min":2.0259999999999998},"min":{"1min":0.41499999999999998,"5min":0.38300000000000001,"15min":0.32900000000000001},"max":{"1min":8.0299999999999994,"5min":8.0660000000000007,"15min":16.265000000000001},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.302,"5min":1.2529999999999999,"15min":1.339},"min":{"1min":0.378,"5min":0.318,"15min":0.318},"max":{"1min":3.8119999999999998,"5min":7.1429999999999998,"15min":12.045999999999999},"last":0.58699999999999997}]}]},{"osd":0,"up_from":93,"seq":399431958781,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3672588,"kb_used_data":2929220,"kb_used_omap":8,"kb_used_meta":743351,"kb_avail":90072564,"statfs":{"total":95995035648,"available":92234305536,"internally_reserved":0,"allocated":2999521280,"data_stored":2997304887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":761192298},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:32:37 2023","interfaces":[{"interface":"back","average":{"1min":7.5819999999999999,"5min":3.4860000000000002,"15min":4.1769999999999996},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":121.688,"5min":121.688,"15min":169.77199999999999},"last":3.133},{"interface":"front","average":{"1min":6.8739999999999997,"5min":3.5529999999999999,"15min":4.2850000000000001},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":122.93300000000001,"5min":122.93300000000001,"15min":169.52099999999999},"last":3.5640000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.2850000000000001,"15min":2.8919999999999999},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2359999999999998,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.415,"15min":1.694},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.3380000000000001,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:32:37 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.45,"15min":3.5419999999999998},"min":{"1min":0.315,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":8.548,"5min":10.292999999999999,"15min":146.87299999999999},"last":1.0269999999999999},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.6259999999999999,"15min":3.5750000000000002},"min":{"1min":0.29299999999999998,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":3.5379999999999998,"5min":22.303000000000001,"15min":145.48099999999999},"last":0.54900000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":2.109,"15min":2.5339999999999998},"min":{"1min":0.30199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.2060000000000004,"5min":11.474,"15min":82.825000000000003},"last":0.90300000000000002},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.4119999999999999,"15min":1.7529999999999999},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.444,"5min":7.2279999999999998,"15min":75.930999999999997},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":2.21,"15min":2.9689999999999999},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2510000000000003,"5min":13.564,"15min":93.685000000000002},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.841,"5min":1.633,"15min":2.246},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4139999999999997,"5min":13.486000000000001,"15min":93.546000000000006},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3290000000000002,"15min":2.3140000000000001},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.308},"max":{"1min":10.183999999999999,"5min":12.955,"15min":16.366},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.6830000000000001,"15min":1.6799999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":6.5060000000000002,"5min":14.208,"15min":22.263000000000002},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.8500000000000001,"15min":2},"min":{"1min":0.42599999999999999,"5min":0.40200000000000002,"15min":0.29999999999999999},"max":{"1min":3.0110000000000001,"5min":22.338999999999999,"15min":32.773000000000003},"last":0.75600000000000001},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.1389999999999998,"15min":2.2890000000000001},"min":{"1min":0.47899999999999998,"5min":0.307,"15min":0.307},"max":{"1min":8.7550000000000008,"5min":10.901,"15min":16.381},"last":1.002}]}]},{"osd":2,"up_from":0,"seq":1498943586344,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468199,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019249,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4304528,"kb_used_data":3908616,"kb_used_omap":0,"kb_used_meta":395840,"kb_avail":89440624,"statfs":{"total":95995035648,"available":91587198976,"internally_reserved":0,"allocated":4002422784,"data_stored":4000967894,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":405340160},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":11.644,"5min":3.5510000000000002,"15min":4.1360000000000001},"min":{"1min":0.216,"5min":0.20699999999999999,"15min":0.19700000000000001},"max":{"1min":101.44499999999999,"5min":101.44499999999999,"15min":150.63499999999999},"last":9.1839999999999993},{"interface":"front","average":{"1min":11.930999999999999,"5min":3.7890000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":101.465,"5min":101.465,"15min":150.672},"last":8.3399999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":5.7779999999999996,"5min":3.2109999999999999,"15min":3.4060000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":94.197999999999993,"5min":94.197999999999993,"15min":107.346},"last":9.0719999999999992},{"interface":"front","average":{"1min":6.359,"5min":3.508,"15min":3.6499999999999999},"min":{"1min":0.26100000000000001,"5min":0.253,"15min":0.22900000000000001},"max":{"1min":94.460999999999999,"5min":94.460999999999999,"15min":107.33499999999999},"last":6.4580000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":1.883,"15min":2.0739999999999998},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.0449999999999999,"5min":8.0449999999999999,"15min":8.0449999999999999},"last":6.5300000000000002},{"interface":"front","average":{"1min":1.4990000000000001,"5min":1.48,"15min":1.52},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9080000000000004,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":2.0649999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.663,"15min":2.472},"min":{"1min":0.36399999999999999,"5min":0.314,"15min":0.29699999999999999},"max":{"1min":4.202,"5min":48.216999999999999,"15min":102.896},"last":9.1560000000000006},{"interface":"front","average":{"1min":1.423,"5min":1.8069999999999999,"15min":2.589},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":7.875,"5min":48.232999999999997,"15min":112.667},"last":2.101}]},{"osd":5,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":1.2669999999999999,"15min":1.0860000000000001},"min":{"1min":0.30299999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.0220000000000002,"5min":13.407,"15min":28.399000000000001},"last":6.4930000000000003},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2809999999999999,"15min":1.153},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":11.048999999999999,"5min":14.212,"15min":28.350999999999999},"last":2.1280000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.034,"15min":0.89000000000000001},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.5439999999999996,"5min":7.5439999999999996,"15min":7.8550000000000004},"last":9.1349999999999998},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.216,"15min":0.91700000000000004},"min":{"1min":0.34699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.1720000000000006,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":7.4450000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":1.03,"15min":0.95399999999999996},"min":{"1min":0.376,"5min":0.307,"15min":0.27500000000000002},"max":{"1min":8.1440000000000001,"5min":8.1440000000000001,"15min":8.577},"last":7.2249999999999996},{"interface":"front","average":{"1min":1.23,"5min":0.89400000000000002,"15min":0.89600000000000002},"min":{"1min":0.32600000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.633,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":9.1110000000000007}]}]},{"osd":5,"up_from":38,"seq":163208757617,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3657604,"kb_used_data":3240124,"kb_used_omap":0,"kb_used_meta":417344,"kb_avail":90087548,"statfs":{"total":95995035648,"available":92249649152,"internally_reserved":0,"allocated":3317886976,"data_stored":3316440091,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":427360256},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":13.648999999999999,"5min":3.9780000000000002,"15min":4.5839999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":124.892,"5min":124.892,"15min":163.54499999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":13.523,"5min":3.528,"15min":4.3230000000000004},"min":{"1min":0.34999999999999998,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":123.807,"5min":123.807,"15min":162.81800000000001},"last":0.53800000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":3.7639999999999998,"15min":3.819},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":7.1440000000000001,"5min":113.514,"15min":113.514},"last":0.60299999999999998},{"interface":"front","average":{"1min":2.323,"5min":3.988,"15min":3.8599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":8.4420000000000002,"5min":113.42700000000001,"15min":113.42700000000001},"last":0.437}]},{"osd":2,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6059999999999999,"5min":2.1720000000000002,"15min":2.331},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":8.5359999999999996,"5min":10.451000000000001,"15min":10.451000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.1459999999999999,"15min":2.4009999999999998},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.5180000000000007,"5min":8.5180000000000007,"15min":8.5180000000000007},"last":0.82899999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.6539999999999999,"15min":3.7719999999999998},"min":{"1min":0.36099999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":8.5459999999999994,"5min":9.3369999999999997,"15min":132.59},"last":0.753},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.7070000000000001,"15min":3.71},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":7.8390000000000004,"5min":7.8390000000000004,"15min":129.381},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9849999999999999,"5min":1.3620000000000001,"15min":1.508},"min":{"1min":0.27700000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":36.343000000000004,"5min":36.343000000000004,"15min":126.679},"last":0.36399999999999999},{"interface":"front","average":{"1min":3.9740000000000002,"5min":1.377,"15min":1.512},"min":{"1min":0.35899999999999999,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":36.421999999999997,"5min":36.421999999999997,"15min":126.78100000000001},"last":0.77600000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:32:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":1.8999999999999999,"15min":1.2350000000000001},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.5820000000000007,"5min":70.581999999999994,"15min":70.581999999999994},"last":0.80300000000000005},{"interface":"front","average":{"1min":2.0059999999999998,"5min":2.0499999999999998,"15min":1.2410000000000001},"min":{"1min":0.28699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":7.25,"5min":71.093000000000004,"15min":71.093000000000004},"last":0.66800000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":3.7650000000000001,"5min":1.24,"15min":1.107},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":40.548000000000002,"5min":40.548000000000002,"15min":40.548000000000002},"last":0.63500000000000001},{"interface":"front","average":{"1min":3.6970000000000001,"5min":1.3440000000000001,"15min":0.98899999999999999},"min":{"1min":0.38,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":41.073,"5min":41.073,"15min":41.073},"last":0.46500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2978152448,"data_stored":2978152448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3012038656,"data_stored":3012034584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3959816192,"data_stored":3959816192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3289976832,"data_stored":3289972760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2655719424,"data_stored":2655715352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":291569664,"data_stored":291569664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:33:07.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:33:07.747 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:07.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:07.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:07.859 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:07.879 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:07.938 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:08.033 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:08.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:08.135 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:08.199 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:08.300 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:08.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:08.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:08.355 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:08.407 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:08.433 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:08.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 308/242970 objects degraded (0.127%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:33:08.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 308/242970 objects degraded (0.127%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:33:08.541 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:08.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 308/242970 objects degraded (0.127%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:33:08.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:08.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:08.652 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:08.692 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:08.768 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:08.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:08.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:08.873 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:08.917 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:08.991 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:08.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:08.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:09.097 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:09.167 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:09.273 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:09.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:09.383 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:09.440 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:09 smithi070 ceph-mon[192652]: pgmap v1054: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 308/242970 objects degraded (0.127%); 208/242970 objects misplaced (0.086%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:09.440 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:09.446 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:09.544 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:09.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:09 smithi026 ceph-mon[223014]: pgmap v1054: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 308/242970 objects degraded (0.127%); 208/242970 objects misplaced (0.086%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:09.548 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:09.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:09 smithi026 ceph-mon[215775]: pgmap v1054: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 966 op/s; 308/242970 objects degraded (0.127%); 208/242970 objects misplaced (0.086%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:09.549 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:09.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:09.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:09.651 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:09.698 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:09.792 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:33:09.793 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:09.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:09.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:09.821 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:33:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:33:09] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.43.0" 2023-09-23T22:33:09.900 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:09.967 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:10.243 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:10.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:10.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:10.349 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:10.402 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:10.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:10 smithi026 ceph-mon[215775]: pgmap v1055: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 977 op/s; 308/239040 objects degraded (0.129%); 208/239040 objects misplaced (0.087%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:10.508 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:33:10.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:10 smithi026 ceph-mon[223014]: pgmap v1055: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 977 op/s; 308/239040 objects degraded (0.129%); 208/239040 objects misplaced (0.087%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:10.509 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:33:10.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:10 smithi070 ceph-mon[192652]: pgmap v1055: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.9 GiB data, 17 GiB used, 430 GiB / 447 GiB avail; 0 B/s wr, 977 op/s; 308/239040 objects degraded (0.129%); 208/239040 objects misplaced (0.087%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:33:10.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:33:11.020 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:33:11.052 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:33:11.079 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:33:11.116 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:33:11.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:33:11.204 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:33:11.242 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:33:11.275 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:33:11.283 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:11.283 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:33:11.283 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:33:11.283 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:33:11.283 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:33:11.284 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:33:11.284 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:33:11.284 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:33:11.284 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:33:11.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:11.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:11.317 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4236748412' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:11.317 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4236748412' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:11.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:11.441 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:11.564 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:11.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:11.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:11.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4236748412' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:11.669 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:11.712 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:11.715 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:11.826 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:11.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:11.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:11.936 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:11.984 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:12.110 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:33:12.110 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:33:12.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.110 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45842.0:123471 8.4 e406/373 8:2788a077:::benchmark_data_smithi026_497658_object24579:head v 406'7730, mlcod=406'7727)", 2023-09-23T22:33:12.110 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:12.101723+0000", 2023-09-23T22:33:12.110 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0050729590000000001, 2023-09-23T22:33:12.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005104188, 2023-09-23T22:33:12.110 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101723+0000", 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101723+0000", 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101722+0000", 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101730+0000", 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0339999999999997e-06 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101752+0000", 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2356000000000001e-05 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101759+0000", 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1239999999999999e-06 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:12.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.105312+0000", 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035523360000000001 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.105329+0000", 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6983999999999999e-05 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45842.0:123470 8.e e406/395 8:78c9946e:::benchmark_data_smithi026_497658_object24578:head v 406'7602, mlcod=406'7600)", 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:12.101796+0000", 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0049993670000000002, 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050968339999999997, 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101796+0000", 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101796+0000", 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101796+0000", 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101801+0000", 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3720000000000004e-06 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101809+0000", 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3930000000000008e-06 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:12.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101814+0000", 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8300000000000003e-06 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101842+0000", 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7574999999999999e-05 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101859+0000", 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6722e-05 2023-09-23T22:33:12.118 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45842.0:123472 8.c e406/388 8:343edd03:::benchmark_data_smithi026_497658_object24580:head v 406'7736, mlcod=406'7734)", 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:12.101824+0000", 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0049719409999999997, 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051345430000000001, 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:12.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101824+0000", 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101824+0000", 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:12.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101823+0000", 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101827+0000", 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6990000000000001e-06 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101833+0000", 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7060000000000001e-06 2023-09-23T22:33:12.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101836+0000", 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.619e-06 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.102510+0000", 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00067394299999999998 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.102525+0000", 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4839e-05 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45842.0:123474 8.e e406/395 8:756a12fb:::benchmark_data_smithi026_497658_object24582:head v 406'7603, mlcod=406'7600)", 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:12.102418+0000", 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0043777659999999999, 2023-09-23T22:33:12.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0046004289999999996, 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.102418+0000", 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.102418+0000", 2023-09-23T22:33:12.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.102417+0000", 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.102428+0000", 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1068e-05 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.102438+0000", 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0182e-05 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.102444+0000", 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4920000000000001e-06 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104516+0000", 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020723730000000002 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104535+0000", 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9097999999999999e-05 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45842.0:123475 8.f e406/373 8:f461486a:::benchmark_data_smithi026_497658_object24583:head v 406'7907, mlcod=406'7905)", 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:12.104554+0000", 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0022410960000000001, 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025222159999999999, 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104554+0000", 2023-09-23T22:33:12.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104554+0000", 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104554+0000", 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104563+0000", 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0669999999999998e-06 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104574+0000", 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1450999999999999e-05 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:12.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104581+0000", 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0910000000000004e-06 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.105027+0000", 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000446021 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.105045+0000", 2023-09-23T22:33:12.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7748e-05 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45842.0:123461 8.0 e406/373 8:0972f9ff:::benchmark_data_smithi026_497658_object24569:head v 406'7485, mlcod=406'7481)", 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:12.093725+0000", 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.013070498, 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013409977999999999, 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:12.132 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.093725+0000", 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.093725+0000", 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.093725+0000", 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967296 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.093728+0000", 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.146e-06 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.093734+0000", 2023-09-23T22:33:12.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7470000000000002e-06 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.093737+0000", 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3780000000000001e-06 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.099020+0000", 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052826069999999999 2023-09-23T22:33:12.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.099042+0000", 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1997e-05 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45842.0:123464 8.0 e406/373 8:0401ccde:::benchmark_data_smithi026_497658_object24572:head v 406'7486, mlcod=406'7482)", 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:12.098337+0000", 2023-09-23T22:33:12.136 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0084581819999999999, 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0088552390000000009, 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098337+0000", 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098337+0000", 2023-09-23T22:33:12.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098337+0000", 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098341+0000", 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6849999999999999e-06 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098347+0000", 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2169999999999999e-06 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098350+0000", 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1470000000000001e-06 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:12.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101914+0000", 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035641000000000002 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.101933+0000", 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9099000000000001e-05 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.45842.0:123466 8.8 e406/399 8:12bb1923:::benchmark_data_smithi026_497658_object24574:head v 406'7697, mlcod=406'7695)", 2023-09-23T22:33:12.140 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:12.098358+0000", 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0084373539999999993, 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0088916689999999996, 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098358+0000", 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:12.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098358+0000", 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098358+0000", 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967296 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098361+0000", 2023-09-23T22:33:12.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3919999999999999e-06 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098367+0000", 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5439999999999998e-06 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.098370+0000", 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4640000000000002e-06 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104596+0000", 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062255410000000002 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:12.104614+0000", 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8465999999999999e-05 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:12.144 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T22:33:12.145 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 8 2023-09-23T22:33:12.145 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:33:12.279 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:33:12.280 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 186/228531 objects degraded (0.081%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1376,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":406,"num_osds":8,"num_up_osds":8,"osd_up_since":1695508188,"num_in_osds":5,"osd_in_since":1695508287,"num_remapped_pgs":1},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":97},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait","count":1}],"num_pgs":99,"num_pools":5,"num_objects":76177,"data_bytes":4995225547,"bytes_used":17615372288,"bytes_avail":462359805952,"bytes_total":479975178240,"degraded_objects":186,"degraded_total":228531,"degraded_ratio":0.00081389395749373176,"recovering_objects_per_sec":68,"recovering_bytes_per_sec":3662330,"recovering_keys_per_sec":0,"num_objects_recovered":681,"num_bytes_recovered":36634624,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":969},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":24,"modified":"2023-09-23T22:30:47.531589+0000","services":{}},"progress_events":{"5b5ab3a4-2f21-4d0a-896f-44b2b3c7eda7":{"message":"Global Recovery Event (75s)\n [===========================.] (remaining: 1s)","progress":0.97979795932769775,"add_to_ceph_s":true},"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:33:12.496 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:12 smithi070 ceph-mon[192652]: pgmap v1056: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 969 op/s; 186/228531 objects degraded (0.081%); 3.5 MiB/s, 68 objects/s recovering 2023-09-23T22:33:12.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:12 smithi026 ceph-mon[215775]: pgmap v1056: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 969 op/s; 186/228531 objects degraded (0.081%); 3.5 MiB/s, 68 objects/s recovering 2023-09-23T22:33:12.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:12 smithi026 ceph-mon[223014]: pgmap v1056: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 969 op/s; 186/228531 objects degraded (0.081%); 3.5 MiB/s, 68 objects/s recovering 2023-09-23T22:33:12.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:12.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:12.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:12.793 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:12.832 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:13.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/879919912' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:33:13.422 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 186/228531 objects degraded (0.081%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:33:13.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/879919912' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:33:13.423 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 186/228531 objects degraded (0.081%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:33:13.499 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:13.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:13.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:13.604 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T22:33:13.605 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:33:13.605 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:13.610 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/879919912' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:33:13.610 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 186/228531 objects degraded (0.081%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T22:33:13.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:33:13.662 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:14.003 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:14.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:14.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:14.109 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:14.156 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:14.477 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:14.639 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:14.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:14.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:14.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:14.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:14.836 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:14.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:14.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:14.848 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:14.940 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:14.971 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:15.073 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:15.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:15.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:33:15.178 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:15.227 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:15.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[223014]: pgmap v1057: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 969 op/s; 186/228531 objects degraded (0.081%); 3.5 MiB/s, 68 objects/s recovering 2023-09-23T22:33:15.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[223014]: osdmap e407: 8 total, 8 up, 5 in 2023-09-23T22:33:15.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[223014]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[223014]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[223014]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[223014]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[223014]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[215775]: pgmap v1057: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 969 op/s; 186/228531 objects degraded (0.081%); 3.5 MiB/s, 68 objects/s recovering 2023-09-23T22:33:15.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[215775]: osdmap e407: 8 total, 8 up, 5 in 2023-09-23T22:33:15.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[215775]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[215775]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[215775]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[215775]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:15 smithi026 ceph-mon[215775]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:15 smithi070 ceph-mon[192652]: pgmap v1057: 99 pgs: 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 4.7 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 969 op/s; 186/228531 objects degraded (0.081%); 3.5 MiB/s, 68 objects/s recovering 2023-09-23T22:33:15.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:15 smithi070 ceph-mon[192652]: osdmap e407: 8 total, 8 up, 5 in 2023-09-23T22:33:15.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:15 smithi070 ceph-mon[192652]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:15 smithi070 ceph-mon[192652]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:15 smithi070 ceph-mon[192652]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:15 smithi070 ceph-mon[192652]: failed to encode map e407 with expected crc 2023-09-23T22:33:15.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:15 smithi070 ceph-mon[192652]: failed to encode map e407 with expected crc 2023-09-23T22:33:16.179 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:33:16.286 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:16.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:16.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:16.390 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:16.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[215775]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[215775]: osdmap e408: 8 total, 8 up, 5 in 2023-09-23T22:33:16.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[215775]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[215775]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[215775]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4261847855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:16.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:16.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[223014]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[223014]: osdmap e408: 8 total, 8 up, 5 in 2023-09-23T22:33:16.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[223014]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[223014]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[223014]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4261847855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:16.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:16 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:16.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:16.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:16 smithi070 ceph-mon[192652]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:16 smithi070 ceph-mon[192652]: osdmap e408: 8 total, 8 up, 5 in 2023-09-23T22:33:16.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:16 smithi070 ceph-mon[192652]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:16 smithi070 ceph-mon[192652]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:16 smithi070 ceph-mon[192652]: failed to encode map e408 with expected crc 2023-09-23T22:33:16.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4261847855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:16.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:16 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:16.911 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:33:16.911 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:33:16.911 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:33:16.911 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:33:16.911 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:33:17.032 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:17.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:17.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:17.137 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:17.187 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:17.334 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:17.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:17 smithi070 ceph-mon[192652]: pgmap v1060: 99 pgs: 1 peering, 98 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s; 1.6 MiB/s, 64 objects/s recovering 2023-09-23T22:33:17.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:17 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 186/228531 objects degraded (0.081%), 1 pg degraded, 1 pg undersized) 2023-09-23T22:33:17.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:17 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:17.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:17 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:33:17.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:17 smithi070 ceph-mon[192652]: osdmap e409: 8 total, 8 up, 5 in 2023-09-23T22:33:17.378 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:17 smithi070 ceph-mon[192652]: failed to encode map e409 with expected crc 2023-09-23T22:33:17.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[223014]: pgmap v1060: 99 pgs: 1 peering, 98 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s; 1.6 MiB/s, 64 objects/s recovering 2023-09-23T22:33:17.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 186/228531 objects degraded (0.081%), 1 pg degraded, 1 pg undersized) 2023-09-23T22:33:17.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:17.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:33:17.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[223014]: osdmap e409: 8 total, 8 up, 5 in 2023-09-23T22:33:17.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[223014]: failed to encode map e409 with expected crc 2023-09-23T22:33:17.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[215775]: pgmap v1060: 99 pgs: 1 peering, 98 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 1.07k op/s; 1.6 MiB/s, 64 objects/s recovering 2023-09-23T22:33:17.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 186/228531 objects degraded (0.081%), 1 pg degraded, 1 pg undersized) 2023-09-23T22:33:17.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:17.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:33:17.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[215775]: osdmap e409: 8 total, 8 up, 5 in 2023-09-23T22:33:17.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:17 smithi026 ceph-mon[215775]: failed to encode map e409 with expected crc 2023-09-23T22:33:18.257 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:18.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:18.310 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:33:18.329 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:33:18.331 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:33:18.375 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:18.418 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:18.428 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:18 smithi070 ceph-mon[192652]: failed to encode map e409 with expected crc 2023-09-23T22:33:18.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:18 smithi026 ceph-mon[223014]: failed to encode map e409 with expected crc 2023-09-23T22:33:18.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:18 smithi026 ceph-mon[215775]: failed to encode map e409 with expected crc 2023-09-23T22:33:19.149 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:33:19.191 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:19.378 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:19 smithi026 ceph-mon[215775]: pgmap v1062: 99 pgs: 1 peering, 98 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 0 B/s, 30 objects/s recovering 2023-09-23T22:33:19.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:19 smithi026 ceph-mon[223014]: pgmap v1062: 99 pgs: 1 peering, 98 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 0 B/s, 30 objects/s recovering 2023-09-23T22:33:19.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:19 smithi070 ceph-mon[192652]: pgmap v1062: 99 pgs: 1 peering, 98 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 619 op/s; 0 B/s, 30 objects/s recovering 2023-09-23T22:33:19.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:33:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:33:19] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.43.0" 2023-09-23T22:33:20.117 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:20.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:20.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:20.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:20.224 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1062,"stamp":"2023-09-23T22:33:17.591789+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4751431627,"num_objects":72457,"num_object_clones":0,"num_object_copies":217371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72457,"num_whiteouts":0,"num_read":5983,"num_read_kb":12185,"num_write":126258,"num_write_kb":6341622,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11360,"num_bytes_recovered":724271467,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85053,"ondisk_log_size":85053,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":16402612,"kb_used_data":13973324,"kb_used_omap":15,"kb_used_meta":2429232,"kb_avail":452323148,"statfs":{"total":479975178240,"available":463178903552,"internally_reserved":0,"allocated":14308683776,"data_stored":14299962689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2487533790},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,5,3,7,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-243793920,"num_objects":-3720,"num_object_clones":0,"num_object_copies":-11160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-186,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3720,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":186,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001710"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":434,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243104+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:33:06.243104+0000","last_peered":"2023-09-23T22:33:06.243104+0000","last_clean":"2023-09-23T22:33:06.243104+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:33:06.243104+0000","last_undegraded":"2023-09-23T22:33:06.243104+0000","last_fullsized":"2023-09-23T22:33:06.243104+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":124,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.246026+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:33:06.246026+0000","last_peered":"2023-09-23T22:33:06.246026+0000","last_clean":"2023-09-23T22:33:06.246026+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:33:06.246026+0000","last_undegraded":"2023-09-23T22:33:06.246026+0000","last_fullsized":"2023-09-23T22:33:06.246026+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":124,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244452+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:33:06.244452+0000","last_peered":"2023-09-23T22:33:06.244452+0000","last_clean":"2023-09-23T22:33:06.244452+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:33:06.244452+0000","last_undegraded":"2023-09-23T22:33:06.244452+0000","last_fullsized":"2023-09-23T22:33:06.244452+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":296,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.163361+0000","last_change":"2023-09-23T22:31:28.120861+0000","last_active":"2023-09-23T22:33:14.163361+0000","last_peered":"2023-09-23T22:33:14.163361+0000","last_clean":"2023-09-23T22:33:14.163361+0000","last_became_active":"2023-09-23T22:31:28.120394+0000","last_became_peered":"2023-09-23T22:31:28.120394+0000","last_unstale":"2023-09-23T22:33:14.163361+0000","last_undegraded":"2023-09-23T22:33:14.163361+0000","last_fullsized":"2023-09-23T22:33:14.163361+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:19.470258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":462,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239330+0000","last_change":"2023-09-23T22:31:28.114528+0000","last_active":"2023-09-23T22:33:06.239330+0000","last_peered":"2023-09-23T22:33:06.239330+0000","last_clean":"2023-09-23T22:33:06.239330+0000","last_became_active":"2023-09-23T22:31:28.114177+0000","last_became_peered":"2023-09-23T22:31:28.114177+0000","last_unstale":"2023-09-23T22:33:06.239330+0000","last_undegraded":"2023-09-23T22:33:06.239330+0000","last_fullsized":"2023-09-23T22:33:06.239330+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:46.680433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":88,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236659+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:33:06.236659+0000","last_peered":"2023-09-23T22:33:06.236659+0000","last_clean":"2023-09-23T22:33:06.236659+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:33:06.236659+0000","last_undegraded":"2023-09-23T22:33:06.236659+0000","last_fullsized":"2023-09-23T22:33:06.236659+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":323,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.240143+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:33:06.240143+0000","last_peered":"2023-09-23T22:33:06.240143+0000","last_clean":"2023-09-23T22:33:06.240143+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:33:06.240143+0000","last_undegraded":"2023-09-23T22:33:06.240143+0000","last_fullsized":"2023-09-23T22:33:06.240143+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":88,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.235951+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:33:06.235951+0000","last_peered":"2023-09-23T22:33:06.235951+0000","last_clean":"2023-09-23T22:33:06.235951+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:33:06.235951+0000","last_undegraded":"2023-09-23T22:33:06.235951+0000","last_fullsized":"2023-09-23T22:33:06.235951+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":407,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244119+0000","last_change":"2023-09-23T22:31:28.116916+0000","last_active":"2023-09-23T22:33:06.244119+0000","last_peered":"2023-09-23T22:33:06.244119+0000","last_clean":"2023-09-23T22:33:06.244119+0000","last_became_active":"2023-09-23T22:31:28.116577+0000","last_became_peered":"2023-09-23T22:31:28.116577+0000","last_unstale":"2023-09-23T22:33:06.244119+0000","last_undegraded":"2023-09-23T22:33:06.244119+0000","last_fullsized":"2023-09-23T22:33:06.244119+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:33.092789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":98,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.237132+0000","last_change":"2023-09-23T22:31:28.122684+0000","last_active":"2023-09-23T22:33:06.237132+0000","last_peered":"2023-09-23T22:33:06.237132+0000","last_clean":"2023-09-23T22:33:06.237132+0000","last_became_active":"2023-09-23T22:31:28.122427+0000","last_became_peered":"2023-09-23T22:31:28.122427+0000","last_unstale":"2023-09-23T22:33:06.237132+0000","last_undegraded":"2023-09-23T22:33:06.237132+0000","last_fullsized":"2023-09-23T22:33:06.237132+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:15.559367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":154,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243568+0000","last_change":"2023-09-23T22:31:28.153199+0000","last_active":"2023-09-23T22:33:06.243568+0000","last_peered":"2023-09-23T22:33:06.243568+0000","last_clean":"2023-09-23T22:33:06.243568+0000","last_became_active":"2023-09-23T22:31:28.129257+0000","last_became_peered":"2023-09-23T22:31:28.129257+0000","last_unstale":"2023-09-23T22:33:06.243568+0000","last_undegraded":"2023-09-23T22:33:06.243568+0000","last_fullsized":"2023-09-23T22:33:06.243568+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:03.743836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":56,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244978+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:33:06.244978+0000","last_peered":"2023-09-23T22:33:06.244978+0000","last_clean":"2023-09-23T22:33:06.244978+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:33:06.244978+0000","last_undegraded":"2023-09-23T22:33:06.244978+0000","last_fullsized":"2023-09-23T22:33:06.244978+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":491,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243409+0000","last_change":"2023-09-23T22:31:28.145916+0000","last_active":"2023-09-23T22:33:06.243409+0000","last_peered":"2023-09-23T22:33:06.243409+0000","last_clean":"2023-09-23T22:33:06.243409+0000","last_became_active":"2023-09-23T22:31:28.120703+0000","last_became_peered":"2023-09-23T22:31:28.120703+0000","last_unstale":"2023-09-23T22:33:06.243409+0000","last_undegraded":"2023-09-23T22:33:06.243409+0000","last_fullsized":"2023-09-23T22:33:06.243409+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:07.819247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":405,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242900+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:33:06.242900+0000","last_peered":"2023-09-23T22:33:06.242900+0000","last_clean":"2023-09-23T22:33:06.242900+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:33:06.242900+0000","last_undegraded":"2023-09-23T22:33:06.242900+0000","last_fullsized":"2023-09-23T22:33:06.242900+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":398,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242978+0000","last_change":"2023-09-23T22:31:28.149149+0000","last_active":"2023-09-23T22:33:06.242978+0000","last_peered":"2023-09-23T22:33:06.242978+0000","last_clean":"2023-09-23T22:33:06.242978+0000","last_became_active":"2023-09-23T22:31:28.121120+0000","last_became_peered":"2023-09-23T22:31:28.121120+0000","last_unstale":"2023-09-23T22:33:06.242978+0000","last_undegraded":"2023-09-23T22:33:06.242978+0000","last_fullsized":"2023-09-23T22:33:06.242978+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.434137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":369,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242288+0000","last_change":"2023-09-23T22:31:28.119520+0000","last_active":"2023-09-23T22:33:06.242288+0000","last_peered":"2023-09-23T22:33:06.242288+0000","last_clean":"2023-09-23T22:33:06.242288+0000","last_became_active":"2023-09-23T22:31:28.118810+0000","last_became_peered":"2023-09-23T22:31:28.118810+0000","last_unstale":"2023-09-23T22:33:06.242288+0000","last_undegraded":"2023-09-23T22:33:06.242288+0000","last_fullsized":"2023-09-23T22:33:06.242288+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:36.007062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":313,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239638+0000","last_change":"2023-09-23T22:31:28.114537+0000","last_active":"2023-09-23T22:33:06.239638+0000","last_peered":"2023-09-23T22:33:06.239638+0000","last_clean":"2023-09-23T22:33:06.239638+0000","last_became_active":"2023-09-23T22:31:28.114176+0000","last_became_peered":"2023-09-23T22:31:28.114176+0000","last_unstale":"2023-09-23T22:33:06.239638+0000","last_undegraded":"2023-09-23T22:33:06.239638+0000","last_fullsized":"2023-09-23T22:33:06.239638+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.324173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":416,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244166+0000","last_change":"2023-09-23T22:31:28.132494+0000","last_active":"2023-09-23T22:33:06.244166+0000","last_peered":"2023-09-23T22:33:06.244166+0000","last_clean":"2023-09-23T22:33:06.244166+0000","last_became_active":"2023-09-23T22:31:28.132197+0000","last_became_peered":"2023-09-23T22:31:28.132197+0000","last_unstale":"2023-09-23T22:33:06.244166+0000","last_undegraded":"2023-09-23T22:33:06.244166+0000","last_fullsized":"2023-09-23T22:33:06.244166+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:36.181016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":350,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243193+0000","last_change":"2023-09-23T22:31:28.119156+0000","last_active":"2023-09-23T22:33:06.243193+0000","last_peered":"2023-09-23T22:33:06.243193+0000","last_clean":"2023-09-23T22:33:06.243193+0000","last_became_active":"2023-09-23T22:31:28.118382+0000","last_became_peered":"2023-09-23T22:31:28.118382+0000","last_unstale":"2023-09-23T22:33:06.243193+0000","last_undegraded":"2023-09-23T22:33:06.243193+0000","last_fullsized":"2023-09-23T22:33:06.243193+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:10.140079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":303,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.240966+0000","last_change":"2023-09-23T22:31:28.121397+0000","last_active":"2023-09-23T22:33:06.240966+0000","last_peered":"2023-09-23T22:33:06.240966+0000","last_clean":"2023-09-23T22:33:06.240966+0000","last_became_active":"2023-09-23T22:31:28.120847+0000","last_became_peered":"2023-09-23T22:31:28.120847+0000","last_unstale":"2023-09-23T22:33:06.240966+0000","last_undegraded":"2023-09-23T22:33:06.240966+0000","last_fullsized":"2023-09-23T22:33:06.240966+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:42.407658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":98,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.235812+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:33:06.235812+0000","last_peered":"2023-09-23T22:33:06.235812+0000","last_clean":"2023-09-23T22:33:06.235812+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:33:06.235812+0000","last_undegraded":"2023-09-23T22:33:06.235812+0000","last_fullsized":"2023-09-23T22:33:06.235812+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":98,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236491+0000","last_change":"2023-09-23T22:31:28.128965+0000","last_active":"2023-09-23T22:33:06.236491+0000","last_peered":"2023-09-23T22:33:06.236491+0000","last_clean":"2023-09-23T22:33:06.236491+0000","last_became_active":"2023-09-23T22:31:28.128224+0000","last_became_peered":"2023-09-23T22:31:28.128224+0000","last_unstale":"2023-09-23T22:33:06.236491+0000","last_undegraded":"2023-09-23T22:33:06.236491+0000","last_fullsized":"2023-09-23T22:33:06.236491+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:11.301143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"407'7844","reported_seq":7886,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.232081+0000","last_change":"2023-09-23T22:31:08.226781+0000","last_active":"2023-09-23T22:33:14.232081+0000","last_peered":"2023-09-23T22:33:14.232081+0000","last_clean":"2023-09-23T22:33:14.232081+0000","last_became_active":"2023-09-23T22:31:08.226511+0000","last_became_peered":"2023-09-23T22:31:08.226511+0000","last_unstale":"2023-09-23T22:33:14.232081+0000","last_undegraded":"2023-09-23T22:33:14.232081+0000","last_fullsized":"2023-09-23T22:33:14.232081+0000","mapping_epoch":373,"log_start":"385'2300","ondisk_log_start":"385'2300","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5544,"log_dups_size":0,"ondisk_log_size":5544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:03.735360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295043072,"num_objects":4502,"num_object_clones":0,"num_object_copies":13506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7844,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":416,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243589+0000","last_change":"2023-09-23T22:31:28.139498+0000","last_active":"2023-09-23T22:33:06.243589+0000","last_peered":"2023-09-23T22:33:06.243589+0000","last_clean":"2023-09-23T22:33:06.243589+0000","last_became_active":"2023-09-23T22:31:28.113206+0000","last_became_peered":"2023-09-23T22:31:28.113206+0000","last_unstale":"2023-09-23T22:33:06.243589+0000","last_undegraded":"2023-09-23T22:33:06.243589+0000","last_fullsized":"2023-09-23T22:33:06.243589+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:11.905312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":409,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242057+0000","last_change":"2023-09-23T22:31:28.131730+0000","last_active":"2023-09-23T22:33:06.242057+0000","last_peered":"2023-09-23T22:33:06.242057+0000","last_clean":"2023-09-23T22:33:06.242057+0000","last_became_active":"2023-09-23T22:31:28.131383+0000","last_became_peered":"2023-09-23T22:31:28.131383+0000","last_unstale":"2023-09-23T22:33:06.242057+0000","last_undegraded":"2023-09-23T22:33:06.242057+0000","last_fullsized":"2023-09-23T22:33:06.242057+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:56.993827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"407'7890","reported_seq":11135,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.464355+0000","last_change":"2023-09-23T22:32:00.147475+0000","last_active":"2023-09-23T22:33:14.464355+0000","last_peered":"2023-09-23T22:33:14.464355+0000","last_clean":"2023-09-23T22:33:14.464355+0000","last_became_active":"2023-09-23T22:32:00.121320+0000","last_became_peered":"2023-09-23T22:32:00.121320+0000","last_unstale":"2023-09-23T22:33:14.464355+0000","last_undegraded":"2023-09-23T22:33:14.464355+0000","last_fullsized":"2023-09-23T22:33:14.464355+0000","mapping_epoch":392,"log_start":"389'2800","ondisk_log_start":"389'2800","created":373,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5090,"log_dups_size":0,"ondisk_log_size":5090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:17:59.934876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298582016,"num_objects":4556,"num_object_clones":0,"num_object_copies":13668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7890,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1591,"num_bytes_recovered":104267776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":358,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243622+0000","last_change":"2023-09-23T22:31:28.125280+0000","last_active":"2023-09-23T22:33:06.243622+0000","last_peered":"2023-09-23T22:33:06.243622+0000","last_clean":"2023-09-23T22:33:06.243622+0000","last_became_active":"2023-09-23T22:31:28.110856+0000","last_became_peered":"2023-09-23T22:31:28.110856+0000","last_unstale":"2023-09-23T22:33:06.243622+0000","last_undegraded":"2023-09-23T22:33:06.243622+0000","last_fullsized":"2023-09-23T22:33:06.243622+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:32.197056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":434,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243123+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:33:06.243123+0000","last_peered":"2023-09-23T22:33:06.243123+0000","last_clean":"2023-09-23T22:33:06.243123+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:33:06.243123+0000","last_undegraded":"2023-09-23T22:33:06.243123+0000","last_fullsized":"2023-09-23T22:33:06.243123+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"406'7870","reported_seq":11151,"reported_epoch":407,"state":"peering","last_fresh":"2023-09-23T22:33:14.162808+0000","last_change":"2023-09-23T22:33:14.163005+0000","last_active":"2023-09-23T22:33:14.162808+0000","last_peered":"2023-09-23T22:33:14.141818+0000","last_clean":"2023-09-23T22:31:27.059725+0000","last_became_active":"2023-09-23T22:31:29.178921+0000","last_became_peered":"2023-09-23T22:31:29.178921+0000","last_unstale":"2023-09-23T22:33:14.162808+0000","last_undegraded":"2023-09-23T22:33:14.162808+0000","last_fullsized":"2023-09-23T22:33:14.162808+0000","mapping_epoch":407,"log_start":"385'2500","ondisk_log_start":"385'2500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5370,"log_dups_size":0,"ondisk_log_size":5370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:41.006523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7870,"num_write_kb":396352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1614,"num_bytes_recovered":85590016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,6,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":374,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244279+0000","last_change":"2023-09-23T22:31:28.136321+0000","last_active":"2023-09-23T22:33:06.244279+0000","last_peered":"2023-09-23T22:33:06.244279+0000","last_clean":"2023-09-23T22:33:06.244279+0000","last_became_active":"2023-09-23T22:31:28.131102+0000","last_became_peered":"2023-09-23T22:31:28.131102+0000","last_unstale":"2023-09-23T22:33:06.244279+0000","last_undegraded":"2023-09-23T22:33:06.244279+0000","last_fullsized":"2023-09-23T22:33:06.244279+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:24.768099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":98,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236130+0000","last_change":"2023-09-23T22:31:28.125138+0000","last_active":"2023-09-23T22:33:06.236130+0000","last_peered":"2023-09-23T22:33:06.236130+0000","last_clean":"2023-09-23T22:33:06.236130+0000","last_became_active":"2023-09-23T22:31:28.124716+0000","last_became_peered":"2023-09-23T22:31:28.124716+0000","last_unstale":"2023-09-23T22:33:06.236130+0000","last_undegraded":"2023-09-23T22:33:06.236130+0000","last_fullsized":"2023-09-23T22:33:06.236130+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.718931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"407'7932","reported_seq":7974,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.492469+0000","last_change":"2023-09-23T22:31:08.225591+0000","last_active":"2023-09-23T22:33:14.492469+0000","last_peered":"2023-09-23T22:33:14.492469+0000","last_clean":"2023-09-23T22:33:14.492469+0000","last_became_active":"2023-09-23T22:31:08.225286+0000","last_became_peered":"2023-09-23T22:31:08.225286+0000","last_unstale":"2023-09-23T22:33:14.492469+0000","last_undegraded":"2023-09-23T22:33:14.492469+0000","last_fullsized":"2023-09-23T22:33:14.492469+0000","mapping_epoch":373,"log_start":"389'2900","ondisk_log_start":"389'2900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:14.895634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297795584,"num_objects":4544,"num_object_clones":0,"num_object_copies":13632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7932,"num_write_kb":399232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":437,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242441+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:33:06.242441+0000","last_peered":"2023-09-23T22:33:06.242441+0000","last_clean":"2023-09-23T22:33:06.242441+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:33:06.242441+0000","last_undegraded":"2023-09-23T22:33:06.242441+0000","last_fullsized":"2023-09-23T22:33:06.242441+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":293,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.240084+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:33:06.240084+0000","last_peered":"2023-09-23T22:33:06.240084+0000","last_clean":"2023-09-23T22:33:06.240084+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:33:06.240084+0000","last_undegraded":"2023-09-23T22:33:06.240084+0000","last_fullsized":"2023-09-23T22:33:06.240084+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"407'7616","reported_seq":7658,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.474723+0000","last_change":"2023-09-23T22:31:08.226199+0000","last_active":"2023-09-23T22:33:14.474723+0000","last_peered":"2023-09-23T22:33:14.474723+0000","last_clean":"2023-09-23T22:33:14.474723+0000","last_became_active":"2023-09-23T22:31:08.225930+0000","last_became_peered":"2023-09-23T22:31:08.225930+0000","last_unstale":"2023-09-23T22:33:14.474723+0000","last_undegraded":"2023-09-23T22:33:14.474723+0000","last_fullsized":"2023-09-23T22:33:14.474723+0000","mapping_epoch":373,"log_start":"385'2100","ondisk_log_start":"385'2100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5516,"log_dups_size":0,"ondisk_log_size":5516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:16.294443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288882688,"num_objects":4408,"num_object_clones":0,"num_object_copies":13224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7616,"num_write_kb":384768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":752,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243621+0000","last_change":"2023-09-23T22:31:31.125903+0000","last_active":"2023-09-23T22:33:06.243621+0000","last_peered":"2023-09-23T22:33:06.243621+0000","last_clean":"2023-09-23T22:33:06.243621+0000","last_became_active":"2023-09-23T22:31:31.125630+0000","last_became_peered":"2023-09-23T22:31:31.125630+0000","last_unstale":"2023-09-23T22:33:06.243621+0000","last_undegraded":"2023-09-23T22:33:06.243621+0000","last_fullsized":"2023-09-23T22:33:06.243621+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:24.405922+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":239,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.240164+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:33:06.240164+0000","last_peered":"2023-09-23T22:33:06.240164+0000","last_clean":"2023-09-23T22:33:06.240164+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:33:06.240164+0000","last_undegraded":"2023-09-23T22:33:06.240164+0000","last_fullsized":"2023-09-23T22:33:06.240164+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":415,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243243+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:33:06.243243+0000","last_peered":"2023-09-23T22:33:06.243243+0000","last_clean":"2023-09-23T22:33:06.243243+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:33:06.243243+0000","last_undegraded":"2023-09-23T22:33:06.243243+0000","last_fullsized":"2023-09-23T22:33:06.243243+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"407'7890","reported_seq":7934,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.426901+0000","last_change":"2023-09-23T22:31:08.227045+0000","last_active":"2023-09-23T22:33:14.426901+0000","last_peered":"2023-09-23T22:33:14.426901+0000","last_clean":"2023-09-23T22:33:14.426901+0000","last_became_active":"2023-09-23T22:31:08.226758+0000","last_became_peered":"2023-09-23T22:31:08.226758+0000","last_unstale":"2023-09-23T22:33:14.426901+0000","last_undegraded":"2023-09-23T22:33:14.426901+0000","last_fullsized":"2023-09-23T22:33:14.426901+0000","mapping_epoch":373,"log_start":"390'3100","ondisk_log_start":"390'3100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4790,"log_dups_size":0,"ondisk_log_size":4790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:55.118636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299499520,"num_objects":4570,"num_object_clones":0,"num_object_copies":13710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4570,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7890,"num_write_kb":398720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":788,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242144+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:33:06.242144+0000","last_peered":"2023-09-23T22:33:06.242144+0000","last_clean":"2023-09-23T22:33:06.242144+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:33:06.242144+0000","last_undegraded":"2023-09-23T22:33:06.242144+0000","last_fullsized":"2023-09-23T22:33:06.242144+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":88,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236204+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:33:06.236204+0000","last_peered":"2023-09-23T22:33:06.236204+0000","last_clean":"2023-09-23T22:33:06.236204+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:33:06.236204+0000","last_undegraded":"2023-09-23T22:33:06.236204+0000","last_fullsized":"2023-09-23T22:33:06.236204+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":442,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243440+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:33:06.243440+0000","last_peered":"2023-09-23T22:33:06.243440+0000","last_clean":"2023-09-23T22:33:06.243440+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:33:06.243440+0000","last_undegraded":"2023-09-23T22:33:06.243440+0000","last_fullsized":"2023-09-23T22:33:06.243440+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"407'7890","reported_seq":7932,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.449904+0000","last_change":"2023-09-23T22:31:08.225049+0000","last_active":"2023-09-23T22:33:14.449904+0000","last_peered":"2023-09-23T22:33:14.449904+0000","last_clean":"2023-09-23T22:33:14.449904+0000","last_became_active":"2023-09-23T22:31:08.224765+0000","last_became_peered":"2023-09-23T22:31:08.224765+0000","last_unstale":"2023-09-23T22:33:14.449904+0000","last_undegraded":"2023-09-23T22:33:14.449904+0000","last_fullsized":"2023-09-23T22:33:14.449904+0000","mapping_epoch":373,"log_start":"390'3100","ondisk_log_start":"390'3100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4790,"log_dups_size":0,"ondisk_log_size":4790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.842171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295698432,"num_objects":4512,"num_object_clones":0,"num_object_copies":13536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7890,"num_write_kb":396864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":398,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242897+0000","last_change":"2023-09-23T22:31:28.119358+0000","last_active":"2023-09-23T22:33:06.242897+0000","last_peered":"2023-09-23T22:33:06.242897+0000","last_clean":"2023-09-23T22:33:06.242897+0000","last_became_active":"2023-09-23T22:31:28.118823+0000","last_became_peered":"2023-09-23T22:31:28.118823+0000","last_unstale":"2023-09-23T22:33:06.242897+0000","last_undegraded":"2023-09-23T22:33:06.242897+0000","last_fullsized":"2023-09-23T22:33:06.242897+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:25.483176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":423,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243494+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:33:06.243494+0000","last_peered":"2023-09-23T22:33:06.243494+0000","last_clean":"2023-09-23T22:33:06.243494+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:33:06.243494+0000","last_undegraded":"2023-09-23T22:33:06.243494+0000","last_fullsized":"2023-09-23T22:33:06.243494+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"407'7913","reported_seq":7957,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.417988+0000","last_change":"2023-09-23T22:31:08.226959+0000","last_active":"2023-09-23T22:33:14.417988+0000","last_peered":"2023-09-23T22:33:14.417988+0000","last_clean":"2023-09-23T22:33:14.417988+0000","last_became_active":"2023-09-23T22:31:08.226652+0000","last_became_peered":"2023-09-23T22:31:08.226652+0000","last_unstale":"2023-09-23T22:33:14.417988+0000","last_undegraded":"2023-09-23T22:33:14.417988+0000","last_fullsized":"2023-09-23T22:33:14.417988+0000","mapping_epoch":373,"log_start":"390'3100","ondisk_log_start":"390'3100","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4813,"log_dups_size":0,"ondisk_log_size":4813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:22.758103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296812544,"num_objects":4529,"num_object_clones":0,"num_object_copies":13587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7913,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":761,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244458+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:33:06.244458+0000","last_peered":"2023-09-23T22:33:06.244458+0000","last_clean":"2023-09-23T22:33:06.244458+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:33:06.244458+0000","last_undegraded":"2023-09-23T22:33:06.244458+0000","last_fullsized":"2023-09-23T22:33:06.244458+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":124,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243072+0000","last_change":"2023-09-23T22:31:28.120600+0000","last_active":"2023-09-23T22:33:06.243072+0000","last_peered":"2023-09-23T22:33:06.243072+0000","last_clean":"2023-09-23T22:33:06.243072+0000","last_became_active":"2023-09-23T22:31:28.120281+0000","last_became_peered":"2023-09-23T22:31:28.120281+0000","last_unstale":"2023-09-23T22:33:06.243072+0000","last_undegraded":"2023-09-23T22:33:06.243072+0000","last_fullsized":"2023-09-23T22:33:06.243072+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:10.594022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":124,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.246443+0000","last_change":"2023-09-23T22:31:28.111270+0000","last_active":"2023-09-23T22:33:06.246443+0000","last_peered":"2023-09-23T22:33:06.246443+0000","last_clean":"2023-09-23T22:33:06.246443+0000","last_became_active":"2023-09-23T22:31:28.110921+0000","last_became_peered":"2023-09-23T22:31:28.110921+0000","last_unstale":"2023-09-23T22:33:06.246443+0000","last_undegraded":"2023-09-23T22:33:06.246443+0000","last_fullsized":"2023-09-23T22:33:06.246443+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:05.417671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"407'7719","reported_seq":10854,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.446739+0000","last_change":"2023-09-23T22:32:15.139562+0000","last_active":"2023-09-23T22:33:14.446739+0000","last_peered":"2023-09-23T22:33:14.446739+0000","last_clean":"2023-09-23T22:33:14.446739+0000","last_became_active":"2023-09-23T22:32:15.135437+0000","last_became_peered":"2023-09-23T22:32:15.135437+0000","last_unstale":"2023-09-23T22:33:14.446739+0000","last_undegraded":"2023-09-23T22:33:14.446739+0000","last_fullsized":"2023-09-23T22:33:14.446739+0000","mapping_epoch":395,"log_start":"385'2300","ondisk_log_start":"385'2300","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5419,"log_dups_size":0,"ondisk_log_size":5419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:36.457421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295370752,"num_objects":4507,"num_object_clones":0,"num_object_copies":13521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7719,"num_write_kb":391232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1537,"num_bytes_recovered":100728832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":682,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236594+0000","last_change":"2023-09-23T22:31:31.129491+0000","last_active":"2023-09-23T22:33:06.236594+0000","last_peered":"2023-09-23T22:33:06.236594+0000","last_clean":"2023-09-23T22:33:06.236594+0000","last_became_active":"2023-09-23T22:31:31.129163+0000","last_became_peered":"2023-09-23T22:31:31.129163+0000","last_unstale":"2023-09-23T22:33:06.236594+0000","last_undegraded":"2023-09-23T22:33:06.236594+0000","last_fullsized":"2023-09-23T22:33:06.236594+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.141229+0000","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":783,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239970+0000","last_change":"2023-09-23T22:31:31.129492+0000","last_active":"2023-09-23T22:33:06.239970+0000","last_peered":"2023-09-23T22:33:06.239970+0000","last_clean":"2023-09-23T22:33:06.239970+0000","last_became_active":"2023-09-23T22:31:31.128706+0000","last_became_peered":"2023-09-23T22:31:31.128706+0000","last_unstale":"2023-09-23T22:33:06.239970+0000","last_undegraded":"2023-09-23T22:33:06.239970+0000","last_fullsized":"2023-09-23T22:33:06.239970+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:16.180152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":474,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244072+0000","last_change":"2023-09-23T22:31:28.130412+0000","last_active":"2023-09-23T22:33:06.244072+0000","last_peered":"2023-09-23T22:33:06.244072+0000","last_clean":"2023-09-23T22:33:06.244072+0000","last_became_active":"2023-09-23T22:31:28.130127+0000","last_became_peered":"2023-09-23T22:31:28.130127+0000","last_unstale":"2023-09-23T22:33:06.244072+0000","last_undegraded":"2023-09-23T22:33:06.244072+0000","last_fullsized":"2023-09-23T22:33:06.244072+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.088651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244817+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:33:06.244817+0000","last_peered":"2023-09-23T22:33:06.244817+0000","last_clean":"2023-09-23T22:33:06.244817+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:33:06.244817+0000","last_undegraded":"2023-09-23T22:33:06.244817+0000","last_fullsized":"2023-09-23T22:33:06.244817+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"407'7809","reported_seq":11080,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.445498+0000","last_change":"2023-09-23T22:33:13.261657+0000","last_active":"2023-09-23T22:33:14.445498+0000","last_peered":"2023-09-23T22:33:14.445498+0000","last_clean":"2023-09-23T22:33:14.445498+0000","last_became_active":"2023-09-23T22:32:31.163888+0000","last_became_peered":"2023-09-23T22:32:31.163888+0000","last_unstale":"2023-09-23T22:33:14.445498+0000","last_undegraded":"2023-09-23T22:33:14.445498+0000","last_fullsized":"2023-09-23T22:33:14.445498+0000","mapping_epoch":399,"log_start":"385'2300","ondisk_log_start":"385'2300","created":373,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5509,"log_dups_size":0,"ondisk_log_size":5509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:18.004864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293535744,"num_objects":4479,"num_object_clones":0,"num_object_copies":13437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7809,"num_write_kb":393216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1603,"num_bytes_recovered":105054208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":751,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243890+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:33:06.243890+0000","last_peered":"2023-09-23T22:33:06.243890+0000","last_clean":"2023-09-23T22:33:06.243890+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:33:06.243890+0000","last_undegraded":"2023-09-23T22:33:06.243890+0000","last_fullsized":"2023-09-23T22:33:06.243890+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1285,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244596+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:33:06.244596+0000","last_peered":"2023-09-23T22:33:06.244596+0000","last_clean":"2023-09-23T22:33:06.244596+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:33:06.244596+0000","last_undegraded":"2023-09-23T22:33:06.244596+0000","last_fullsized":"2023-09-23T22:33:06.244596+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":446,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242913+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:33:06.242913+0000","last_peered":"2023-09-23T22:33:06.242913+0000","last_clean":"2023-09-23T22:33:06.242913+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:33:06.242913+0000","last_undegraded":"2023-09-23T22:33:06.242913+0000","last_fullsized":"2023-09-23T22:33:06.242913+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":98,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236344+0000","last_change":"2023-09-23T22:31:28.123596+0000","last_active":"2023-09-23T22:33:06.236344+0000","last_peered":"2023-09-23T22:33:06.236344+0000","last_clean":"2023-09-23T22:33:06.236344+0000","last_became_active":"2023-09-23T22:31:28.123179+0000","last_became_peered":"2023-09-23T22:31:28.123179+0000","last_unstale":"2023-09-23T22:33:06.236344+0000","last_undegraded":"2023-09-23T22:33:06.236344+0000","last_fullsized":"2023-09-23T22:33:06.236344+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:53.106243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.b","version":"407'7700","reported_seq":10744,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.418191+0000","last_change":"2023-09-23T22:31:44.145615+0000","last_active":"2023-09-23T22:33:14.418191+0000","last_peered":"2023-09-23T22:33:14.418191+0000","last_clean":"2023-09-23T22:33:14.418191+0000","last_became_active":"2023-09-23T22:31:44.142103+0000","last_became_peered":"2023-09-23T22:31:44.142103+0000","last_unstale":"2023-09-23T22:33:14.418191+0000","last_undegraded":"2023-09-23T22:33:14.418191+0000","last_fullsized":"2023-09-23T22:33:14.418191+0000","mapping_epoch":387,"log_start":"390'2900","ondisk_log_start":"390'2900","created":373,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4800,"log_dups_size":0,"ondisk_log_size":4800,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:15.488007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300285952,"num_objects":4582,"num_object_clones":0,"num_object_copies":13746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7700,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1492,"num_bytes_recovered":97779712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":757,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239519+0000","last_change":"2023-09-23T22:31:31.130599+0000","last_active":"2023-09-23T22:33:06.239519+0000","last_peered":"2023-09-23T22:33:06.239519+0000","last_clean":"2023-09-23T22:33:06.239519+0000","last_became_active":"2023-09-23T22:31:31.127784+0000","last_became_peered":"2023-09-23T22:31:31.127784+0000","last_unstale":"2023-09-23T22:33:06.239519+0000","last_undegraded":"2023-09-23T22:33:06.239519+0000","last_fullsized":"2023-09-23T22:33:06.239519+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:28.788741+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":624,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239398+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:33:06.239398+0000","last_peered":"2023-09-23T22:33:06.239398+0000","last_clean":"2023-09-23T22:33:06.239398+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:33:06.239398+0000","last_undegraded":"2023-09-23T22:33:06.239398+0000","last_fullsized":"2023-09-23T22:33:06.239398+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":88,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236029+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:33:06.236029+0000","last_peered":"2023-09-23T22:33:06.236029+0000","last_clean":"2023-09-23T22:33:06.236029+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:33:06.236029+0000","last_undegraded":"2023-09-23T22:33:06.236029+0000","last_fullsized":"2023-09-23T22:33:06.236029+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":56,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.246309+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:33:06.246309+0000","last_peered":"2023-09-23T22:33:06.246309+0000","last_clean":"2023-09-23T22:33:06.246309+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:33:06.246309+0000","last_undegraded":"2023-09-23T22:33:06.246309+0000","last_fullsized":"2023-09-23T22:33:06.246309+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"407'7732","reported_seq":7774,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.503899+0000","last_change":"2023-09-23T22:31:08.227185+0000","last_active":"2023-09-23T22:33:14.503899+0000","last_peered":"2023-09-23T22:33:14.503899+0000","last_clean":"2023-09-23T22:33:14.503899+0000","last_became_active":"2023-09-23T22:31:08.226918+0000","last_became_peered":"2023-09-23T22:31:08.226918+0000","last_unstale":"2023-09-23T22:33:14.503899+0000","last_undegraded":"2023-09-23T22:33:14.503899+0000","last_fullsized":"2023-09-23T22:33:14.503899+0000","mapping_epoch":373,"log_start":"389'2700","ondisk_log_start":"389'2700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5032,"log_dups_size":0,"ondisk_log_size":5032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:54.257936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298975232,"num_objects":4562,"num_object_clones":0,"num_object_copies":13686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7732,"num_write_kb":393408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1169,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.241939+0000","last_change":"2023-09-23T22:31:31.127033+0000","last_active":"2023-09-23T22:33:06.241939+0000","last_peered":"2023-09-23T22:33:06.241939+0000","last_clean":"2023-09-23T22:33:06.241939+0000","last_became_active":"2023-09-23T22:31:31.126601+0000","last_became_peered":"2023-09-23T22:31:31.126601+0000","last_unstale":"2023-09-23T22:33:06.241939+0000","last_undegraded":"2023-09-23T22:33:06.241939+0000","last_fullsized":"2023-09-23T22:33:06.241939+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:00.619278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":792,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.157601+0000","last_change":"2023-09-23T22:31:31.126756+0000","last_active":"2023-09-23T22:33:14.157601+0000","last_peered":"2023-09-23T22:33:14.157601+0000","last_clean":"2023-09-23T22:33:14.157601+0000","last_became_active":"2023-09-23T22:31:31.126409+0000","last_became_peered":"2023-09-23T22:31:31.126409+0000","last_unstale":"2023-09-23T22:33:14.157601+0000","last_undegraded":"2023-09-23T22:33:14.157601+0000","last_fullsized":"2023-09-23T22:33:14.157601+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.605240+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":288,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244509+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:33:06.244509+0000","last_peered":"2023-09-23T22:33:06.244509+0000","last_clean":"2023-09-23T22:33:06.244509+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:33:06.244509+0000","last_undegraded":"2023-09-23T22:33:06.244509+0000","last_fullsized":"2023-09-23T22:33:06.244509+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":98,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.237304+0000","last_change":"2023-09-23T22:31:28.124597+0000","last_active":"2023-09-23T22:33:06.237304+0000","last_peered":"2023-09-23T22:33:06.237304+0000","last_clean":"2023-09-23T22:33:06.237304+0000","last_became_active":"2023-09-23T22:31:28.124245+0000","last_became_peered":"2023-09-23T22:31:28.124245+0000","last_unstale":"2023-09-23T22:33:06.237304+0000","last_undegraded":"2023-09-23T22:33:06.237304+0000","last_fullsized":"2023-09-23T22:33:06.237304+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.558632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"407'7854","reported_seq":7897,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.417984+0000","last_change":"2023-09-23T22:31:08.223664+0000","last_active":"2023-09-23T22:33:14.417984+0000","last_peered":"2023-09-23T22:33:14.417984+0000","last_clean":"2023-09-23T22:33:14.417984+0000","last_became_active":"2023-09-23T22:31:08.223323+0000","last_became_peered":"2023-09-23T22:31:08.223323+0000","last_unstale":"2023-09-23T22:33:14.417984+0000","last_undegraded":"2023-09-23T22:33:14.417984+0000","last_fullsized":"2023-09-23T22:33:14.417984+0000","mapping_epoch":373,"log_start":"390'3000","ondisk_log_start":"390'3000","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4854,"log_dups_size":0,"ondisk_log_size":4854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:38.536861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296615936,"num_objects":4526,"num_object_clones":0,"num_object_copies":13578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7854,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":639,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.237023+0000","last_change":"2023-09-23T22:31:21.362707+0000","last_active":"2023-09-23T22:33:06.237023+0000","last_peered":"2023-09-23T22:33:06.237023+0000","last_clean":"2023-09-23T22:33:06.237023+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:33:06.237023+0000","last_undegraded":"2023-09-23T22:33:06.237023+0000","last_fullsized":"2023-09-23T22:33:06.237023+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:58.622607+0000","scrub_duration":0.0034076229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":786,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243150+0000","last_change":"2023-09-23T22:31:28.147724+0000","last_active":"2023-09-23T22:33:06.243150+0000","last_peered":"2023-09-23T22:33:06.243150+0000","last_clean":"2023-09-23T22:33:06.243150+0000","last_became_active":"2023-09-23T22:31:28.120238+0000","last_became_peered":"2023-09-23T22:31:28.120238+0000","last_unstale":"2023-09-23T22:33:06.243150+0000","last_undegraded":"2023-09-23T22:33:06.243150+0000","last_fullsized":"2023-09-23T22:33:06.243150+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:53.259496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":302,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239491+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:33:06.239491+0000","last_peered":"2023-09-23T22:33:06.239491+0000","last_clean":"2023-09-23T22:33:06.239491+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:33:06.239491+0000","last_undegraded":"2023-09-23T22:33:06.239491+0000","last_fullsized":"2023-09-23T22:33:06.239491+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":295,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.240902+0000","last_change":"2023-09-23T22:31:28.121081+0000","last_active":"2023-09-23T22:33:06.240902+0000","last_peered":"2023-09-23T22:33:06.240902+0000","last_clean":"2023-09-23T22:33:06.240902+0000","last_became_active":"2023-09-23T22:31:28.120399+0000","last_became_peered":"2023-09-23T22:31:28.120399+0000","last_unstale":"2023-09-23T22:33:06.240902+0000","last_undegraded":"2023-09-23T22:33:06.240902+0000","last_fullsized":"2023-09-23T22:33:06.240902+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:11.267296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"407'8033","reported_seq":8076,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.395216+0000","last_change":"2023-09-23T22:31:08.224259+0000","last_active":"2023-09-23T22:33:14.395216+0000","last_peered":"2023-09-23T22:33:14.395216+0000","last_clean":"2023-09-23T22:33:14.395216+0000","last_became_active":"2023-09-23T22:31:08.224062+0000","last_became_peered":"2023-09-23T22:31:08.224062+0000","last_unstale":"2023-09-23T22:33:14.395216+0000","last_undegraded":"2023-09-23T22:33:14.395216+0000","last_fullsized":"2023-09-23T22:33:14.395216+0000","mapping_epoch":373,"log_start":"390'3200","ondisk_log_start":"390'3200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4833,"log_dups_size":0,"ondisk_log_size":4833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:18.330753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304939008,"num_objects":4653,"num_object_clones":0,"num_object_copies":13959,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4653,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8033,"num_write_kb":405952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":757,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236811+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:33:06.236811+0000","last_peered":"2023-09-23T22:33:06.236811+0000","last_clean":"2023-09-23T22:33:06.236811+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:33:06.236811+0000","last_undegraded":"2023-09-23T22:33:06.236811+0000","last_fullsized":"2023-09-23T22:33:06.236811+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":334,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242419+0000","last_change":"2023-09-23T22:31:28.152186+0000","last_active":"2023-09-23T22:33:06.242419+0000","last_peered":"2023-09-23T22:33:06.242419+0000","last_clean":"2023-09-23T22:33:06.242419+0000","last_became_active":"2023-09-23T22:31:28.111292+0000","last_became_peered":"2023-09-23T22:31:28.111292+0000","last_unstale":"2023-09-23T22:33:06.242419+0000","last_undegraded":"2023-09-23T22:33:06.242419+0000","last_fullsized":"2023-09-23T22:33:06.242419+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:06.097398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":333,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242515+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:33:06.242515+0000","last_peered":"2023-09-23T22:33:06.242515+0000","last_clean":"2023-09-23T22:33:06.242515+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:33:06.242515+0000","last_undegraded":"2023-09-23T22:33:06.242515+0000","last_fullsized":"2023-09-23T22:33:06.242515+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":275,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.240009+0000","last_change":"2023-09-23T22:31:28.106985+0000","last_active":"2023-09-23T22:33:06.240009+0000","last_peered":"2023-09-23T22:33:06.240009+0000","last_clean":"2023-09-23T22:33:06.240009+0000","last_became_active":"2023-09-23T22:31:28.105451+0000","last_became_peered":"2023-09-23T22:31:28.105451+0000","last_unstale":"2023-09-23T22:33:06.240009+0000","last_undegraded":"2023-09-23T22:33:06.240009+0000","last_fullsized":"2023-09-23T22:33:06.240009+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:43.614928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"407'7867","reported_seq":11131,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.449881+0000","last_change":"2023-09-23T22:31:45.135267+0000","last_active":"2023-09-23T22:33:14.449881+0000","last_peered":"2023-09-23T22:33:14.449881+0000","last_clean":"2023-09-23T22:33:14.449881+0000","last_became_active":"2023-09-23T22:31:45.131633+0000","last_became_peered":"2023-09-23T22:31:45.131633+0000","last_unstale":"2023-09-23T22:33:14.449881+0000","last_undegraded":"2023-09-23T22:33:14.449881+0000","last_fullsized":"2023-09-23T22:33:14.449881+0000","mapping_epoch":388,"log_start":"385'2500","ondisk_log_start":"385'2500","created":373,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5367,"log_dups_size":0,"ondisk_log_size":5367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:00.859364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295895040,"num_objects":4515,"num_object_clones":0,"num_object_copies":13545,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7867,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1601,"num_bytes_recovered":104923136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":761,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243305+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:33:06.243305+0000","last_peered":"2023-09-23T22:33:06.243305+0000","last_clean":"2023-09-23T22:33:06.243305+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:33:06.243305+0000","last_undegraded":"2023-09-23T22:33:06.243305+0000","last_fullsized":"2023-09-23T22:33:06.243305+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":239,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239968+0000","last_change":"2023-09-23T22:31:28.131347+0000","last_active":"2023-09-23T22:33:06.239968+0000","last_peered":"2023-09-23T22:33:06.239968+0000","last_clean":"2023-09-23T22:33:06.239968+0000","last_became_active":"2023-09-23T22:31:28.114169+0000","last_became_peered":"2023-09-23T22:31:28.114169+0000","last_unstale":"2023-09-23T22:33:06.239968+0000","last_undegraded":"2023-09-23T22:33:06.239968+0000","last_fullsized":"2023-09-23T22:33:06.239968+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:30.007254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":903,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.246320+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:33:06.246320+0000","last_peered":"2023-09-23T22:33:06.246320+0000","last_clean":"2023-09-23T22:33:06.246320+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:33:06.246320+0000","last_undegraded":"2023-09-23T22:33:06.246320+0000","last_fullsized":"2023-09-23T22:33:06.246320+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":347,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239233+0000","last_change":"2023-09-23T22:31:28.114550+0000","last_active":"2023-09-23T22:33:06.239233+0000","last_peered":"2023-09-23T22:33:06.239233+0000","last_clean":"2023-09-23T22:33:06.239233+0000","last_became_active":"2023-09-23T22:31:28.114217+0000","last_became_peered":"2023-09-23T22:31:28.114217+0000","last_unstale":"2023-09-23T22:33:06.239233+0000","last_undegraded":"2023-09-23T22:33:06.239233+0000","last_fullsized":"2023-09-23T22:33:06.239233+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:13.616772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.d","version":"407'7792","reported_seq":11024,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T22:33:14.451705+0000","last_change":"2023-09-23T22:32:51.714615+0000","last_active":"2023-09-23T22:33:14.451705+0000","last_peered":"2023-09-23T22:33:14.451705+0000","last_clean":"2023-09-23T22:33:14.451705+0000","last_became_active":"2023-09-23T22:32:51.673119+0000","last_became_peered":"2023-09-23T22:32:51.673119+0000","last_unstale":"2023-09-23T22:33:14.451705+0000","last_undegraded":"2023-09-23T22:33:14.451705+0000","last_fullsized":"2023-09-23T22:33:14.451705+0000","mapping_epoch":403,"log_start":"385'2300","ondisk_log_start":"385'2300","created":373,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5492,"log_dups_size":0,"ondisk_log_size":5492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:58.214128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292749336,"num_objects":4468,"num_object_clones":0,"num_object_copies":13404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4468,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7792,"num_write_kb":392257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":103809024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":687,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243778+0000","last_change":"2023-09-23T22:31:45.133529+0000","last_active":"2023-09-23T22:33:06.243778+0000","last_peered":"2023-09-23T22:33:06.243778+0000","last_clean":"2023-09-23T22:33:06.243778+0000","last_became_active":"2023-09-23T22:31:45.133117+0000","last_became_peered":"2023-09-23T22:31:45.133117+0000","last_unstale":"2023-09-23T22:33:06.243778+0000","last_undegraded":"2023-09-23T22:33:06.243778+0000","last_fullsized":"2023-09-23T22:33:06.243778+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.591643+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.591643+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:28.957050+0000","scrub_duration":0.0073638979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":118,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236376+0000","last_change":"2023-09-23T22:31:28.156716+0000","last_active":"2023-09-23T22:33:06.236376+0000","last_peered":"2023-09-23T22:33:06.236376+0000","last_clean":"2023-09-23T22:33:06.236376+0000","last_became_active":"2023-09-23T22:31:28.128617+0000","last_became_peered":"2023-09-23T22:31:28.128617+0000","last_unstale":"2023-09-23T22:33:06.236376+0000","last_undegraded":"2023-09-23T22:33:06.236376+0000","last_fullsized":"2023-09-23T22:33:06.236376+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:34.465027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":271,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.240760+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:33:06.240760+0000","last_peered":"2023-09-23T22:33:06.240760+0000","last_clean":"2023-09-23T22:33:06.240760+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:33:06.240760+0000","last_undegraded":"2023-09-23T22:33:06.240760+0000","last_fullsized":"2023-09-23T22:33:06.240760+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":380,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242992+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:33:06.242992+0000","last_peered":"2023-09-23T22:33:06.242992+0000","last_clean":"2023-09-23T22:33:06.242992+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:33:06.242992+0000","last_undegraded":"2023-09-23T22:33:06.242992+0000","last_fullsized":"2023-09-23T22:33:06.242992+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":67,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239935+0000","last_change":"2023-09-23T22:31:28.111728+0000","last_active":"2023-09-23T22:33:06.239935+0000","last_peered":"2023-09-23T22:33:06.239935+0000","last_clean":"2023-09-23T22:33:06.239935+0000","last_became_active":"2023-09-23T22:31:28.111369+0000","last_became_peered":"2023-09-23T22:31:28.111369+0000","last_unstale":"2023-09-23T22:33:06.239935+0000","last_undegraded":"2023-09-23T22:33:06.239935+0000","last_fullsized":"2023-09-23T22:33:06.239935+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:42.837575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":158,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:09.423092+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:33:09.423092+0000","last_peered":"2023-09-23T22:33:09.423092+0000","last_clean":"2023-09-23T22:33:09.423092+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:33:09.423092+0000","last_undegraded":"2023-09-23T22:33:09.423092+0000","last_fullsized":"2023-09-23T22:33:09.423092+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":397,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242367+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:33:06.242367+0000","last_peered":"2023-09-23T22:33:06.242367+0000","last_clean":"2023-09-23T22:33:06.242367+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:33:06.242367+0000","last_undegraded":"2023-09-23T22:33:06.242367+0000","last_fullsized":"2023-09-23T22:33:06.242367+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":432,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243002+0000","last_change":"2023-09-23T22:31:28.111351+0000","last_active":"2023-09-23T22:33:06.243002+0000","last_peered":"2023-09-23T22:33:06.243002+0000","last_clean":"2023-09-23T22:33:06.243002+0000","last_became_active":"2023-09-23T22:31:28.111006+0000","last_became_peered":"2023-09-23T22:31:28.111006+0000","last_unstale":"2023-09-23T22:33:06.243002+0000","last_undegraded":"2023-09-23T22:33:06.243002+0000","last_fullsized":"2023-09-23T22:33:06.243002+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:51.435513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":422,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.244182+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:33:06.244182+0000","last_peered":"2023-09-23T22:33:06.244182+0000","last_clean":"2023-09-23T22:33:06.244182+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:33:06.244182+0000","last_undegraded":"2023-09-23T22:33:06.244182+0000","last_fullsized":"2023-09-23T22:33:06.244182+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":271,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.243072+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:33:06.243072+0000","last_peered":"2023-09-23T22:33:06.243072+0000","last_clean":"2023-09-23T22:33:06.243072+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:33:06.243072+0000","last_undegraded":"2023-09-23T22:33:06.243072+0000","last_fullsized":"2023-09-23T22:33:06.243072+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":98,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.236664+0000","last_change":"2023-09-23T22:31:28.126203+0000","last_active":"2023-09-23T22:33:06.236664+0000","last_peered":"2023-09-23T22:33:06.236664+0000","last_clean":"2023-09-23T22:33:06.236664+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:31:28.125529+0000","last_unstale":"2023-09-23T22:33:06.236664+0000","last_undegraded":"2023-09-23T22:33:06.236664+0000","last_fullsized":"2023-09-23T22:33:06.236664+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:47.359771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":407,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.239587+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:33:06.239587+0000","last_peered":"2023-09-23T22:33:06.239587+0000","last_clean":"2023-09-23T22:33:06.239587+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:33:06.239587+0000","last_undegraded":"2023-09-23T22:33:06.239587+0000","last_fullsized":"2023-09-23T22:33:06.239587+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":345,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.240061+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:33:06.240061+0000","last_peered":"2023-09-23T22:33:06.240061+0000","last_clean":"2023-09-23T22:33:06.240061+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:33:06.240061+0000","last_undegraded":"2023-09-23T22:33:06.240061+0000","last_fullsized":"2023-09-23T22:33:06.240061+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":465,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T22:33:06.242677+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:33:06.242677+0000","last_peered":"2023-09-23T22:33:06.242677+0000","last_clean":"2023-09-23T22:33:06.242677+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:33:06.242677+0000","last_undegraded":"2023-09-23T22:33:06.242677+0000","last_fullsized":"2023-09-23T22:33:06.242677+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":31404,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4746641432,"num_objects":72429,"num_object_clones":0,"num_object_copies":217287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72429,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":125351,"num_write_kb":6328897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11022,"num_bytes_recovered":702152704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14502801408,"data_stored":14502789192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82251,"ondisk_log_size":82251,"up":48,"acting":48,"num_store_stats":6},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1115744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234346,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495982,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2520744,"kb_used_data":2337632,"kb_used_omap":0,"kb_used_meta":183104,"kb_avail":91224408,"statfs":{"total":95995035648,"available":93413793792,"internally_reserved":0,"allocated":2393735168,"data_stored":2392272336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":187498496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":7.2089999999999996,"5min":5.6589999999999998,"15min":2.9900000000000002},"min":{"1min":0.22900000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":57.936999999999998,"5min":213.23699999999999,"15min":213.23699999999999},"last":1.1899999999999999},{"interface":"front","average":{"1min":7.1070000000000002,"5min":5.992,"15min":4.0529999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":58.526000000000003,"5min":213.304,"15min":213.304},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":5.3650000000000002,"5min":1.8620000000000001,"15min":3.5110000000000001},"min":{"1min":0.27700000000000002,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":58.143999999999998,"5min":58.143999999999998,"15min":129.964},"last":0.495},{"interface":"front","average":{"1min":5.2050000000000001,"5min":2.0750000000000002,"15min":3.8039999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":63.073999999999998,"5min":63.073999999999998,"15min":131.18600000000001},"last":0.40100000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:33:00 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":1.927,"15min":1.911},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.2990000000000004,"5min":6.6559999999999997,"15min":6.6559999999999997},"last":0.377},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.016,"15min":2.105},"min":{"1min":0.51300000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.1109999999999998,"5min":9.7319999999999993,"15min":9.7319999999999993},"last":0.46700000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.5940000000000001,"15min":3.145},"min":{"1min":0.56899999999999995,"5min":0.32500000000000001,"15min":0.316},"max":{"1min":6.165,"5min":6.8609999999999998,"15min":125.125},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.2829999999999999,"5min":1.4119999999999999,"15min":2.8610000000000002},"min":{"1min":0.33900000000000002,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":9.2569999999999997,"5min":9.2569999999999997,"15min":76.406999999999996},"last":0.61399999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.127,"15min":2.6299999999999999},"min":{"1min":0.29299999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5010000000000003,"5min":6.8650000000000002,"15min":305.30200000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":4.1020000000000003,"5min":1.782,"15min":2.8319999999999999},"min":{"1min":0.317,"5min":0.29299999999999998,"15min":0.255},"max":{"1min":11.561,"5min":11.561,"15min":305.40300000000002},"last":0.41299999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":2.726,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.148,"5min":88.194999999999993,"15min":88.194999999999993},"last":0.45600000000000002},{"interface":"front","average":{"1min":3.1909999999999998,"5min":2.2839999999999998,"15min":1.5860000000000001},"min":{"1min":0.44600000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.2859999999999996,"5min":88.025000000000006,"15min":88.025000000000006},"last":0.44800000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:32:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":0.95399999999999996,"15min":1.0980000000000001},"min":{"1min":0.38800000000000001,"5min":0.314,"15min":0.314},"max":{"1min":9.2270000000000003,"5min":9.2270000000000003,"15min":82.801000000000002},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.4630000000000001,"5min":1.105,"15min":1.1459999999999999},"min":{"1min":0.41999999999999998,"5min":0.33300000000000002,"15min":0.29699999999999999},"max":{"1min":8.7690000000000001,"5min":8.7690000000000001,"15min":83.052999999999997},"last":0.55900000000000005}]}]},{"osd":1,"up_from":98,"seq":420906795254,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3308916,"kb_used_data":2641132,"kb_used_omap":7,"kb_used_meta":667768,"kb_avail":90436236,"statfs":{"total":95995035648,"available":92606705664,"internally_reserved":0,"allocated":2704519168,"data_stored":2702387731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":683794804},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":6.3769999999999998,"5min":5.8250000000000002,"15min":5.1369999999999996},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":107.202,"5min":202.73500000000001,"15min":202.73500000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":7.0449999999999999,"5min":6.3120000000000003,"15min":5.9349999999999996},"min":{"1min":0.24399999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":107.426,"5min":202.80799999999999,"15min":202.80799999999999},"last":0.45200000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":0.94999999999999996,"15min":1.0209999999999999},"min":{"1min":0.23599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":3.8820000000000001,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":1.2210000000000001},{"interface":"front","average":{"1min":1.24,"5min":1.8999999999999999,"15min":2.214},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.9859999999999998,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":0.48799999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2669999999999999,"5min":1.732,"15min":4.4980000000000002},"min":{"1min":0.30599999999999999,"5min":0.246,"15min":0.246},"max":{"1min":10.587999999999999,"5min":10.587999999999999,"15min":160.499},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.653,"5min":1.46,"15min":4.1890000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.223},"max":{"1min":5.1920000000000002,"5min":8.1590000000000007,"15min":160.465},"last":1.2350000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.92200000000000004,"15min":2.2440000000000002},"min":{"1min":0.29499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.3730000000000002,"5min":7.0380000000000003,"15min":234.79900000000001},"last":1.5900000000000001},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.478,"15min":2.919},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.4729999999999999,"5min":9.093,"15min":234.81700000000001},"last":1.605}]},{"osd":5,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":1.6859999999999999,"15min":2.5},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":7.4450000000000003,"5min":9.2759999999999998,"15min":196.79900000000001},"last":2.2490000000000001},{"interface":"front","average":{"1min":1.611,"5min":1.294,"15min":2.3239999999999998},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.29099999999999998},"max":{"1min":5.2889999999999997,"5min":6.8049999999999997,"15min":196.68299999999999},"last":3.4940000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":2.2559999999999998,"15min":2.1379999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":5.2350000000000003,"5min":77.033000000000001,"15min":77.033000000000001},"last":2.8500000000000001},{"interface":"front","average":{"1min":1.681,"5min":1.9630000000000001,"15min":1.583},"min":{"1min":0.442,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":8.3640000000000008,"5min":76.965999999999994,"15min":76.965999999999994},"last":3.0049999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":1.663,"15min":2.0259999999999998},"min":{"1min":0.41499999999999998,"5min":0.38300000000000001,"15min":0.32900000000000001},"max":{"1min":8.0299999999999994,"5min":8.0660000000000007,"15min":16.265000000000001},"last":1.798},{"interface":"front","average":{"1min":1.302,"5min":1.2529999999999999,"15min":1.339},"min":{"1min":0.378,"5min":0.318,"15min":0.318},"max":{"1min":3.8119999999999998,"5min":7.1429999999999998,"15min":12.045999999999999},"last":2.8340000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958783,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3363812,"kb_used_data":2616348,"kb_used_omap":8,"kb_used_meta":747447,"kb_avail":90381340,"statfs":{"total":95995035648,"available":92550492160,"internally_reserved":0,"allocated":2679140352,"data_stored":2676915959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":765386602},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:32:37 2023","interfaces":[{"interface":"back","average":{"1min":7.5819999999999999,"5min":3.4860000000000002,"15min":4.1769999999999996},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":121.688,"5min":121.688,"15min":169.77199999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":6.8739999999999997,"5min":3.5529999999999999,"15min":4.2850000000000001},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":122.93300000000001,"5min":122.93300000000001,"15min":169.52099999999999},"last":0.245}]},{"osd":2,"last update":"Sat Sep 23 22:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.2850000000000001,"15min":2.8919999999999999},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2359999999999998,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.415,"15min":1.694},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.3380000000000001,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.26200000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:32:37 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.45,"15min":3.5419999999999998},"min":{"1min":0.315,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":8.548,"5min":10.292999999999999,"15min":146.87299999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.6259999999999999,"15min":3.5750000000000002},"min":{"1min":0.29299999999999998,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":3.5379999999999998,"5min":22.303000000000001,"15min":145.48099999999999},"last":0.33800000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":2.109,"15min":2.5339999999999998},"min":{"1min":0.30199999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.2060000000000004,"5min":11.474,"15min":82.825000000000003},"last":0.499},{"interface":"front","average":{"1min":1.6799999999999999,"5min":1.4119999999999999,"15min":1.7529999999999999},"min":{"1min":0.29599999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.444,"5min":7.2279999999999998,"15min":75.930999999999997},"last":0.439}]},{"osd":5,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":2.21,"15min":2.9689999999999999},"min":{"1min":0.32400000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2510000000000003,"5min":13.564,"15min":93.685000000000002},"last":1.1479999999999999},{"interface":"front","average":{"1min":1.841,"5min":1.633,"15min":2.246},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.4139999999999997,"5min":13.486000000000001,"15min":93.546000000000006},"last":0.38400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":2.593,"5min":2.3290000000000002,"15min":2.3140000000000001},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.308},"max":{"1min":10.183999999999999,"5min":12.955,"15min":16.366},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.8540000000000001,"5min":1.6830000000000001,"15min":1.6799999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":6.5060000000000002,"5min":14.208,"15min":22.263000000000002},"last":1.5109999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:32:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":1.8500000000000001,"15min":2},"min":{"1min":0.42599999999999999,"5min":0.40200000000000002,"15min":0.29999999999999999},"max":{"1min":3.0110000000000001,"5min":22.338999999999999,"15min":32.773000000000003},"last":4.6980000000000004},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.1389999999999998,"15min":2.2890000000000001},"min":{"1min":0.47899999999999998,"5min":0.307,"15min":0.307},"max":{"1min":8.7550000000000008,"5min":10.901,"15min":16.381},"last":4.6050000000000004}]}]},{"osd":2,"up_from":0,"seq":1498943586347,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468201,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019251,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3890848,"kb_used_data":3485080,"kb_used_omap":0,"kb_used_meta":405760,"kb_avail":89854304,"statfs":{"total":95995035648,"available":92010807296,"internally_reserved":0,"allocated":3568721920,"data_stored":3567266705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":415498240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":11.644,"5min":3.5510000000000002,"15min":4.1360000000000001},"min":{"1min":0.216,"5min":0.20699999999999999,"15min":0.19700000000000001},"max":{"1min":101.44499999999999,"5min":101.44499999999999,"15min":150.63499999999999},"last":0.627},{"interface":"front","average":{"1min":11.930999999999999,"5min":3.7890000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":101.465,"5min":101.465,"15min":150.672},"last":0.61099999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":5.7779999999999996,"5min":3.2109999999999999,"15min":3.4060000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":94.197999999999993,"5min":94.197999999999993,"15min":107.346},"last":4.0069999999999997},{"interface":"front","average":{"1min":6.359,"5min":3.508,"15min":3.6499999999999999},"min":{"1min":0.26100000000000001,"5min":0.253,"15min":0.22900000000000001},"max":{"1min":94.460999999999999,"5min":94.460999999999999,"15min":107.33499999999999},"last":0.57399999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":1.883,"15min":2.0739999999999998},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.0449999999999999,"5min":8.0449999999999999,"15min":8.0449999999999999},"last":3.952},{"interface":"front","average":{"1min":1.4990000000000001,"5min":1.48,"15min":1.52},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9080000000000004,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":2.0819999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.663,"15min":2.472},"min":{"1min":0.36399999999999999,"5min":0.314,"15min":0.29699999999999999},"max":{"1min":4.202,"5min":48.216999999999999,"15min":102.896},"last":1.349},{"interface":"front","average":{"1min":1.423,"5min":1.8069999999999999,"15min":2.589},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":7.875,"5min":48.232999999999997,"15min":112.667},"last":2.266}]},{"osd":5,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":1.2669999999999999,"15min":1.0860000000000001},"min":{"1min":0.30299999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.0220000000000002,"5min":13.407,"15min":28.399000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2809999999999999,"15min":1.153},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":11.048999999999999,"5min":14.212,"15min":28.350999999999999},"last":14.016999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.034,"15min":0.89000000000000001},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.5439999999999996,"5min":7.5439999999999996,"15min":7.8550000000000004},"last":5.0949999999999998},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.216,"15min":0.91700000000000004},"min":{"1min":0.34699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.1720000000000006,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":4.0229999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":1.03,"15min":0.95399999999999996},"min":{"1min":0.376,"5min":0.307,"15min":0.27500000000000002},"max":{"1min":8.1440000000000001,"5min":8.1440000000000001,"15min":8.577},"last":3.9950000000000001},{"interface":"front","average":{"1min":1.23,"5min":0.89400000000000002,"15min":0.89600000000000002},"min":{"1min":0.32600000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.633,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":0.42699999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757619,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3318292,"kb_used_data":2893132,"kb_used_omap":0,"kb_used_meta":425152,"kb_avail":90426860,"statfs":{"total":95995035648,"available":92597104640,"internally_reserved":0,"allocated":2962567168,"data_stored":2961119958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":435355648},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":13.648999999999999,"5min":3.9780000000000002,"15min":4.5839999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":124.892,"5min":124.892,"15min":163.54499999999999},"last":0.249},{"interface":"front","average":{"1min":13.523,"5min":3.528,"15min":4.3230000000000004},"min":{"1min":0.34999999999999998,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":123.807,"5min":123.807,"15min":162.81800000000001},"last":3.0289999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":3.7639999999999998,"15min":3.819},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":7.1440000000000001,"5min":113.514,"15min":113.514},"last":2.9950000000000001},{"interface":"front","average":{"1min":2.323,"5min":3.988,"15min":3.8599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":8.4420000000000002,"5min":113.42700000000001,"15min":113.42700000000001},"last":3.5139999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6059999999999999,"5min":2.1720000000000002,"15min":2.331},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":8.5359999999999996,"5min":10.451000000000001,"15min":10.451000000000001},"last":3.5470000000000002},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.1459999999999999,"15min":2.4009999999999998},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.5180000000000007,"5min":8.5180000000000007,"15min":8.5180000000000007},"last":0.375}]},{"osd":3,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.6539999999999999,"15min":3.7719999999999998},"min":{"1min":0.36099999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":8.5459999999999994,"5min":9.3369999999999997,"15min":132.59},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.7070000000000001,"15min":3.71},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":7.8390000000000004,"5min":7.8390000000000004,"15min":129.381},"last":3.0819999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9849999999999999,"5min":1.3620000000000001,"15min":1.508},"min":{"1min":0.27700000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":36.343000000000004,"5min":36.343000000000004,"15min":126.679},"last":3.8029999999999999},{"interface":"front","average":{"1min":3.9740000000000002,"5min":1.377,"15min":1.512},"min":{"1min":0.35899999999999999,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":36.421999999999997,"5min":36.421999999999997,"15min":126.78100000000001},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 22:32:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":1.8999999999999999,"15min":1.2350000000000001},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.5820000000000007,"5min":70.581999999999994,"15min":70.581999999999994},"last":0.624},{"interface":"front","average":{"1min":2.0059999999999998,"5min":2.0499999999999998,"15min":1.2410000000000001},"min":{"1min":0.28699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":7.25,"5min":71.093000000000004,"15min":71.093000000000004},"last":3.137}]},{"osd":7,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":3.7650000000000001,"5min":1.24,"15min":1.107},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":40.548000000000002,"5min":40.548000000000002,"15min":40.548000000000002},"last":3.1099999999999999},{"interface":"front","average":{"1min":3.6970000000000001,"5min":1.3440000000000001,"15min":0.98899999999999999},"min":{"1min":0.38,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":41.073,"5min":41.073,"15min":41.073},"last":3.5760000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2673737728,"data_stored":2673737728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2674135040,"data_stored":2674130968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3557883904,"data_stored":3557883904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2953252864,"data_stored":2953248792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2380140544,"data_stored":2380136472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":263651328,"data_stored":263651328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:33:20.285 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:20 smithi070 ceph-mon[192652]: from='client.36072 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:33:20.285 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:20.285 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:20.292 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:20.378 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:20 smithi026 ceph-mon[223014]: from='client.36072 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:33:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:20.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:20 smithi026 ceph-mon[215775]: from='client.36072 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:33:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:20.385 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:20.601 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:33:20.639 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:33:20.673 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:33:20.708 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:33:20.759 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:33:20.789 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:33:20.830 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:33:20.866 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:33:20.871 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:33:20.871 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:33:20.871 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:33:20.871 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:33:20.871 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:33:20.871 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:33:20.871 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:33:20.872 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:33:20.918 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:20.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:20.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:21.027 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:21.126 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:21.204 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:21 smithi070 ceph-mon[192652]: pgmap v1063: 99 pgs: 1 peering, 98 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 761 op/s; 0 B/s, 30 objects/s recovering 2023-09-23T22:33:21.204 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:21 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1091991638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:21.204 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:21.204 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:21.288 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:21.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:21.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:21.394 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:21.421 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:21 smithi026 ceph-mon[215775]: pgmap v1063: 99 pgs: 1 peering, 98 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 761 op/s; 0 B/s, 30 objects/s recovering 2023-09-23T22:33:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:21 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1091991638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:21.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:21 smithi026 ceph-mon[223014]: pgmap v1063: 99 pgs: 1 peering, 98 active+clean; 4.4 GiB data, 16 GiB used, 431 GiB / 447 GiB avail; 0 B/s wr, 761 op/s; 0 B/s, 30 objects/s recovering 2023-09-23T22:33:21.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:21 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1091991638' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:21.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:21.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:21.597 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:21.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:21.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:21.669 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:33:21.669 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:33:21.713 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:21.761 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:22.420 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:22.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:22.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:22.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:22.562 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:22.666 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:33:22.792 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:22.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:22.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:22.903 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:22.950 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:23.085 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:23.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:23.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:23.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:23.231 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:33:23.232 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:23.357 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:23.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:23.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:23 smithi026 ceph-mon[215775]: pgmap v1064: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 0 B/s, 25 objects/s recovering 2023-09-23T22:33:23.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:33:23.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:33:23.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:23.365 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:23 smithi026 ceph-mon[223014]: pgmap v1064: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 0 B/s, 25 objects/s recovering 2023-09-23T22:33:23.365 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:33:23.365 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:33:23.365 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:23.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:23.369 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:23.467 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:23.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:23 smithi070 ceph-mon[192652]: pgmap v1064: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 1.03k op/s; 0 B/s, 25 objects/s recovering 2023-09-23T22:33:23.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:33:23.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:33:23.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:23.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:23.725 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:23.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:23.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:23.824 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:23.837 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:23.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:23.998 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:24.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:24.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:24.105 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:24.163 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:24.223 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:33:24.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1098130292' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:24.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:24 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:24.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:24.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:24.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1098130292' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:24.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:24 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:24.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:24.430 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:24.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:33:24.504 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:24.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:24.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:24.509 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1098130292' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:24.509 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:24 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:24.509 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:24.509 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:24.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:24.656 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:24.793 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:24.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:24.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:24.900 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:24.967 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:25.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[215775]: pgmap v1065: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 574 op/s 2023-09-23T22:33:25.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:25.218 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[215775]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:33:25.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[215775]: osdmap e410: 8 total, 8 up, 5 in 2023-09-23T22:33:25.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[215775]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[215775]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.219 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[215775]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.281 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:25.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:25.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:25.401 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:25.443 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[223014]: pgmap v1065: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 574 op/s 2023-09-23T22:33:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[223014]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:33:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[223014]: osdmap e410: 8 total, 8 up, 5 in 2023-09-23T22:33:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[223014]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[223014]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:25 smithi026 ceph-mon[223014]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:25 smithi070 ceph-mon[192652]: pgmap v1065: 99 pgs: 99 active+clean; 4.2 GiB data, 15 GiB used, 432 GiB / 447 GiB avail; 0 B/s wr, 574 op/s 2023-09-23T22:33:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:25 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:25 smithi070 ceph-mon[192652]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:25 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:33:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:25 smithi070 ceph-mon[192652]: osdmap e410: 8 total, 8 up, 5 in 2023-09-23T22:33:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:25 smithi070 ceph-mon[192652]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:25 smithi070 ceph-mon[192652]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:25 smithi070 ceph-mon[192652]: failed to encode map e410 with expected crc 2023-09-23T22:33:25.974 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:33:25.975 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:33:26.077 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:26.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:26.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:26.185 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:26.249 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:26.294 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:26.359 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:26 smithi026 ceph-mon[215775]: failed to encode map e410 with expected crc 2023-09-23T22:33:26.359 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:26 smithi026 ceph-mon[223014]: failed to encode map e410 with expected crc 2023-09-23T22:33:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:26.360 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:26.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:26.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:26 smithi070 ceph-mon[192652]: failed to encode map e410 with expected crc 2023-09-23T22:33:26.461 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:26.490 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:26.533 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:26.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:26.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:26.637 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:26.732 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:27.083 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:33:27.084 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:27.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:27.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:27.199 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:27.277 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:27.296 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:33:27.298 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:33:27.320 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:27.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:27 smithi070 ceph-mon[192652]: pgmap v1067: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 899 op/s 2023-09-23T22:33:27.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:27.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:27.426 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:27.457 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:27 smithi026 ceph-mon[215775]: pgmap v1067: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 899 op/s 2023-09-23T22:33:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:27 smithi026 ceph-mon[223014]: pgmap v1067: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 899 op/s 2023-09-23T22:33:27.516 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:27.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:27.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:27.588 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:27.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:27.660 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:28.135 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:33:28.522 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:28.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:28.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:28.604 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1067,"stamp":"2023-09-23T22:33:25.594194+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4195817419,"num_objects":63979,"num_object_clones":0,"num_object_copies":191937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63979,"num_whiteouts":0,"num_read":5983,"num_read_kb":12185,"num_write":134736,"num_write_kb":6341622,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11360,"num_bytes_recovered":724271467,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84931,"ondisk_log_size":84931,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":5,"num_per_pool_osds":5,"num_per_pool_omap_osds":5,"kb":468725760,"kb_used":14857712,"kb_used_data":12404360,"kb_used_omap":15,"kb_used_meta":2452912,"kb_avail":453868048,"statfs":{"total":479975178240,"available":464760881152,"internally_reserved":0,"allocated":12702064640,"data_stored":12693322037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":16162,"internal_metadata":2511782110},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-555614208,"num_objects":-8478,"num_object_clones":0,"num_object_copies":-25434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8478,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.429174"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":437,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.183367+0000","last_change":"2023-09-23T22:28:05.055232+0000","last_active":"2023-09-23T22:33:16.183367+0000","last_peered":"2023-09-23T22:33:16.183367+0000","last_clean":"2023-09-23T22:33:16.183367+0000","last_became_active":"2023-09-23T22:28:05.054995+0000","last_became_peered":"2023-09-23T22:28:05.054995+0000","last_unstale":"2023-09-23T22:33:16.183367+0000","last_undegraded":"2023-09-23T22:33:16.183367+0000","last_fullsized":"2023-09-23T22:33:16.183367+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:04.623050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":127,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.168054+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:33:16.168054+0000","last_peered":"2023-09-23T22:33:16.168054+0000","last_clean":"2023-09-23T22:33:16.168054+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:33:16.168054+0000","last_undegraded":"2023-09-23T22:33:16.168054+0000","last_fullsized":"2023-09-23T22:33:16.168054+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":127,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.167682+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:33:16.167682+0000","last_peered":"2023-09-23T22:33:16.167682+0000","last_clean":"2023-09-23T22:33:16.167682+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:33:16.167682+0000","last_undegraded":"2023-09-23T22:33:16.167682+0000","last_fullsized":"2023-09-23T22:33:16.167682+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":299,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178781+0000","last_change":"2023-09-23T22:31:28.120861+0000","last_active":"2023-09-23T22:33:16.178781+0000","last_peered":"2023-09-23T22:33:16.178781+0000","last_clean":"2023-09-23T22:33:16.178781+0000","last_became_active":"2023-09-23T22:31:28.120394+0000","last_became_peered":"2023-09-23T22:31:28.120394+0000","last_unstale":"2023-09-23T22:33:16.178781+0000","last_undegraded":"2023-09-23T22:33:16.178781+0000","last_fullsized":"2023-09-23T22:33:16.178781+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:19.470258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":465,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.182240+0000","last_change":"2023-09-23T22:31:28.114528+0000","last_active":"2023-09-23T22:33:16.182240+0000","last_peered":"2023-09-23T22:33:16.182240+0000","last_clean":"2023-09-23T22:33:16.182240+0000","last_became_active":"2023-09-23T22:31:28.114177+0000","last_became_peered":"2023-09-23T22:31:28.114177+0000","last_unstale":"2023-09-23T22:33:16.182240+0000","last_undegraded":"2023-09-23T22:33:16.182240+0000","last_fullsized":"2023-09-23T22:33:16.182240+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:46.680433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.193376+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:33:16.193376+0000","last_peered":"2023-09-23T22:33:16.193376+0000","last_clean":"2023-09-23T22:33:16.193376+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:33:16.193376+0000","last_undegraded":"2023-09-23T22:33:16.193376+0000","last_fullsized":"2023-09-23T22:33:16.193376+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":326,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178200+0000","last_change":"2023-09-23T22:29:01.670381+0000","last_active":"2023-09-23T22:33:16.178200+0000","last_peered":"2023-09-23T22:33:16.178200+0000","last_clean":"2023-09-23T22:33:16.178200+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:33:16.178200+0000","last_undegraded":"2023-09-23T22:33:16.178200+0000","last_fullsized":"2023-09-23T22:33:16.178200+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:43.604444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":91,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.185297+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:33:16.185297+0000","last_peered":"2023-09-23T22:33:16.185297+0000","last_clean":"2023-09-23T22:33:16.185297+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:33:16.185297+0000","last_undegraded":"2023-09-23T22:33:16.185297+0000","last_fullsized":"2023-09-23T22:33:16.185297+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":410,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.174859+0000","last_change":"2023-09-23T22:31:28.116916+0000","last_active":"2023-09-23T22:33:16.174859+0000","last_peered":"2023-09-23T22:33:16.174859+0000","last_clean":"2023-09-23T22:33:16.174859+0000","last_became_active":"2023-09-23T22:31:28.116577+0000","last_became_peered":"2023-09-23T22:31:28.116577+0000","last_unstale":"2023-09-23T22:33:16.174859+0000","last_undegraded":"2023-09-23T22:33:16.174859+0000","last_fullsized":"2023-09-23T22:33:16.174859+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:33.092789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":101,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.188736+0000","last_change":"2023-09-23T22:31:28.122684+0000","last_active":"2023-09-23T22:33:16.188736+0000","last_peered":"2023-09-23T22:33:16.188736+0000","last_clean":"2023-09-23T22:33:16.188736+0000","last_became_active":"2023-09-23T22:31:28.122427+0000","last_became_peered":"2023-09-23T22:31:28.122427+0000","last_unstale":"2023-09-23T22:33:16.188736+0000","last_undegraded":"2023-09-23T22:33:16.188736+0000","last_fullsized":"2023-09-23T22:33:16.188736+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:15.559367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":157,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.181840+0000","last_change":"2023-09-23T22:31:28.153199+0000","last_active":"2023-09-23T22:33:16.181840+0000","last_peered":"2023-09-23T22:33:16.181840+0000","last_clean":"2023-09-23T22:33:16.181840+0000","last_became_active":"2023-09-23T22:31:28.129257+0000","last_became_peered":"2023-09-23T22:31:28.129257+0000","last_unstale":"2023-09-23T22:33:16.181840+0000","last_undegraded":"2023-09-23T22:33:16.181840+0000","last_fullsized":"2023-09-23T22:33:16.181840+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:03.743836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.169304+0000","last_change":"2023-09-23T22:30:35.118944+0000","last_active":"2023-09-23T22:33:16.169304+0000","last_peered":"2023-09-23T22:33:16.169304+0000","last_clean":"2023-09-23T22:33:16.169304+0000","last_became_active":"2023-09-23T22:30:35.118608+0000","last_became_peered":"2023-09-23T22:30:35.118608+0000","last_unstale":"2023-09-23T22:33:16.169304+0000","last_undegraded":"2023-09-23T22:33:16.169304+0000","last_fullsized":"2023-09-23T22:33:16.169304+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:45.333663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":494,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.182843+0000","last_change":"2023-09-23T22:31:28.145916+0000","last_active":"2023-09-23T22:33:16.182843+0000","last_peered":"2023-09-23T22:33:16.182843+0000","last_clean":"2023-09-23T22:33:16.182843+0000","last_became_active":"2023-09-23T22:31:28.120703+0000","last_became_peered":"2023-09-23T22:31:28.120703+0000","last_unstale":"2023-09-23T22:33:16.182843+0000","last_undegraded":"2023-09-23T22:33:16.182843+0000","last_fullsized":"2023-09-23T22:33:16.182843+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:07.819247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":408,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178844+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:33:16.178844+0000","last_peered":"2023-09-23T22:33:16.178844+0000","last_clean":"2023-09-23T22:33:16.178844+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:33:16.178844+0000","last_undegraded":"2023-09-23T22:33:16.178844+0000","last_fullsized":"2023-09-23T22:33:16.178844+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":401,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178931+0000","last_change":"2023-09-23T22:31:28.149149+0000","last_active":"2023-09-23T22:33:16.178931+0000","last_peered":"2023-09-23T22:33:16.178931+0000","last_clean":"2023-09-23T22:33:16.178931+0000","last_became_active":"2023-09-23T22:31:28.121120+0000","last_became_peered":"2023-09-23T22:31:28.121120+0000","last_unstale":"2023-09-23T22:33:16.178931+0000","last_undegraded":"2023-09-23T22:33:16.178931+0000","last_fullsized":"2023-09-23T22:33:16.178931+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:19.434137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":372,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.176762+0000","last_change":"2023-09-23T22:31:28.119520+0000","last_active":"2023-09-23T22:33:16.176762+0000","last_peered":"2023-09-23T22:33:16.176762+0000","last_clean":"2023-09-23T22:33:16.176762+0000","last_became_active":"2023-09-23T22:31:28.118810+0000","last_became_peered":"2023-09-23T22:31:28.118810+0000","last_unstale":"2023-09-23T22:33:16.176762+0000","last_undegraded":"2023-09-23T22:33:16.176762+0000","last_fullsized":"2023-09-23T22:33:16.176762+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:36.007062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":316,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177484+0000","last_change":"2023-09-23T22:31:28.114537+0000","last_active":"2023-09-23T22:33:16.177484+0000","last_peered":"2023-09-23T22:33:16.177484+0000","last_clean":"2023-09-23T22:33:16.177484+0000","last_became_active":"2023-09-23T22:31:28.114176+0000","last_became_peered":"2023-09-23T22:31:28.114176+0000","last_unstale":"2023-09-23T22:33:16.177484+0000","last_undegraded":"2023-09-23T22:33:16.177484+0000","last_fullsized":"2023-09-23T22:33:16.177484+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:01.324173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":419,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.175762+0000","last_change":"2023-09-23T22:31:28.132494+0000","last_active":"2023-09-23T22:33:16.175762+0000","last_peered":"2023-09-23T22:33:16.175762+0000","last_clean":"2023-09-23T22:33:16.175762+0000","last_became_active":"2023-09-23T22:31:28.132197+0000","last_became_peered":"2023-09-23T22:31:28.132197+0000","last_unstale":"2023-09-23T22:33:16.175762+0000","last_undegraded":"2023-09-23T22:33:16.175762+0000","last_fullsized":"2023-09-23T22:33:16.175762+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:36.181016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":353,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178238+0000","last_change":"2023-09-23T22:31:28.119156+0000","last_active":"2023-09-23T22:33:16.178238+0000","last_peered":"2023-09-23T22:33:16.178238+0000","last_clean":"2023-09-23T22:33:16.178238+0000","last_became_active":"2023-09-23T22:31:28.118382+0000","last_became_peered":"2023-09-23T22:31:28.118382+0000","last_unstale":"2023-09-23T22:33:16.178238+0000","last_undegraded":"2023-09-23T22:33:16.178238+0000","last_fullsized":"2023-09-23T22:33:16.178238+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:10.140079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":306,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178931+0000","last_change":"2023-09-23T22:31:28.121397+0000","last_active":"2023-09-23T22:33:16.178931+0000","last_peered":"2023-09-23T22:33:16.178931+0000","last_clean":"2023-09-23T22:33:16.178931+0000","last_became_active":"2023-09-23T22:31:28.120847+0000","last_became_peered":"2023-09-23T22:31:28.120847+0000","last_unstale":"2023-09-23T22:33:16.178931+0000","last_undegraded":"2023-09-23T22:33:16.178931+0000","last_fullsized":"2023-09-23T22:33:16.178931+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:42.407658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":101,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.185080+0000","last_change":"2023-09-23T22:29:01.645385+0000","last_active":"2023-09-23T22:33:16.185080+0000","last_peered":"2023-09-23T22:33:16.185080+0000","last_clean":"2023-09-23T22:33:16.185080+0000","last_became_active":"2023-09-23T22:29:01.597647+0000","last_became_peered":"2023-09-23T22:29:01.597647+0000","last_unstale":"2023-09-23T22:33:16.185080+0000","last_undegraded":"2023-09-23T22:33:16.185080+0000","last_fullsized":"2023-09-23T22:33:16.185080+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.173780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":101,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.193231+0000","last_change":"2023-09-23T22:31:28.128965+0000","last_active":"2023-09-23T22:33:16.193231+0000","last_peered":"2023-09-23T22:33:16.193231+0000","last_clean":"2023-09-23T22:33:16.193231+0000","last_became_active":"2023-09-23T22:31:28.128224+0000","last_became_peered":"2023-09-23T22:31:28.128224+0000","last_unstale":"2023-09-23T22:33:16.193231+0000","last_undegraded":"2023-09-23T22:33:16.193231+0000","last_fullsized":"2023-09-23T22:33:16.193231+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:11.301143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.4","version":"410'8397","reported_seq":8442,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.976700+0000","last_change":"2023-09-23T22:31:08.226781+0000","last_active":"2023-09-23T22:33:24.976700+0000","last_peered":"2023-09-23T22:33:24.976700+0000","last_clean":"2023-09-23T22:33:24.976700+0000","last_became_active":"2023-09-23T22:31:08.226511+0000","last_became_peered":"2023-09-23T22:31:08.226511+0000","last_unstale":"2023-09-23T22:33:24.976700+0000","last_undegraded":"2023-09-23T22:33:24.976700+0000","last_fullsized":"2023-09-23T22:33:24.976700+0000","mapping_epoch":373,"log_start":"389'2900","ondisk_log_start":"389'2900","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5497,"log_dups_size":0,"ondisk_log_size":5497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:03.735360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258801664,"num_objects":3949,"num_object_clones":0,"num_object_copies":11847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8397,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":419,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.183057+0000","last_change":"2023-09-23T22:31:28.139498+0000","last_active":"2023-09-23T22:33:16.183057+0000","last_peered":"2023-09-23T22:33:16.183057+0000","last_clean":"2023-09-23T22:33:16.183057+0000","last_became_active":"2023-09-23T22:31:28.113206+0000","last_became_peered":"2023-09-23T22:31:28.113206+0000","last_unstale":"2023-09-23T22:33:16.183057+0000","last_undegraded":"2023-09-23T22:33:16.183057+0000","last_fullsized":"2023-09-23T22:33:16.183057+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:11.905312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":412,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178768+0000","last_change":"2023-09-23T22:31:28.131730+0000","last_active":"2023-09-23T22:33:16.178768+0000","last_peered":"2023-09-23T22:33:16.178768+0000","last_clean":"2023-09-23T22:33:16.178768+0000","last_became_active":"2023-09-23T22:31:28.131383+0000","last_became_peered":"2023-09-23T22:31:28.131383+0000","last_unstale":"2023-09-23T22:33:16.178768+0000","last_undegraded":"2023-09-23T22:33:16.178768+0000","last_fullsized":"2023-09-23T22:33:16.178768+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:56.993827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"410'8414","reported_seq":11662,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:25.008315+0000","last_change":"2023-09-23T22:32:00.147475+0000","last_active":"2023-09-23T22:33:25.008315+0000","last_peered":"2023-09-23T22:33:25.008315+0000","last_clean":"2023-09-23T22:33:25.008315+0000","last_became_active":"2023-09-23T22:32:00.121320+0000","last_became_peered":"2023-09-23T22:32:00.121320+0000","last_unstale":"2023-09-23T22:33:25.008315+0000","last_undegraded":"2023-09-23T22:33:25.008315+0000","last_fullsized":"2023-09-23T22:33:25.008315+0000","mapping_epoch":392,"log_start":"390'3400","ondisk_log_start":"390'3400","created":373,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5014,"log_dups_size":0,"ondisk_log_size":5014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:17:59.934876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264241152,"num_objects":4032,"num_object_clones":0,"num_object_copies":12096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8414,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1591,"num_bytes_recovered":104267776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":361,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.187437+0000","last_change":"2023-09-23T22:31:28.125280+0000","last_active":"2023-09-23T22:33:16.187437+0000","last_peered":"2023-09-23T22:33:16.187437+0000","last_clean":"2023-09-23T22:33:16.187437+0000","last_became_active":"2023-09-23T22:31:28.110856+0000","last_became_peered":"2023-09-23T22:31:28.110856+0000","last_unstale":"2023-09-23T22:33:16.187437+0000","last_undegraded":"2023-09-23T22:33:16.187437+0000","last_fullsized":"2023-09-23T22:33:16.187437+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:32.197056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":437,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178171+0000","last_change":"2023-09-23T22:29:01.604844+0000","last_active":"2023-09-23T22:33:16.178171+0000","last_peered":"2023-09-23T22:33:16.178171+0000","last_clean":"2023-09-23T22:33:16.178171+0000","last_became_active":"2023-09-23T22:29:01.604110+0000","last_became_peered":"2023-09-23T22:29:01.604110+0000","last_unstale":"2023-09-23T22:33:16.178171+0000","last_undegraded":"2023-09-23T22:33:16.178171+0000","last_fullsized":"2023-09-23T22:33:16.178171+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.034641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"410'8433","reported_seq":11726,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.946667+0000","last_change":"2023-09-23T22:33:15.177158+0000","last_active":"2023-09-23T22:33:24.946667+0000","last_peered":"2023-09-23T22:33:24.946667+0000","last_clean":"2023-09-23T22:33:24.946667+0000","last_became_active":"2023-09-23T22:33:15.164505+0000","last_became_peered":"2023-09-23T22:33:15.164505+0000","last_unstale":"2023-09-23T22:33:24.946667+0000","last_undegraded":"2023-09-23T22:33:24.946667+0000","last_fullsized":"2023-09-23T22:33:24.946667+0000","mapping_epoch":407,"log_start":"390'3000","ondisk_log_start":"390'3000","created":373,"last_epoch_clean":408,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5433,"log_dups_size":0,"ondisk_log_size":5433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:51.116827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259063808,"num_objects":3953,"num_object_clones":0,"num_object_copies":11859,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3953,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8433,"num_write_kb":396352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1614,"num_bytes_recovered":85590016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":377,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.175854+0000","last_change":"2023-09-23T22:31:28.136321+0000","last_active":"2023-09-23T22:33:16.175854+0000","last_peered":"2023-09-23T22:33:16.175854+0000","last_clean":"2023-09-23T22:33:16.175854+0000","last_became_active":"2023-09-23T22:31:28.131102+0000","last_became_peered":"2023-09-23T22:31:28.131102+0000","last_unstale":"2023-09-23T22:33:16.175854+0000","last_undegraded":"2023-09-23T22:33:16.175854+0000","last_fullsized":"2023-09-23T22:33:16.175854+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:24.768099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":101,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.187988+0000","last_change":"2023-09-23T22:31:28.125138+0000","last_active":"2023-09-23T22:33:16.187988+0000","last_peered":"2023-09-23T22:33:16.187988+0000","last_clean":"2023-09-23T22:33:16.187988+0000","last_became_active":"2023-09-23T22:31:28.124716+0000","last_became_peered":"2023-09-23T22:31:28.124716+0000","last_unstale":"2023-09-23T22:33:16.187988+0000","last_undegraded":"2023-09-23T22:33:16.187988+0000","last_fullsized":"2023-09-23T22:33:16.187988+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.718931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"410'8427","reported_seq":8472,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.980711+0000","last_change":"2023-09-23T22:31:08.225591+0000","last_active":"2023-09-23T22:33:24.980711+0000","last_peered":"2023-09-23T22:33:24.980711+0000","last_clean":"2023-09-23T22:33:24.980711+0000","last_became_active":"2023-09-23T22:31:08.225286+0000","last_became_peered":"2023-09-23T22:31:08.225286+0000","last_unstale":"2023-09-23T22:33:24.980711+0000","last_undegraded":"2023-09-23T22:33:24.980711+0000","last_fullsized":"2023-09-23T22:33:24.980711+0000","mapping_epoch":373,"log_start":"390'3400","ondisk_log_start":"390'3400","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5027,"log_dups_size":0,"ondisk_log_size":5027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:14.895634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265355264,"num_objects":4049,"num_object_clones":0,"num_object_copies":12147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8427,"num_write_kb":399232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":440,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.176912+0000","last_change":"2023-09-23T22:29:01.581676+0000","last_active":"2023-09-23T22:33:16.176912+0000","last_peered":"2023-09-23T22:33:16.176912+0000","last_clean":"2023-09-23T22:33:16.176912+0000","last_became_active":"2023-09-23T22:29:01.581305+0000","last_became_peered":"2023-09-23T22:29:01.581305+0000","last_unstale":"2023-09-23T22:33:16.176912+0000","last_undegraded":"2023-09-23T22:33:16.176912+0000","last_fullsized":"2023-09-23T22:33:16.176912+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:35.680140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":296,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177757+0000","last_change":"2023-09-23T22:27:52.100327+0000","last_active":"2023-09-23T22:33:16.177757+0000","last_peered":"2023-09-23T22:33:16.177757+0000","last_clean":"2023-09-23T22:33:16.177757+0000","last_became_active":"2023-09-23T22:27:52.100050+0000","last_became_peered":"2023-09-23T22:27:52.100050+0000","last_unstale":"2023-09-23T22:33:16.177757+0000","last_undegraded":"2023-09-23T22:33:16.177757+0000","last_fullsized":"2023-09-23T22:33:16.177757+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:29.861503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"410'8130","reported_seq":8175,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:25.042387+0000","last_change":"2023-09-23T22:31:08.226199+0000","last_active":"2023-09-23T22:33:25.042387+0000","last_peered":"2023-09-23T22:33:25.042387+0000","last_clean":"2023-09-23T22:33:25.042387+0000","last_became_active":"2023-09-23T22:31:08.225930+0000","last_became_peered":"2023-09-23T22:31:08.225930+0000","last_unstale":"2023-09-23T22:33:25.042387+0000","last_undegraded":"2023-09-23T22:33:25.042387+0000","last_fullsized":"2023-09-23T22:33:25.042387+0000","mapping_epoch":373,"log_start":"386'2600","ondisk_log_start":"386'2600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5530,"log_dups_size":0,"ondisk_log_size":5530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:16.294443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":255197184,"num_objects":3894,"num_object_clones":0,"num_object_copies":11682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8130,"num_write_kb":384768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":755,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.174449+0000","last_change":"2023-09-23T22:31:31.125903+0000","last_active":"2023-09-23T22:33:16.174449+0000","last_peered":"2023-09-23T22:33:16.174449+0000","last_clean":"2023-09-23T22:33:16.174449+0000","last_became_active":"2023-09-23T22:31:31.125630+0000","last_became_peered":"2023-09-23T22:31:31.125630+0000","last_unstale":"2023-09-23T22:33:16.174449+0000","last_undegraded":"2023-09-23T22:33:16.174449+0000","last_fullsized":"2023-09-23T22:33:16.174449+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:24.405922+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":242,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177854+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:33:16.177854+0000","last_peered":"2023-09-23T22:33:16.177854+0000","last_clean":"2023-09-23T22:33:16.177854+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:33:16.177854+0000","last_undegraded":"2023-09-23T22:33:16.177854+0000","last_fullsized":"2023-09-23T22:33:16.177854+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":418,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.182534+0000","last_change":"2023-09-23T22:29:01.661335+0000","last_active":"2023-09-23T22:33:16.182534+0000","last_peered":"2023-09-23T22:33:16.182534+0000","last_clean":"2023-09-23T22:33:16.182534+0000","last_became_active":"2023-09-23T22:29:01.660998+0000","last_became_peered":"2023-09-23T22:29:01.660998+0000","last_unstale":"2023-09-23T22:33:16.182534+0000","last_undegraded":"2023-09-23T22:33:16.182534+0000","last_fullsized":"2023-09-23T22:33:16.182534+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:58.512333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"410'8385","reported_seq":8432,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.398289+0000","last_change":"2023-09-23T22:31:08.227045+0000","last_active":"2023-09-23T22:33:24.398289+0000","last_peered":"2023-09-23T22:33:24.398289+0000","last_clean":"2023-09-23T22:33:24.398289+0000","last_became_active":"2023-09-23T22:31:08.226758+0000","last_became_peered":"2023-09-23T22:31:08.226758+0000","last_unstale":"2023-09-23T22:33:24.398289+0000","last_undegraded":"2023-09-23T22:33:24.398289+0000","last_fullsized":"2023-09-23T22:33:24.398289+0000","mapping_epoch":373,"log_start":"391'3600","ondisk_log_start":"391'3600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4785,"log_dups_size":0,"ondisk_log_size":4785,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:05:55.118636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267059200,"num_objects":4075,"num_object_clones":0,"num_object_copies":12225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8385,"num_write_kb":398720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":791,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178976+0000","last_change":"2023-09-23T22:29:19.097992+0000","last_active":"2023-09-23T22:33:16.178976+0000","last_peered":"2023-09-23T22:33:16.178976+0000","last_clean":"2023-09-23T22:33:16.178976+0000","last_became_active":"2023-09-23T22:29:19.097594+0000","last_became_peered":"2023-09-23T22:29:19.097594+0000","last_unstale":"2023-09-23T22:33:16.178976+0000","last_undegraded":"2023-09-23T22:33:16.178976+0000","last_fullsized":"2023-09-23T22:33:16.178976+0000","mapping_epoch":340,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":341,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:33.767096+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":91,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.188120+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:33:16.188120+0000","last_peered":"2023-09-23T22:33:16.188120+0000","last_clean":"2023-09-23T22:33:16.188120+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:33:16.188120+0000","last_undegraded":"2023-09-23T22:33:16.188120+0000","last_fullsized":"2023-09-23T22:33:16.188120+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":445,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.179429+0000","last_change":"2023-09-23T22:29:01.603793+0000","last_active":"2023-09-23T22:33:16.179429+0000","last_peered":"2023-09-23T22:33:16.179429+0000","last_clean":"2023-09-23T22:33:16.179429+0000","last_became_active":"2023-09-23T22:29:01.603259+0000","last_became_peered":"2023-09-23T22:29:01.603259+0000","last_unstale":"2023-09-23T22:33:16.179429+0000","last_undegraded":"2023-09-23T22:33:16.179429+0000","last_fullsized":"2023-09-23T22:33:16.179429+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:36.287629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"410'8465","reported_seq":8510,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:25.024394+0000","last_change":"2023-09-23T22:31:08.225049+0000","last_active":"2023-09-23T22:33:25.024394+0000","last_peered":"2023-09-23T22:33:25.024394+0000","last_clean":"2023-09-23T22:33:25.024394+0000","last_became_active":"2023-09-23T22:31:08.224765+0000","last_became_peered":"2023-09-23T22:31:08.224765+0000","last_unstale":"2023-09-23T22:33:25.024394+0000","last_undegraded":"2023-09-23T22:33:25.024394+0000","last_fullsized":"2023-09-23T22:33:25.024394+0000","mapping_epoch":373,"log_start":"393'3700","ondisk_log_start":"393'3700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4765,"log_dups_size":0,"ondisk_log_size":4765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.842171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258015232,"num_objects":3937,"num_object_clones":0,"num_object_copies":11811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8465,"num_write_kb":396864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":401,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177983+0000","last_change":"2023-09-23T22:31:28.119358+0000","last_active":"2023-09-23T22:33:16.177983+0000","last_peered":"2023-09-23T22:33:16.177983+0000","last_clean":"2023-09-23T22:33:16.177983+0000","last_became_active":"2023-09-23T22:31:28.118823+0000","last_became_peered":"2023-09-23T22:31:28.118823+0000","last_unstale":"2023-09-23T22:33:16.177983+0000","last_undegraded":"2023-09-23T22:33:16.177983+0000","last_fullsized":"2023-09-23T22:33:16.177983+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:25.483176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,5],"acting":[4,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":426,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.187261+0000","last_change":"2023-09-23T22:30:35.118937+0000","last_active":"2023-09-23T22:33:16.187261+0000","last_peered":"2023-09-23T22:33:16.187261+0000","last_clean":"2023-09-23T22:33:16.187261+0000","last_became_active":"2023-09-23T22:30:35.118646+0000","last_became_peered":"2023-09-23T22:30:35.118646+0000","last_unstale":"2023-09-23T22:33:16.187261+0000","last_undegraded":"2023-09-23T22:33:16.187261+0000","last_fullsized":"2023-09-23T22:33:16.187261+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.340847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"410'8392","reported_seq":8439,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.413873+0000","last_change":"2023-09-23T22:31:08.226959+0000","last_active":"2023-09-23T22:33:24.413873+0000","last_peered":"2023-09-23T22:33:24.413873+0000","last_clean":"2023-09-23T22:33:24.413873+0000","last_became_active":"2023-09-23T22:31:08.226652+0000","last_became_peered":"2023-09-23T22:31:08.226652+0000","last_unstale":"2023-09-23T22:33:24.413873+0000","last_undegraded":"2023-09-23T22:33:24.413873+0000","last_fullsized":"2023-09-23T22:33:24.413873+0000","mapping_epoch":373,"log_start":"391'3600","ondisk_log_start":"391'3600","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4792,"log_dups_size":0,"ondisk_log_size":4792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:22.758103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265420800,"num_objects":4050,"num_object_clones":0,"num_object_copies":12150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8392,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":764,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.174587+0000","last_change":"2023-09-23T22:29:04.061254+0000","last_active":"2023-09-23T22:33:16.174587+0000","last_peered":"2023-09-23T22:33:16.174587+0000","last_clean":"2023-09-23T22:33:16.174587+0000","last_became_active":"2023-09-23T22:29:04.059430+0000","last_became_peered":"2023-09-23T22:29:04.059430+0000","last_unstale":"2023-09-23T22:33:16.174587+0000","last_undegraded":"2023-09-23T22:33:16.174587+0000","last_fullsized":"2023-09-23T22:33:16.174587+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:03.727332+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":127,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.167683+0000","last_change":"2023-09-23T22:31:28.120600+0000","last_active":"2023-09-23T22:33:16.167683+0000","last_peered":"2023-09-23T22:33:16.167683+0000","last_clean":"2023-09-23T22:33:16.167683+0000","last_became_active":"2023-09-23T22:31:28.120281+0000","last_became_peered":"2023-09-23T22:31:28.120281+0000","last_unstale":"2023-09-23T22:33:16.167683+0000","last_undegraded":"2023-09-23T22:33:16.167683+0000","last_fullsized":"2023-09-23T22:33:16.167683+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:10.594022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":127,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.168340+0000","last_change":"2023-09-23T22:31:28.111270+0000","last_active":"2023-09-23T22:33:16.168340+0000","last_peered":"2023-09-23T22:33:16.168340+0000","last_clean":"2023-09-23T22:33:16.168340+0000","last_became_active":"2023-09-23T22:31:28.110921+0000","last_became_peered":"2023-09-23T22:31:28.110921+0000","last_unstale":"2023-09-23T22:33:16.168340+0000","last_undegraded":"2023-09-23T22:33:16.168340+0000","last_fullsized":"2023-09-23T22:33:16.168340+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:05.417671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"410'8284","reported_seq":11423,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.986162+0000","last_change":"2023-09-23T22:32:15.139562+0000","last_active":"2023-09-23T22:33:24.986162+0000","last_peered":"2023-09-23T22:33:24.986162+0000","last_clean":"2023-09-23T22:33:24.986162+0000","last_became_active":"2023-09-23T22:32:15.135437+0000","last_became_peered":"2023-09-23T22:32:15.135437+0000","last_unstale":"2023-09-23T22:33:24.986162+0000","last_undegraded":"2023-09-23T22:33:24.986162+0000","last_fullsized":"2023-09-23T22:33:24.986162+0000","mapping_epoch":395,"log_start":"390'2900","ondisk_log_start":"390'2900","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5384,"log_dups_size":0,"ondisk_log_size":5384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:36.457421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258342912,"num_objects":3942,"num_object_clones":0,"num_object_copies":11826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8284,"num_write_kb":391232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1537,"num_bytes_recovered":100728832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":685,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.193307+0000","last_change":"2023-09-23T22:31:31.129491+0000","last_active":"2023-09-23T22:33:16.193307+0000","last_peered":"2023-09-23T22:33:16.193307+0000","last_clean":"2023-09-23T22:33:16.193307+0000","last_became_active":"2023-09-23T22:31:31.129163+0000","last_became_peered":"2023-09-23T22:31:31.129163+0000","last_unstale":"2023-09-23T22:33:16.193307+0000","last_undegraded":"2023-09-23T22:33:16.193307+0000","last_fullsized":"2023-09-23T22:33:16.193307+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:19.141229+0000","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":786,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178124+0000","last_change":"2023-09-23T22:31:31.129492+0000","last_active":"2023-09-23T22:33:16.178124+0000","last_peered":"2023-09-23T22:33:16.178124+0000","last_clean":"2023-09-23T22:33:16.178124+0000","last_became_active":"2023-09-23T22:31:31.128706+0000","last_became_peered":"2023-09-23T22:31:31.128706+0000","last_unstale":"2023-09-23T22:33:16.178124+0000","last_undegraded":"2023-09-23T22:33:16.178124+0000","last_fullsized":"2023-09-23T22:33:16.178124+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:02:16.180152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":477,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.175595+0000","last_change":"2023-09-23T22:31:28.130412+0000","last_active":"2023-09-23T22:33:16.175595+0000","last_peered":"2023-09-23T22:33:16.175595+0000","last_clean":"2023-09-23T22:33:16.175595+0000","last_became_active":"2023-09-23T22:31:28.130127+0000","last_became_peered":"2023-09-23T22:31:28.130127+0000","last_unstale":"2023-09-23T22:33:16.175595+0000","last_undegraded":"2023-09-23T22:33:16.175595+0000","last_fullsized":"2023-09-23T22:33:16.175595+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.088651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":118,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.169173+0000","last_change":"2023-09-23T22:27:52.095935+0000","last_active":"2023-09-23T22:33:16.169173+0000","last_peered":"2023-09-23T22:33:16.169173+0000","last_clean":"2023-09-23T22:33:16.169173+0000","last_became_active":"2023-09-23T22:27:52.093483+0000","last_became_peered":"2023-09-23T22:27:52.093483+0000","last_unstale":"2023-09-23T22:33:16.169173+0000","last_undegraded":"2023-09-23T22:33:16.169173+0000","last_fullsized":"2023-09-23T22:33:16.169173+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:05.461980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"410'8364","reported_seq":11637,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:25.034586+0000","last_change":"2023-09-23T22:33:13.261657+0000","last_active":"2023-09-23T22:33:25.034586+0000","last_peered":"2023-09-23T22:33:25.034586+0000","last_clean":"2023-09-23T22:33:25.034586+0000","last_became_active":"2023-09-23T22:32:31.163888+0000","last_became_peered":"2023-09-23T22:32:31.163888+0000","last_unstale":"2023-09-23T22:33:25.034586+0000","last_undegraded":"2023-09-23T22:33:25.034586+0000","last_fullsized":"2023-09-23T22:33:25.034586+0000","mapping_epoch":399,"log_start":"389'2900","ondisk_log_start":"389'2900","created":373,"last_epoch_clean":406,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5464,"log_dups_size":0,"ondisk_log_size":5464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:18.004864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257163264,"num_objects":3924,"num_object_clones":0,"num_object_copies":11772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8364,"num_write_kb":393216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1603,"num_bytes_recovered":105054208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":754,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.174628+0000","last_change":"2023-09-23T22:28:08.057498+0000","last_active":"2023-09-23T22:33:16.174628+0000","last_peered":"2023-09-23T22:33:16.174628+0000","last_clean":"2023-09-23T22:33:16.174628+0000","last_became_active":"2023-09-23T22:28:08.057170+0000","last_became_peered":"2023-09-23T22:28:08.057170+0000","last_unstale":"2023-09-23T22:33:16.174628+0000","last_undegraded":"2023-09-23T22:33:16.174628+0000","last_fullsized":"2023-09-23T22:33:16.174628+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":314,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:19.898738+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1288,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.174808+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:33:16.174808+0000","last_peered":"2023-09-23T22:33:16.174808+0000","last_clean":"2023-09-23T22:33:16.174808+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:33:16.174808+0000","last_undegraded":"2023-09-23T22:33:16.174808+0000","last_fullsized":"2023-09-23T22:33:16.174808+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":449,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178774+0000","last_change":"2023-09-23T22:29:01.601806+0000","last_active":"2023-09-23T22:33:16.178774+0000","last_peered":"2023-09-23T22:33:16.178774+0000","last_clean":"2023-09-23T22:33:16.178774+0000","last_became_active":"2023-09-23T22:29:01.601190+0000","last_became_peered":"2023-09-23T22:29:01.601190+0000","last_unstale":"2023-09-23T22:33:16.178774+0000","last_undegraded":"2023-09-23T22:33:16.178774+0000","last_fullsized":"2023-09-23T22:33:16.178774+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:17.309138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":101,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.191730+0000","last_change":"2023-09-23T22:31:28.123596+0000","last_active":"2023-09-23T22:33:16.191730+0000","last_peered":"2023-09-23T22:33:16.191730+0000","last_clean":"2023-09-23T22:33:16.191730+0000","last_became_active":"2023-09-23T22:31:28.123179+0000","last_became_peered":"2023-09-23T22:31:28.123179+0000","last_unstale":"2023-09-23T22:33:16.191730+0000","last_undegraded":"2023-09-23T22:33:16.191730+0000","last_fullsized":"2023-09-23T22:33:16.191730+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:53.106243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.b","version":"410'8262","reported_seq":11308,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:25.024199+0000","last_change":"2023-09-23T22:31:44.145615+0000","last_active":"2023-09-23T22:33:25.024199+0000","last_peered":"2023-09-23T22:33:25.024199+0000","last_clean":"2023-09-23T22:33:25.024199+0000","last_became_active":"2023-09-23T22:31:44.142103+0000","last_became_peered":"2023-09-23T22:31:44.142103+0000","last_unstale":"2023-09-23T22:33:25.024199+0000","last_undegraded":"2023-09-23T22:33:25.024199+0000","last_fullsized":"2023-09-23T22:33:25.024199+0000","mapping_epoch":387,"log_start":"391'3500","ondisk_log_start":"391'3500","created":373,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4762,"log_dups_size":0,"ondisk_log_size":4762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:15.488007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263454720,"num_objects":4020,"num_object_clones":0,"num_object_copies":12060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8262,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1492,"num_bytes_recovered":97779712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":760,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177420+0000","last_change":"2023-09-23T22:31:31.130599+0000","last_active":"2023-09-23T22:33:16.177420+0000","last_peered":"2023-09-23T22:33:16.177420+0000","last_clean":"2023-09-23T22:33:16.177420+0000","last_became_active":"2023-09-23T22:31:31.127784+0000","last_became_peered":"2023-09-23T22:31:31.127784+0000","last_unstale":"2023-09-23T22:33:16.177420+0000","last_undegraded":"2023-09-23T22:33:16.177420+0000","last_fullsized":"2023-09-23T22:33:16.177420+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:28.788741+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":627,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177268+0000","last_change":"2023-09-23T22:29:09.079640+0000","last_active":"2023-09-23T22:33:16.177268+0000","last_peered":"2023-09-23T22:33:16.177268+0000","last_clean":"2023-09-23T22:33:16.177268+0000","last_became_active":"2023-09-23T22:29:01.597111+0000","last_became_peered":"2023-09-23T22:29:01.597111+0000","last_unstale":"2023-09-23T22:33:16.177268+0000","last_undegraded":"2023-09-23T22:33:16.177268+0000","last_fullsized":"2023-09-23T22:33:16.177268+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":334,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:49.008670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":91,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.185381+0000","last_change":"2023-09-23T22:29:01.584309+0000","last_active":"2023-09-23T22:33:16.185381+0000","last_peered":"2023-09-23T22:33:16.185381+0000","last_clean":"2023-09-23T22:33:16.185381+0000","last_became_active":"2023-09-23T22:29:01.584048+0000","last_became_peered":"2023-09-23T22:29:01.584048+0000","last_unstale":"2023-09-23T22:33:16.185381+0000","last_undegraded":"2023-09-23T22:33:16.185381+0000","last_fullsized":"2023-09-23T22:33:16.185381+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:26.666756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":59,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.168199+0000","last_change":"2023-09-23T22:30:35.116363+0000","last_active":"2023-09-23T22:33:16.168199+0000","last_peered":"2023-09-23T22:33:16.168199+0000","last_clean":"2023-09-23T22:33:16.168199+0000","last_became_active":"2023-09-23T22:30:35.115969+0000","last_became_peered":"2023-09-23T22:30:35.115969+0000","last_unstale":"2023-09-23T22:33:16.168199+0000","last_undegraded":"2023-09-23T22:33:16.168199+0000","last_fullsized":"2023-09-23T22:33:16.168199+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:22.817311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"410'8285","reported_seq":8330,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.983256+0000","last_change":"2023-09-23T22:31:08.227185+0000","last_active":"2023-09-23T22:33:24.983256+0000","last_peered":"2023-09-23T22:33:24.983256+0000","last_clean":"2023-09-23T22:33:24.983256+0000","last_became_active":"2023-09-23T22:31:08.226918+0000","last_became_peered":"2023-09-23T22:31:08.226918+0000","last_unstale":"2023-09-23T22:33:24.983256+0000","last_undegraded":"2023-09-23T22:33:24.983256+0000","last_fullsized":"2023-09-23T22:33:24.983256+0000","mapping_epoch":373,"log_start":"390'3200","ondisk_log_start":"390'3200","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5085,"log_dups_size":0,"ondisk_log_size":5085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:54.257936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262733824,"num_objects":4009,"num_object_clones":0,"num_object_copies":12027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8285,"num_write_kb":393408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1172,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.174843+0000","last_change":"2023-09-23T22:31:31.127033+0000","last_active":"2023-09-23T22:33:16.174843+0000","last_peered":"2023-09-23T22:33:16.174843+0000","last_clean":"2023-09-23T22:33:16.174843+0000","last_became_active":"2023-09-23T22:31:31.126601+0000","last_became_peered":"2023-09-23T22:31:31.126601+0000","last_unstale":"2023-09-23T22:33:16.174843+0000","last_undegraded":"2023-09-23T22:33:16.174843+0000","last_fullsized":"2023-09-23T22:33:16.174843+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:43:00.619278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":795,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.182347+0000","last_change":"2023-09-23T22:31:31.126756+0000","last_active":"2023-09-23T22:33:16.182347+0000","last_peered":"2023-09-23T22:33:16.182347+0000","last_clean":"2023-09-23T22:33:16.182347+0000","last_became_active":"2023-09-23T22:31:31.126409+0000","last_became_peered":"2023-09-23T22:31:31.126409+0000","last_unstale":"2023-09-23T22:33:16.182347+0000","last_undegraded":"2023-09-23T22:33:16.182347+0000","last_fullsized":"2023-09-23T22:33:16.182347+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:56.605240+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":291,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.174664+0000","last_change":"2023-09-23T22:29:01.602185+0000","last_active":"2023-09-23T22:33:16.174664+0000","last_peered":"2023-09-23T22:33:16.174664+0000","last_clean":"2023-09-23T22:33:16.174664+0000","last_became_active":"2023-09-23T22:29:01.601877+0000","last_became_peered":"2023-09-23T22:29:01.601877+0000","last_unstale":"2023-09-23T22:33:16.174664+0000","last_undegraded":"2023-09-23T22:33:16.174664+0000","last_fullsized":"2023-09-23T22:33:16.174664+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:47.080273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":101,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.194012+0000","last_change":"2023-09-23T22:31:28.124597+0000","last_active":"2023-09-23T22:33:16.194012+0000","last_peered":"2023-09-23T22:33:16.194012+0000","last_clean":"2023-09-23T22:33:16.194012+0000","last_became_active":"2023-09-23T22:31:28.124245+0000","last_became_peered":"2023-09-23T22:31:28.124245+0000","last_unstale":"2023-09-23T22:33:16.194012+0000","last_undegraded":"2023-09-23T22:33:16.194012+0000","last_fullsized":"2023-09-23T22:33:16.194012+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.558632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"410'8342","reported_seq":8388,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.411579+0000","last_change":"2023-09-23T22:31:08.223664+0000","last_active":"2023-09-23T22:33:24.411579+0000","last_peered":"2023-09-23T22:33:24.411579+0000","last_clean":"2023-09-23T22:33:24.411579+0000","last_became_active":"2023-09-23T22:31:08.223323+0000","last_became_peered":"2023-09-23T22:31:08.223323+0000","last_unstale":"2023-09-23T22:33:24.411579+0000","last_undegraded":"2023-09-23T22:33:24.411579+0000","last_fullsized":"2023-09-23T22:33:24.411579+0000","mapping_epoch":373,"log_start":"391'3500","ondisk_log_start":"391'3500","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4842,"log_dups_size":0,"ondisk_log_size":4842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:38.536861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264634368,"num_objects":4038,"num_object_clones":0,"num_object_copies":12114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8342,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":642,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.188383+0000","last_change":"2023-09-23T22:31:21.362707+0000","last_active":"2023-09-23T22:33:16.188383+0000","last_peered":"2023-09-23T22:33:16.188383+0000","last_clean":"2023-09-23T22:33:16.188383+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:33:16.188383+0000","last_undegraded":"2023-09-23T22:33:16.188383+0000","last_fullsized":"2023-09-23T22:33:16.188383+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:58.622607+0000","scrub_duration":0.0034076229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":789,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.167903+0000","last_change":"2023-09-23T22:31:28.147724+0000","last_active":"2023-09-23T22:33:16.167903+0000","last_peered":"2023-09-23T22:33:16.167903+0000","last_clean":"2023-09-23T22:33:16.167903+0000","last_became_active":"2023-09-23T22:31:28.120238+0000","last_became_peered":"2023-09-23T22:31:28.120238+0000","last_unstale":"2023-09-23T22:33:16.167903+0000","last_undegraded":"2023-09-23T22:33:16.167903+0000","last_fullsized":"2023-09-23T22:33:16.167903+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:53.259496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":305,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177268+0000","last_change":"2023-09-23T22:27:27.281589+0000","last_active":"2023-09-23T22:33:16.177268+0000","last_peered":"2023-09-23T22:33:16.177268+0000","last_clean":"2023-09-23T22:33:16.177268+0000","last_became_active":"2023-09-23T22:27:27.281135+0000","last_became_peered":"2023-09-23T22:27:27.281135+0000","last_unstale":"2023-09-23T22:33:16.177268+0000","last_undegraded":"2023-09-23T22:33:16.177268+0000","last_fullsized":"2023-09-23T22:33:16.177268+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:31.520391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":298,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178854+0000","last_change":"2023-09-23T22:31:28.121081+0000","last_active":"2023-09-23T22:33:16.178854+0000","last_peered":"2023-09-23T22:33:16.178854+0000","last_clean":"2023-09-23T22:33:16.178854+0000","last_became_active":"2023-09-23T22:31:28.120399+0000","last_became_peered":"2023-09-23T22:31:28.120399+0000","last_unstale":"2023-09-23T22:33:16.178854+0000","last_undegraded":"2023-09-23T22:33:16.178854+0000","last_fullsized":"2023-09-23T22:33:16.178854+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:11.267296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"410'8559","reported_seq":8605,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.413377+0000","last_change":"2023-09-23T22:31:08.224259+0000","last_active":"2023-09-23T22:33:24.413377+0000","last_peered":"2023-09-23T22:33:24.413377+0000","last_clean":"2023-09-23T22:33:24.413377+0000","last_became_active":"2023-09-23T22:31:08.224062+0000","last_became_peered":"2023-09-23T22:31:08.224062+0000","last_unstale":"2023-09-23T22:33:24.413377+0000","last_undegraded":"2023-09-23T22:33:24.413377+0000","last_fullsized":"2023-09-23T22:33:24.413377+0000","mapping_epoch":373,"log_start":"391'3700","ondisk_log_start":"391'3700","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4859,"log_dups_size":0,"ondisk_log_size":4859,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:18.330753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270467072,"num_objects":4127,"num_object_clones":0,"num_object_copies":12381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8559,"num_write_kb":405952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":760,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.192218+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:33:16.192218+0000","last_peered":"2023-09-23T22:33:16.192218+0000","last_clean":"2023-09-23T22:33:16.192218+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:33:16.192218+0000","last_undegraded":"2023-09-23T22:33:16.192218+0000","last_fullsized":"2023-09-23T22:33:16.192218+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":337,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178178+0000","last_change":"2023-09-23T22:31:28.152186+0000","last_active":"2023-09-23T22:33:16.178178+0000","last_peered":"2023-09-23T22:33:16.178178+0000","last_clean":"2023-09-23T22:33:16.178178+0000","last_became_active":"2023-09-23T22:31:28.111292+0000","last_became_peered":"2023-09-23T22:31:28.111292+0000","last_unstale":"2023-09-23T22:33:16.178178+0000","last_undegraded":"2023-09-23T22:33:16.178178+0000","last_fullsized":"2023-09-23T22:33:16.178178+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:06.097398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":336,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.176995+0000","last_change":"2023-09-23T22:28:41.358596+0000","last_active":"2023-09-23T22:33:16.176995+0000","last_peered":"2023-09-23T22:33:16.176995+0000","last_clean":"2023-09-23T22:33:16.176995+0000","last_became_active":"2023-09-23T22:28:41.358322+0000","last_became_peered":"2023-09-23T22:28:41.358322+0000","last_unstale":"2023-09-23T22:33:16.176995+0000","last_undegraded":"2023-09-23T22:33:16.176995+0000","last_fullsized":"2023-09-23T22:33:16.176995+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":324,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:45.600784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":278,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177759+0000","last_change":"2023-09-23T22:31:28.106985+0000","last_active":"2023-09-23T22:33:16.177759+0000","last_peered":"2023-09-23T22:33:16.177759+0000","last_clean":"2023-09-23T22:33:16.177759+0000","last_became_active":"2023-09-23T22:31:28.105451+0000","last_became_peered":"2023-09-23T22:31:28.105451+0000","last_unstale":"2023-09-23T22:33:16.177759+0000","last_undegraded":"2023-09-23T22:33:16.177759+0000","last_fullsized":"2023-09-23T22:33:16.177759+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:43.614928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"410'8367","reported_seq":11633,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:24.972699+0000","last_change":"2023-09-23T22:31:45.135267+0000","last_active":"2023-09-23T22:33:24.972699+0000","last_peered":"2023-09-23T22:33:24.972699+0000","last_clean":"2023-09-23T22:33:24.972699+0000","last_became_active":"2023-09-23T22:31:45.131633+0000","last_became_peered":"2023-09-23T22:31:45.131633+0000","last_unstale":"2023-09-23T22:33:24.972699+0000","last_undegraded":"2023-09-23T22:33:24.972699+0000","last_fullsized":"2023-09-23T22:33:24.972699+0000","mapping_epoch":388,"log_start":"390'3000","ondisk_log_start":"390'3000","created":373,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5367,"log_dups_size":0,"ondisk_log_size":5367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:27:00.859364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263127040,"num_objects":4015,"num_object_clones":0,"num_object_copies":12045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8367,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1601,"num_bytes_recovered":104923136,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":764,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.179296+0000","last_change":"2023-09-23T22:29:04.061152+0000","last_active":"2023-09-23T22:33:16.179296+0000","last_peered":"2023-09-23T22:33:16.179296+0000","last_clean":"2023-09-23T22:33:16.179296+0000","last_became_active":"2023-09-23T22:29:04.060777+0000","last_became_peered":"2023-09-23T22:29:04.060777+0000","last_unstale":"2023-09-23T22:33:16.179296+0000","last_undegraded":"2023-09-23T22:33:16.179296+0000","last_fullsized":"2023-09-23T22:33:16.179296+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:55.435518+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":242,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178096+0000","last_change":"2023-09-23T22:31:28.131347+0000","last_active":"2023-09-23T22:33:16.178096+0000","last_peered":"2023-09-23T22:33:16.178096+0000","last_clean":"2023-09-23T22:33:16.178096+0000","last_became_active":"2023-09-23T22:31:28.114169+0000","last_became_peered":"2023-09-23T22:31:28.114169+0000","last_unstale":"2023-09-23T22:33:16.178096+0000","last_undegraded":"2023-09-23T22:33:16.178096+0000","last_fullsized":"2023-09-23T22:33:16.178096+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:30.007254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":906,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.169757+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:33:16.169757+0000","last_peered":"2023-09-23T22:33:16.169757+0000","last_clean":"2023-09-23T22:33:16.169757+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:33:16.169757+0000","last_undegraded":"2023-09-23T22:33:16.169757+0000","last_fullsized":"2023-09-23T22:33:16.169757+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":350,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.182094+0000","last_change":"2023-09-23T22:31:28.114550+0000","last_active":"2023-09-23T22:33:16.182094+0000","last_peered":"2023-09-23T22:33:16.182094+0000","last_clean":"2023-09-23T22:33:16.182094+0000","last_became_active":"2023-09-23T22:31:28.114217+0000","last_became_peered":"2023-09-23T22:31:28.114217+0000","last_unstale":"2023-09-23T22:33:16.182094+0000","last_undegraded":"2023-09-23T22:33:16.182094+0000","last_fullsized":"2023-09-23T22:33:16.182094+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:13.616772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.d","version":"410'8323","reported_seq":11558,"reported_epoch":410,"state":"active+clean","last_fresh":"2023-09-23T22:33:25.122860+0000","last_change":"2023-09-23T22:32:51.714615+0000","last_active":"2023-09-23T22:33:25.122860+0000","last_peered":"2023-09-23T22:33:25.122860+0000","last_clean":"2023-09-23T22:33:25.122860+0000","last_became_active":"2023-09-23T22:32:51.673119+0000","last_became_peered":"2023-09-23T22:32:51.673119+0000","last_unstale":"2023-09-23T22:33:25.122860+0000","last_undegraded":"2023-09-23T22:33:25.122860+0000","last_fullsized":"2023-09-23T22:33:25.122860+0000","mapping_epoch":403,"log_start":"389'2800","ondisk_log_start":"389'2800","created":373,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5523,"log_dups_size":0,"ondisk_log_size":5523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:58.214128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257949720,"num_objects":3937,"num_object_clones":0,"num_object_copies":11811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3937,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8323,"num_write_kb":392257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1584,"num_bytes_recovered":103809024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":690,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177068+0000","last_change":"2023-09-23T22:31:45.133529+0000","last_active":"2023-09-23T22:33:16.177068+0000","last_peered":"2023-09-23T22:33:16.177068+0000","last_clean":"2023-09-23T22:33:16.177068+0000","last_became_active":"2023-09-23T22:31:45.133117+0000","last_became_peered":"2023-09-23T22:31:45.133117+0000","last_unstale":"2023-09-23T22:33:16.177068+0000","last_undegraded":"2023-09-23T22:33:16.177068+0000","last_fullsized":"2023-09-23T22:33:16.177068+0000","mapping_epoch":388,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.591643+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.591643+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:13:28.957050+0000","scrub_duration":0.0073638979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":121,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.193156+0000","last_change":"2023-09-23T22:31:28.156716+0000","last_active":"2023-09-23T22:33:16.193156+0000","last_peered":"2023-09-23T22:33:16.193156+0000","last_clean":"2023-09-23T22:33:16.193156+0000","last_became_active":"2023-09-23T22:31:28.128617+0000","last_became_peered":"2023-09-23T22:31:28.128617+0000","last_unstale":"2023-09-23T22:33:16.193156+0000","last_undegraded":"2023-09-23T22:33:16.193156+0000","last_fullsized":"2023-09-23T22:33:16.193156+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:34.465027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":274,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178695+0000","last_change":"2023-09-23T22:27:52.179484+0000","last_active":"2023-09-23T22:33:16.178695+0000","last_peered":"2023-09-23T22:33:16.178695+0000","last_clean":"2023-09-23T22:33:16.178695+0000","last_became_active":"2023-09-23T22:27:52.107547+0000","last_became_peered":"2023-09-23T22:27:52.107547+0000","last_unstale":"2023-09-23T22:33:16.178695+0000","last_undegraded":"2023-09-23T22:33:16.178695+0000","last_fullsized":"2023-09-23T22:33:16.178695+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:10:07.655359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":383,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178054+0000","last_change":"2023-09-23T22:29:01.605026+0000","last_active":"2023-09-23T22:33:16.178054+0000","last_peered":"2023-09-23T22:33:16.178054+0000","last_clean":"2023-09-23T22:33:16.178054+0000","last_became_active":"2023-09-23T22:29:01.604433+0000","last_became_peered":"2023-09-23T22:29:01.604433+0000","last_unstale":"2023-09-23T22:33:16.178054+0000","last_undegraded":"2023-09-23T22:33:16.178054+0000","last_fullsized":"2023-09-23T22:33:16.178054+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:26.853673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":70,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177589+0000","last_change":"2023-09-23T22:31:28.111728+0000","last_active":"2023-09-23T22:33:16.177589+0000","last_peered":"2023-09-23T22:33:16.177589+0000","last_clean":"2023-09-23T22:33:16.177589+0000","last_became_active":"2023-09-23T22:31:28.111369+0000","last_became_peered":"2023-09-23T22:31:28.111369+0000","last_unstale":"2023-09-23T22:33:16.177589+0000","last_undegraded":"2023-09-23T22:33:16.177589+0000","last_fullsized":"2023-09-23T22:33:16.177589+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:42.837575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":161,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.167229+0000","last_change":"2023-09-23T22:28:05.145371+0000","last_active":"2023-09-23T22:33:16.167229+0000","last_peered":"2023-09-23T22:33:16.167229+0000","last_clean":"2023-09-23T22:33:16.167229+0000","last_became_active":"2023-09-23T22:28:05.144513+0000","last_became_peered":"2023-09-23T22:28:05.144513+0000","last_unstale":"2023-09-23T22:33:16.167229+0000","last_undegraded":"2023-09-23T22:33:16.167229+0000","last_fullsized":"2023-09-23T22:33:16.167229+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:00.745694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":400,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.176842+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:33:16.176842+0000","last_peered":"2023-09-23T22:33:16.176842+0000","last_clean":"2023-09-23T22:33:16.176842+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:33:16.176842+0000","last_undegraded":"2023-09-23T22:33:16.176842+0000","last_fullsized":"2023-09-23T22:33:16.176842+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":435,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.179025+0000","last_change":"2023-09-23T22:31:28.111351+0000","last_active":"2023-09-23T22:33:16.179025+0000","last_peered":"2023-09-23T22:33:16.179025+0000","last_clean":"2023-09-23T22:33:16.179025+0000","last_became_active":"2023-09-23T22:31:28.111006+0000","last_became_peered":"2023-09-23T22:31:28.111006+0000","last_unstale":"2023-09-23T22:33:16.179025+0000","last_undegraded":"2023-09-23T22:33:16.179025+0000","last_fullsized":"2023-09-23T22:33:16.179025+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:51.435513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":425,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.175680+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:33:16.175680+0000","last_peered":"2023-09-23T22:33:16.175680+0000","last_clean":"2023-09-23T22:33:16.175680+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:33:16.175680+0000","last_undegraded":"2023-09-23T22:33:16.175680+0000","last_fullsized":"2023-09-23T22:33:16.175680+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":274,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.183285+0000","last_change":"2023-09-23T22:29:01.599552+0000","last_active":"2023-09-23T22:33:16.183285+0000","last_peered":"2023-09-23T22:33:16.183285+0000","last_clean":"2023-09-23T22:33:16.183285+0000","last_became_active":"2023-09-23T22:29:01.599257+0000","last_became_peered":"2023-09-23T22:29:01.599257+0000","last_unstale":"2023-09-23T22:33:16.183285+0000","last_undegraded":"2023-09-23T22:33:16.183285+0000","last_fullsized":"2023-09-23T22:33:16.183285+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:12.363120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":101,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.192090+0000","last_change":"2023-09-23T22:31:28.126203+0000","last_active":"2023-09-23T22:33:16.192090+0000","last_peered":"2023-09-23T22:33:16.192090+0000","last_clean":"2023-09-23T22:33:16.192090+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:31:28.125529+0000","last_unstale":"2023-09-23T22:33:16.192090+0000","last_undegraded":"2023-09-23T22:33:16.192090+0000","last_fullsized":"2023-09-23T22:33:16.192090+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:47.359771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":411,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:19.988692+0000","last_change":"2023-09-23T22:28:05.149400+0000","last_active":"2023-09-23T22:33:19.988692+0000","last_peered":"2023-09-23T22:33:19.988692+0000","last_clean":"2023-09-23T22:33:19.988692+0000","last_became_active":"2023-09-23T22:28:05.148306+0000","last_became_peered":"2023-09-23T22:28:05.148306+0000","last_unstale":"2023-09-23T22:33:19.988692+0000","last_undegraded":"2023-09-23T22:33:19.988692+0000","last_fullsized":"2023-09-23T22:33:19.988692+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:05.100846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":348,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.178153+0000","last_change":"2023-09-23T22:29:01.685712+0000","last_active":"2023-09-23T22:33:16.178153+0000","last_peered":"2023-09-23T22:33:16.178153+0000","last_clean":"2023-09-23T22:33:16.178153+0000","last_became_active":"2023-09-23T22:29:01.669963+0000","last_became_peered":"2023-09-23T22:29:01.669963+0000","last_unstale":"2023-09-23T22:33:16.178153+0000","last_undegraded":"2023-09-23T22:33:16.178153+0000","last_fullsized":"2023-09-23T22:33:16.178153+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:37.688251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":468,"reported_epoch":409,"state":"active+clean","last_fresh":"2023-09-23T22:33:16.177985+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:33:16.177985+0000","last_peered":"2023-09-23T22:33:16.177985+0000","last_clean":"2023-09-23T22:33:16.177985+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:33:16.177985+0000","last_undegraded":"2023-09-23T22:33:16.177985+0000","last_fullsized":"2023-09-23T22:33:16.177985+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":102,"num_bytes_recovered":31404,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":21496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26780,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4191027224,"num_objects":63951,"num_object_clones":0,"num_object_copies":191853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63951,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133829,"num_write_kb":6328897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11022,"num_bytes_recovered":702152704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12768260096,"data_stored":12768247880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82129,"ondisk_log_size":82129,"up":48,"acting":48,"num_store_stats":6},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1115744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":232,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234348,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568495984,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2263032,"kb_used_data":2081456,"kb_used_omap":0,"kb_used_meta":181568,"kb_avail":91482120,"statfs":{"total":95995035648,"available":93677690880,"internally_reserved":0,"allocated":2131410944,"data_stored":2129947758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":185925632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:33:22 2023","interfaces":[{"interface":"back","average":{"1min":7.9219999999999997,"5min":7.0460000000000003,"15min":3.4390000000000001},"min":{"1min":0.23499999999999999,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":131.73400000000001,"5min":213.23699999999999,"15min":213.23699999999999},"last":2.4380000000000002},{"interface":"front","average":{"1min":7.8220000000000001,"5min":7.3879999999999999,"15min":4.468},"min":{"1min":0.34399999999999997,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":132.452,"5min":213.304,"15min":213.304},"last":2.3929999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":5.3650000000000002,"5min":1.8620000000000001,"15min":3.5110000000000001},"min":{"1min":0.27700000000000002,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":58.143999999999998,"5min":58.143999999999998,"15min":129.964},"last":2.637},{"interface":"front","average":{"1min":5.2050000000000001,"5min":2.0750000000000002,"15min":3.8039999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":63.073999999999998,"5min":63.073999999999998,"15min":131.18600000000001},"last":2.423}]},{"osd":2,"last update":"Sat Sep 23 22:33:00 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":1.927,"15min":1.911},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.2990000000000004,"5min":6.6559999999999997,"15min":6.6559999999999997},"last":0.39800000000000002},{"interface":"front","average":{"1min":2.6659999999999999,"5min":2.016,"15min":2.105},"min":{"1min":0.51300000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.1109999999999998,"5min":9.7319999999999993,"15min":9.7319999999999993},"last":2.4969999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.6339999999999999,"5min":1.5940000000000001,"15min":3.145},"min":{"1min":0.56899999999999995,"5min":0.32500000000000001,"15min":0.316},"max":{"1min":6.165,"5min":6.8609999999999998,"15min":125.125},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.2829999999999999,"5min":1.4119999999999999,"15min":2.8610000000000002},"min":{"1min":0.33900000000000002,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":9.2569999999999997,"5min":9.2569999999999997,"15min":76.406999999999996},"last":2.4809999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.127,"15min":2.6299999999999999},"min":{"1min":0.29299999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5010000000000003,"5min":6.8650000000000002,"15min":305.30200000000002},"last":2.855},{"interface":"front","average":{"1min":4.1020000000000003,"5min":1.782,"15min":2.8319999999999999},"min":{"1min":0.317,"5min":0.29299999999999998,"15min":0.255},"max":{"1min":11.561,"5min":11.561,"15min":305.40300000000002},"last":0.95699999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":2.726,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.148,"5min":88.194999999999993,"15min":88.194999999999993},"last":2.4590000000000001},{"interface":"front","average":{"1min":3.1909999999999998,"5min":2.2839999999999998,"15min":1.5860000000000001},"min":{"1min":0.44600000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.2859999999999996,"5min":88.025000000000006,"15min":88.025000000000006},"last":4.04}]},{"osd":7,"last update":"Sat Sep 23 22:33:22 2023","interfaces":[{"interface":"back","average":{"1min":2.5720000000000001,"5min":1.369,"15min":1.2090000000000001},"min":{"1min":0.39200000000000002,"5min":0.33800000000000002,"15min":0.314},"max":{"1min":8.9700000000000006,"5min":9.2270000000000003,"15min":82.801000000000002},"last":2.8149999999999999},{"interface":"front","average":{"1min":3.4060000000000001,"5min":1.679,"15min":1.3160000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.29699999999999999},"max":{"1min":10.324999999999999,"5min":10.324999999999999,"15min":83.052999999999997},"last":4.9820000000000002}]}]},{"osd":1,"up_from":98,"seq":420906795256,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3036440,"kb_used_data":2350480,"kb_used_omap":7,"kb_used_meta":685816,"kb_avail":90708712,"statfs":{"total":95995035648,"available":92885721088,"internally_reserved":0,"allocated":2406891520,"data_stored":2404748033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":702275956},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":6.3769999999999998,"5min":5.8250000000000002,"15min":5.1369999999999996},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":107.202,"5min":202.73500000000001,"15min":202.73500000000001},"last":0.23000000000000001},{"interface":"front","average":{"1min":7.0449999999999999,"5min":6.3120000000000003,"15min":5.9349999999999996},"min":{"1min":0.24399999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":107.426,"5min":202.80799999999999,"15min":202.80799999999999},"last":0.53500000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":0.94999999999999996,"15min":1.0209999999999999},"min":{"1min":0.23599999999999999,"5min":0.216,"15min":0.216},"max":{"1min":3.8820000000000001,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.24,"5min":1.8999999999999999,"15min":2.214},"min":{"1min":0.23799999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.9859999999999998,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":0.46700000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2669999999999999,"5min":1.732,"15min":4.4980000000000002},"min":{"1min":0.30599999999999999,"5min":0.246,"15min":0.246},"max":{"1min":10.587999999999999,"5min":10.587999999999999,"15min":160.499},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.653,"5min":1.46,"15min":4.1890000000000001},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.223},"max":{"1min":5.1920000000000002,"5min":8.1590000000000007,"15min":160.465},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":0.92200000000000004,"15min":2.2440000000000002},"min":{"1min":0.29499999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.3730000000000002,"5min":7.0380000000000003,"15min":234.79900000000001},"last":2.9119999999999999},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.478,"15min":2.919},"min":{"1min":0.36299999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":7.4729999999999999,"5min":9.093,"15min":234.81700000000001},"last":2.9289999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":1.6859999999999999,"15min":2.5},"min":{"1min":0.36299999999999999,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":7.4450000000000003,"5min":9.2759999999999998,"15min":196.79900000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.611,"5min":1.294,"15min":2.3239999999999998},"min":{"1min":0.32900000000000001,"5min":0.313,"15min":0.29099999999999998},"max":{"1min":5.2889999999999997,"5min":6.8049999999999997,"15min":196.68299999999999},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8260000000000001,"5min":2.2559999999999998,"15min":2.1379999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":5.2350000000000003,"5min":77.033000000000001,"15min":77.033000000000001},"last":2.6120000000000001},{"interface":"front","average":{"1min":1.681,"5min":1.9630000000000001,"15min":1.583},"min":{"1min":0.442,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":8.3640000000000008,"5min":76.965999999999994,"15min":76.965999999999994},"last":0.48199999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:32:32 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":1.663,"15min":2.0259999999999998},"min":{"1min":0.41499999999999998,"5min":0.38300000000000001,"15min":0.32900000000000001},"max":{"1min":8.0299999999999994,"5min":8.0660000000000007,"15min":16.265000000000001},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.302,"5min":1.2529999999999999,"15min":1.339},"min":{"1min":0.378,"5min":0.318,"15min":0.318},"max":{"1min":3.8119999999999998,"5min":7.1429999999999998,"15min":12.045999999999999},"last":0.73299999999999998}]}]},{"osd":0,"up_from":93,"seq":399431958785,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3057732,"kb_used_data":2310204,"kb_used_omap":8,"kb_used_meta":747447,"kb_avail":90687420,"statfs":{"total":95995035648,"available":92863918080,"internally_reserved":0,"allocated":2365648896,"data_stored":2363416276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":765386602},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:32:37 2023","interfaces":[{"interface":"back","average":{"1min":7.5819999999999999,"5min":3.4860000000000002,"15min":4.1769999999999996},"min":{"1min":0.215,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":121.688,"5min":121.688,"15min":169.77199999999999},"last":1.002},{"interface":"front","average":{"1min":6.8739999999999997,"5min":3.5529999999999999,"15min":4.2850000000000001},"min":{"1min":0.25,"5min":0.218,"15min":0.218},"max":{"1min":122.93300000000001,"5min":122.93300000000001,"15min":169.52099999999999},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:32:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":2.2850000000000001,"15min":2.8919999999999999},"min":{"1min":0.30099999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.2359999999999998,"5min":22.263000000000002,"15min":22.263000000000002},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.415,"15min":1.694},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.3380000000000001,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:32:37 2023","interfaces":[{"interface":"back","average":{"1min":1.827,"5min":1.45,"15min":3.5419999999999998},"min":{"1min":0.315,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":8.548,"5min":10.292999999999999,"15min":146.87299999999999},"last":0.313},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.6259999999999999,"15min":3.5750000000000002},"min":{"1min":0.29299999999999998,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":3.5379999999999998,"5min":22.303000000000001,"15min":145.48099999999999},"last":0.45700000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:33:15 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":2.1890000000000001,"15min":2.4900000000000002},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.27000000000000002},"max":{"1min":4.5819999999999999,"5min":11.474,"15min":82.825000000000003},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.355,"5min":1.4390000000000001,"15min":1.768},"min":{"1min":0.379,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.8659999999999997,"5min":7.2279999999999998,"15min":75.930999999999997},"last":0.48399999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:33:15 2023","interfaces":[{"interface":"back","average":{"1min":2.6890000000000001,"5min":2.4119999999999999,"15min":2.988},"min":{"1min":0.438,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.085,"5min":13.564,"15min":93.685000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.8560000000000001,"5min":1.7529999999999999,"15min":2.286},"min":{"1min":0.38400000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.7889999999999997,"5min":13.486000000000001,"15min":93.546000000000006},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:33:15 2023","interfaces":[{"interface":"back","average":{"1min":3.1000000000000001,"5min":2.6099999999999999,"15min":2.3679999999999999},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.308},"max":{"1min":12.579000000000001,"5min":12.955,"15min":16.366},"last":0.56999999999999995},{"interface":"front","average":{"1min":2,"5min":1.8240000000000001,"15min":1.7310000000000001},"min":{"1min":0.46400000000000002,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":7.4770000000000003,"5min":14.208,"15min":22.263000000000002},"last":0.53300000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:33:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8999999999999999,"5min":1.931,"15min":2.0099999999999998},"min":{"1min":0.45700000000000002,"5min":0.40200000000000002,"15min":0.29999999999999999},"max":{"1min":10.377000000000001,"5min":22.338999999999999,"15min":32.773000000000003},"last":0.79300000000000004},{"interface":"front","average":{"1min":2.1499999999999999,"5min":2.2250000000000001,"15min":2.2799999999999998},"min":{"1min":0.5,"5min":0.34200000000000003,"15min":0.307},"max":{"1min":4.7229999999999999,"5min":10.666,"15min":16.381},"last":0.78100000000000003}]}]},{"osd":2,"up_from":0,"seq":1498943586348,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":0,"seq":463856468203,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019253,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3513712,"kb_used_data":3100008,"kb_used_omap":0,"kb_used_meta":413696,"kb_avail":90231440,"statfs":{"total":95995035648,"available":92396994560,"internally_reserved":0,"allocated":3174408192,"data_stored":3172952623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":423624704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":11.644,"5min":3.5510000000000002,"15min":4.1360000000000001},"min":{"1min":0.216,"5min":0.20699999999999999,"15min":0.19700000000000001},"max":{"1min":101.44499999999999,"5min":101.44499999999999,"15min":150.63499999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":11.930999999999999,"5min":3.7890000000000001,"15min":4.2850000000000001},"min":{"1min":0.22900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":101.465,"5min":101.465,"15min":150.672},"last":0.41999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":5.7779999999999996,"5min":3.2109999999999999,"15min":3.4060000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.21299999999999999},"max":{"1min":94.197999999999993,"5min":94.197999999999993,"15min":107.346},"last":0.33000000000000002},{"interface":"front","average":{"1min":6.359,"5min":3.508,"15min":3.6499999999999999},"min":{"1min":0.26100000000000001,"5min":0.253,"15min":0.22900000000000001},"max":{"1min":94.460999999999999,"5min":94.460999999999999,"15min":107.33499999999999},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0310000000000001,"5min":1.883,"15min":2.0739999999999998},"min":{"1min":0.309,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":8.0449999999999999,"5min":8.0449999999999999,"15min":8.0449999999999999},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.4990000000000001,"5min":1.48,"15min":1.52},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":7.9080000000000004,"5min":8.1159999999999997,"15min":8.1159999999999997},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":1.663,"15min":2.472},"min":{"1min":0.36399999999999999,"5min":0.314,"15min":0.29699999999999999},"max":{"1min":4.202,"5min":48.216999999999999,"15min":102.896},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.423,"5min":1.8069999999999999,"15min":2.589},"min":{"1min":0.34100000000000003,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":7.875,"5min":48.232999999999997,"15min":112.667},"last":0.65300000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:32:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0390000000000001,"5min":1.2669999999999999,"15min":1.0860000000000001},"min":{"1min":0.30299999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.0220000000000002,"5min":13.407,"15min":28.399000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2809999999999999,"15min":1.153},"min":{"1min":0.33800000000000002,"5min":0.29499999999999998,"15min":0.27900000000000003},"max":{"1min":11.048999999999999,"5min":14.212,"15min":28.350999999999999},"last":0.35799999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:32:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.034,"15min":0.89000000000000001},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.5439999999999996,"5min":7.5439999999999996,"15min":7.8550000000000004},"last":0.99199999999999999},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.216,"15min":0.91700000000000004},"min":{"1min":0.34699999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.1720000000000006,"5min":8.1720000000000006,"15min":8.1720000000000006},"last":0.94299999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":1.03,"15min":0.95399999999999996},"min":{"1min":0.376,"5min":0.307,"15min":0.27500000000000002},"max":{"1min":8.1440000000000001,"5min":8.1440000000000001,"15min":8.577},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.23,"5min":0.89400000000000002,"15min":0.89600000000000002},"min":{"1min":0.32600000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.633,"5min":3.9319999999999999,"15min":7.8109999999999999},"last":0.56000000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757621,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2986796,"kb_used_data":2562212,"kb_used_omap":0,"kb_used_meta":424384,"kb_avail":90758356,"statfs":{"total":95995035648,"available":92936556544,"internally_reserved":0,"allocated":2623705088,"data_stored":2622257347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":434569216},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":13.648999999999999,"5min":3.9780000000000002,"15min":4.5839999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":124.892,"5min":124.892,"15min":163.54499999999999},"last":7.0940000000000003},{"interface":"front","average":{"1min":13.523,"5min":3.528,"15min":4.3230000000000004},"min":{"1min":0.34999999999999998,"5min":0.23499999999999999,"15min":0.22900000000000001},"max":{"1min":123.807,"5min":123.807,"15min":162.81800000000001},"last":0.28699999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":3.7639999999999998,"15min":3.819},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.23799999999999999},"max":{"1min":7.1440000000000001,"5min":113.514,"15min":113.514},"last":0.254},{"interface":"front","average":{"1min":2.323,"5min":3.988,"15min":3.8599999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.252},"max":{"1min":8.4420000000000002,"5min":113.42700000000001,"15min":113.42700000000001},"last":0.38900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:32:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6059999999999999,"5min":2.1720000000000002,"15min":2.331},"min":{"1min":0.33600000000000002,"5min":0.316,"15min":0.316},"max":{"1min":8.5359999999999996,"5min":10.451000000000001,"15min":10.451000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.1459999999999999,"15min":2.4009999999999998},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.5180000000000007,"5min":8.5180000000000007,"15min":8.5180000000000007},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:32:34 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.6539999999999999,"15min":3.7719999999999998},"min":{"1min":0.36099999999999999,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":8.5459999999999994,"5min":9.3369999999999997,"15min":132.59},"last":0.48599999999999999},{"interface":"front","average":{"1min":2.4910000000000001,"5min":1.7070000000000001,"15min":3.71},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.28100000000000003},"max":{"1min":7.8390000000000004,"5min":7.8390000000000004,"15min":129.381},"last":1.7569999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:32:31 2023","interfaces":[{"interface":"back","average":{"1min":3.9849999999999999,"5min":1.3620000000000001,"15min":1.508},"min":{"1min":0.27700000000000002,"5min":0.26200000000000001,"15min":0.24099999999999999},"max":{"1min":36.343000000000004,"5min":36.343000000000004,"15min":126.679},"last":0.36099999999999999},{"interface":"front","average":{"1min":3.9740000000000002,"5min":1.377,"15min":1.512},"min":{"1min":0.35899999999999999,"5min":0.26200000000000001,"15min":0.22600000000000001},"max":{"1min":36.421999999999997,"5min":36.421999999999997,"15min":126.78100000000001},"last":0.432}]},{"osd":6,"last update":"Sat Sep 23 22:32:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":1.8999999999999999,"15min":1.2350000000000001},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.5820000000000007,"5min":70.581999999999994,"15min":70.581999999999994},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.0059999999999998,"5min":2.0499999999999998,"15min":1.2410000000000001},"min":{"1min":0.28699999999999998,"5min":0.248,"15min":0.248},"max":{"1min":7.25,"5min":71.093000000000004,"15min":71.093000000000004},"last":3.2080000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:32:22 2023","interfaces":[{"interface":"back","average":{"1min":3.7650000000000001,"5min":1.24,"15min":1.107},"min":{"1min":0.372,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":40.548000000000002,"5min":40.548000000000002,"15min":40.548000000000002},"last":0.60099999999999998},{"interface":"front","average":{"1min":3.6970000000000001,"5min":1.3440000000000001,"15min":0.98899999999999999},"min":{"1min":0.38,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":41.073,"5min":41.073,"15min":41.073},"last":0.64300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2351366144,"data_stored":2351366144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2380533760,"data_stored":2380529688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3132162048,"data_stored":3132162048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2600275968,"data_stored":2600271896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2099253248,"data_stored":2099249176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":204668928,"data_stored":204668928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:33:28.609 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T22:33:28.610 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:33:28.610 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:33:28.610 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.506535 2023-09-23T22:33:28.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0.5065354717283584 2023-09-23T22:33:28.633 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:28.660 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:28.702 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:28.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:28.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:28.805 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:28.907 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:28.988 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:29.047 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:33:29.077 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:33:29.111 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:33:29.141 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:33:29.174 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:29.180 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:33:29.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:29.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:29.218 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:33:29.243 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:29 smithi070 ceph-mon[192652]: from='client.36090 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:33:29.243 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:29 smithi070 ceph-mon[192652]: pgmap v1068: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 847 op/s 2023-09-23T22:33:29.243 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:29 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/498558899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:29.253 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:33:29.291 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:29.296 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:33:29.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:33:29.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:33:29.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:33:29.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:33:29.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:33:29.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:33:29.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:33:29.301 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:33:29.334 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:29.353 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:29.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:29 smithi026 ceph-mon[215775]: from='client.36090 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:33:29.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:29 smithi026 ceph-mon[215775]: pgmap v1068: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 847 op/s 2023-09-23T22:33:29.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:29 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/498558899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:29.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:29 smithi026 ceph-mon[223014]: from='client.36090 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:33:29.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:29 smithi026 ceph-mon[223014]: pgmap v1068: 99 pgs: 99 active+clean; 3.9 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 847 op/s 2023-09-23T22:33:29.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:29 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/498558899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:29.817 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:29.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:29.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:29.879 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:33:29.879 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:33:29.880 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:33:29.880 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:33:29.880 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:33:29.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:29.948 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:33:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:33:29] "GET /metrics HTTP/1.1" 200 36366 "" "Prometheus/2.43.0" 2023-09-23T22:33:29.976 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:30.602 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:30.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:30.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:30.706 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:30.749 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:30.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:30 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:30.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:30 smithi070 ceph-mon[192652]: pgmap v1069: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 877 op/s 2023-09-23T22:33:30.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:30 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:30.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:30 smithi026 ceph-mon[215775]: pgmap v1069: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 877 op/s 2023-09-23T22:33:30.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:30 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:30.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:30 smithi026 ceph-mon[223014]: pgmap v1069: 99 pgs: 99 active+clean; 3.8 GiB data, 14 GiB used, 433 GiB / 447 GiB avail; 0 B/s wr, 877 op/s 2023-09-23T22:33:31.003 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:33:31.230 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:31.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:31.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:31.337 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:31.424 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:31.533 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:31.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:31.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:31.646 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:31.696 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:31.712 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:32.035 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:32.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:32.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:32.138 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:32.172 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:32.250 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:32.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:32.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:32.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:32.406 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:32.635 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:32.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:32.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:32 smithi026 ceph-mon[215775]: pgmap v1070: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T22:33:32.656 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:32 smithi026 ceph-mon[223014]: pgmap v1070: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T22:33:32.737 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:32.776 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:32.830 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:32.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:32.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:32 smithi070 ceph-mon[192652]: pgmap v1070: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T22:33:32.934 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:32.968 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:33.097 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:33.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:33.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:33.115 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:33:33.206 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:33.277 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:33.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:33.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:33.495 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:33.546 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:33.636 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:33.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:33.698 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.4 primary-affinity to 0.506535 (8331962) 2023-09-23T22:33:33.707 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:33:33.731 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:33.741 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:33.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:33 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3081188521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:33.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:33 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:33.791 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:33 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3696928284' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.5065354717283584}]: dispatch 2023-09-23T22:33:33.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:33 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.5065354717283584}]: dispatch 2023-09-23T22:33:33.792 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:33 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3081188521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:33.792 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:33 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:33.792 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:33 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3696928284' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.5065354717283584}]: dispatch 2023-09-23T22:33:33.792 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:33 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.5065354717283584}]: dispatch 2023-09-23T22:33:33.802 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:34.055 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:33:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:33:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:33:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:33:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:7732 8.d 8:b5d16800:::benchmark_data_smithi026_497658_object7731:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T22:33:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:17.762412+0000", 2023-09-23T22:33:34.055 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 136.28923426, 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.29224014799999998, 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 7732 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.762412+0000", 2023-09-23T22:33:34.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.762412+0000", 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.762411+0000", 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.762460+0000", 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9576000000000001e-05 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.762472+0000", 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1703e-05 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.762479+0000", 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7660000000000001e-06 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.762991+0000", 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00051220699999999998 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.763110+0000", 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119059 2023-09-23T22:33:34.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 6,7", 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.763242+0000", 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000132168 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764555+0000", 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013129299999999999 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053937+0000", 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.289382472 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053966+0000", 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8493e-05 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.054603+0000", 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00063701700000000003 2023-09-23T22:33:34.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.054652+0000", 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8773999999999998e-05 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:7734 8.8 8:14431331:::benchmark_data_smithi026_497658_object7733:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T22:33:34.062 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:17.764721+0000", 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 136.28692511700001, 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.28976928899999999, 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 7734 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764721+0000", 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764721+0000", 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764720+0000", 2023-09-23T22:33:34.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764792+0000", 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2683999999999997e-05 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764803+0000", 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0878e-05 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764810+0000", 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4910000000000002e-06 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764987+0000", 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017767999999999999 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765127+0000", 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013978300000000001 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765250+0000", 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012266999999999999 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053926+0000", 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.28867568300000002 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.054401+0000", 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00047491299999999998 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.054407+0000", 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9959999999999999e-06 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.054450+0000", 2023-09-23T22:33:34.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3135e-05 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.054490+0000", 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0346999999999998e-05 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:7737 8.0 8:0756bb7d:::benchmark_data_smithi026_497658_object7736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T22:33:34.069 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:17.764914+0000", 2023-09-23T22:33:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 136.28673173799999, 2023-09-23T22:33:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.29174983300000001, 2023-09-23T22:33:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.070 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 7737 2023-09-23T22:33:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764914+0000", 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764914+0000", 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764913+0000", 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764955+0000", 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1542000000000003e-05 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764965+0000", 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0249e-05 2023-09-23T22:33:34.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.764971+0000", 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1700000000000002e-06 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765031+0000", 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0062000000000002e-05 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765144+0000", 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112388 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765276+0000", 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013237899999999999 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.054526+0000", 2023-09-23T22:33:34.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.28924989899999998 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.054678+0000", 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015176499999999999 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.056574+0000", 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018958169999999999 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.056621+0000", 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7774e-05 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.056664+0000", 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2585e-05 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.077 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.45842.0:7732 8.d e378/373 ondisk, result = 0)", 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:17.765318+0000", 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 136.286328174, 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.28862414600000003, 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765318+0000", 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765318+0000", 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765317+0000", 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:34.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765321+0000", 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5549999999999998e-06 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765330+0000", 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6330000000000008e-06 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.765335+0000", 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1849999999999997e-06 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053923+0000", 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.28858844 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053931+0000", 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8949999999999998e-06 2023-09-23T22:33:34.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053942+0000", 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0862999999999999e-05 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.45842.0:7734 8.8 e378/373 ondisk, result = 0)", 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:17.766424+0000", 2023-09-23T22:33:34.082 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 136.285222035, 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.28750435699999999, 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.766424+0000", 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.766424+0000", 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.766423+0000", 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.766428+0000", 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3760000000000001e-06 2023-09-23T22:33:34.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053869+0000", 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.28744199599999998 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053880+0000", 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0883000000000001e-05 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053908+0000", 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7226e-05 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053917+0000", 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9530000000000005e-06 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.053928+0000", 2023-09-23T22:33:34.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1595e-05 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:103920 8.d 8:be863593:::benchmark_data_smithi026_497658_object5028:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:50.785854+0000", 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 43.265791999000001, 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92969025199999999, 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.087 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103920 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.785854+0000", 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.785854+0000", 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.785853+0000", 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.785858+0000", 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.933e-06 2023-09-23T22:33:34.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.785869+0000", 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0764999999999999e-05 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.785873+0000", 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4129999999999999e-06 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.785930+0000", 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7018999999999997e-05 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.785942+0000", 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1936000000000001e-05 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.687655+0000", 2023-09-23T22:33:34.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90171282399999997 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.687729+0000", 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4548000000000005e-05 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.687796+0000", 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6264000000000001e-05 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.696627+0000", 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0088312579999999998 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715406+0000", 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018778934000000001 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715471+0000", 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4843000000000002e-05 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715509+0000", 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8395000000000001e-05 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715544+0000", 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5094e-05 2023-09-23T22:33:34.094 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:103933 8.d 8:b0d34731:::benchmark_data_smithi026_497658_object5041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:50.799254+0000", 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 43.252391461000002, 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91645436700000005, 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.095 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103933 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.799254+0000", 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.799254+0000", 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.799253+0000", 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.799261+0000", 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.072e-06 2023-09-23T22:33:34.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.799277+0000", 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6375000000000002e-05 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.799284+0000", 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.153e-06 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.799356+0000", 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2087000000000003e-05 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.799369+0000", 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2785999999999999e-05 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.689396+0000", 2023-09-23T22:33:34.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.89002742700000004 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.689521+0000", 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000125038 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.689660+0000", 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013874699999999999 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.699263+0000", 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096027500000000002 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715436+0000", 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016172570000000001 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715637+0000", 2023-09-23T22:33:34.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000201948 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715675+0000", 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7147000000000002e-05 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715709+0000", 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4119000000000003e-05 2023-09-23T22:33:34.102 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:103960 8.d 8:b8b3f100:::benchmark_data_smithi026_497658_object5068:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:50.838665+0000", 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 43.212980348000002, 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87721954099999999, 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.103 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103960 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.838665+0000", 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.838665+0000", 2023-09-23T22:33:34.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.838664+0000", 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.838670+0000", 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.37e-06 2023-09-23T22:33:34.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.838685+0000", 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4678000000000001e-05 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.838691+0000", 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2999999999999998e-06 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.838755+0000", 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3509000000000005e-05 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.838763+0000", 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4840000000000004e-06 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.691478+0000", 2023-09-23T22:33:34.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85271510699999997 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.691589+0000", 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000110401 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.691700+0000", 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111204 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.699272+0000", 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075720919999999999 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715593+0000", 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016321307 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715814+0000", 2023-09-23T22:33:34.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00022097100000000001 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715852+0000", 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7290999999999999e-05 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715885+0000", 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3390999999999999e-05 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.110 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:103963 8.d 8:bbace04d:::benchmark_data_smithi026_497658_object5071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:50.842116+0000", 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 43.209529994, 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87390602399999995, 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.111 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 103963 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.842116+0000", 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.842116+0000", 2023-09-23T22:33:34.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.842115+0000", 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.842122+0000", 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2760000000000003e-06 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.842138+0000", 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5673999999999999e-05 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.842144+0000", 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1290000000000001e-06 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.842200+0000", 2023-09-23T22:33:34.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5958000000000002e-05 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.842209+0000", 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2900000000000008e-06 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.694165+0000", 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.851955672 2023-09-23T22:33:34.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.694263+0000", 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8554999999999994e-05 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.694375+0000", 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011131700000000001 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.704923+0000", 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010548655000000001 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715762+0000", 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010838373 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715936+0000", 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017386100000000001 2023-09-23T22:33:34.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.715988+0000", 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2281000000000003e-05 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716022+0000", 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3932e-05 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:104019 8.d 8:bb19d41e:::benchmark_data_smithi026_497658_object5127:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:50.911007+0000", 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 43.140638948000003, 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.805175062, 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.119 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104019 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.911007+0000", 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.911007+0000", 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.911006+0000", 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.911011+0000", 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1880000000000001e-06 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.911025+0000", 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3762e-05 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.911031+0000", 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4489999999999996e-06 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.911088+0000", 2023-09-23T22:33:34.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6875e-05 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.911098+0000", 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7180000000000008e-06 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.696656+0000", 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.78555765700000002 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.696772+0000", 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000116551 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.696915+0000", 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000143323 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.709841+0000", 2023-09-23T22:33:34.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01292513 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716072+0000", 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062315779999999998 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716112+0000", 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0130999999999999e-05 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716150+0000", 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8056999999999998e-05 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716182+0000", 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1600999999999998e-05 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:104027 8.d 8:b16a5428:::benchmark_data_smithi026_497658_object5135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:50.919846+0000", 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 43.131800075000001, 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79648360399999996, 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104027 2023-09-23T22:33:34.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.919846+0000", 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.919846+0000", 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.919845+0000", 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.919848+0000", 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1320000000000001e-06 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.919854+0000", 2023-09-23T22:33:34.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0209999999999996e-06 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.919857+0000", 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.72e-06 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.919900+0000", 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3024e-05 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.919907+0000", 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4910000000000002e-06 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.699320+0000", 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.77941321200000002 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.699421+0000", 2023-09-23T22:33:34.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100814 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.699533+0000", 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111826 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.712041+0000", 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012508162 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716225+0000", 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041844680000000002 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716265+0000", 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9625000000000001e-05 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716299+0000", 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4317999999999999e-05 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716329+0000", 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0091999999999999e-05 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:104059 8.d 8:bf892c46:::benchmark_data_smithi026_497658_object5167:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.134 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:50.980887+0000", 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 43.070759236999997, 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.735593045, 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104059 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.980887+0000", 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.980887+0000", 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.980885+0000", 2023-09-23T22:33:34.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.980892+0000", 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3350000000000003e-06 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.980908+0000", 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5879000000000002e-05 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.980914+0000", 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3779999999999998e-06 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.980980+0000", 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6105000000000005e-05 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:50.980989+0000", 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1549999999999996e-06 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.702059+0000", 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.72106950000000003 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.702176+0000", 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000117454 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.702305+0000", 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012920799999999999 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.714424+0000", 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01211894 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716373+0000", 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019483980000000001 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716413+0000", 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0037999999999999e-05 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716448+0000", 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5459000000000001e-05 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716480+0000", 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1442999999999997e-05 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:104075 8.d 8:b7199aef:::benchmark_data_smithi026_497658_object5183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:51.004967+0000", 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 43.046678943000003, 2023-09-23T22:33:34.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.71170150300000001, 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104075 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.004967+0000", 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.004967+0000", 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.004966+0000", 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.004971+0000", 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0869999999999997e-06 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.004982+0000", 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0686999999999999e-05 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.004986+0000", 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6890000000000003e-06 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.005045+0000", 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8959000000000003e-05 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.005055+0000", 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.465e-06 2023-09-23T22:33:34.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.704973+0000", 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.69991869699999998 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.705056+0000", 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2133999999999997e-05 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.705136+0000", 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0389000000000006e-05 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.714430+0000", 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0092943169999999999 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716519+0000", 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020891099999999999 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716597+0000", 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7384000000000003e-05 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716637+0000", 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0652999999999999e-05 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716668+0000", 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0939000000000003e-05 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:104108 8.d 8:bbf82b53:::benchmark_data_smithi026_497658_object5216:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:51.056572+0000", 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 42.995074193000001, 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.66020927200000001, 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.150 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104108 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.056572+0000", 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.056572+0000", 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.056570+0000", 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.152 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.056576+0000", 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5520000000000004e-06 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.056587+0000", 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1308000000000001e-05 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.056591+0000", 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4190000000000004e-06 2023-09-23T22:33:34.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.056650+0000", 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8872000000000001e-05 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.056662+0000", 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1936999999999999e-05 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.706971+0000", 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.65030880700000004 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.707093+0000", 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000121635 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.707232+0000", 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000139758 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.714435+0000", 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072027430000000002 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716560+0000", 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021251159999999998 2023-09-23T22:33:34.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716713+0000", 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015256100000000001 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716750+0000", 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6782999999999997e-05 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.157 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716781+0000", 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1352000000000001e-05 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:104144 8.d 8:ba8cd8a7:::benchmark_data_smithi026_497658_object5252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:51.134627+0000", 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 42.917018810999998, 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.582341891, 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.158 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104144 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.134627+0000", 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.134627+0000", 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.134626+0000", 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.134632+0000", 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8719999999999999e-06 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.134646+0000", 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3983e-05 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.134652+0000", 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2149999999999998e-06 2023-09-23T22:33:34.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.134717+0000", 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5532999999999998e-05 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.134728+0000", 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1015e-05 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.709867+0000", 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57513862299999996 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.710003+0000", 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013656200000000001 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.710146+0000", 2023-09-23T22:33:34.163 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014284000000000001 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.714691+0000", 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0045445720000000002 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716823+0000", 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021323879999999998 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716898+0000", 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4558e-05 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716934+0000", 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.646e-05 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716969+0000", 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4609e-05 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:104146 8.d 8:bda36310:::benchmark_data_smithi026_497658_object5254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:51.142138+0000", 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 42.909508287000001, 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.57493107499999996, 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.166 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104146 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.142138+0000", 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.167 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.142138+0000", 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.142136+0000", 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.142142+0000", 2023-09-23T22:33:34.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2510000000000002e-06 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.142155+0000", 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2899e-05 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.142160+0000", 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.011e-06 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.142207+0000", 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6944000000000003e-05 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.142213+0000", 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8759999999999999e-06 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.170 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.712081+0000", 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.56986795599999995 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.712196+0000", 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114613 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.712329+0000", 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013303100000000001 2023-09-23T22:33:34.171 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.714814+0000", 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024853430000000001 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.716863+0000", 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020484689999999998 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.172 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.717002+0000", 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013957999999999999 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.717040+0000", 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7147000000000002e-05 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:51.717069+0000", 2023-09-23T22:33:34.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9017999999999999e-05 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:108466 8.0 8:0bc84b91:::benchmark_data_smithi026_497658_object9574:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:56.155984+0000", 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 37.895662256000001, 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.34326947899999999, 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.174 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 108466 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.155984+0000", 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.155984+0000", 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.155983+0000", 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.155987+0000", 2023-09-23T22:33:34.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4800000000000001e-06 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.155995+0000", 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8210000000000008e-06 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.156000+0000", 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9389999999999996e-06 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.156042+0000", 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2009999999999999e-05 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.156148+0000", 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010573099999999999 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.156256+0000", 2023-09-23T22:33:34.178 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000107769 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.158845+0000", 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025895559999999998 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.168684+0000", 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0098386129999999995 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.499140+0000", 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33045592499999998 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.499204+0000", 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4406000000000006e-05 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.499253+0000", 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8770000000000002e-05 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:108487 8.d 8:b9766b14:::benchmark_data_smithi026_497658_object9595:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e405)", 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:56.180249+0000", 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 37.871396461000003, 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.32763400799999998, 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.181 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 108487 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.180249+0000", 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.182 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.180249+0000", 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.180248+0000", 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.180252+0000", 2023-09-23T22:33:34.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7019999999999999e-06 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.180262+0000", 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6180000000000001e-06 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.180266+0000", 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3189999999999996e-06 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.180321+0000", 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4530999999999997e-05 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.180458+0000", 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013720900000000001 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:33:34.185 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.180590+0000", 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000132325 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.183645+0000", 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030549029999999999 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.183681+0000", 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5958999999999999e-05 2023-09-23T22:33:34.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.507790+0000", 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.32410910599999998 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.507842+0000", 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1629000000000003e-05 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.507883+0000", 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1711000000000001e-05 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:129674 8.4 8:2155023b:::benchmark_data_smithi026_497658_object30782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:20.309552+0000", 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 13.742094074000001, 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.28603285899999997, 2023-09-23T22:33:34.188 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 129674 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.309552+0000", 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.189 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.309552+0000", 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.309551+0000", 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.309554+0000", 2023-09-23T22:33:34.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1860000000000001e-06 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.309562+0000", 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4839999999999998e-06 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.309566+0000", 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4129999999999999e-06 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.315392+0000", 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005825177 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.315503+0000", 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000111493 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:33:34.192 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.315628+0000", 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012461699999999999 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.321707+0000", 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0060792219999999996 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.330060+0000", 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083529889999999999 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.595492+0000", 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26543205199999997 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.595539+0000", 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7264e-05 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.194 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.595585+0000", 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5456e-05 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:129679 8.4 8:2fca0694:::benchmark_data_smithi026_497658_object30787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:20.311767+0000", 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 13.739878624999999, 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.28394527200000003, 2023-09-23T22:33:34.195 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 129679 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.311767+0000", 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.311767+0000", 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.311766+0000", 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:34.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.311772+0000", 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7299999999999999e-06 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.311784+0000", 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1725999999999999e-05 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.311790+0000", 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2999999999999998e-06 2023-09-23T22:33:34.198 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.318503+0000", 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006712132 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.318603+0000", 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100814 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.199 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.318707+0000", 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010393 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.324850+0000", 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006142486 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.330093+0000", 2023-09-23T22:33:34.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052434659999999996 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.595639+0000", 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26554524800000001 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.595677+0000", 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8546e-05 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:34.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.595712+0000", 2023-09-23T22:33:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5389000000000003e-05 2023-09-23T22:33:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:34.202 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:34.202 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:33:34.203 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:33 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3081188521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:34.203 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:33 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:34.203 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:33 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3696928284' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.5065354717283584}]: dispatch 2023-09-23T22:33:34.203 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:33 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.5065354717283584}]: dispatch 2023-09-23T22:33:34.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:34.204 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:34.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:34.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:34.334 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:34.378 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:34.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:34.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:34.481 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:34.764 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:34.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[223014]: pgmap v1071: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T22:33:34.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:33:34.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:33:34.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[223014]: failed to encode map e411 with expected crc 2023-09-23T22:33:34.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[223014]: failed to encode map e411 with expected crc 2023-09-23T22:33:34.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:33:34.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.5065354717283584}]': finished 2023-09-23T22:33:34.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[223014]: osdmap e411: 8 total, 8 up, 5 in 2023-09-23T22:33:34.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[215775]: pgmap v1071: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T22:33:34.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:33:34.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:33:34.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[215775]: failed to encode map e411 with expected crc 2023-09-23T22:33:34.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[215775]: failed to encode map e411 with expected crc 2023-09-23T22:33:34.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:33:34.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.5065354717283584}]': finished 2023-09-23T22:33:34.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:34 smithi026 ceph-mon[215775]: osdmap e411: 8 total, 8 up, 5 in 2023-09-23T22:33:35.055 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:35.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:34 smithi070 ceph-mon[192652]: pgmap v1071: 99 pgs: 99 active+clean; 3.6 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T22:33:35.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:34 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:33:35.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:34 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:33:35.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:34 smithi070 ceph-mon[192652]: failed to encode map e411 with expected crc 2023-09-23T22:33:35.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:34 smithi070 ceph-mon[192652]: failed to encode map e411 with expected crc 2023-09-23T22:33:35.059 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:34 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:33:35.060 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:34 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.5065354717283584}]': finished 2023-09-23T22:33:35.060 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:34 smithi070 ceph-mon[192652]: osdmap e411: 8 total, 8 up, 5 in 2023-09-23T22:33:35.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:35.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:35.099 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:33:35.161 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:35.283 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:35.410 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:35.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:35.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:35.529 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:35.591 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:35.685 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:35 smithi070 ceph-mon[192652]: osdmap e412: 8 total, 8 up, 5 in 2023-09-23T22:33:35.685 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:35 smithi070 ceph-mon[192652]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.685 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:35 smithi070 ceph-mon[192652]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.685 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:35 smithi070 ceph-mon[192652]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.685 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:35 smithi070 ceph-mon[192652]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.685 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:35 smithi070 ceph-mon[192652]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.741 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:35.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:35.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:35.830 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:35.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:35.885 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:35.949 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:35.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:35.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[223014]: osdmap e412: 8 total, 8 up, 5 in 2023-09-23T22:33:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[223014]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[223014]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[223014]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[223014]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[223014]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[215775]: osdmap e412: 8 total, 8 up, 5 in 2023-09-23T22:33:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[215775]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[215775]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[215775]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[215775]: failed to encode map e412 with expected crc 2023-09-23T22:33:35.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:35 smithi026 ceph-mon[215775]: failed to encode map e412 with expected crc 2023-09-23T22:33:36.053 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:36.094 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:36.455 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:36.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:36.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:36.565 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:36.617 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:36.698 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:36.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:36.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:36.803 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:36.863 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:36.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[215775]: failed to encode map e412 with expected crc 2023-09-23T22:33:36.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[215775]: failed to encode map e412 with expected crc 2023-09-23T22:33:36.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[215775]: failed to encode map e412 with expected crc 2023-09-23T22:33:36.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[215775]: failed to encode map e412 with expected crc 2023-09-23T22:33:36.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[215775]: pgmap v1074: 99 pgs: 4 peering, 95 active+clean; 3.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s 2023-09-23T22:33:36.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T22:33:36.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[223014]: failed to encode map e412 with expected crc 2023-09-23T22:33:36.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[223014]: failed to encode map e412 with expected crc 2023-09-23T22:33:36.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[223014]: failed to encode map e412 with expected crc 2023-09-23T22:33:36.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[223014]: failed to encode map e412 with expected crc 2023-09-23T22:33:36.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[223014]: pgmap v1074: 99 pgs: 4 peering, 95 active+clean; 3.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s 2023-09-23T22:33:36.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:36 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T22:33:36.994 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:36.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:36.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:37.099 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:37.099 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:33:37.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:36 smithi070 ceph-mon[192652]: failed to encode map e412 with expected crc 2023-09-23T22:33:37.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:36 smithi070 ceph-mon[192652]: failed to encode map e412 with expected crc 2023-09-23T22:33:37.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:36 smithi070 ceph-mon[192652]: failed to encode map e412 with expected crc 2023-09-23T22:33:37.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:36 smithi070 ceph-mon[192652]: failed to encode map e412 with expected crc 2023-09-23T22:33:37.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:36 smithi070 ceph-mon[192652]: pgmap v1074: 99 pgs: 4 peering, 95 active+clean; 3.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 1.10k op/s 2023-09-23T22:33:37.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:36 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T22:33:37.127 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:33:37.153 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:33:37.171 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:33:37.177 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:37.190 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:33:37.212 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:33:37.231 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:33:37.241 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:33:37.245 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:37.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:37.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:37.274 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:33:37.281 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:33:37.281 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:33:37.281 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:33:37.281 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:33:37.281 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:33:37.281 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:33:37.281 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:33:37.281 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:33:37.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:37.434 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:37.558 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:37.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:37.563 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:37.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:37.728 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:37.804 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:37.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:37.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:37 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2082117299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:37.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:37.941 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:37 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2082117299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:37.942 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:37 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2082117299' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:37.945 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:38.230 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:38.231 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:33:38.231 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:33:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:38.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:38.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:38.399 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:38.487 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:38.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:38.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:38.511 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:38.592 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:38.662 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:38.767 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:38.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:38.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:38.873 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:38.963 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:39.261 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:39 smithi070 ceph-mon[192652]: pgmap v1075: 99 pgs: 4 peering, 95 active+clean; 3.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 962 op/s 2023-09-23T22:33:39.261 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:39.261 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:39.277 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:33:39.282 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:39.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:39.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:39.387 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:39.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:39 smithi026 ceph-mon[223014]: pgmap v1075: 99 pgs: 4 peering, 95 active+clean; 3.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 962 op/s 2023-09-23T22:33:39.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:39.419 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:39.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:39 smithi026 ceph-mon[215775]: pgmap v1075: 99 pgs: 4 peering, 95 active+clean; 3.4 GiB data, 13 GiB used, 434 GiB / 447 GiB avail; 0 B/s wr, 962 op/s 2023-09-23T22:33:39.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:39.419 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:39.423 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:39.495 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:39.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:39.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:39.601 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:39.660 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:39.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:33:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:33:39] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2023-09-23T22:33:39.769 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:39.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:39.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:39.882 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:39.899 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:39.982 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:40.087 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:40.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:40.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:40.197 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:40.252 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 4]}]: dispatch 2023-09-23T22:33:40.278 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:40.294 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6] out_osds: [3, 2, 7] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:33:40.294 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:33:40.295 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-23T22:33:40.295 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-23T22:33:40.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 4]}]: dispatch 2023-09-23T22:33:40.354 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 4]}]: dispatch 2023-09-23T22:33:40.391 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:40.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:33:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:40.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:40.496 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:40.548 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:40.753 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:40.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:40.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:40.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:40.917 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:40.954 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:40.988 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:41.060 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:41.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:41.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:41.164 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:41.173 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:33:41.191 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:41 smithi070 ceph-mon[192652]: pgmap v1076: 99 pgs: 99 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 688 op/s 2023-09-23T22:33:41.191 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:41 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T22:33:41.191 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:41 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:33:41.191 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:41 smithi070 ceph-mon[192652]: failed to encode map e413 with expected crc 2023-09-23T22:33:41.191 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:41 smithi070 ceph-mon[192652]: failed to encode map e413 with expected crc 2023-09-23T22:33:41.191 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 4]}]': finished 2023-09-23T22:33:41.191 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:41 smithi070 ceph-mon[192652]: osdmap e413: 8 total, 8 up, 5 in 2023-09-23T22:33:41.191 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2488912266' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:41.191 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:41 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:41.193 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:41.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[215775]: pgmap v1076: 99 pgs: 99 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 688 op/s 2023-09-23T22:33:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T22:33:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:33:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[215775]: failed to encode map e413 with expected crc 2023-09-23T22:33:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[215775]: failed to encode map e413 with expected crc 2023-09-23T22:33:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 4]}]': finished 2023-09-23T22:33:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[215775]: osdmap e413: 8 total, 8 up, 5 in 2023-09-23T22:33:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2488912266' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:41.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[223014]: pgmap v1076: 99 pgs: 99 active+clean; 3.3 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 688 op/s 2023-09-23T22:33:41.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T22:33:41.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:33:41.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[223014]: failed to encode map e413 with expected crc 2023-09-23T22:33:41.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[223014]: failed to encode map e413 with expected crc 2023-09-23T22:33:41.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 4]}]': finished 2023-09-23T22:33:41.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[223014]: osdmap e413: 8 total, 8 up, 5 in 2023-09-23T22:33:41.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2488912266' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:41.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:41 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:33:41.523 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:41.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:41.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:41.628 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:41.672 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:42.271 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:33:42.272 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:33:42.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:42 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:42.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:42 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:33:42.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:42 smithi070 ceph-mon[192652]: osdmap e414: 8 total, 8 up, 5 in 2023-09-23T22:33:42.300 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:42 smithi070 ceph-mon[192652]: failed to encode map e414 with expected crc 2023-09-23T22:33:42.301 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:42 smithi070 ceph-mon[192652]: failed to encode map e414 with expected crc 2023-09-23T22:33:42.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:42.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:33:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[215775]: osdmap e414: 8 total, 8 up, 5 in 2023-09-23T22:33:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[215775]: failed to encode map e414 with expected crc 2023-09-23T22:33:42.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[215775]: failed to encode map e414 with expected crc 2023-09-23T22:33:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:33:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[223014]: osdmap e414: 8 total, 8 up, 5 in 2023-09-23T22:33:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[223014]: failed to encode map e414 with expected crc 2023-09-23T22:33:42.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:42 smithi026 ceph-mon[223014]: failed to encode map e414 with expected crc 2023-09-23T22:33:43.251 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:43.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:43.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:43.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:33:43.356 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:43.398 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:43.435 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:43 smithi070 ceph-mon[192652]: pgmap v1079: 99 pgs: 99 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.13k op/s 2023-09-23T22:33:43.435 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:43 smithi070 ceph-mon[192652]: failed to encode map e414 with expected crc 2023-09-23T22:33:43.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:43 smithi026 ceph-mon[215775]: pgmap v1079: 99 pgs: 99 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.13k op/s 2023-09-23T22:33:43.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:43 smithi026 ceph-mon[215775]: failed to encode map e414 with expected crc 2023-09-23T22:33:43.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:43 smithi026 ceph-mon[223014]: pgmap v1079: 99 pgs: 99 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 1.13k op/s 2023-09-23T22:33:43.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:43 smithi026 ceph-mon[223014]: failed to encode map e414 with expected crc 2023-09-23T22:33:43.527 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:43.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:43.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:43.634 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:43.703 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:43.753 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:43.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:43.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:43.857 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:43.928 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:43.975 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:44.039 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:44.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:44.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:44.151 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:44.211 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:44.354 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:44.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:44.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:44.463 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:44.525 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:44.616 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:44.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:44.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:44.725 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:44.794 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:44.912 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:44.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:44.962 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:33:44.999 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:33:45.020 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:45.032 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:33:45.062 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:33:45.072 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:45.117 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:45.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:45.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:45.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:45 smithi026 ceph-mon[223014]: pgmap v1080: 99 pgs: 99 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T22:33:45.185 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3603664127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:45.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3778265005' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T22:33:45.186 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:45 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T22:33:45.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:45 smithi026 conmon[215751]: 2023-09-23T22:33:45.038+0000 7f4580f04700 -1 mon.a@0(leader).osd e414 definitely_dead 0 2023-09-23T22:33:45.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:45 smithi026 ceph-mon[215775]: pgmap v1080: 99 pgs: 99 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T22:33:45.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3603664127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:45.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3778265005' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T22:33:45.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:45 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T22:33:45.190 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.2. 2023-09-23T22:33:45.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:45.250 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:45.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:33:45.316 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:33:45.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:33:45.355 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:33:45.374 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:33:45.376 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:33:45.376 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:33:45.376 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:33:45.376 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:33:45.376 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:33:45.376 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:33:45.376 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:33:45.376 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:33:45.406 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:45.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:45.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:45.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:45.538 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:45 smithi070 ceph-mon[192652]: pgmap v1080: 99 pgs: 99 active+clean; 3.1 GiB data, 12 GiB used, 435 GiB / 447 GiB avail; 0 B/s wr, 567 op/s 2023-09-23T22:33:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3603664127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3778265005' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T22:33:45.620 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:45 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T22:33:46.102 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:33:46.102 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:33:46.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:33:46.102 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:33:46.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:119346 7.2 7.d96857c2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:29:00.616173+0000", 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 285.48328239699998, 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95561031100000005, 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119346 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.103 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616173+0000", 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616173+0000", 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616172+0000", 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616176+0000", 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4700000000000004e-06 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616185+0000", 2023-09-23T22:33:46.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5620000000000004e-06 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616191+0000", 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4189999999999999e-06 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616252+0000", 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0748999999999997e-05 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616261+0000", 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3499999999999997e-06 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:01.571770+0000", 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95550986299999996 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:01.571783+0000", 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2985e-05 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:173118 7.c e364/351 7:3149f6a3:::benchmark_data_smithi026_410749_object82781:head v 364'10620, mlcod=270'5124)", 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:39.811876+0000", 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 186.28757972400001, 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23435493099999999, 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.108 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.811876+0000", 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.811876+0000", 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.811875+0000", 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.811881+0000", 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8490000000000002e-06 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.811889+0000", 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2519999999999999e-06 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.811893+0000", 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.036e-06 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.811930+0000", 2023-09-23T22:33:46.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6973e-05 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.811949+0000", 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9244000000000001e-05 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046187+0000", 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23423728399999999 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046231+0000", 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3992999999999997e-05 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:173132 7.d e364/329 7:b1cabeb1:::benchmark_data_smithi026_410749_object82795:head v 364'10847, mlcod=364'10845)", 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:39.813906+0000", 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 186.28554905199999, 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.237242183, 2023-09-23T22:33:46.113 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.813906+0000", 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.813906+0000", 2023-09-23T22:33:46.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.813905+0000", 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.813913+0000", 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5689999999999999e-06 2023-09-23T22:33:46.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.813925+0000", 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2099000000000001e-05 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.813931+0000", 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5860000000000004e-06 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046209+0000", 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23227859300000001 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046232+0000", 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3010000000000002e-05 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.051112+0000", 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048798820000000003 2023-09-23T22:33:46.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.051149+0000", 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6402000000000001e-05 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGRecoveryDelete(7.0 e364,329 7:0e89f604:::benchmark_data_smithi026_410749_object15850:head,289'6680)", 2023-09-23T22:33:46.118 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:39.815169+0000", 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 186.28428667099999, 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23112137399999999, 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.815169+0000", 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.815169+0000", 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.815168+0000", 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.815174+0000", 2023-09-23T22:33:46.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3949999999999998e-06 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046140+0000", 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23096538999999999 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046156+0000", 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6521e-05 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046207+0000", 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0337999999999998e-05 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046229+0000", 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2427999999999999e-05 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046290+0000", 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0982e-05 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.35406.0:173131 7.3 e364/350 7:c99f1de3:::benchmark_data_smithi026_410749_object82794:head v 364'10868, mlcod=276'5462)", 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:30:39.815483+0000", 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 186.283972398, 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23837618199999999, 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.123 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.815483+0000", 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.815483+0000", 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.815482+0000", 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:39.815490+0000", 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3019999999999997e-06 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046141+0000", 2023-09-23T22:33:46.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.230651143 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046154+0000", 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3532e-05 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046209+0000", 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4517e-05 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.046231+0000", 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2439999999999999e-05 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.053818+0000", 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075870870000000002 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:30:40.053859+0000", 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0902999999999998e-05 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:7752 8.9 8:93bbf6df:::benchmark_data_smithi026_497658_object7751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:17.824665+0000", 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 148.27479000599999, 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23194152000000001, 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.128 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 7752 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.824665+0000", 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.824665+0000", 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.824665+0000", 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.824719+0000", 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4601000000000001e-05 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.824729+0000", 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7750000000000004e-06 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.824734+0000", 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3839999999999999e-06 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.824782+0000", 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8761000000000003e-05 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.824879+0000", 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6582999999999993e-05 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:33:46.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.824971+0000", 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2108000000000006e-05 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.825568+0000", 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059686299999999999 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:17.825966+0000", 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039771400000000002 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.056511+0000", 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23054582600000001 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.056568+0000", 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6895999999999999e-05 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.134 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:18.056607+0000", 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8690999999999999e-05 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:25708 8.5 8:aef67ecb:::benchmark_data_smithi026_497658_object25707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:29.435811+0000", 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 136.66364427400001, 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.27992180300000002, 2023-09-23T22:33:46.135 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 25708 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.435811+0000", 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.435811+0000", 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.435810+0000", 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.435902+0000", 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1681999999999994e-05 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.435933+0000", 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0905999999999999e-05 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.435972+0000", 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8996999999999999e-05 2023-09-23T22:33:46.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.436029+0000", 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7451999999999998e-05 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.437435+0000", 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001405782 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.437595+0000", 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000159796 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.440221+0000", 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002626654 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441297+0000", 2023-09-23T22:33:46.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001075681 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.714321+0000", 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.27302378799999999 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.714372+0000", 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1816e-05 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.715733+0000", 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013604979999999999 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:25710 8.9 8:93a64f24:::benchmark_data_smithi026_497658_object25709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:29.435999+0000", 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 136.66345675599999, 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.27846054599999998, 2023-09-23T22:33:46.142 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 25710 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.435999+0000", 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.435999+0000", 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.435998+0000", 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.436045+0000", 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6511999999999998e-05 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.436054+0000", 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3780000000000007e-06 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.436059+0000", 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.417e-06 2023-09-23T22:33:46.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.436107+0000", 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7141000000000001e-05 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.436242+0000", 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000135766 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.436385+0000", 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000142848 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.437569+0000", 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011838389999999999 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.438959+0000", 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001389726 2023-09-23T22:33:46.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.714350+0000", 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.27539072399999998 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.714410+0000", 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0591000000000003e-05 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.714459+0000", 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9135000000000003e-05 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:25720 8.9 8:951492e3:::benchmark_data_smithi026_497658_object25719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:29.441256+0000", 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 136.65819954, 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.273398684, 2023-09-23T22:33:46.149 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 25720 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441256+0000", 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.150 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441256+0000", 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441255+0000", 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.151 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441284+0000", 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9403999999999998e-05 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441293+0000", 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8859999999999993e-06 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441298+0000", 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7090000000000001e-06 2023-09-23T22:33:46.152 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441353+0000", 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5364000000000001e-05 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441544+0000", 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000190972 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.153 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.441707+0000", 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016316599999999999 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.444185+0000", 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024782649999999999 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.444517+0000", 2023-09-23T22:33:46.154 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000331935 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.714531+0000", 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.270013426 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.714617+0000", 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5796000000000003e-05 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.155 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.714655+0000", 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7954999999999998e-05 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:25730 8.5 8:a4ed6e7e:::benchmark_data_smithi026_497658_object25729:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e382)", 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:31:29.447413+0000", 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 136.65204265099999, 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.27218463900000001, 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.156 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 25730 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.447413+0000", 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.157 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.447413+0000", 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.447411+0000", 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.158 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.448802+0000", 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001390344 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.448815+0000", 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3599e-05 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.448822+0000", 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3060000000000003e-06 2023-09-23T22:33:46.159 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.448874+0000", 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2524000000000001e-05 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.449019+0000", 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014504700000000001 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:33:46.160 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.449189+0000", 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017000199999999999 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.450396+0000", 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001207308 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.451537+0000", 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011403089999999999 2023-09-23T22:33:46.161 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.718196+0000", 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26665878999999998 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.718250+0000", 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4701999999999999e-05 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.162 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:31:29.719597+0000", 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013472079999999999 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:88224 8.7 8:e39d3ed9:::benchmark_data_smithi026_497658_object88223:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e400)", 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:32.943025+0000", 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 73.156430150999995, 2023-09-23T22:33:46.163 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.21816126499999999, 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 88224 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.943025+0000", 2023-09-23T22:33:46.164 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.943025+0000", 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.943024+0000", 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.165 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.943114+0000", 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0027999999999999e-05 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.943131+0000", 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6966e-05 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.166 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.943138+0000", 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7340000000000002e-06 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.943203+0000", 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5137999999999999e-05 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.943314+0000", 2023-09-23T22:33:46.167 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011100800000000001 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.943445+0000", 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013127500000000001 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.946620+0000", 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031745499999999999 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.168 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:32.952490+0000", 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005870475 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:33.161088+0000", 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20859778000000001 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.169 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:33.161146+0000", 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8093000000000001e-05 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:33.161187+0000", 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0648000000000001e-05 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.170 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:108484 8.7 8:e27b1fa8:::benchmark_data_smithi026_497658_object9592:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:32:56.173745+0000", 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 49.925710473999999, 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.336067279, 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 108484 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.171 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.173745+0000", 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.173745+0000", 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.172 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.173743+0000", 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.173752+0000", 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0960000000000007e-06 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.173 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.173775+0000", 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2086000000000002e-05 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.173783+0000", 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7649999999999999e-06 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.173840+0000", 2023-09-23T22:33:46.174 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6357000000000003e-05 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.173986+0000", 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014628800000000001 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.174113+0000", 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012703700000000001 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.175 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.176699+0000", 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025865279999999998 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.179670+0000", 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029707380000000001 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.509720+0000", 2023-09-23T22:33:46.176 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.330050024 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.509769+0000", 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8868e-05 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:32:56.509812+0000", 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3115000000000003e-05 2023-09-23T22:33:46.177 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:125935 8.7 8:e11e2c1c:::benchmark_data_smithi026_497658_object27043:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:15.853751+0000", 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 30.245704732, 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22605503800000001, 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.178 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 125935 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.853751+0000", 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.853751+0000", 2023-09-23T22:33:46.179 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.853749+0000", 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.853756+0000", 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.08e-06 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.180 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.853769+0000", 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2629999999999999e-05 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.853775+0000", 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0639999999999999e-06 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.181 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.856002+0000", 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022270049999999998 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.856118+0000", 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011671399999999999 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.856229+0000", 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011099099999999999 2023-09-23T22:33:46.182 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.858806+0000", 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025770659999999998 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.860555+0000", 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017484690000000001 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.183 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:16.079711+0000", 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.21915582 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:16.079761+0000", 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0068000000000002e-05 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:16.079806+0000", 2023-09-23T22:33:46.184 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5034999999999998e-05 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:125943 8.5 8:a57eb58f:::benchmark_data_smithi026_497658_object27051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:15.857206+0000", 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 30.242249569999998, 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.21551087799999999, 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.185 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 125943 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.857206+0000", 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.186 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.857206+0000", 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.857204+0000", 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.857211+0000", 2023-09-23T22:33:46.187 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8299999999999998e-06 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.857223+0000", 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2475e-05 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.857230+0000", 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3409999999999999e-06 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.188 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.857285+0000", 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4818000000000003e-05 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.857391+0000", 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106622 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.857497+0000", 2023-09-23T22:33:46.189 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106036 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.860381+0000", 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028838919999999999 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:15.862321+0000", 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019400050000000001 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.190 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:16.072620+0000", 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.21029859200000001 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:16.072671+0000", 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1116999999999999e-05 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:16.072717+0000", 2023-09-23T22:33:46.191 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5942999999999997e-05 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:129699 8.9 8:96d8ee30:::benchmark_data_smithi026_497658_object30807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:20.336521+0000", 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 25.762934122000001, 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26996454600000003, 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.192 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 129699 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336521+0000", 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.193 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336521+0000", 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336521+0000", 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336528+0000", 2023-09-23T22:33:46.194 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6799999999999993e-06 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336537+0000", 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.799e-06 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336542+0000", 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9110000000000001e-06 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.195 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336588+0000", 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6152999999999997e-05 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336701+0000", 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112564 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:33:46.196 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336808+0000", 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000107015 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.340907+0000", 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040995249999999997 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.349811+0000", 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0089034230000000006 2023-09-23T22:33:46.197 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.606403+0000", 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.25659210999999998 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.606447+0000", 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3869999999999998e-05 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.198 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.606486+0000", 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9199000000000002e-05 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:129700 8.9 8:97ba4df5:::benchmark_data_smithi026_497658_object30808:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:20.336739+0000", 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 25.762716559000001, 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26983967199999997, 2023-09-23T22:33:46.199 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 129700 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336739+0000", 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.200 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336739+0000", 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336738+0000", 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.201 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336746+0000", 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0839999999999992e-06 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336758+0000", 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2371000000000001e-05 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336766+0000", 2023-09-23T22:33:46.202 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3660000000000003e-06 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.338865+0000", 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020996959999999999 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.338965+0000", 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9155999999999997e-05 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.203 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.339035+0000", 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0007999999999998e-05 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.342648+0000", 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036129209999999998 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.204 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.349838+0000", 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007190554 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.606516+0000", 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.25667775199999998 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.606549+0000", 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2787000000000003e-05 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.205 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.606579+0000", 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9899999999999998e-05 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:129702 8.9 8:9aaa8ff6:::benchmark_data_smithi026_497658_object30810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:20.336779+0000", 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 25.762676908, 2023-09-23T22:33:46.206 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.26988300900000001, 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 129702 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336779+0000", 2023-09-23T22:33:46.207 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336779+0000", 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336778+0000", 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.208 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336781+0000", 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2760000000000001e-06 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336789+0000", 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4930000000000001e-06 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.209 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.336794+0000", 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2619999999999998e-06 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.340940+0000", 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041456649999999998 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.341047+0000", 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000107577 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.210 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,5", 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.341156+0000", 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010892599999999999 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.342758+0000", 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016015300000000001 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.211 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.349872+0000", 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0071144770000000001 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.606604+0000", 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.25673161999999999 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.606634+0000", 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9944e-05 2023-09-23T22:33:46.212 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:20.606662+0000", 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7724000000000001e-05 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:140939 8.7 8:e3a2cec8:::benchmark_data_smithi026_497658_object42047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:32.741040+0000", 2023-09-23T22:33:46.213 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 13.358415917, 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.21747103000000001, 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 140939 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.214 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.741040+0000", 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.741040+0000", 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.741038+0000", 2023-09-23T22:33:46.215 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.741044+0000", 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5260000000000001e-06 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.937492+0000", 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19644827400000001 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.216 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.937501+0000", 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7260000000000008e-06 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.937551+0000", 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9938999999999997e-05 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.217 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.937672+0000", 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000120838 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.937784+0000", 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0001125 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.940486+0000", 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002701972 2023-09-23T22:33:46.218 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.940605+0000", 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011946 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.958409+0000", 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017803103000000001 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.219 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.958459+0000", 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0269999999999998e-05 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:32.958511+0000", 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1742e-05 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.220 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:142385 8.7 8:e0eab33d:::benchmark_data_smithi026_497658_object43493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:34.518191+0000", 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 11.581264915, 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.21801918000000001, 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 142385 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.221 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518191+0000", 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518191+0000", 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.222 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518189+0000", 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518197+0000", 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6529999999999994e-06 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518209+0000", 2023-09-23T22:33:46.223 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2191999999999999e-05 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518215+0000", 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3609999999999997e-06 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.520161+0000", 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019453540000000001 2023-09-23T22:33:46.224 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.520265+0000", 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104275 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.520370+0000", 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000105167 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.225 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.528356+0000", 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0079863699999999996 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.528624+0000", 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026745600000000001 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.226 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.736124+0000", 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20750016499999999 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.736169+0000", 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5371999999999999e-05 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.736210+0000", 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0243000000000001e-05 2023-09-23T22:33:46.227 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:142386 8.7 8:eafbcfb1:::benchmark_data_smithi026_497658_object43494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:33:34.518399+0000", 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 11.581056432, 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.21791106599999999, 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:33:46.228 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 142386 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518399+0000", 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:33:46.229 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518399+0000", 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518398+0000", 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518403+0000", 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6410000000000001e-06 2023-09-23T22:33:46.230 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518412+0000", 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5359999999999998e-06 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.518418+0000", 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5350000000000004e-06 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:33:46.231 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.522829+0000", 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044109450000000003 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.522909+0000", 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0457000000000006e-05 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.523033+0000", 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012403300000000001 2023-09-23T22:33:46.232 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.528628+0000", 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055948630000000003 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.733536+0000", 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20490740800000001 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.233 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.736242+0000", 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027067110000000001 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.736278+0000", 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5210999999999999e-05 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:33:34.736310+0000", 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2465000000000002e-05 2023-09-23T22:33:46.234 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.235 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.235 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.235 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:33:46.235 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:33:46.235 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:33:46.407 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:46.408 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:46 smithi026 ceph-mon[215775]: failed to encode map e415 with expected crc 2023-09-23T22:33:46.409 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:46 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T22:33:46.409 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:46 smithi026 ceph-mon[215775]: osdmap e415: 8 total, 8 up, 6 in 2023-09-23T22:33:46.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:46 smithi026 ceph-mon[223014]: failed to encode map e415 with expected crc 2023-09-23T22:33:46.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:46 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T22:33:46.409 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:46 smithi026 ceph-mon[223014]: osdmap e415: 8 total, 8 up, 6 in 2023-09-23T22:33:46.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:46.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:46.455 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-23T22:33:46.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:46.539 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:46.588 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:46.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:46.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:46.620 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:46 smithi070 ceph-mon[192652]: failed to encode map e415 with expected crc 2023-09-23T22:33:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:46 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T22:33:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:46 smithi070 ceph-mon[192652]: osdmap e415: 8 total, 8 up, 6 in 2023-09-23T22:33:46.695 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:46.722 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:46.767 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:46.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:46.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:46.837 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:33:46.870 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:46.909 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:46.955 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:46.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:46.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:47.059 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:47.086 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:47.131 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:47.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:47.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:47.183 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:47.238 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:33:47.265 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:47.316 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:33:47.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:47.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:33:47.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:47 smithi070 ceph-mon[192652]: pgmap v1082: 99 pgs: 7 peering, 92 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.32k op/s 2023-09-23T22:33:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:47 smithi070 ceph-mon[192652]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:47 smithi070 ceph-mon[192652]: osdmap e416: 8 total, 8 up, 6 in 2023-09-23T22:33:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:47 smithi070 ceph-mon[192652]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:47 smithi070 ceph-mon[192652]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:47 smithi070 ceph-mon[192652]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.419 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:47.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[215775]: pgmap v1082: 99 pgs: 7 peering, 92 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.32k op/s 2023-09-23T22:33:47.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[215775]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[215775]: osdmap e416: 8 total, 8 up, 6 in 2023-09-23T22:33:47.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[215775]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[215775]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.446 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[215775]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.446 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[223014]: pgmap v1082: 99 pgs: 7 peering, 92 active+clean; 2.8 GiB data, 11 GiB used, 436 GiB / 447 GiB avail; 0 B/s wr, 1.32k op/s 2023-09-23T22:33:47.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[223014]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[223014]: osdmap e416: 8 total, 8 up, 6 in 2023-09-23T22:33:47.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[223014]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[223014]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:47 smithi026 ceph-mon[223014]: failed to encode map e416 with expected crc 2023-09-23T22:33:47.448 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:47.495 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:47.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:47.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:47.599 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:47.678 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:47.905 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:47.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:47.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:48.014 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:48.170 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:48.478 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:48.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:48.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:48.484 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: osdmap e417: 8 total, 8 up, 6 in 2023-09-23T22:33:48.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.1 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9551 2023-09-23T22:33:48.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.a starting backfill to osd.2 from (0'0,0'0] MAX to 414'9513 2023-09-23T22:33:48.485 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.e starting backfill to osd.2 from (0'0,0'0] MAX to 414'9396 2023-09-23T22:33:48.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.7 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9530 2023-09-23T22:33:48.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 414'9708 2023-09-23T22:33:48.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.8 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9487 2023-09-23T22:33:48.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.6 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9584 2023-09-23T22:33:48.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.d starting backfill to osd.2 from (0'0,0'0] MAX to 414'9406 2023-09-23T22:33:48.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.4 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9494 2023-09-23T22:33:48.486 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: 8.b starting backfill to osd.2 from (0'0,0'0] MAX to 414'9368 2023-09-23T22:33:48.487 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:48 smithi070 ceph-mon[192652]: osdmap e418: 8 total, 8 up, 6 in 2023-09-23T22:33:48.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: osdmap e417: 8 total, 8 up, 6 in 2023-09-23T22:33:48.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.1 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9551 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.a starting backfill to osd.2 from (0'0,0'0] MAX to 414'9513 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.e starting backfill to osd.2 from (0'0,0'0] MAX to 414'9396 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.7 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9530 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 414'9708 2023-09-23T22:33:48.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.8 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9487 2023-09-23T22:33:48.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.6 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9584 2023-09-23T22:33:48.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.d starting backfill to osd.2 from (0'0,0'0] MAX to 414'9406 2023-09-23T22:33:48.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.4 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9494 2023-09-23T22:33:48.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: 8.b starting backfill to osd.2 from (0'0,0'0] MAX to 414'9368 2023-09-23T22:33:48.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[223014]: osdmap e418: 8 total, 8 up, 6 in 2023-09-23T22:33:48.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: osdmap e417: 8 total, 8 up, 6 in 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.1 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9551 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.a starting backfill to osd.2 from (0'0,0'0] MAX to 414'9513 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.e starting backfill to osd.2 from (0'0,0'0] MAX to 414'9396 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.7 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9530 2023-09-23T22:33:48.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: failed to encode map e417 with expected crc 2023-09-23T22:33:48.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.f starting backfill to osd.2 from (0'0,0'0] MAX to 414'9708 2023-09-23T22:33:48.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.8 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9487 2023-09-23T22:33:48.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.6 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9584 2023-09-23T22:33:48.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.d starting backfill to osd.2 from (0'0,0'0] MAX to 414'9406 2023-09-23T22:33:48.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.4 starting backfill to osd.2 from (0'0,0'0] MAX to 414'9494 2023-09-23T22:33:48.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: 8.b starting backfill to osd.2 from (0'0,0'0] MAX to 414'9368 2023-09-23T22:33:48.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:48 smithi026 ceph-mon[215775]: osdmap e418: 8 total, 8 up, 6 in 2023-09-23T22:33:48.524 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:33:48.584 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:48.632 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:48.728 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:48.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:48.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:48.843 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:48.917 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:49.049 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:49.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:49.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:49.157 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:49.189 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:33:49.221 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:33:49.221 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:33:49.222 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:33:49.222 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:33:49.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[215775]: pgmap v1085: 99 pgs: 7 peering, 92 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 743 op/s; 11 B/s, 3 keys/s, 2 objects/s recovering 2023-09-23T22:33:49.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[215775]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[215775]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[215775]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[215775]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3738960810' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:49.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:49.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3738960810' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:33:49.229 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[215775]: osdmap e419: 8 total, 8 up, 6 in 2023-09-23T22:33:49.231 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:49.349 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:49.475 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:49.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[223014]: pgmap v1085: 99 pgs: 7 peering, 92 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 743 op/s; 11 B/s, 3 keys/s, 2 objects/s recovering 2023-09-23T22:33:49.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[223014]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[223014]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[223014]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[223014]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3738960810' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:49.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:49.477 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3738960810' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:33:49.478 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:49 smithi026 ceph-mon[223014]: osdmap e419: 8 total, 8 up, 6 in 2023-09-23T22:33:49.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:49.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:49.579 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:49.636 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:49 smithi070 ceph-mon[192652]: pgmap v1085: 99 pgs: 7 peering, 92 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 743 op/s; 11 B/s, 3 keys/s, 2 objects/s recovering 2023-09-23T22:33:49.636 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:49 smithi070 ceph-mon[192652]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.636 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:49 smithi070 ceph-mon[192652]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.636 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:49 smithi070 ceph-mon[192652]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.636 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:49 smithi070 ceph-mon[192652]: failed to encode map e418 with expected crc 2023-09-23T22:33:49.637 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:49 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3738960810' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:33:49.637 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:49 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:49.637 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:49 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3738960810' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:33:49.637 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:49 smithi070 ceph-mon[192652]: osdmap e419: 8 total, 8 up, 6 in 2023-09-23T22:33:49.669 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:49.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:33:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:33:49] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2023-09-23T22:33:49.778 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:49.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:49.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:49.887 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:49.915 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:50.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[223014]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[223014]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.225 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[223014]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.380 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:33:50.466 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:50.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:50.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:50.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[215775]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[215775]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[215775]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[215775]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[215775]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[215775]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[215775]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:50.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[223014]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[223014]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[223014]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[223014]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:50 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:50.572 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:50.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:50 smithi070 ceph-mon[192652]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:50 smithi070 ceph-mon[192652]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:50 smithi070 ceph-mon[192652]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:50 smithi070 ceph-mon[192652]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:50 smithi070 ceph-mon[192652]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:50 smithi070 ceph-mon[192652]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:50 smithi070 ceph-mon[192652]: failed to encode map e419 with expected crc 2023-09-23T22:33:50.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:50 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:50.653 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:50.755 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:50.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:50.870 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:50.953 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:51.053 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:51.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:51.070 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:51.097 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:51.170 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:51.228 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:51.229 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:51 smithi070 ceph-mon[192652]: pgmap v1088: 99 pgs: 3 active+remapped+backfill_wait, 7 peering, 89 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 8482/136758 objects misplaced (6.202%); 16 B/s, 4 keys/s, 3 objects/s recovering 2023-09-23T22:33:51.294 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:51.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:51.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:51.397 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:51.430 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:51.456 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:33:51.456 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:33:51.456 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T22:33:51.456 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:33:51.456 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:33:51.456 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T22:33:51.473 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:51.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:51.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:51.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:51 smithi026 ceph-mon[215775]: pgmap v1088: 99 pgs: 3 active+remapped+backfill_wait, 7 peering, 89 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 8482/136758 objects misplaced (6.202%); 16 B/s, 4 keys/s, 3 objects/s recovering 2023-09-23T22:33:51.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:51 smithi026 ceph-mon[223014]: pgmap v1088: 99 pgs: 3 active+remapped+backfill_wait, 7 peering, 89 active+clean; 2.8 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 134 op/s; 8482/136758 objects misplaced (6.202%); 16 B/s, 4 keys/s, 3 objects/s recovering 2023-09-23T22:33:51.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:51.640 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:51.900 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:51.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:51.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:52.009 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:52.088 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:52.194 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:52.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:52.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:52.228 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:52.303 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:52.334 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:52.507 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:52.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:52.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:52.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:33:52.617 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:52.673 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:33:52.692 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:52.710 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:33:52.754 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:33:52.791 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:52.797 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:33:52.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:52.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:52.838 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:33:52.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:52.907 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:33:52.929 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:33:52.979 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:33:52.988 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:33:52.988 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:33:52.988 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:33:52.988 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:33:52.988 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:33:52.988 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:33:52.989 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:33:52.989 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:33:52.996 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:53.609 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:33:53.610 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:33:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:53 smithi070 ceph-mon[192652]: pgmap v1089: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 384 op/s; 27226/132120 objects misplaced (20.607%); 4.4 MiB/s, 3 keys/s, 79 objects/s recovering 2023-09-23T22:33:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:53 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1834111947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:53.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:53 smithi026 ceph-mon[215775]: pgmap v1089: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 384 op/s; 27226/132120 objects misplaced (20.607%); 4.4 MiB/s, 3 keys/s, 79 objects/s recovering 2023-09-23T22:33:53.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:53 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1834111947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:53.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:53 smithi026 ceph-mon[223014]: pgmap v1089: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 384 op/s; 27226/132120 objects misplaced (20.607%); 4.4 MiB/s, 3 keys/s, 79 objects/s recovering 2023-09-23T22:33:53.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:53 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1834111947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:33:54.112 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:33:54.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:54.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:33:54.219 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:54.248 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:54.596 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:54.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:54.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:54.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:54.779 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:33:54.802 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:54.860 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:54.919 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:54.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:54.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:55.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:33:55.124 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:55.283 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:33:55.283 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:55 smithi026 ceph-mon[215775]: pgmap v1090: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 324 op/s; 27226/132120 objects misplaced (20.607%); 3.7 MiB/s, 3 keys/s, 66 objects/s recovering 2023-09-23T22:33:55.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:55.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:55.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:55 smithi026 ceph-mon[223014]: pgmap v1090: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 324 op/s; 27226/132120 objects misplaced (20.607%); 3.7 MiB/s, 3 keys/s, 66 objects/s recovering 2023-09-23T22:33:55.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:55.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:55.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:55.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:33:55.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:55 smithi070 ceph-mon[192652]: pgmap v1090: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.7 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 324 op/s; 27226/132120 objects misplaced (20.607%); 3.7 MiB/s, 3 keys/s, 66 objects/s recovering 2023-09-23T22:33:55.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:33:55.377 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:33:55.390 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:33:55.391 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "" 2023-09-23T22:33:55.391 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:33:55.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:33:55.467 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:33:55.477 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:56.395 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:33:56.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:56.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:33:56.501 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:33:56.555 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:56.723 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:33:56.947 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:33:56.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:56.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:33:57.051 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:33:57.186 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:57.333 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:33:57.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:57.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:33:57.379 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:57 smithi070 ceph-mon[192652]: pgmap v1091: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 24324/120891 objects misplaced (20.121%); 7.9 MiB/s, 130 objects/s recovering 2023-09-23T22:33:57.397 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:57 smithi026 ceph-mon[215775]: pgmap v1091: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 24324/120891 objects misplaced (20.121%); 7.9 MiB/s, 130 objects/s recovering 2023-09-23T22:33:57.399 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:57 smithi026 ceph-mon[223014]: pgmap v1091: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 24324/120891 objects misplaced (20.121%); 7.9 MiB/s, 130 objects/s recovering 2023-09-23T22:33:57.440 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:33:57.544 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:57.722 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:33:57.747 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:33:57.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:57.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:33:57.855 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:57.941 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:58.037 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:33:58.037 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:33:58.039 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:58.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:58.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:58.125 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:33:58.144 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:33:58.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3324588764' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:58.205 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:58.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:58.312 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:33:58.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:33:58.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:33:58.418 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:33:58.434 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3324588764' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:58.434 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:58.439 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3324588764' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:58.439 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:33:58.505 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:33:59.349 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:33:59.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:59 smithi070 ceph-mon[192652]: pgmap v1092: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 24324/120891 objects misplaced (20.121%); 6.7 MiB/s, 110 objects/s recovering 2023-09-23T22:33:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:59 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:59 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:33:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:59 smithi070 ceph-mon[192652]: osdmap e420: 8 total, 8 up, 6 in 2023-09-23T22:33:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:59 smithi070 ceph-mon[192652]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:59 smithi070 ceph-mon[192652]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:59 smithi070 ceph-mon[192652]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:33:59 smithi070 ceph-mon[192652]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[215775]: pgmap v1092: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 24324/120891 objects misplaced (20.121%); 6.7 MiB/s, 110 objects/s recovering 2023-09-23T22:33:59.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:59.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:33:59.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[215775]: osdmap e420: 8 total, 8 up, 6 in 2023-09-23T22:33:59.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[215775]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[215775]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[215775]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[215775]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[223014]: pgmap v1092: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 24324/120891 objects misplaced (20.121%); 6.7 MiB/s, 110 objects/s recovering 2023-09-23T22:33:59.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:33:59.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:33:59.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[223014]: osdmap e420: 8 total, 8 up, 6 in 2023-09-23T22:33:59.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[223014]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[223014]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[223014]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:33:59 smithi026 ceph-mon[223014]: failed to encode map e420 with expected crc 2023-09-23T22:33:59.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:33:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:33:59] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2023-09-23T22:33:59.989 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:00.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:00.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:00.108 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:00.166 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:00.278 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:00.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:00.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:00.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:00.477 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:00.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:00 smithi070 ceph-mon[192652]: pgmap v1094: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.4 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 23670/118257 objects misplaced (20.016%); 6.3 MiB/s, 104 objects/s recovering 2023-09-23T22:34:00.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:00 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:00.628 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:00.707 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:00.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:00.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:00 smithi026 ceph-mon[223014]: pgmap v1094: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.4 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 23670/118257 objects misplaced (20.016%); 6.3 MiB/s, 104 objects/s recovering 2023-09-23T22:34:00.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:00 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:00.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:00.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:00 smithi026 ceph-mon[215775]: pgmap v1094: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.4 GiB data, 9.9 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 23670/118257 objects misplaced (20.016%); 6.3 MiB/s, 104 objects/s recovering 2023-09-23T22:34:00.720 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:00 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:00.807 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:34:00.821 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:00.843 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:34:00.880 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:34:00.886 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:00.903 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:34:00.942 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:34:00.978 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:34:00.995 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:01.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:01.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:01.022 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:34:01.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:34:01.063 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:34:01.063 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:34:01.063 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:34:01.063 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:34:01.063 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:34:01.064 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:34:01.064 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:34:01.064 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:34:01.106 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:01.172 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:01.279 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:01.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:01.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:01.386 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:01.437 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:01.590 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:01.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:01.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:01.639 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1987799776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:01.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:01.763 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:01.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1987799776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:01.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1987799776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:01.996 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:34:02.074 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:34:02.074 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:34:02.075 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T22:34:02.075 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:34:02.075 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:34:02.075 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T22:34:02.629 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:34:02.639 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:02 smithi026 ceph-mon[223014]: pgmap v1095: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 728 op/s; 21335/110271 objects misplaced (19.348%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T22:34:02.919 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:02 smithi070 ceph-mon[192652]: pgmap v1095: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 728 op/s; 21335/110271 objects misplaced (19.348%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T22:34:02.963 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:02.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:02.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:02.972 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:02 smithi026 ceph-mon[215775]: pgmap v1095: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.2 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 728 op/s; 21335/110271 objects misplaced (19.348%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T22:34:03.071 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:03.138 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:03.171 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:03.666 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:03.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:03.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:03.770 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:03.833 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:03.877 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:34:03.877 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:34:03.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.877 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45842.0:164335 8.f e420/416 8:f9036356:::benchmark_data_smithi026_497658_object65443:head v 420'10516, mlcod=420'10513)", 2023-09-23T22:34:03.878 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:03.861458+0000", 2023-09-23T22:34:03.878 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001610155, 2023-09-23T22:34:03.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001639859, 2023-09-23T22:34:03.878 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:03.878 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:34:03.878 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.861458+0000", 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.861458+0000", 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:03.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:03.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.861457+0000", 2023-09-23T22:34:03.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:34:03.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.861464+0000", 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1189999999999999e-06 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.861474+0000", 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0125e-05 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:03.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.861481+0000", 2023-09-23T22:34:03.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5729999999999996e-06 2023-09-23T22:34:03.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:03.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.861868+0000", 2023-09-23T22:34:03.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038674400000000001 2023-09-23T22:34:03.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:03.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.861887+0000", 2023-09-23T22:34:03.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9811999999999999e-05 2023-09-23T22:34:03.883 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:03.883 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:03.883 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:03.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.883 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.45842.0:164336 8.6 e420/416)", 2023-09-23T22:34:03.884 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:03.862294+0000", 2023-09-23T22:34:03.884 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.00077408000000000002, 2023-09-23T22:34:03.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000873828, 2023-09-23T22:34:03.884 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:03.884 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:34:03.884 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:03.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:03.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.862294+0000", 2023-09-23T22:34:03.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:03.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:03.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.862294+0000", 2023-09-23T22:34:03.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:03.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:03.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.862293+0000", 2023-09-23T22:34:03.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:34:03.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:03.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.862300+0000", 2023-09-23T22:34:03.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8940000000000004e-06 2023-09-23T22:34:03.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:03.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.862315+0000", 2023-09-23T22:34:03.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5308e-05 2023-09-23T22:34:03.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:03.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:03.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:03.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:03.862322+0000", 2023-09-23T22:34:03.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1570000000000003e-06 2023-09-23T22:34:03.890 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:03.891 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:03.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:03.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:03.891 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:34:03.891 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 2 2023-09-23T22:34:03.892 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:34:03.907 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:03.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:03.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:04.022 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:04.064 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:04.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:04.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:04.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:04.484 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:04.518 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:04.618 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:04.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:04.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:04.733 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:04.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:04.802 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:34:04.813 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:04.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:04.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:04.918 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:04.954 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:05.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:05 smithi026 ceph-mon[215775]: pgmap v1096: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 728 op/s; 21335/110271 objects misplaced (19.348%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T22:34:05.163 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:05 smithi026 ceph-mon[223014]: pgmap v1096: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 728 op/s; 21335/110271 objects misplaced (19.348%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T22:34:05.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:05 smithi070 ceph-mon[192652]: pgmap v1096: 99 pgs: 1 active+remapped+backfilling, 9 active+remapped+backfill_wait, 89 active+clean; 2.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 728 op/s; 21335/110271 objects misplaced (19.348%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T22:34:05.662 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:34:05.712 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:05.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:05.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:05.817 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:05.847 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:05.892 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:05.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:05.921 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:34:05.922 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:34:05.922 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:34:05.995 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:06.041 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:06.128 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:06.165 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:06.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:06.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:06.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:06.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:06 smithi026 ceph-mon[215775]: failed to encode map e421 with expected crc 2023-09-23T22:34:06.338 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:06 smithi026 ceph-mon[215775]: osdmap e421: 8 total, 8 up, 6 in 2023-09-23T22:34:06.339 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:06 smithi026 ceph-mon[215775]: failed to encode map e421 with expected crc 2023-09-23T22:34:06.339 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:06 smithi026 ceph-mon[223014]: failed to encode map e421 with expected crc 2023-09-23T22:34:06.339 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:06 smithi026 ceph-mon[223014]: osdmap e421: 8 total, 8 up, 6 in 2023-09-23T22:34:06.339 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:06 smithi026 ceph-mon[223014]: failed to encode map e421 with expected crc 2023-09-23T22:34:06.348 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:06.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:06 smithi070 ceph-mon[192652]: failed to encode map e421 with expected crc 2023-09-23T22:34:06.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:06 smithi070 ceph-mon[192652]: osdmap e421: 8 total, 8 up, 6 in 2023-09-23T22:34:06.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:06 smithi070 ceph-mon[192652]: failed to encode map e421 with expected crc 2023-09-23T22:34:06.489 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:06.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:06.596 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:06.650 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:06.709 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:06.855 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:06.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:06.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:06.966 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:07.023 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:07.118 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:07.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:07.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:07.216 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:34:07.228 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:07.288 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:07 smithi070 ceph-mon[192652]: pgmap v1098: 99 pgs: 1 peering, 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 89 active+clean; 2.0 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 947 op/s; 18342/98136 objects misplaced (18.690%); 13 MiB/s, 202 objects/s recovering 2023-09-23T22:34:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:07 smithi070 ceph-mon[192652]: failed to encode map e422 with expected crc 2023-09-23T22:34:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:07 smithi070 ceph-mon[192652]: failed to encode map e422 with expected crc 2023-09-23T22:34:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:07 smithi070 ceph-mon[192652]: osdmap e422: 8 total, 8 up, 6 in 2023-09-23T22:34:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:07 smithi070 ceph-mon[192652]: failed to encode map e422 with expected crc 2023-09-23T22:34:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2232578569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:07.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:07 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:07.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[215775]: pgmap v1098: 99 pgs: 1 peering, 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 89 active+clean; 2.0 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 947 op/s; 18342/98136 objects misplaced (18.690%); 13 MiB/s, 202 objects/s recovering 2023-09-23T22:34:07.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[215775]: failed to encode map e422 with expected crc 2023-09-23T22:34:07.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[215775]: failed to encode map e422 with expected crc 2023-09-23T22:34:07.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[215775]: osdmap e422: 8 total, 8 up, 6 in 2023-09-23T22:34:07.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[215775]: failed to encode map e422 with expected crc 2023-09-23T22:34:07.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2232578569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:07.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:07.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[223014]: pgmap v1098: 99 pgs: 1 peering, 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 89 active+clean; 2.0 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 947 op/s; 18342/98136 objects misplaced (18.690%); 13 MiB/s, 202 objects/s recovering 2023-09-23T22:34:07.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[223014]: failed to encode map e422 with expected crc 2023-09-23T22:34:07.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[223014]: failed to encode map e422 with expected crc 2023-09-23T22:34:07.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[223014]: osdmap e422: 8 total, 8 up, 6 in 2023-09-23T22:34:07.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[223014]: failed to encode map e422 with expected crc 2023-09-23T22:34:07.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2232578569' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:07.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:07 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:07.995 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:08.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:08.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:08.106 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:08.171 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:08.492 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:34:08.492 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:34:08.492 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[215775]: failed to encode map e423 with expected crc 2023-09-23T22:34:08.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:34:08.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[215775]: osdmap e423: 8 total, 8 up, 6 in 2023-09-23T22:34:08.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[215775]: failed to encode map e423 with expected crc 2023-09-23T22:34:08.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:34:08.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:34:08.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[223014]: failed to encode map e423 with expected crc 2023-09-23T22:34:08.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:34:08.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[223014]: osdmap e423: 8 total, 8 up, 6 in 2023-09-23T22:34:08.493 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:08 smithi026 ceph-mon[223014]: failed to encode map e423 with expected crc 2023-09-23T22:34:08.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:08 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:34:08.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:08 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:34:08.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:08 smithi070 ceph-mon[192652]: failed to encode map e423 with expected crc 2023-09-23T22:34:08.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:08 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:34:08.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:08 smithi070 ceph-mon[192652]: osdmap e423: 8 total, 8 up, 6 in 2023-09-23T22:34:08.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:08 smithi070 ceph-mon[192652]: failed to encode map e423 with expected crc 2023-09-23T22:34:08.877 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:08.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:08.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:08.984 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:09.052 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:09.216 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:09.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:09.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:09.327 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:09.382 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:09.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:09 smithi026 ceph-mon[223014]: pgmap v1101: 99 pgs: 1 peering, 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 89 active+clean; 2.0 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 18342/98136 objects misplaced (18.690%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T22:34:09.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:09.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:09.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:09 smithi026 ceph-mon[215775]: pgmap v1101: 99 pgs: 1 peering, 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 89 active+clean; 2.0 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 18342/98136 objects misplaced (18.690%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T22:34:09.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:09.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:09.480 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:09.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:09.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:09.489 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:09 smithi070 ceph-mon[192652]: pgmap v1101: 99 pgs: 1 peering, 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 89 active+clean; 2.0 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 18342/98136 objects misplaced (18.690%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T22:34:09.489 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:09.490 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:09.532 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:34:09.578 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:34:09.590 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:09.611 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:34:09.636 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:09.643 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:34:09.708 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:09.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:09.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:09.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:34:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:34:09] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-09-23T22:34:09.814 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:09.843 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:09.888 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:09.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:09.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:34:09.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:34:09.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:34:09.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:34:09.915 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:34:09.915 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:34:09.915 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:34:09.915 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:34:09.916 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:34:09.916 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:34:09.916 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:34:09.916 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:34:09.992 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:10.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:10.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/229120368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:10.384 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/229120368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:10.595 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:10.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:10.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:10.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/229120368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:10.701 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:10.730 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:10.951 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:34:11.127 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:11.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:11.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:11.234 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:11.261 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:11 smithi026 ceph-mon[215775]: pgmap v1102: 99 pgs: 1 peering, 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 89 active+clean; 1.9 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 17689/95547 objects misplaced (18.513%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T22:34:11.261 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:11 smithi026 ceph-mon[223014]: pgmap v1102: 99 pgs: 1 peering, 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 89 active+clean; 1.9 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 17689/95547 objects misplaced (18.513%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T22:34:11.274 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:11.366 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:11.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:11.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:11.481 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:11.575 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:11.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:11 smithi070 ceph-mon[192652]: pgmap v1102: 99 pgs: 1 peering, 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 89 active+clean; 1.9 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 17689/95547 objects misplaced (18.513%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T22:34:11.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:11.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:11.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:11.842 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:34:11.842 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:34:11.845 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:34:11.846 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:34:11.846 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T22:34:11.846 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2023-09-23T22:34:11.846 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0 2023-09-23T22:34:11.861 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:11.880 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:11.974 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:12.111 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:12.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:12.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:12.218 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:12.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:12.871 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:12.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:12.962 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:34:12.976 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:13.122 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:13.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:34:13.281 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:13.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:13.390 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:13.434 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:13.481 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:13.492 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:13 smithi026 ceph-mon[223014]: pgmap v1103: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.8 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 889 op/s; 15576/87621 objects misplaced (17.777%); 15 MiB/s, 242 objects/s recovering 2023-09-23T22:34:13.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:13 smithi026 ceph-mon[215775]: pgmap v1103: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.8 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 889 op/s; 15576/87621 objects misplaced (17.777%); 15 MiB/s, 242 objects/s recovering 2023-09-23T22:34:13.519 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:13.526 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:13 smithi070 ceph-mon[192652]: pgmap v1103: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.8 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 889 op/s; 15576/87621 objects misplaced (17.777%); 15 MiB/s, 242 objects/s recovering 2023-09-23T22:34:13.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:13.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:13.627 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:13.678 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:13.781 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:13.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:13.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:13.895 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:13.949 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:13.989 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:13.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:13.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:14.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:14.125 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:14.614 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:14.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:14.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:14.725 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:14.797 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:15.077 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:15.120 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:15.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:15.126 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:15.227 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:15.287 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:15 smithi070 ceph-mon[192652]: pgmap v1104: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.8 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 437 op/s; 15576/87621 objects misplaced (17.777%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:34:15.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:15 smithi026 ceph-mon[215775]: pgmap v1104: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.8 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 437 op/s; 15576/87621 objects misplaced (17.777%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:34:15.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:15 smithi026 ceph-mon[223014]: pgmap v1104: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.8 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 437 op/s; 15576/87621 objects misplaced (17.777%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:34:15.834 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:15.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:15.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:15.939 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:15.963 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:16.025 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:16.227 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:16.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:16.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:16.339 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:16.390 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:16.490 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:16.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:16.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:16.600 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:16.645 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:16.716 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:16.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:16.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:16.827 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:16.904 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:17.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:17 smithi026 ceph-mon[223014]: pgmap v1105: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.6 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 13133/76560 objects misplaced (17.154%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T22:34:17.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:17 smithi026 ceph-mon[215775]: pgmap v1105: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.6 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 13133/76560 objects misplaced (17.154%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T22:34:17.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:17 smithi070 ceph-mon[192652]: pgmap v1105: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.6 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 13133/76560 objects misplaced (17.154%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T22:34:18.310 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:34:18.313 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.0 primary-affinity to 0 (802) 2023-09-23T22:34:18.434 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:18.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:18.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:18.541 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:18.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3545966147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:34:18.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2825587387' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T22:34:18.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:18 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T22:34:18.566 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:18.607 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:18.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:18.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:18.711 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:18.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3545966147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:34:18.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2825587387' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T22:34:18.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:18 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T22:34:18.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3545966147' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:34:18.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2825587387' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T22:34:18.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:18 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T22:34:18.738 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:18.784 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:18.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:18.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:18.889 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:18.916 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:18.961 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:18.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:18.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:19.066 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:19.096 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:19.364 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[215775]: pgmap v1106: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.6 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 689 op/s; 13133/76560 objects misplaced (17.154%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:34:19.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:34:19.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[215775]: failed to encode map e424 with expected crc 2023-09-23T22:34:19.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3545966147' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:34:19.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T22:34:19.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[215775]: osdmap e424: 8 total, 8 up, 6 in 2023-09-23T22:34:19.365 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[215775]: failed to encode map e424 with expected crc 2023-09-23T22:34:19.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[223014]: pgmap v1106: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.6 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 689 op/s; 13133/76560 objects misplaced (17.154%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:34:19.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:34:19.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[223014]: failed to encode map e424 with expected crc 2023-09-23T22:34:19.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3545966147' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:34:19.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T22:34:19.366 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[223014]: osdmap e424: 8 total, 8 up, 6 in 2023-09-23T22:34:19.367 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:19 smithi026 ceph-mon[223014]: failed to encode map e424 with expected crc 2023-09-23T22:34:19.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:19 smithi070 ceph-mon[192652]: pgmap v1106: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.6 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 689 op/s; 13133/76560 objects misplaced (17.154%); 8.4 MiB/s, 133 objects/s recovering 2023-09-23T22:34:19.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:19 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:34:19.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:19 smithi070 ceph-mon[192652]: failed to encode map e424 with expected crc 2023-09-23T22:34:19.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3545966147' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:34:19.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T22:34:19.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:19 smithi070 ceph-mon[192652]: osdmap e424: 8 total, 8 up, 6 in 2023-09-23T22:34:19.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:19 smithi070 ceph-mon[192652]: failed to encode map e424 with expected crc 2023-09-23T22:34:19.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:34:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:34:19] "GET /metrics HTTP/1.1" 200 36380 "" "Prometheus/2.43.0" 2023-09-23T22:34:19.787 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:19.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:19.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:19.893 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:19.961 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:20.084 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:20.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:20.196 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:20.264 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:20.364 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:20.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:20.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:20.468 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:20.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:20 smithi070 ceph-mon[192652]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:20 smithi070 ceph-mon[192652]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:20 smithi070 ceph-mon[192652]: osdmap e425: 8 total, 8 up, 6 in 2023-09-23T22:34:20.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:20 smithi070 ceph-mon[192652]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:20 smithi070 ceph-mon[192652]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:20 smithi070 ceph-mon[192652]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:20 smithi070 ceph-mon[192652]: 8.e continuing backfill to osd.2 from (394'4100,423'10816] MIN to 423'10816 2023-09-23T22:34:20.536 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:20 smithi070 ceph-mon[192652]: pgmap v1109: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.5 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 12529/74139 objects misplaced (16.899%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:34:20.546 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:20.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[223014]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[223014]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[223014]: osdmap e425: 8 total, 8 up, 6 in 2023-09-23T22:34:20.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[223014]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[223014]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[223014]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[223014]: 8.e continuing backfill to osd.2 from (394'4100,423'10816] MIN to 423'10816 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[223014]: pgmap v1109: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.5 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 12529/74139 objects misplaced (16.899%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[215775]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[215775]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[215775]: osdmap e425: 8 total, 8 up, 6 in 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[215775]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[215775]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[215775]: failed to encode map e425 with expected crc 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[215775]: 8.e continuing backfill to osd.2 from (394'4100,423'10816] MIN to 423'10816 2023-09-23T22:34:20.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:20 smithi026 ceph-mon[215775]: pgmap v1109: 99 pgs: 1 active+remapped+backfilling, 8 active+remapped+backfill_wait, 90 active+clean; 1.5 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 561 op/s; 12529/74139 objects misplaced (16.899%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:34:20.673 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:20.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:20.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:20.784 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:20.880 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:20.995 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:21.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:21.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:21.036 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:34:21.036 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:34:21.103 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:21.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:34:21.186 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:21.207 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:34:21.249 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:34:21.285 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:34:21.305 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:21 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/143809421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:21.308 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:21.318 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:34:21.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:21.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:21.351 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:34:21.383 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:34:21.403 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:34:21.404 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:34:21.404 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:34:21.404 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:34:21.404 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:34:21.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:34:21.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:34:21.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:34:21.405 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:34:21.420 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:21.446 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:21.491 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:21.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:21.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:21.595 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:21.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:21 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/143809421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:21.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:21 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/143809421' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:21.629 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:22.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:22.083 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:22.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:22.184 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:22.210 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:22.250 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:22.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:22.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:22.354 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:22.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:22 smithi026 ceph-mon[223014]: pgmap v1110: 99 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 91 active+clean; 1.4 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 10458/66873 objects misplaced (15.639%); 12 MiB/s, 198 objects/s recovering 2023-09-23T22:34:22.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:22 smithi026 ceph-mon[215775]: pgmap v1110: 99 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 91 active+clean; 1.4 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 10458/66873 objects misplaced (15.639%); 12 MiB/s, 198 objects/s recovering 2023-09-23T22:34:22.422 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:22.450 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:34:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:22 smithi070 ceph-mon[192652]: pgmap v1110: 99 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 91 active+clean; 1.4 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 10458/66873 objects misplaced (15.639%); 12 MiB/s, 198 objects/s recovering 2023-09-23T22:34:22.718 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:22.724 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:34:22.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:22.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:22.828 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:22.932 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:23.070 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:23.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:23.175 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:23.235 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:23.268 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:23.329 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:23.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:23.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:23.452 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:23.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:23.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:23.536 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:23.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:34:23.656 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:23.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:23.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:23.766 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:23.836 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:23.883 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:24.096 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:24.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:24.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:24.200 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:24.251 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:24.380 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:24.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:24.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:24.489 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:24.550 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:24.645 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:24.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:24.731 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:34:24.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:24.851 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:24.950 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:34:24.950 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:34:24.950 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.21613948324516474 2023-09-23T22:34:24.950 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 0.21613948324516474 2023-09-23T22:34:25.102 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:25.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:25.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:25.213 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:25.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:25 smithi026 ceph-mon[223014]: pgmap v1111: 99 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 91 active+clean; 1.4 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 10458/66873 objects misplaced (15.639%); 8.5 MiB/s, 133 objects/s recovering 2023-09-23T22:34:25.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:25.325 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:25.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:25 smithi026 ceph-mon[215775]: pgmap v1111: 99 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 91 active+clean; 1.4 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 10458/66873 objects misplaced (15.639%); 8.5 MiB/s, 133 objects/s recovering 2023-09-23T22:34:25.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:25.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:25.336 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:25.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:25 smithi070 ceph-mon[192652]: pgmap v1111: 99 pgs: 1 active+remapped+backfilling, 7 active+remapped+backfill_wait, 91 active+clean; 1.4 GiB data, 7.0 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 10458/66873 objects misplaced (15.639%); 8.5 MiB/s, 133 objects/s recovering 2023-09-23T22:34:25.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:25.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:25.463 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:25.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:25.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:25.576 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:25.620 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:25.726 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:25.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:25.830 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:25.884 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:25.987 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:25.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:25.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:26.093 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:26.122 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:26.172 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:26.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:26.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:26.276 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:26.307 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:26.325 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:34:26.325 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:34:26.325 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:34:26.325 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:34:26.361 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:26.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:26.467 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:26.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:26 smithi026 ceph-mon[215775]: osdmap e426: 8 total, 8 up, 6 in 2023-09-23T22:34:26.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:26 smithi026 ceph-mon[215775]: failed to encode map e426 with expected crc 2023-09-23T22:34:26.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:26 smithi026 ceph-mon[215775]: failed to encode map e426 with expected crc 2023-09-23T22:34:26.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:26 smithi026 ceph-mon[215775]: pgmap v1113: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 1.1 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 973 op/s; 8070/55140 objects misplaced (14.635%); 14 MiB/s, 228 objects/s recovering 2023-09-23T22:34:26.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:26 smithi026 ceph-mon[223014]: osdmap e426: 8 total, 8 up, 6 in 2023-09-23T22:34:26.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:26 smithi026 ceph-mon[223014]: failed to encode map e426 with expected crc 2023-09-23T22:34:26.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:26 smithi026 ceph-mon[223014]: failed to encode map e426 with expected crc 2023-09-23T22:34:26.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:26 smithi026 ceph-mon[223014]: pgmap v1113: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 1.1 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 973 op/s; 8070/55140 objects misplaced (14.635%); 14 MiB/s, 228 objects/s recovering 2023-09-23T22:34:26.496 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:26.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:26 smithi070 ceph-mon[192652]: osdmap e426: 8 total, 8 up, 6 in 2023-09-23T22:34:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:26 smithi070 ceph-mon[192652]: failed to encode map e426 with expected crc 2023-09-23T22:34:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:26 smithi070 ceph-mon[192652]: failed to encode map e426 with expected crc 2023-09-23T22:34:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:26 smithi070 ceph-mon[192652]: pgmap v1113: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 1.1 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 973 op/s; 8070/55140 objects misplaced (14.635%); 14 MiB/s, 228 objects/s recovering 2023-09-23T22:34:27.095 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:27.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:27.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:27.203 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:27.264 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:27.351 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:27.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:27.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:27.458 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:27 smithi026 ceph-mon[215775]: failed to encode map e427 with expected crc 2023-09-23T22:34:27.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:27 smithi026 ceph-mon[215775]: osdmap e427: 8 total, 8 up, 6 in 2023-09-23T22:34:27.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:27 smithi026 ceph-mon[215775]: failed to encode map e427 with expected crc 2023-09-23T22:34:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:27 smithi026 ceph-mon[223014]: failed to encode map e427 with expected crc 2023-09-23T22:34:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:27 smithi026 ceph-mon[223014]: osdmap e427: 8 total, 8 up, 6 in 2023-09-23T22:34:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:27 smithi026 ceph-mon[223014]: failed to encode map e427 with expected crc 2023-09-23T22:34:27.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:27 smithi070 ceph-mon[192652]: failed to encode map e427 with expected crc 2023-09-23T22:34:27.529 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:27 smithi070 ceph-mon[192652]: osdmap e427: 8 total, 8 up, 6 in 2023-09-23T22:34:27.529 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:27 smithi070 ceph-mon[192652]: failed to encode map e427 with expected crc 2023-09-23T22:34:27.535 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:27.722 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:27.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:27.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:27.835 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:27.847 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:34:27.863 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:27.865 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:27.887 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:27.969 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:27.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:27.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:28.076 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:28.127 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:28.226 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:28.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:28.335 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:28.386 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:28.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:28.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:28.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:28.590 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:28.630 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:28.710 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:28.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:28.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:28.818 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:28.939 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:29.174 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:28 smithi026 ceph-mon[223014]: pgmap v1115: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 1.1 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 791 op/s; 8070/55140 objects misplaced (14.635%); 13 MiB/s, 209 objects/s recovering 2023-09-23T22:34:29.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:28 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:29.175 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:28 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:29.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:28 smithi026 ceph-mon[215775]: pgmap v1115: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 1.1 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 791 op/s; 8070/55140 objects misplaced (14.635%); 13 MiB/s, 209 objects/s recovering 2023-09-23T22:34:29.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:28 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:29.175 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:28 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:29.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:28 smithi070 ceph-mon[192652]: pgmap v1115: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 1.1 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 791 op/s; 8070/55140 objects misplaced (14.635%); 13 MiB/s, 209 objects/s recovering 2023-09-23T22:34:29.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:28 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:29.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:28 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:29.919 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:34:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:34:29] "GET /metrics HTTP/1.1" 200 36381 "" "Prometheus/2.43.0" 2023-09-23T22:34:29.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:29 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3369158149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:29.919 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:29 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:29.919 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:29 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3369158149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:29.920 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:29 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:29.931 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:34:29.999 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:30.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:30.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:30.109 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:30.205 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:29 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3369158149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:29 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:30.815 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:30.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:30.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:30.921 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:31.029 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:31.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[223014]: pgmap v1116: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 1.1 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 579 op/s; 7523/52968 objects misplaced (14.203%); 4.7 MiB/s, 73 objects/s recovering 2023-09-23T22:34:31.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:34:31.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:34:31.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[223014]: osdmap e428: 8 total, 8 up, 6 in 2023-09-23T22:34:31.105 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[223014]: failed to encode map e428 with expected crc 2023-09-23T22:34:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[215775]: pgmap v1116: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 1.1 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 579 op/s; 7523/52968 objects misplaced (14.203%); 4.7 MiB/s, 73 objects/s recovering 2023-09-23T22:34:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:34:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:34:31.106 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[215775]: osdmap e428: 8 total, 8 up, 6 in 2023-09-23T22:34:31.107 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:30 smithi026 ceph-mon[215775]: failed to encode map e428 with expected crc 2023-09-23T22:34:31.190 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:31.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:31.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:31.295 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:31.333 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:31.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:30 smithi070 ceph-mon[192652]: pgmap v1116: 99 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 6 active+remapped+backfill_wait, 91 active+clean; 1.1 GiB data, 6.2 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 579 op/s; 7523/52968 objects misplaced (14.203%); 4.7 MiB/s, 73 objects/s recovering 2023-09-23T22:34:31.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:30 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:34:31.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:30 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:34:31.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:30 smithi070 ceph-mon[192652]: osdmap e428: 8 total, 8 up, 6 in 2023-09-23T22:34:31.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:30 smithi070 ceph-mon[192652]: failed to encode map e428 with expected crc 2023-09-23T22:34:31.413 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:31.446 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:31.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:31.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:31.550 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:34:31.550 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:31.574 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:34:31.585 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:31.599 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:34:31.628 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:34:31.654 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:34:31.681 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:34:31.704 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:34:31.705 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:31.709 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:31.730 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:34:31.732 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:34:31.732 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:34:31.732 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:34:31.732 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:34:31.732 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:34:31.732 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:34:31.732 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:34:31.732 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:34:31.810 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:31.847 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:31.949 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 0.216139 (3754) 2023-09-23T22:34:31.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[215775]: osdmap e429: 8 total, 8 up, 6 in 2023-09-23T22:34:31.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[215775]: failed to encode map e429 with expected crc 2023-09-23T22:34:31.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[215775]: failed to encode map e429 with expected crc 2023-09-23T22:34:31.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[215775]: failed to encode map e429 with expected crc 2023-09-23T22:34:31.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[215775]: failed to encode map e429 with expected crc 2023-09-23T22:34:31.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1490613547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:31.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/562105321' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.21613948324516474}]: dispatch 2023-09-23T22:34:31.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.21613948324516474}]: dispatch 2023-09-23T22:34:31.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[223014]: osdmap e429: 8 total, 8 up, 6 in 2023-09-23T22:34:31.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[223014]: failed to encode map e429 with expected crc 2023-09-23T22:34:31.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[223014]: failed to encode map e429 with expected crc 2023-09-23T22:34:31.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[223014]: failed to encode map e429 with expected crc 2023-09-23T22:34:31.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[223014]: failed to encode map e429 with expected crc 2023-09-23T22:34:31.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1490613547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:31.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/562105321' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.21613948324516474}]: dispatch 2023-09-23T22:34:31.980 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:31 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.21613948324516474}]: dispatch 2023-09-23T22:34:32.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:31 smithi070 ceph-mon[192652]: osdmap e429: 8 total, 8 up, 6 in 2023-09-23T22:34:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:31 smithi070 ceph-mon[192652]: failed to encode map e429 with expected crc 2023-09-23T22:34:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:31 smithi070 ceph-mon[192652]: failed to encode map e429 with expected crc 2023-09-23T22:34:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:31 smithi070 ceph-mon[192652]: failed to encode map e429 with expected crc 2023-09-23T22:34:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:31 smithi070 ceph-mon[192652]: failed to encode map e429 with expected crc 2023-09-23T22:34:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1490613547' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/562105321' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.21613948324516474}]: dispatch 2023-09-23T22:34:32.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:31 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.21613948324516474}]: dispatch 2023-09-23T22:34:32.903 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:32.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:32.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:32.969 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:34:33.007 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:33.056 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:33.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[223014]: pgmap v1119: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 966 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 5868/46224 objects misplaced (12.695%); 11 MiB/s, 171 objects/s recovering 2023-09-23T22:34:33.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[223014]: failed to encode map e430 with expected crc 2023-09-23T22:34:33.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.21613948324516474}]': finished 2023-09-23T22:34:33.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[223014]: osdmap e430: 8 total, 8 up, 6 in 2023-09-23T22:34:33.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[223014]: failed to encode map e430 with expected crc 2023-09-23T22:34:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[215775]: pgmap v1119: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 966 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 5868/46224 objects misplaced (12.695%); 11 MiB/s, 171 objects/s recovering 2023-09-23T22:34:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[215775]: failed to encode map e430 with expected crc 2023-09-23T22:34:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.21613948324516474}]': finished 2023-09-23T22:34:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[215775]: osdmap e430: 8 total, 8 up, 6 in 2023-09-23T22:34:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:32 smithi026 ceph-mon[215775]: failed to encode map e430 with expected crc 2023-09-23T22:34:33.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:32 smithi070 ceph-mon[192652]: pgmap v1119: 99 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 5 active+remapped+backfill_wait, 92 active+clean; 966 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 5868/46224 objects misplaced (12.695%); 11 MiB/s, 171 objects/s recovering 2023-09-23T22:34:33.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:32 smithi070 ceph-mon[192652]: failed to encode map e430 with expected crc 2023-09-23T22:34:33.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:32 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.21613948324516474}]': finished 2023-09-23T22:34:33.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:32 smithi070 ceph-mon[192652]: osdmap e430: 8 total, 8 up, 6 in 2023-09-23T22:34:33.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:32 smithi070 ceph-mon[192652]: failed to encode map e430 with expected crc 2023-09-23T22:34:33.739 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:33.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:33.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:33.843 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:33.892 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:33.936 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:33.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:33.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:34.044 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:34.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:34.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[215775]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[215775]: osdmap e431: 8 total, 8 up, 6 in 2023-09-23T22:34:34.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[215775]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[215775]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[215775]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[215775]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[223014]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[223014]: osdmap e431: 8 total, 8 up, 6 in 2023-09-23T22:34:34.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[223014]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[223014]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[223014]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.217 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:33 smithi026 ceph-mon[223014]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:33 smithi070 ceph-mon[192652]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:33 smithi070 ceph-mon[192652]: osdmap e431: 8 total, 8 up, 6 in 2023-09-23T22:34:34.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:33 smithi070 ceph-mon[192652]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:33 smithi070 ceph-mon[192652]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:33 smithi070 ceph-mon[192652]: failed to encode map e431 with expected crc 2023-09-23T22:34:34.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:33 smithi070 ceph-mon[192652]: failed to encode map e431 with expected crc 2023-09-23T22:34:35.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: pgmap v1122: 99 pgs: 5 peering, 4 unknown, 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 84 active+clean; 959 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 5868/45921 objects misplaced (12.778%); 16 MiB/s, 255 objects/s recovering 2023-09-23T22:34:35.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: osdmap e432: 8 total, 8 up, 6 in 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: 8.d starting backfill to osd.1 from (396'4700,425'11149] MIN to 429'11370 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: 8.c starting backfill to osd.6 from (0'0,0'0] MAX to 429'11495 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: 8.9 starting backfill to osd.4 from (0'0,0'0] MAX to 429'11411 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 429'11563 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: 8.a continuing backfill to osd.2 from (402'6300,429'11457] MIN to 429'11457 2023-09-23T22:34:35.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[223014]: 8.0 starting backfill to osd.6 from (0'0,0'0] MAX to 429'11156 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: pgmap v1122: 99 pgs: 5 peering, 4 unknown, 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 84 active+clean; 959 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 5868/45921 objects misplaced (12.778%); 16 MiB/s, 255 objects/s recovering 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: osdmap e432: 8 total, 8 up, 6 in 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: 8.d starting backfill to osd.1 from (396'4700,425'11149] MIN to 429'11370 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: 8.c starting backfill to osd.6 from (0'0,0'0] MAX to 429'11495 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: 8.9 starting backfill to osd.4 from (0'0,0'0] MAX to 429'11411 2023-09-23T22:34:35.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 429'11563 2023-09-23T22:34:35.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: 8.a continuing backfill to osd.2 from (402'6300,429'11457] MIN to 429'11457 2023-09-23T22:34:35.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:34 smithi026 ceph-mon[215775]: 8.0 starting backfill to osd.6 from (0'0,0'0] MAX to 429'11156 2023-09-23T22:34:35.310 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:35.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:35.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: pgmap v1122: 99 pgs: 5 peering, 4 unknown, 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 84 active+clean; 959 MiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 587 op/s; 5868/45921 objects misplaced (12.778%); 16 MiB/s, 255 objects/s recovering 2023-09-23T22:34:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: osdmap e432: 8 total, 8 up, 6 in 2023-09-23T22:34:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: 8.d starting backfill to osd.1 from (396'4700,425'11149] MIN to 429'11370 2023-09-23T22:34:35.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: 8.c starting backfill to osd.6 from (0'0,0'0] MAX to 429'11495 2023-09-23T22:34:35.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: 8.9 starting backfill to osd.4 from (0'0,0'0] MAX to 429'11411 2023-09-23T22:34:35.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: 8.3 starting backfill to osd.2 from (0'0,0'0] MAX to 429'11563 2023-09-23T22:34:35.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: failed to encode map e432 with expected crc 2023-09-23T22:34:35.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: 8.a continuing backfill to osd.2 from (402'6300,429'11457] MIN to 429'11457 2023-09-23T22:34:35.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:34 smithi070 ceph-mon[192652]: 8.0 starting backfill to osd.6 from (0'0,0'0] MAX to 429'11156 2023-09-23T22:34:35.416 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:35.495 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:35.554 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:34:35.577 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:35.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:35.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:35.640 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:35.685 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:35.729 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:35.800 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:35.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:35.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:35.911 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:35.948 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:35.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[215775]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.019 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:36.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:36.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:35 smithi070 ceph-mon[192652]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:35 smithi070 ceph-mon[192652]: osdmap e433: 8 total, 8 up, 6 in 2023-09-23T22:34:36.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:35 smithi070 ceph-mon[192652]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:35 smithi070 ceph-mon[192652]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:35 smithi070 ceph-mon[192652]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.124 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:36.206 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:36.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[223014]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[223014]: osdmap e433: 8 total, 8 up, 6 in 2023-09-23T22:34:36.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[223014]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[223014]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[223014]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[215775]: osdmap e433: 8 total, 8 up, 6 in 2023-09-23T22:34:36.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[215775]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[215775]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:35 smithi026 ceph-mon[215775]: failed to encode map e433 with expected crc 2023-09-23T22:34:36.438 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:36.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:36.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:36.551 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:36.590 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:36.619 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:34:36.620 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:34:36.679 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:36.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:36.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:36.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:36.845 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:36.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:36.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:36.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:37.040 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:37.094 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:36 smithi070 ceph-mon[192652]: pgmap v1125: 99 pgs: 7 peering, 4 unknown, 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 79 active+clean; 865 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 416 op/s; 6456/41418 objects misplaced (15.587%); 35 MiB/s, 563 objects/s recovering 2023-09-23T22:34:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:36 smithi070 ceph-mon[192652]: failed to encode map e434 with expected crc 2023-09-23T22:34:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:36 smithi070 ceph-mon[192652]: osdmap e434: 8 total, 8 up, 6 in 2023-09-23T22:34:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:36 smithi070 ceph-mon[192652]: failed to encode map e434 with expected crc 2023-09-23T22:34:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:36 smithi070 ceph-mon[192652]: failed to encode map e434 with expected crc 2023-09-23T22:34:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:37.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:36 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[215775]: pgmap v1125: 99 pgs: 7 peering, 4 unknown, 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 79 active+clean; 865 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 416 op/s; 6456/41418 objects misplaced (15.587%); 35 MiB/s, 563 objects/s recovering 2023-09-23T22:34:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[215775]: failed to encode map e434 with expected crc 2023-09-23T22:34:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[215775]: osdmap e434: 8 total, 8 up, 6 in 2023-09-23T22:34:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[215775]: failed to encode map e434 with expected crc 2023-09-23T22:34:37.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[215775]: failed to encode map e434 with expected crc 2023-09-23T22:34:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:37.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[223014]: pgmap v1125: 99 pgs: 7 peering, 4 unknown, 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 79 active+clean; 865 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 416 op/s; 6456/41418 objects misplaced (15.587%); 35 MiB/s, 563 objects/s recovering 2023-09-23T22:34:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[223014]: failed to encode map e434 with expected crc 2023-09-23T22:34:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[223014]: osdmap e434: 8 total, 8 up, 6 in 2023-09-23T22:34:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[223014]: failed to encode map e434 with expected crc 2023-09-23T22:34:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[223014]: failed to encode map e434 with expected crc 2023-09-23T22:34:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:37.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:36 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:37 smithi070 ceph-mon[192652]: failed to encode map e434 with expected crc 2023-09-23T22:34:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:37 smithi070 ceph-mon[192652]: failed to encode map e434 with expected crc 2023-09-23T22:34:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:37 smithi070 ceph-mon[192652]: failed to encode map e434 with expected crc 2023-09-23T22:34:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:37 smithi070 ceph-mon[192652]: osdmap e435: 8 total, 8 up, 6 in 2023-09-23T22:34:38.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:37 smithi070 ceph-mon[192652]: failed to encode map e435 with expected crc 2023-09-23T22:34:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[215775]: failed to encode map e434 with expected crc 2023-09-23T22:34:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[215775]: failed to encode map e434 with expected crc 2023-09-23T22:34:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[215775]: failed to encode map e434 with expected crc 2023-09-23T22:34:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[215775]: osdmap e435: 8 total, 8 up, 6 in 2023-09-23T22:34:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[215775]: failed to encode map e435 with expected crc 2023-09-23T22:34:38.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[223014]: failed to encode map e434 with expected crc 2023-09-23T22:34:38.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[223014]: failed to encode map e434 with expected crc 2023-09-23T22:34:38.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[223014]: failed to encode map e434 with expected crc 2023-09-23T22:34:38.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[223014]: osdmap e435: 8 total, 8 up, 6 in 2023-09-23T22:34:38.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:37 smithi026 ceph-mon[223014]: failed to encode map e435 with expected crc 2023-09-23T22:34:38.899 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:34:38.899 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:34:38.899 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-09-23T22:34:38.899 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:34:39.022 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:39.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:39.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:39.056 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:34:39.120 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:39.131 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[215775]: pgmap v1128: 99 pgs: 3 peering, 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 87 active+clean; 848 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 460 op/s; 6456/40590 objects misplaced (15.905%); 37 MiB/s, 599 objects/s recovering 2023-09-23T22:34:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[215775]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[215775]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[215775]: osdmap e436: 8 total, 8 up, 6 in 2023-09-23T22:34:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[215775]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[215775]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:39.200 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:39.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[223014]: pgmap v1128: 99 pgs: 3 peering, 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 87 active+clean; 848 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 460 op/s; 6456/40590 objects misplaced (15.905%); 37 MiB/s, 599 objects/s recovering 2023-09-23T22:34:39.202 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[223014]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[223014]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[223014]: osdmap e436: 8 total, 8 up, 6 in 2023-09-23T22:34:39.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[223014]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[223014]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:39.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:39.203 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:38 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:39.208 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:39.356 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:39.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:39.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:38 smithi070 ceph-mon[192652]: pgmap v1128: 99 pgs: 3 peering, 2 active+remapped+backfilling, 7 active+remapped+backfill_wait, 87 active+clean; 848 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 460 op/s; 6456/40590 objects misplaced (15.905%); 37 MiB/s, 599 objects/s recovering 2023-09-23T22:34:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:38 smithi070 ceph-mon[192652]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:38 smithi070 ceph-mon[192652]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:38 smithi070 ceph-mon[192652]: osdmap e436: 8 total, 8 up, 6 in 2023-09-23T22:34:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:38 smithi070 ceph-mon[192652]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:38 smithi070 ceph-mon[192652]: failed to encode map e436 with expected crc 2023-09-23T22:34:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:34:39.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:34:39.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:38 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:39.468 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:39.525 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:39.724 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:34:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:34:39] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.43.0" 2023-09-23T22:34:39.949 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:39.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:39.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:40.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:40.126 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[223014]: failed to encode map e437 with expected crc 2023-09-23T22:34:40.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[223014]: osdmap e437: 8 total, 8 up, 6 in 2023-09-23T22:34:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[223014]: failed to encode map e437 with expected crc 2023-09-23T22:34:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[223014]: failed to encode map e437 with expected crc 2023-09-23T22:34:40.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[215775]: failed to encode map e437 with expected crc 2023-09-23T22:34:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[215775]: osdmap e437: 8 total, 8 up, 6 in 2023-09-23T22:34:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[215775]: failed to encode map e437 with expected crc 2023-09-23T22:34:40.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[215775]: failed to encode map e437 with expected crc 2023-09-23T22:34:40.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:40.241 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:40.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:40.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:40.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:40 smithi070 ceph-mon[192652]: failed to encode map e437 with expected crc 2023-09-23T22:34:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:40 smithi070 ceph-mon[192652]: osdmap e437: 8 total, 8 up, 6 in 2023-09-23T22:34:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:40 smithi070 ceph-mon[192652]: failed to encode map e437 with expected crc 2023-09-23T22:34:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:40 smithi070 ceph-mon[192652]: failed to encode map e437 with expected crc 2023-09-23T22:34:40.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:40.398 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:40.476 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:40.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:40.581 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:40.610 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:40.656 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:40.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:40.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:40.760 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:40.828 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:41.050 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:34:41.165 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:41.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:41.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:41.309 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[215775]: pgmap v1131: 99 pgs: 3 peering, 3 active+remapped+backfilling, 6 active+remapped+backfill_wait, 87 active+clean; 809 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 5635/38721 objects misplaced (14.553%); 5.3 MiB/s, 86 objects/s recovering 2023-09-23T22:34:41.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[215775]: failed to encode map e438 with expected crc 2023-09-23T22:34:41.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[215775]: failed to encode map e438 with expected crc 2023-09-23T22:34:41.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[215775]: osdmap e438: 8 total, 8 up, 6 in 2023-09-23T22:34:41.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[215775]: failed to encode map e438 with expected crc 2023-09-23T22:34:41.310 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4232379615' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:34:41.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[223014]: pgmap v1131: 99 pgs: 3 peering, 3 active+remapped+backfilling, 6 active+remapped+backfill_wait, 87 active+clean; 809 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 5635/38721 objects misplaced (14.553%); 5.3 MiB/s, 86 objects/s recovering 2023-09-23T22:34:41.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[223014]: failed to encode map e438 with expected crc 2023-09-23T22:34:41.310 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[223014]: failed to encode map e438 with expected crc 2023-09-23T22:34:41.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[223014]: osdmap e438: 8 total, 8 up, 6 in 2023-09-23T22:34:41.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[223014]: failed to encode map e438 with expected crc 2023-09-23T22:34:41.311 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4232379615' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:34:41.318 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:41 smithi070 ceph-mon[192652]: pgmap v1131: 99 pgs: 3 peering, 3 active+remapped+backfilling, 6 active+remapped+backfill_wait, 87 active+clean; 809 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 224 op/s; 5635/38721 objects misplaced (14.553%); 5.3 MiB/s, 86 objects/s recovering 2023-09-23T22:34:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:41 smithi070 ceph-mon[192652]: failed to encode map e438 with expected crc 2023-09-23T22:34:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:41 smithi070 ceph-mon[192652]: failed to encode map e438 with expected crc 2023-09-23T22:34:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:41 smithi070 ceph-mon[192652]: osdmap e438: 8 total, 8 up, 6 in 2023-09-23T22:34:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:41 smithi070 ceph-mon[192652]: failed to encode map e438 with expected crc 2023-09-23T22:34:41.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4232379615' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:34:41.405 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:41.959 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:41.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:41.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:42.065 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:42.105 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:42.116 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:42.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:42.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:42.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:42.220 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:34:42.221 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4232379615' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:34:42.221 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[223014]: osdmap e439: 8 total, 8 up, 6 in 2023-09-23T22:34:42.221 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[223014]: failed to encode map e439 with expected crc 2023-09-23T22:34:42.221 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[223014]: failed to encode map e439 with expected crc 2023-09-23T22:34:42.221 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[223014]: 1.4 scrub starts 2023-09-23T22:34:42.221 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:34:42.221 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4232379615' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:34:42.221 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[215775]: osdmap e439: 8 total, 8 up, 6 in 2023-09-23T22:34:42.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[215775]: failed to encode map e439 with expected crc 2023-09-23T22:34:42.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[215775]: failed to encode map e439 with expected crc 2023-09-23T22:34:42.222 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:42 smithi026 ceph-mon[215775]: 1.4 scrub starts 2023-09-23T22:34:42.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:42.327 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:42 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:34:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4232379615' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:34:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:42 smithi070 ceph-mon[192652]: osdmap e439: 8 total, 8 up, 6 in 2023-09-23T22:34:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:42 smithi070 ceph-mon[192652]: failed to encode map e439 with expected crc 2023-09-23T22:34:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:42 smithi070 ceph-mon[192652]: failed to encode map e439 with expected crc 2023-09-23T22:34:42.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:42 smithi070 ceph-mon[192652]: 1.4 scrub starts 2023-09-23T22:34:42.464 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:42.504 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:42.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:42.554 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:34:42.580 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:34:42.607 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:34:42.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:42.634 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:34:42.634 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:42.660 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:34:42.678 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:42.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:42.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:42.690 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:34:42.720 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:34:42.746 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:34:42.747 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:34:42.747 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:34:42.748 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:34:42.748 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:34:42.748 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:34:42.748 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:34:42.748 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:34:42.748 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:34:42.782 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:42.808 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:42.852 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:42.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:42.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:42.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:43.001 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:43.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[215775]: pgmap v1134: 99 pgs: 1 peering, 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 91 active+clean; 728 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 4531/34836 objects misplaced (13.007%); 16 MiB/s, 259 objects/s recovering 2023-09-23T22:34:43.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[215775]: failed to encode map e440 with expected crc 2023-09-23T22:34:43.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[215775]: osdmap e440: 8 total, 8 up, 6 in 2023-09-23T22:34:43.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[215775]: failed to encode map e440 with expected crc 2023-09-23T22:34:43.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[215775]: failed to encode map e440 with expected crc 2023-09-23T22:34:43.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[215775]: 1.4 scrub starts 2023-09-23T22:34:43.042 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2835356884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:43.043 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[223014]: pgmap v1134: 99 pgs: 1 peering, 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 91 active+clean; 728 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 4531/34836 objects misplaced (13.007%); 16 MiB/s, 259 objects/s recovering 2023-09-23T22:34:43.043 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[223014]: failed to encode map e440 with expected crc 2023-09-23T22:34:43.043 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[223014]: osdmap e440: 8 total, 8 up, 6 in 2023-09-23T22:34:43.043 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[223014]: failed to encode map e440 with expected crc 2023-09-23T22:34:43.043 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[223014]: failed to encode map e440 with expected crc 2023-09-23T22:34:43.043 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[223014]: 1.4 scrub starts 2023-09-23T22:34:43.043 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2835356884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:43.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:43 smithi070 ceph-mon[192652]: pgmap v1134: 99 pgs: 1 peering, 2 active+remapped+backfilling, 5 active+remapped+backfill_wait, 91 active+clean; 728 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 479 op/s; 4531/34836 objects misplaced (13.007%); 16 MiB/s, 259 objects/s recovering 2023-09-23T22:34:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:43 smithi070 ceph-mon[192652]: failed to encode map e440 with expected crc 2023-09-23T22:34:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:43 smithi070 ceph-mon[192652]: osdmap e440: 8 total, 8 up, 6 in 2023-09-23T22:34:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:43 smithi070 ceph-mon[192652]: failed to encode map e440 with expected crc 2023-09-23T22:34:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:43 smithi070 ceph-mon[192652]: failed to encode map e440 with expected crc 2023-09-23T22:34:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:43 smithi070 ceph-mon[192652]: 1.4 scrub starts 2023-09-23T22:34:43.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2835356884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:44.117 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:34:44.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:44 smithi070 ceph-mon[192652]: failed to encode map e441 with expected crc 2023-09-23T22:34:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:44 smithi070 ceph-mon[192652]: osdmap e441: 8 total, 8 up, 6 in 2023-09-23T22:34:44.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:44 smithi070 ceph-mon[192652]: failed to encode map e441 with expected crc 2023-09-23T22:34:44.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:44 smithi026 ceph-mon[215775]: failed to encode map e441 with expected crc 2023-09-23T22:34:44.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:44 smithi026 ceph-mon[215775]: osdmap e441: 8 total, 8 up, 6 in 2023-09-23T22:34:44.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:44 smithi026 ceph-mon[215775]: failed to encode map e441 with expected crc 2023-09-23T22:34:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:44 smithi026 ceph-mon[223014]: failed to encode map e441 with expected crc 2023-09-23T22:34:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:44 smithi026 ceph-mon[223014]: osdmap e441: 8 total, 8 up, 6 in 2023-09-23T22:34:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:44 smithi026 ceph-mon[223014]: failed to encode map e441 with expected crc 2023-09-23T22:34:45.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: pgmap v1137: 99 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 93 active+clean; 691 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 3678/33069 objects misplaced (11.122%); 25 MiB/s, 396 objects/s recovering 2023-09-23T22:34:45.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: osdmap e442: 8 total, 8 up, 6 in 2023-09-23T22:34:45.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.060 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[223014]: osdmap e443: 8 total, 8 up, 6 in 2023-09-23T22:34:45.075 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:45.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:45.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:45.180 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:45.241 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: pgmap v1137: 99 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 93 active+clean; 691 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 3678/33069 objects misplaced (11.122%); 25 MiB/s, 396 objects/s recovering 2023-09-23T22:34:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: osdmap e442: 8 total, 8 up, 6 in 2023-09-23T22:34:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:45 smithi070 ceph-mon[192652]: osdmap e443: 8 total, 8 up, 6 in 2023-09-23T22:34:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: pgmap v1137: 99 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 93 active+clean; 691 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 470 op/s; 3678/33069 objects misplaced (11.122%); 25 MiB/s, 396 objects/s recovering 2023-09-23T22:34:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: osdmap e442: 8 total, 8 up, 6 in 2023-09-23T22:34:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: failed to encode map e442 with expected crc 2023-09-23T22:34:45.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:45 smithi026 ceph-mon[215775]: osdmap e443: 8 total, 8 up, 6 in 2023-09-23T22:34:45.758 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:45.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:45.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:45.888 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:45.903 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:45.928 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:45.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:45.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:46.033 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:46.071 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:46.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:46 smithi070 ceph-mon[192652]: failed to encode map e443 with expected crc 2023-09-23T22:34:46.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:46 smithi070 ceph-mon[192652]: failed to encode map e443 with expected crc 2023-09-23T22:34:46.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:46 smithi070 ceph-mon[192652]: failed to encode map e443 with expected crc 2023-09-23T22:34:46.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:46 smithi070 ceph-mon[192652]: osdmap e444: 8 total, 8 up, 6 in 2023-09-23T22:34:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:46 smithi026 ceph-mon[223014]: failed to encode map e443 with expected crc 2023-09-23T22:34:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:46 smithi026 ceph-mon[223014]: failed to encode map e443 with expected crc 2023-09-23T22:34:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:46 smithi026 ceph-mon[223014]: failed to encode map e443 with expected crc 2023-09-23T22:34:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:46 smithi026 ceph-mon[223014]: osdmap e444: 8 total, 8 up, 6 in 2023-09-23T22:34:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:46 smithi026 ceph-mon[215775]: failed to encode map e443 with expected crc 2023-09-23T22:34:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:46 smithi026 ceph-mon[215775]: failed to encode map e443 with expected crc 2023-09-23T22:34:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:46 smithi026 ceph-mon[215775]: failed to encode map e443 with expected crc 2023-09-23T22:34:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:46 smithi026 ceph-mon[215775]: osdmap e444: 8 total, 8 up, 6 in 2023-09-23T22:34:47.195 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:47.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:47.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:47.307 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:47 smithi070 ceph-mon[192652]: pgmap v1140: 99 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 574 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 947/27438 objects misplaced (3.451%); 45 MiB/s, 726 objects/s recovering 2023-09-23T22:34:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:47 smithi070 ceph-mon[192652]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:47 smithi070 ceph-mon[192652]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:47 smithi070 ceph-mon[192652]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:47 smithi070 ceph-mon[192652]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:47 smithi070 ceph-mon[192652]: osdmap e445: 8 total, 8 up, 6 in 2023-09-23T22:34:47.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[215775]: pgmap v1140: 99 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 574 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 947/27438 objects misplaced (3.451%); 45 MiB/s, 726 objects/s recovering 2023-09-23T22:34:47.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[215775]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[215775]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.380 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[215775]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[215775]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[215775]: osdmap e445: 8 total, 8 up, 6 in 2023-09-23T22:34:47.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[223014]: pgmap v1140: 99 pgs: 1 peering, 1 active+remapped+backfilling, 2 active+remapped+backfill_wait, 95 active+clean; 574 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 616 op/s; 947/27438 objects misplaced (3.451%); 45 MiB/s, 726 objects/s recovering 2023-09-23T22:34:47.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[223014]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[223014]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[223014]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[223014]: failed to encode map e444 with expected crc 2023-09-23T22:34:47.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:47 smithi026 ceph-mon[223014]: osdmap e445: 8 total, 8 up, 6 in 2023-09-23T22:34:47.383 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:47.482 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:34:47.484 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:34:47.758 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:34:47.890 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:47.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:47.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:47.994 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:48.021 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:48.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:48 smithi070 ceph-mon[192652]: failed to encode map e445 with expected crc 2023-09-23T22:34:48.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:48 smithi070 ceph-mon[192652]: failed to encode map e445 with expected crc 2023-09-23T22:34:48.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:48 smithi070 ceph-mon[192652]: failed to encode map e445 with expected crc 2023-09-23T22:34:48.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:48 smithi026 ceph-mon[223014]: failed to encode map e445 with expected crc 2023-09-23T22:34:48.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:48 smithi026 ceph-mon[223014]: failed to encode map e445 with expected crc 2023-09-23T22:34:48.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:48 smithi026 ceph-mon[223014]: failed to encode map e445 with expected crc 2023-09-23T22:34:48.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:48 smithi026 ceph-mon[215775]: failed to encode map e445 with expected crc 2023-09-23T22:34:48.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:48 smithi026 ceph-mon[215775]: failed to encode map e445 with expected crc 2023-09-23T22:34:48.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:48 smithi026 ceph-mon[215775]: failed to encode map e445 with expected crc 2023-09-23T22:34:48.605 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:48.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:48.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:48.719 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:48.744 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:48.822 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:48.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:48.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:48.928 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:34:48.957 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1142,"stamp":"2023-09-23T22:34:47.049783+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":602073457,"num_objects":9146,"num_object_clones":0,"num_object_copies":27438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":947,"num_objects_unfound":0,"num_objects_dirty":9146,"num_whiteouts":0,"num_read":6027,"num_read_kb":12268,"num_write":189590,"num_write_kb":6341842,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24457,"num_bytes_recovered":1581574049,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85316,"ondisk_log_size":85316,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5019304,"kb_used_data":2222392,"kb_used_omap":31,"kb_used_meta":2796832,"kb_avail":557451608,"statfs":{"total":575970213888,"available":570830446592,"internally_reserved":0,"allocated":2275729408,"data_stored":2264126921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31788,"internal_metadata":2863956948},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-123011072,"num_objects":-1877,"num_object_clones":0,"num_object_copies":-5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2731,"num_objects_unfound":0,"num_objects_dirty":-1877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1877,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2147,"num_bytes_recovered":140705792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.004376"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":480,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.057086+0000","last_change":"2023-09-23T22:33:46.209288+0000","last_active":"2023-09-23T22:34:44.057086+0000","last_peered":"2023-09-23T22:34:44.057086+0000","last_clean":"2023-09-23T22:34:44.057086+0000","last_became_active":"2023-09-23T22:33:46.209051+0000","last_became_peered":"2023-09-23T22:33:46.209051+0000","last_unstale":"2023-09-23T22:34:44.057086+0000","last_undegraded":"2023-09-23T22:34:44.057086+0000","last_fullsized":"2023-09-23T22:34:44.057086+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:41.235601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":159,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.052250+0000","last_change":"2023-09-23T22:29:01.584265+0000","last_active":"2023-09-23T22:34:43.052250+0000","last_peered":"2023-09-23T22:34:43.052250+0000","last_clean":"2023-09-23T22:34:43.052250+0000","last_became_active":"2023-09-23T22:29:01.583551+0000","last_became_peered":"2023-09-23T22:29:01.583551+0000","last_unstale":"2023-09-23T22:34:43.052250+0000","last_undegraded":"2023-09-23T22:34:43.052250+0000","last_fullsized":"2023-09-23T22:34:43.052250+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:15.918693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":159,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.050808+0000","last_change":"2023-09-23T22:29:01.595317+0000","last_active":"2023-09-23T22:34:43.050808+0000","last_peered":"2023-09-23T22:34:43.050808+0000","last_clean":"2023-09-23T22:34:43.050808+0000","last_became_active":"2023-09-23T22:29:01.595020+0000","last_became_peered":"2023-09-23T22:29:01.595020+0000","last_unstale":"2023-09-23T22:34:43.050808+0000","last_undegraded":"2023-09-23T22:34:43.050808+0000","last_fullsized":"2023-09-23T22:34:43.050808+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:18.471652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":298,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.052464+0000","last_change":"2023-09-23T22:34:32.965464+0000","last_active":"2023-09-23T22:34:43.052464+0000","last_peered":"2023-09-23T22:34:43.052464+0000","last_clean":"2023-09-23T22:34:43.052464+0000","last_became_active":"2023-09-23T22:34:32.965039+0000","last_became_peered":"2023-09-23T22:34:32.965039+0000","last_unstale":"2023-09-23T22:34:43.052464+0000","last_undegraded":"2023-09-23T22:34:43.052464+0000","last_fullsized":"2023-09-23T22:34:43.052464+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:16.002987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":500,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.050997+0000","last_change":"2023-09-23T22:34:19.426816+0000","last_active":"2023-09-23T22:34:43.050997+0000","last_peered":"2023-09-23T22:34:43.050997+0000","last_clean":"2023-09-23T22:34:43.050997+0000","last_became_active":"2023-09-23T22:34:19.426413+0000","last_became_peered":"2023-09-23T22:34:19.426413+0000","last_unstale":"2023-09-23T22:34:43.050997+0000","last_undegraded":"2023-09-23T22:34:43.050997+0000","last_fullsized":"2023-09-23T22:34:43.050997+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:58.025095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":124,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055538+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:34:44.055538+0000","last_peered":"2023-09-23T22:34:44.055538+0000","last_clean":"2023-09-23T22:34:44.055538+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:34:44.055538+0000","last_undegraded":"2023-09-23T22:34:44.055538+0000","last_fullsized":"2023-09-23T22:34:44.055538+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":264,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055384+0000","last_change":"2023-09-23T22:34:32.965370+0000","last_active":"2023-09-23T22:34:44.055384+0000","last_peered":"2023-09-23T22:34:44.055384+0000","last_clean":"2023-09-23T22:34:44.055384+0000","last_became_active":"2023-09-23T22:34:32.965095+0000","last_became_peered":"2023-09-23T22:34:32.965095+0000","last_unstale":"2023-09-23T22:34:44.055384+0000","last_undegraded":"2023-09-23T22:34:44.055384+0000","last_fullsized":"2023-09-23T22:34:44.055384+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:15.925188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":124,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054055+0000","last_change":"2023-09-23T22:29:01.598342+0000","last_active":"2023-09-23T22:34:44.054055+0000","last_peered":"2023-09-23T22:34:44.054055+0000","last_clean":"2023-09-23T22:34:44.054055+0000","last_became_active":"2023-09-23T22:29:01.597144+0000","last_became_peered":"2023-09-23T22:29:01.597144+0000","last_unstale":"2023-09-23T22:34:44.054055+0000","last_undegraded":"2023-09-23T22:34:44.054055+0000","last_fullsized":"2023-09-23T22:34:44.054055+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:57.556517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":440,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.052584+0000","last_change":"2023-09-23T22:34:32.974206+0000","last_active":"2023-09-23T22:34:43.052584+0000","last_peered":"2023-09-23T22:34:43.052584+0000","last_clean":"2023-09-23T22:34:43.052584+0000","last_became_active":"2023-09-23T22:34:32.973949+0000","last_became_peered":"2023-09-23T22:34:32.973949+0000","last_unstale":"2023-09-23T22:34:43.052584+0000","last_undegraded":"2023-09-23T22:34:43.052584+0000","last_fullsized":"2023-09-23T22:34:43.052584+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:04:51.871254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":134,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054162+0000","last_change":"2023-09-23T22:31:28.122684+0000","last_active":"2023-09-23T22:34:44.054162+0000","last_peered":"2023-09-23T22:34:44.054162+0000","last_clean":"2023-09-23T22:34:44.054162+0000","last_became_active":"2023-09-23T22:31:28.122427+0000","last_became_peered":"2023-09-23T22:31:28.122427+0000","last_unstale":"2023-09-23T22:34:44.054162+0000","last_undegraded":"2023-09-23T22:34:44.054162+0000","last_fullsized":"2023-09-23T22:34:44.054162+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:51:15.559367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":144,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055591+0000","last_change":"2023-09-23T22:34:19.321223+0000","last_active":"2023-09-23T22:34:44.055591+0000","last_peered":"2023-09-23T22:34:44.055591+0000","last_clean":"2023-09-23T22:34:44.055591+0000","last_became_active":"2023-09-23T22:34:19.320871+0000","last_became_peered":"2023-09-23T22:34:19.320871+0000","last_unstale":"2023-09-23T22:34:44.055591+0000","last_undegraded":"2023-09-23T22:34:44.055591+0000","last_fullsized":"2023-09-23T22:34:44.055591+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:55.202978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":100,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.054022+0000","last_change":"2023-09-23T22:33:46.207512+0000","last_active":"2023-09-23T22:34:43.054022+0000","last_peered":"2023-09-23T22:34:43.054022+0000","last_clean":"2023-09-23T22:34:43.054022+0000","last_became_active":"2023-09-23T22:33:46.207193+0000","last_became_peered":"2023-09-23T22:33:46.207193+0000","last_unstale":"2023-09-23T22:34:43.054022+0000","last_undegraded":"2023-09-23T22:34:43.054022+0000","last_fullsized":"2023-09-23T22:34:43.054022+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:23.755903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":438,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054615+0000","last_change":"2023-09-23T22:33:46.222562+0000","last_active":"2023-09-23T22:34:44.054615+0000","last_peered":"2023-09-23T22:34:44.054615+0000","last_clean":"2023-09-23T22:34:44.054615+0000","last_became_active":"2023-09-23T22:33:46.207754+0000","last_became_peered":"2023-09-23T22:33:46.207754+0000","last_unstale":"2023-09-23T22:34:44.054615+0000","last_undegraded":"2023-09-23T22:34:44.054615+0000","last_fullsized":"2023-09-23T22:34:44.054615+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:33.798053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":23670,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":441,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055138+0000","last_change":"2023-09-23T22:29:01.601610+0000","last_active":"2023-09-23T22:34:44.055138+0000","last_peered":"2023-09-23T22:34:44.055138+0000","last_clean":"2023-09-23T22:34:44.055138+0000","last_became_active":"2023-09-23T22:29:01.600784+0000","last_became_peered":"2023-09-23T22:29:01.600784+0000","last_unstale":"2023-09-23T22:34:44.055138+0000","last_undegraded":"2023-09-23T22:34:44.055138+0000","last_fullsized":"2023-09-23T22:34:44.055138+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:04.148298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":41,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.048898+0000","last_change":"2023-09-23T22:33:46.216607+0000","last_active":"2023-09-23T22:34:41.048898+0000","last_peered":"2023-09-23T22:34:41.048898+0000","last_clean":"2023-09-23T22:34:41.048898+0000","last_became_active":"2023-09-23T22:33:46.210696+0000","last_became_peered":"2023-09-23T22:33:46.210696+0000","last_unstale":"2023-09-23T22:34:41.048898+0000","last_undegraded":"2023-09-23T22:34:41.048898+0000","last_fullsized":"2023-09-23T22:34:41.048898+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:24.739320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":35,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.049037+0000","last_change":"2023-09-23T22:33:46.209433+0000","last_active":"2023-09-23T22:34:41.049037+0000","last_peered":"2023-09-23T22:34:41.049037+0000","last_clean":"2023-09-23T22:34:41.049037+0000","last_became_active":"2023-09-23T22:33:46.209131+0000","last_became_peered":"2023-09-23T22:33:46.209131+0000","last_unstale":"2023-09-23T22:34:41.049037+0000","last_undegraded":"2023-09-23T22:34:41.049037+0000","last_fullsized":"2023-09-23T22:34:41.049037+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:36.290396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":349,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.047504+0000","last_change":"2023-09-23T22:34:19.428299+0000","last_active":"2023-09-23T22:34:41.047504+0000","last_peered":"2023-09-23T22:34:41.047504+0000","last_clean":"2023-09-23T22:34:41.047504+0000","last_became_active":"2023-09-23T22:34:19.428045+0000","last_became_peered":"2023-09-23T22:34:19.428045+0000","last_unstale":"2023-09-23T22:34:41.047504+0000","last_undegraded":"2023-09-23T22:34:41.047504+0000","last_fullsized":"2023-09-23T22:34:41.047504+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:36.509892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":406,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055878+0000","last_change":"2023-09-23T22:34:32.968461+0000","last_active":"2023-09-23T22:34:44.055878+0000","last_peered":"2023-09-23T22:34:44.055878+0000","last_clean":"2023-09-23T22:34:44.055878+0000","last_became_active":"2023-09-23T22:34:32.968146+0000","last_became_peered":"2023-09-23T22:34:32.968146+0000","last_unstale":"2023-09-23T22:34:44.055878+0000","last_undegraded":"2023-09-23T22:34:44.055878+0000","last_fullsized":"2023-09-23T22:34:44.055878+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:30.629155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":386,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055443+0000","last_change":"2023-09-23T22:31:28.119156+0000","last_active":"2023-09-23T22:34:44.055443+0000","last_peered":"2023-09-23T22:34:44.055443+0000","last_clean":"2023-09-23T22:34:44.055443+0000","last_became_active":"2023-09-23T22:31:28.118382+0000","last_became_peered":"2023-09-23T22:31:28.118382+0000","last_unstale":"2023-09-23T22:34:44.055443+0000","last_undegraded":"2023-09-23T22:34:44.055443+0000","last_fullsized":"2023-09-23T22:34:44.055443+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:10.140079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":340,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.049086+0000","last_change":"2023-09-23T22:34:19.427219+0000","last_active":"2023-09-23T22:34:41.049086+0000","last_peered":"2023-09-23T22:34:41.049086+0000","last_clean":"2023-09-23T22:34:41.049086+0000","last_became_active":"2023-09-23T22:34:19.426801+0000","last_became_peered":"2023-09-23T22:34:19.426801+0000","last_unstale":"2023-09-23T22:34:41.049086+0000","last_undegraded":"2023-09-23T22:34:41.049086+0000","last_fullsized":"2023-09-23T22:34:41.049086+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:25.018907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":154,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053871+0000","last_change":"2023-09-23T22:34:33.228716+0000","last_active":"2023-09-23T22:34:44.053871+0000","last_peered":"2023-09-23T22:34:44.053871+0000","last_clean":"2023-09-23T22:34:44.053871+0000","last_became_active":"2023-09-23T22:34:33.017756+0000","last_became_peered":"2023-09-23T22:34:33.017756+0000","last_unstale":"2023-09-23T22:34:44.053871+0000","last_undegraded":"2023-09-23T22:34:44.053871+0000","last_fullsized":"2023-09-23T22:34:44.053871+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:47.604622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":35,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.049781+0000","last_change":"2023-09-23T22:33:46.211736+0000","last_active":"2023-09-23T22:34:41.049781+0000","last_peered":"2023-09-23T22:34:41.049781+0000","last_clean":"2023-09-23T22:34:41.049781+0000","last_became_active":"2023-09-23T22:33:46.211411+0000","last_became_peered":"2023-09-23T22:33:46.211411+0000","last_unstale":"2023-09-23T22:34:41.049781+0000","last_undegraded":"2023-09-23T22:34:41.049781+0000","last_fullsized":"2023-09-23T22:34:41.049781+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:49.020816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.4","version":"443'11771","reported_seq":13566,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T22:34:45.188630+0000","last_change":"2023-09-23T22:34:40.080056+0000","last_active":"2023-09-23T22:34:45.188630+0000","last_peered":"2023-09-23T22:34:45.188630+0000","last_clean":"2023-09-23T22:34:45.188630+0000","last_became_active":"2023-09-23T22:34:40.019449+0000","last_became_peered":"2023-09-23T22:34:40.019449+0000","last_unstale":"2023-09-23T22:34:45.188630+0000","last_undegraded":"2023-09-23T22:34:45.188630+0000","last_fullsized":"2023-09-23T22:34:45.188630+0000","mapping_epoch":437,"log_start":"397'5000","ondisk_log_start":"397'5000","created":373,"last_epoch_clean":438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":6771,"log_dups_size":0,"ondisk_log_size":6771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:54:18.266818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37683200,"num_objects":575,"num_object_clones":0,"num_object_copies":1725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11771,"num_write_kb":395072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":844,"num_bytes_recovered":55312384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":452,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.056744+0000","last_change":"2023-09-23T22:31:28.139498+0000","last_active":"2023-09-23T22:34:44.056744+0000","last_peered":"2023-09-23T22:34:44.056744+0000","last_clean":"2023-09-23T22:34:44.056744+0000","last_became_active":"2023-09-23T22:31:28.113206+0000","last_became_peered":"2023-09-23T22:31:28.113206+0000","last_unstale":"2023-09-23T22:34:44.056744+0000","last_undegraded":"2023-09-23T22:34:44.056744+0000","last_fullsized":"2023-09-23T22:34:44.056744+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:11.905312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":399,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053515+0000","last_change":"2023-09-23T22:34:32.969728+0000","last_active":"2023-09-23T22:34:44.053515+0000","last_peered":"2023-09-23T22:34:44.053515+0000","last_clean":"2023-09-23T22:34:44.053515+0000","last_became_active":"2023-09-23T22:34:32.969365+0000","last_became_peered":"2023-09-23T22:34:32.969365+0000","last_unstale":"2023-09-23T22:34:44.053515+0000","last_undegraded":"2023-09-23T22:34:44.053515+0000","last_fullsized":"2023-09-23T22:34:44.053515+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:29:52.626082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.5","version":"443'11888","reported_seq":15169,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T22:34:45.231315+0000","last_change":"2023-09-23T22:32:00.147475+0000","last_active":"2023-09-23T22:34:45.231315+0000","last_peered":"2023-09-23T22:34:45.231315+0000","last_clean":"2023-09-23T22:34:45.231315+0000","last_became_active":"2023-09-23T22:32:00.121320+0000","last_became_peered":"2023-09-23T22:32:00.121320+0000","last_unstale":"2023-09-23T22:34:45.231315+0000","last_undegraded":"2023-09-23T22:34:45.231315+0000","last_fullsized":"2023-09-23T22:34:45.231315+0000","mapping_epoch":392,"log_start":"405'6912","ondisk_log_start":"405'6912","created":373,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4976,"log_dups_size":0,"ondisk_log_size":4976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:17:59.934876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36569088,"num_objects":558,"num_object_clones":0,"num_object_copies":1674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11888,"num_write_kb":398272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1591,"num_bytes_recovered":104267776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":271,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.052685+0000","last_change":"2023-09-23T22:33:34.744279+0000","last_active":"2023-09-23T22:34:43.052685+0000","last_peered":"2023-09-23T22:34:43.052685+0000","last_clean":"2023-09-23T22:34:43.052685+0000","last_became_active":"2023-09-23T22:33:34.743934+0000","last_became_peered":"2023-09-23T22:33:34.743934+0000","last_unstale":"2023-09-23T22:34:43.052685+0000","last_undegraded":"2023-09-23T22:34:43.052685+0000","last_fullsized":"2023-09-23T22:34:43.052685+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:48.737827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":389,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.050201+0000","last_change":"2023-09-23T22:33:34.752042+0000","last_active":"2023-09-23T22:34:43.050201+0000","last_peered":"2023-09-23T22:34:43.050201+0000","last_clean":"2023-09-23T22:34:43.050201+0000","last_became_active":"2023-09-23T22:33:34.751766+0000","last_became_peered":"2023-09-23T22:33:34.751766+0000","last_unstale":"2023-09-23T22:34:43.050201+0000","last_undegraded":"2023-09-23T22:34:43.050201+0000","last_fullsized":"2023-09-23T22:34:43.050201+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:02.590988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"443'11849","reported_seq":18751,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T22:34:45.231128+0000","last_change":"2023-09-23T22:34:19.483333+0000","last_active":"2023-09-23T22:34:45.231128+0000","last_peered":"2023-09-23T22:34:45.231128+0000","last_clean":"2023-09-23T22:34:45.231128+0000","last_became_active":"2023-09-23T22:34:19.426217+0000","last_became_peered":"2023-09-23T22:34:19.426217+0000","last_unstale":"2023-09-23T22:34:45.231128+0000","last_undegraded":"2023-09-23T22:34:45.231128+0000","last_fullsized":"2023-09-23T22:34:45.231128+0000","mapping_epoch":424,"log_start":"405'6896","ondisk_log_start":"405'6896","created":373,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4953,"log_dups_size":0,"ondisk_log_size":4953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:37.659668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35192832,"num_objects":537,"num_object_clones":0,"num_object_copies":1611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11849,"num_write_kb":396352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3391,"num_bytes_recovered":202047488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":298,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055930+0000","last_change":"2023-09-23T22:34:32.970001+0000","last_active":"2023-09-23T22:34:44.055930+0000","last_peered":"2023-09-23T22:34:44.055930+0000","last_clean":"2023-09-23T22:34:44.055930+0000","last_became_active":"2023-09-23T22:34:32.963996+0000","last_became_peered":"2023-09-23T22:34:32.963996+0000","last_unstale":"2023-09-23T22:34:44.055930+0000","last_undegraded":"2023-09-23T22:34:44.055930+0000","last_fullsized":"2023-09-23T22:34:44.055930+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:38.625304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":134,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054740+0000","last_change":"2023-09-23T22:31:28.125138+0000","last_active":"2023-09-23T22:34:44.054740+0000","last_peered":"2023-09-23T22:34:44.054740+0000","last_clean":"2023-09-23T22:34:44.054740+0000","last_became_active":"2023-09-23T22:31:28.124716+0000","last_became_peered":"2023-09-23T22:31:28.124716+0000","last_unstale":"2023-09-23T22:34:44.054740+0000","last_undegraded":"2023-09-23T22:34:44.054740+0000","last_fullsized":"2023-09-23T22:34:44.054740+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:26.718931+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"443'11966","reported_seq":16982,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T22:34:45.215790+0000","last_change":"2023-09-23T22:34:06.227305+0000","last_active":"2023-09-23T22:34:45.215790+0000","last_peered":"2023-09-23T22:34:45.215790+0000","last_clean":"2023-09-23T22:34:45.215790+0000","last_became_active":"2023-09-23T22:34:06.183348+0000","last_became_peered":"2023-09-23T22:34:06.183348+0000","last_unstale":"2023-09-23T22:34:45.215790+0000","last_undegraded":"2023-09-23T22:34:45.215790+0000","last_fullsized":"2023-09-23T22:34:45.215790+0000","mapping_epoch":421,"log_start":"405'6977","ondisk_log_start":"405'6977","created":373,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4989,"log_dups_size":0,"ondisk_log_size":4989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:05.041707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33423360,"num_objects":510,"num_object_clones":0,"num_object_copies":1530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11966,"num_write_kb":399232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2456,"num_bytes_recovered":160956416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":375,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054525+0000","last_change":"2023-09-23T22:34:32.968460+0000","last_active":"2023-09-23T22:34:44.054525+0000","last_peered":"2023-09-23T22:34:44.054525+0000","last_clean":"2023-09-23T22:34:44.054525+0000","last_became_active":"2023-09-23T22:34:32.968169+0000","last_became_peered":"2023-09-23T22:34:32.968169+0000","last_unstale":"2023-09-23T22:34:44.054525+0000","last_undegraded":"2023-09-23T22:34:44.054525+0000","last_fullsized":"2023-09-23T22:34:44.054525+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:02.842773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":208,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.051804+0000","last_change":"2023-09-23T22:34:19.322772+0000","last_active":"2023-09-23T22:34:43.051804+0000","last_peered":"2023-09-23T22:34:43.051804+0000","last_clean":"2023-09-23T22:34:43.051804+0000","last_became_active":"2023-09-23T22:34:19.322266+0000","last_became_peered":"2023-09-23T22:34:19.322266+0000","last_unstale":"2023-09-23T22:34:43.051804+0000","last_undegraded":"2023-09-23T22:34:43.051804+0000","last_fullsized":"2023-09-23T22:34:43.051804+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:00.946534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"442'11466","reported_seq":12001,"reported_epoch":442,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T22:34:44.528235+0000","last_change":"2023-09-23T22:34:44.113461+0000","last_active":"2023-09-23T22:34:44.528235+0000","last_peered":"2023-09-23T22:34:44.528235+0000","last_clean":"2023-09-23T22:34:31.431046+0000","last_became_active":"2023-09-23T22:34:34.083256+0000","last_became_peered":"2023-09-23T22:34:34.083256+0000","last_unstale":"2023-09-23T22:34:44.528235+0000","last_undegraded":"2023-09-23T22:34:44.528235+0000","last_fullsized":"2023-09-23T22:34:44.528235+0000","mapping_epoch":431,"log_start":"405'6979","ondisk_log_start":"405'6979","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4487,"log_dups_size":0,"ondisk_log_size":4487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:08.083810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36569088,"num_objects":558,"num_object_clones":0,"num_object_copies":1674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":336,"num_objects_unfound":0,"num_objects_dirty":558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11466,"num_write_kb":384768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":224,"num_bytes_recovered":14680064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,5],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4,5","objects":558}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":748,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.047792+0000","last_change":"2023-09-23T22:33:49.192269+0000","last_active":"2023-09-23T22:34:41.047792+0000","last_peered":"2023-09-23T22:34:41.047792+0000","last_clean":"2023-09-23T22:34:41.047792+0000","last_became_active":"2023-09-23T22:33:49.191968+0000","last_became_peered":"2023-09-23T22:33:49.191968+0000","last_unstale":"2023-09-23T22:34:41.047792+0000","last_undegraded":"2023-09-23T22:34:41.047792+0000","last_fullsized":"2023-09-23T22:34:41.047792+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:44.724416+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":276,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053107+0000","last_change":"2023-09-23T22:27:52.156595+0000","last_active":"2023-09-23T22:34:44.053107+0000","last_peered":"2023-09-23T22:34:44.053107+0000","last_clean":"2023-09-23T22:34:44.053107+0000","last_became_active":"2023-09-23T22:27:52.099614+0000","last_became_peered":"2023-09-23T22:27:52.099614+0000","last_unstale":"2023-09-23T22:34:44.053107+0000","last_undegraded":"2023-09-23T22:34:44.053107+0000","last_fullsized":"2023-09-23T22:34:44.053107+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:36.114410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":372,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054682+0000","last_change":"2023-09-23T22:33:34.762641+0000","last_active":"2023-09-23T22:34:44.054682+0000","last_peered":"2023-09-23T22:34:44.054682+0000","last_clean":"2023-09-23T22:34:44.054682+0000","last_became_active":"2023-09-23T22:33:34.762242+0000","last_became_peered":"2023-09-23T22:33:34.762242+0000","last_unstale":"2023-09-23T22:34:44.054682+0000","last_undegraded":"2023-09-23T22:34:44.054682+0000","last_fullsized":"2023-09-23T22:34:44.054682+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:11.643483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.1","version":"442'11881","reported_seq":13436,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.426548+0000","last_change":"2023-09-23T22:34:43.089108+0000","last_active":"2023-09-23T22:34:44.426548+0000","last_peered":"2023-09-23T22:34:44.426548+0000","last_clean":"2023-09-23T22:34:44.426548+0000","last_became_active":"2023-09-23T22:34:43.055594+0000","last_became_peered":"2023-09-23T22:34:43.055594+0000","last_unstale":"2023-09-23T22:34:44.426548+0000","last_undegraded":"2023-09-23T22:34:44.426548+0000","last_fullsized":"2023-09-23T22:34:44.426548+0000","mapping_epoch":440,"log_start":"405'7371","ondisk_log_start":"405'7371","created":373,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4510,"log_dups_size":0,"ondisk_log_size":4510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:16.050112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37945344,"num_objects":579,"num_object_clones":0,"num_object_copies":1737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11881,"num_write_kb":398720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":725,"num_bytes_recovered":47513600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":834,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.047351+0000","last_change":"2023-09-23T22:34:36.009288+0000","last_active":"2023-09-23T22:34:41.047351+0000","last_peered":"2023-09-23T22:34:41.047351+0000","last_clean":"2023-09-23T22:34:41.047351+0000","last_became_active":"2023-09-23T22:34:36.008883+0000","last_became_peered":"2023-09-23T22:34:36.008883+0000","last_unstale":"2023-09-23T22:34:41.047351+0000","last_undegraded":"2023-09-23T22:34:41.047351+0000","last_fullsized":"2023-09-23T22:34:41.047351+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:58.600657+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054845+0000","last_change":"2023-09-23T22:29:01.594502+0000","last_active":"2023-09-23T22:34:44.054845+0000","last_peered":"2023-09-23T22:34:44.054845+0000","last_clean":"2023-09-23T22:34:44.054845+0000","last_became_active":"2023-09-23T22:29:01.594226+0000","last_became_peered":"2023-09-23T22:29:01.594226+0000","last_unstale":"2023-09-23T22:34:44.054845+0000","last_undegraded":"2023-09-23T22:34:44.054845+0000","last_fullsized":"2023-09-23T22:34:44.054845+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:09.922281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":35,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.049031+0000","last_change":"2023-09-23T22:33:46.208682+0000","last_active":"2023-09-23T22:34:41.049031+0000","last_peered":"2023-09-23T22:34:41.049031+0000","last_clean":"2023-09-23T22:34:41.049031+0000","last_became_active":"2023-09-23T22:33:46.208123+0000","last_became_peered":"2023-09-23T22:33:46.208123+0000","last_unstale":"2023-09-23T22:34:41.049031+0000","last_undegraded":"2023-09-23T22:34:41.049031+0000","last_fullsized":"2023-09-23T22:34:41.049031+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:38.082947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"443'11828","reported_seq":11906,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T22:34:45.236959+0000","last_change":"2023-09-23T22:31:08.225049+0000","last_active":"2023-09-23T22:34:45.236959+0000","last_peered":"2023-09-23T22:34:45.236959+0000","last_clean":"2023-09-23T22:34:45.236959+0000","last_became_active":"2023-09-23T22:31:08.224765+0000","last_became_peered":"2023-09-23T22:31:08.224765+0000","last_unstale":"2023-09-23T22:34:45.236959+0000","last_undegraded":"2023-09-23T22:34:45.236959+0000","last_fullsized":"2023-09-23T22:34:45.236959+0000","mapping_epoch":373,"log_start":"405'6944","ondisk_log_start":"405'6944","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4884,"log_dups_size":0,"ondisk_log_size":4884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.842171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37617664,"num_objects":574,"num_object_clones":0,"num_object_copies":1722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11828,"num_write_kb":396864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":444,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055067+0000","last_change":"2023-09-23T22:33:46.206965+0000","last_active":"2023-09-23T22:34:44.055067+0000","last_peered":"2023-09-23T22:34:44.055067+0000","last_clean":"2023-09-23T22:34:44.055067+0000","last_became_active":"2023-09-23T22:33:46.206638+0000","last_became_peered":"2023-09-23T22:33:46.206638+0000","last_unstale":"2023-09-23T22:34:44.055067+0000","last_undegraded":"2023-09-23T22:34:44.055067+0000","last_fullsized":"2023-09-23T22:34:44.055067+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:18.048059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":414,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053348+0000","last_change":"2023-09-23T22:33:34.760677+0000","last_active":"2023-09-23T22:34:44.053348+0000","last_peered":"2023-09-23T22:34:44.053348+0000","last_clean":"2023-09-23T22:34:44.053348+0000","last_became_active":"2023-09-23T22:33:34.760469+0000","last_became_peered":"2023-09-23T22:33:34.760469+0000","last_unstale":"2023-09-23T22:34:44.053348+0000","last_undegraded":"2023-09-23T22:34:44.053348+0000","last_fullsized":"2023-09-23T22:34:44.053348+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:18:51.602930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.3","version":"442'11909","reported_seq":12451,"reported_epoch":442,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:34:44.443519+0000","last_change":"2023-09-23T22:34:44.112701+0000","last_active":"2023-09-23T22:34:44.443519+0000","last_peered":"2023-09-23T22:34:44.443519+0000","last_clean":"2023-09-23T22:34:31.495881+0000","last_became_active":"2023-09-23T22:34:34.054449+0000","last_became_peered":"2023-09-23T22:34:34.054449+0000","last_unstale":"2023-09-23T22:34:44.443519+0000","last_undegraded":"2023-09-23T22:34:44.443519+0000","last_fullsized":"2023-09-23T22:34:44.443519+0000","mapping_epoch":431,"log_start":"405'7386","ondisk_log_start":"405'7386","created":373,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4523,"log_dups_size":0,"ondisk_log_size":4523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:52.106217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34930688,"num_objects":533,"num_object_clones":0,"num_object_copies":1599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":316,"num_objects_unfound":0,"num_objects_dirty":533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11909,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":225,"num_bytes_recovered":14745600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":533}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":817,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053888+0000","last_change":"2023-09-23T22:33:49.199701+0000","last_active":"2023-09-23T22:34:44.053888+0000","last_peered":"2023-09-23T22:34:44.053888+0000","last_clean":"2023-09-23T22:34:44.053888+0000","last_became_active":"2023-09-23T22:33:49.198936+0000","last_became_peered":"2023-09-23T22:33:49.198936+0000","last_unstale":"2023-09-23T22:34:44.053888+0000","last_undegraded":"2023-09-23T22:34:44.053888+0000","last_fullsized":"2023-09-23T22:34:44.053888+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.214906+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":169,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.051236+0000","last_change":"2023-09-23T22:34:32.964535+0000","last_active":"2023-09-23T22:34:43.051236+0000","last_peered":"2023-09-23T22:34:43.051236+0000","last_clean":"2023-09-23T22:34:43.051236+0000","last_became_active":"2023-09-23T22:34:32.964088+0000","last_became_peered":"2023-09-23T22:34:32.964088+0000","last_unstale":"2023-09-23T22:34:43.051236+0000","last_undegraded":"2023-09-23T22:34:43.051236+0000","last_fullsized":"2023-09-23T22:34:43.051236+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:45.342898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":169,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.052893+0000","last_change":"2023-09-23T22:34:32.984051+0000","last_active":"2023-09-23T22:34:43.052893+0000","last_peered":"2023-09-23T22:34:43.052893+0000","last_clean":"2023-09-23T22:34:43.052893+0000","last_became_active":"2023-09-23T22:34:32.983779+0000","last_became_peered":"2023-09-23T22:34:32.983779+0000","last_unstale":"2023-09-23T22:34:43.052893+0000","last_undegraded":"2023-09-23T22:34:43.052893+0000","last_fullsized":"2023-09-23T22:34:43.052893+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:29.405366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"437'11560","reported_seq":15707,"reported_epoch":437,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T22:34:39.634562+0000","last_change":"2023-09-23T22:34:34.034740+0000","last_active":"2023-09-23T22:34:39.634562+0000","last_peered":"2023-09-23T22:34:39.634562+0000","last_clean":"2023-09-23T22:33:45.054365+0000","last_became_active":"2023-09-23T22:34:19.426351+0000","last_became_peered":"2023-09-23T22:34:19.426351+0000","last_unstale":"2023-09-23T22:34:39.634562+0000","last_undegraded":"2023-09-23T22:34:39.634562+0000","last_fullsized":"2023-09-23T22:34:39.634562+0000","mapping_epoch":424,"log_start":"401'5704","ondisk_log_start":"401'5704","created":373,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":5856,"log_dups_size":0,"ondisk_log_size":5856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:19.391548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43646976,"num_objects":666,"num_object_clones":0,"num_object_copies":1998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":295,"num_objects_unfound":0,"num_objects_dirty":666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11560,"num_write_kb":391232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2018,"num_bytes_recovered":132251648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[0,4,1],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":666}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":758,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055603+0000","last_change":"2023-09-23T22:34:36.010427+0000","last_active":"2023-09-23T22:34:44.055603+0000","last_peered":"2023-09-23T22:34:44.055603+0000","last_clean":"2023-09-23T22:34:44.055603+0000","last_became_active":"2023-09-23T22:34:36.010161+0000","last_became_peered":"2023-09-23T22:34:36.010161+0000","last_unstale":"2023-09-23T22:34:44.055603+0000","last_undegraded":"2023-09-23T22:34:44.055603+0000","last_fullsized":"2023-09-23T22:34:44.055603+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:18.506595+0000","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":822,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.048622+0000","last_change":"2023-09-23T22:34:19.424868+0000","last_active":"2023-09-23T22:34:41.048622+0000","last_peered":"2023-09-23T22:34:41.048622+0000","last_clean":"2023-09-23T22:34:41.048622+0000","last_became_active":"2023-09-23T22:34:19.424312+0000","last_became_peered":"2023-09-23T22:34:19.424312+0000","last_unstale":"2023-09-23T22:34:41.048622+0000","last_undegraded":"2023-09-23T22:34:41.048622+0000","last_fullsized":"2023-09-23T22:34:41.048622+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:22.650850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":505,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.048893+0000","last_change":"2023-09-23T22:34:32.968665+0000","last_active":"2023-09-23T22:34:41.048893+0000","last_peered":"2023-09-23T22:34:41.048893+0000","last_clean":"2023-09-23T22:34:41.048893+0000","last_became_active":"2023-09-23T22:34:32.968352+0000","last_became_peered":"2023-09-23T22:34:32.968352+0000","last_unstale":"2023-09-23T22:34:41.048893+0000","last_undegraded":"2023-09-23T22:34:41.048893+0000","last_fullsized":"2023-09-23T22:34:41.048893+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:45.768172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":160,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.053887+0000","last_change":"2023-09-23T22:34:32.964342+0000","last_active":"2023-09-23T22:34:43.053887+0000","last_peered":"2023-09-23T22:34:43.053887+0000","last_clean":"2023-09-23T22:34:43.053887+0000","last_became_active":"2023-09-23T22:34:32.964166+0000","last_became_peered":"2023-09-23T22:34:32.964166+0000","last_unstale":"2023-09-23T22:34:43.053887+0000","last_undegraded":"2023-09-23T22:34:43.053887+0000","last_fullsized":"2023-09-23T22:34:43.053887+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:34.878333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"440'11726","reported_seq":17115,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T22:34:42.319496+0000","last_change":"2023-09-23T22:34:33.013780+0000","last_active":"2023-09-23T22:34:42.319496+0000","last_peered":"2023-09-23T22:34:42.319496+0000","last_clean":"2023-09-23T22:34:42.319496+0000","last_became_active":"2023-09-23T22:34:32.956906+0000","last_became_peered":"2023-09-23T22:34:32.956906+0000","last_unstale":"2023-09-23T22:34:42.319496+0000","last_undegraded":"2023-09-23T22:34:42.319496+0000","last_fullsized":"2023-09-23T22:34:42.319496+0000","mapping_epoch":430,"log_start":"400'5413","ondisk_log_start":"400'5413","created":373,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":6313,"log_dups_size":0,"ondisk_log_size":6313,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:41.801829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36831232,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11726,"num_write_kb":393216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2635,"num_bytes_recovered":172687360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":793,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.047603+0000","last_change":"2023-09-23T22:34:36.010632+0000","last_active":"2023-09-23T22:34:41.047603+0000","last_peered":"2023-09-23T22:34:41.047603+0000","last_clean":"2023-09-23T22:34:41.047603+0000","last_became_active":"2023-09-23T22:34:36.010058+0000","last_became_peered":"2023-09-23T22:34:36.010058+0000","last_unstale":"2023-09-23T22:34:41.047603+0000","last_undegraded":"2023-09-23T22:34:41.047603+0000","last_fullsized":"2023-09-23T22:34:41.047603+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:41.458035+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1321,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053978+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:34:44.053978+0000","last_peered":"2023-09-23T22:34:44.053978+0000","last_clean":"2023-09-23T22:34:44.053978+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:34:44.053978+0000","last_undegraded":"2023-09-23T22:34:44.053978+0000","last_fullsized":"2023-09-23T22:34:44.053978+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":392,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054330+0000","last_change":"2023-09-23T22:34:19.320033+0000","last_active":"2023-09-23T22:34:44.054330+0000","last_peered":"2023-09-23T22:34:44.054330+0000","last_clean":"2023-09-23T22:34:44.054330+0000","last_became_active":"2023-09-23T22:34:19.319679+0000","last_became_peered":"2023-09-23T22:34:19.319679+0000","last_unstale":"2023-09-23T22:34:44.054330+0000","last_undegraded":"2023-09-23T22:34:44.054330+0000","last_fullsized":"2023-09-23T22:34:44.054330+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:36.951622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":35,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.048792+0000","last_change":"2023-09-23T22:33:46.211031+0000","last_active":"2023-09-23T22:34:41.048792+0000","last_peered":"2023-09-23T22:34:41.048792+0000","last_clean":"2023-09-23T22:34:41.048792+0000","last_became_active":"2023-09-23T22:33:46.210699+0000","last_became_peered":"2023-09-23T22:33:46.210699+0000","last_unstale":"2023-09-23T22:34:41.048792+0000","last_undegraded":"2023-09-23T22:34:41.048792+0000","last_fullsized":"2023-09-23T22:34:41.048792+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:06:26.338584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.b","version":"443'11721","reported_seq":16921,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T22:34:45.225011+0000","last_change":"2023-09-23T22:34:34.033849+0000","last_active":"2023-09-23T22:34:45.225011+0000","last_peered":"2023-09-23T22:34:45.225011+0000","last_clean":"2023-09-23T22:34:45.225011+0000","last_became_active":"2023-09-23T22:34:33.993258+0000","last_became_peered":"2023-09-23T22:34:33.993258+0000","last_unstale":"2023-09-23T22:34:45.225011+0000","last_undegraded":"2023-09-23T22:34:45.225011+0000","last_fullsized":"2023-09-23T22:34:45.225011+0000","mapping_epoch":431,"log_start":"405'6816","ondisk_log_start":"405'6816","created":373,"last_epoch_clean":432,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4905,"log_dups_size":0,"ondisk_log_size":4905,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:24.380159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36765696,"num_objects":561,"num_object_clones":0,"num_object_copies":1683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11721,"num_write_kb":393024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2537,"num_bytes_recovered":166264832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":796,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.047547+0000","last_change":"2023-09-23T22:34:19.430835+0000","last_active":"2023-09-23T22:34:41.047547+0000","last_peered":"2023-09-23T22:34:41.047547+0000","last_clean":"2023-09-23T22:34:41.047547+0000","last_became_active":"2023-09-23T22:34:19.430446+0000","last_became_peered":"2023-09-23T22:34:19.430446+0000","last_unstale":"2023-09-23T22:34:41.047547+0000","last_undegraded":"2023-09-23T22:34:41.047547+0000","last_fullsized":"2023-09-23T22:34:41.047547+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:12.576379+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":61,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.047744+0000","last_change":"2023-09-23T22:33:46.222482+0000","last_active":"2023-09-23T22:34:41.047744+0000","last_peered":"2023-09-23T22:34:41.047744+0000","last_clean":"2023-09-23T22:34:41.047744+0000","last_became_active":"2023-09-23T22:33:46.205871+0000","last_became_peered":"2023-09-23T22:33:46.205871+0000","last_unstale":"2023-09-23T22:34:41.047744+0000","last_undegraded":"2023-09-23T22:34:41.047744+0000","last_fullsized":"2023-09-23T22:34:41.047744+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:54.426538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":35,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.047494+0000","last_change":"2023-09-23T22:33:46.213597+0000","last_active":"2023-09-23T22:34:41.047494+0000","last_peered":"2023-09-23T22:34:41.047494+0000","last_clean":"2023-09-23T22:34:41.047494+0000","last_became_active":"2023-09-23T22:33:46.213294+0000","last_became_peered":"2023-09-23T22:33:46.213294+0000","last_unstale":"2023-09-23T22:34:41.047494+0000","last_undegraded":"2023-09-23T22:34:41.047494+0000","last_fullsized":"2023-09-23T22:34:41.047494+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:29.121139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":100,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.052751+0000","last_change":"2023-09-23T22:34:32.968875+0000","last_active":"2023-09-23T22:34:43.052751+0000","last_peered":"2023-09-23T22:34:43.052751+0000","last_clean":"2023-09-23T22:34:43.052751+0000","last_became_active":"2023-09-23T22:34:32.968493+0000","last_became_peered":"2023-09-23T22:34:32.968493+0000","last_unstale":"2023-09-23T22:34:43.052751+0000","last_undegraded":"2023-09-23T22:34:43.052751+0000","last_fullsized":"2023-09-23T22:34:43.052751+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:22.290739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"443'11764","reported_seq":13493,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T22:34:45.220163+0000","last_change":"2023-09-23T22:34:38.105543+0000","last_active":"2023-09-23T22:34:45.220163+0000","last_peered":"2023-09-23T22:34:45.220163+0000","last_clean":"2023-09-23T22:34:45.220163+0000","last_became_active":"2023-09-23T22:34:38.065409+0000","last_became_peered":"2023-09-23T22:34:38.065409+0000","last_unstale":"2023-09-23T22:34:45.220163+0000","last_undegraded":"2023-09-23T22:34:45.220163+0000","last_fullsized":"2023-09-23T22:34:45.220163+0000","mapping_epoch":435,"log_start":"405'6793","ondisk_log_start":"405'6793","created":373,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4971,"log_dups_size":0,"ondisk_log_size":4971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:39.412374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34734080,"num_objects":530,"num_object_clones":0,"num_object_copies":1590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11764,"num_write_kb":393408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":813,"num_bytes_recovered":53280768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1233,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.048680+0000","last_change":"2023-09-23T22:34:36.009304+0000","last_active":"2023-09-23T22:34:41.048680+0000","last_peered":"2023-09-23T22:34:41.048680+0000","last_clean":"2023-09-23T22:34:41.048680+0000","last_became_active":"2023-09-23T22:34:36.008947+0000","last_became_peered":"2023-09-23T22:34:36.008947+0000","last_unstale":"2023-09-23T22:34:41.048680+0000","last_undegraded":"2023-09-23T22:34:41.048680+0000","last_fullsized":"2023-09-23T22:34:41.048680+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:06.227088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":710,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053972+0000","last_change":"2023-09-23T22:34:19.320750+0000","last_active":"2023-09-23T22:34:44.053972+0000","last_peered":"2023-09-23T22:34:44.053972+0000","last_clean":"2023-09-23T22:34:44.053972+0000","last_became_active":"2023-09-23T22:34:19.320462+0000","last_became_peered":"2023-09-23T22:34:19.320462+0000","last_unstale":"2023-09-23T22:34:44.053972+0000","last_undegraded":"2023-09-23T22:34:44.053972+0000","last_fullsized":"2023-09-23T22:34:44.053972+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:48.441688+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":226,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054941+0000","last_change":"2023-09-23T22:34:32.964756+0000","last_active":"2023-09-23T22:34:44.054941+0000","last_peered":"2023-09-23T22:34:44.054941+0000","last_clean":"2023-09-23T22:34:44.054941+0000","last_became_active":"2023-09-23T22:34:32.964128+0000","last_became_peered":"2023-09-23T22:34:32.964128+0000","last_unstale":"2023-09-23T22:34:44.054941+0000","last_undegraded":"2023-09-23T22:34:44.054941+0000","last_fullsized":"2023-09-23T22:34:44.054941+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:05.113761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":134,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.056059+0000","last_change":"2023-09-23T22:31:28.124597+0000","last_active":"2023-09-23T22:34:44.056059+0000","last_peered":"2023-09-23T22:34:44.056059+0000","last_clean":"2023-09-23T22:34:44.056059+0000","last_became_active":"2023-09-23T22:31:28.124245+0000","last_became_peered":"2023-09-23T22:31:28.124245+0000","last_unstale":"2023-09-23T22:34:44.056059+0000","last_undegraded":"2023-09-23T22:34:44.056059+0000","last_fullsized":"2023-09-23T22:34:44.056059+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.558632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"441'11787","reported_seq":13098,"reported_epoch":442,"state":"peering","last_fresh":"2023-09-23T22:34:44.050812+0000","last_change":"2023-09-23T22:34:44.050974+0000","last_active":"2023-09-23T22:34:44.050812+0000","last_peered":"2023-09-23T22:34:43.797088+0000","last_clean":"2023-09-23T22:34:43.797088+0000","last_became_active":"2023-09-23T22:34:33.974306+0000","last_became_peered":"2023-09-23T22:34:33.974306+0000","last_unstale":"2023-09-23T22:34:44.050812+0000","last_undegraded":"2023-09-23T22:34:44.050812+0000","last_fullsized":"2023-09-23T22:34:44.050812+0000","mapping_epoch":442,"log_start":"405'7280","ondisk_log_start":"405'7280","created":373,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4507,"log_dups_size":0,"ondisk_log_size":4507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:01.505252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38862848,"num_objects":593,"num_object_clones":0,"num_object_copies":1779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11787,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":602,"num_bytes_recovered":39452672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4,5],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":675,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054608+0000","last_change":"2023-09-23T22:31:21.362707+0000","last_active":"2023-09-23T22:34:44.054608+0000","last_peered":"2023-09-23T22:34:44.054608+0000","last_clean":"2023-09-23T22:34:44.054608+0000","last_became_active":"2023-09-23T22:29:04.055622+0000","last_became_peered":"2023-09-23T22:29:04.055622+0000","last_unstale":"2023-09-23T22:34:44.054608+0000","last_undegraded":"2023-09-23T22:34:44.054608+0000","last_fullsized":"2023-09-23T22:34:44.054608+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:58.622607+0000","scrub_duration":0.0034076229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":850,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.051323+0000","last_change":"2023-09-23T22:34:32.986616+0000","last_active":"2023-09-23T22:34:43.051323+0000","last_peered":"2023-09-23T22:34:43.051323+0000","last_clean":"2023-09-23T22:34:43.051323+0000","last_became_active":"2023-09-23T22:34:32.964909+0000","last_became_peered":"2023-09-23T22:34:32.964909+0000","last_unstale":"2023-09-23T22:34:43.051323+0000","last_undegraded":"2023-09-23T22:34:43.051323+0000","last_fullsized":"2023-09-23T22:34:43.051323+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:30.735998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":212,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054987+0000","last_change":"2023-09-23T22:34:19.322711+0000","last_active":"2023-09-23T22:34:44.054987+0000","last_peered":"2023-09-23T22:34:44.054987+0000","last_clean":"2023-09-23T22:34:44.054987+0000","last_became_active":"2023-09-23T22:34:19.322439+0000","last_became_peered":"2023-09-23T22:34:19.322439+0000","last_unstale":"2023-09-23T22:34:44.054987+0000","last_undegraded":"2023-09-23T22:34:44.054987+0000","last_fullsized":"2023-09-23T22:34:44.054987+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:51.411442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":344,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.056515+0000","last_change":"2023-09-23T22:34:19.426441+0000","last_active":"2023-09-23T22:34:44.056515+0000","last_peered":"2023-09-23T22:34:44.056515+0000","last_clean":"2023-09-23T22:34:44.056515+0000","last_became_active":"2023-09-23T22:34:19.426191+0000","last_became_peered":"2023-09-23T22:34:19.426191+0000","last_unstale":"2023-09-23T22:34:44.056515+0000","last_undegraded":"2023-09-23T22:34:44.056515+0000","last_fullsized":"2023-09-23T22:34:44.056515+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:47.541011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.f","version":"440'12051","reported_seq":13772,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T22:34:42.328687+0000","last_change":"2023-09-23T22:34:39.062276+0000","last_active":"2023-09-23T22:34:42.328687+0000","last_peered":"2023-09-23T22:34:42.328687+0000","last_clean":"2023-09-23T22:34:42.328687+0000","last_became_active":"2023-09-23T22:34:39.012980+0000","last_became_peered":"2023-09-23T22:34:39.012980+0000","last_unstale":"2023-09-23T22:34:42.328687+0000","last_undegraded":"2023-09-23T22:34:42.328687+0000","last_fullsized":"2023-09-23T22:34:42.328687+0000","mapping_epoch":436,"log_start":"405'7376","ondisk_log_start":"405'7376","created":373,"last_epoch_clean":437,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4675,"log_dups_size":0,"ondisk_log_size":4675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:07.151932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41615360,"num_objects":635,"num_object_clones":0,"num_object_copies":1905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12051,"num_write_kb":405952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":814,"num_bytes_recovered":53346304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,4],"acting":[2,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":793,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055544+0000","last_change":"2023-09-23T22:29:04.056681+0000","last_active":"2023-09-23T22:34:44.055544+0000","last_peered":"2023-09-23T22:34:44.055544+0000","last_clean":"2023-09-23T22:34:44.055544+0000","last_became_active":"2023-09-23T22:29:04.056323+0000","last_became_peered":"2023-09-23T22:29:04.056323+0000","last_unstale":"2023-09-23T22:34:44.055544+0000","last_undegraded":"2023-09-23T22:34:44.055544+0000","last_fullsized":"2023-09-23T22:34:44.055544+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:45.708088+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":371,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053131+0000","last_change":"2023-09-23T22:31:28.152186+0000","last_active":"2023-09-23T22:34:44.053131+0000","last_peered":"2023-09-23T22:34:44.053131+0000","last_clean":"2023-09-23T22:34:44.053131+0000","last_became_active":"2023-09-23T22:31:28.111292+0000","last_became_peered":"2023-09-23T22:31:28.111292+0000","last_unstale":"2023-09-23T22:34:44.053131+0000","last_undegraded":"2023-09-23T22:34:44.053131+0000","last_fullsized":"2023-09-23T22:34:44.053131+0000","mapping_epoch":380,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:06.097398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":44,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.048880+0000","last_change":"2023-09-23T22:34:32.974362+0000","last_active":"2023-09-23T22:34:41.048880+0000","last_peered":"2023-09-23T22:34:41.048880+0000","last_clean":"2023-09-23T22:34:41.048880+0000","last_became_active":"2023-09-23T22:34:32.973858+0000","last_became_peered":"2023-09-23T22:34:32.973858+0000","last_unstale":"2023-09-23T22:34:41.048880+0000","last_undegraded":"2023-09-23T22:34:41.048880+0000","last_fullsized":"2023-09-23T22:34:41.048880+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:15.264858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":266,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.051511+0000","last_change":"2023-09-23T22:34:32.979376+0000","last_active":"2023-09-23T22:34:43.051511+0000","last_peered":"2023-09-23T22:34:43.051511+0000","last_clean":"2023-09-23T22:34:43.051511+0000","last_became_active":"2023-09-23T22:34:32.979024+0000","last_became_peered":"2023-09-23T22:34:32.979024+0000","last_unstale":"2023-09-23T22:34:43.051511+0000","last_undegraded":"2023-09-23T22:34:43.051511+0000","last_fullsized":"2023-09-23T22:34:43.051511+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:13.975591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.c","version":"443'11815","reported_seq":16334,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T22:34:45.237332+0000","last_change":"2023-09-23T22:34:45.105978+0000","last_active":"2023-09-23T22:34:45.237332+0000","last_peered":"2023-09-23T22:34:45.237332+0000","last_clean":"2023-09-23T22:34:45.237332+0000","last_became_active":"2023-09-23T22:34:45.056789+0000","last_became_peered":"2023-09-23T22:34:45.056789+0000","last_unstale":"2023-09-23T22:34:45.237332+0000","last_undegraded":"2023-09-23T22:34:45.237332+0000","last_fullsized":"2023-09-23T22:34:45.237332+0000","mapping_epoch":442,"log_start":"405'6880","ondisk_log_start":"405'6880","created":373,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:45.732305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37158912,"num_objects":567,"num_object_clones":0,"num_object_copies":1701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11815,"num_write_kb":396224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2197,"num_bytes_recovered":143982592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":817,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055591+0000","last_change":"2023-09-23T22:33:49.195949+0000","last_active":"2023-09-23T22:34:44.055591+0000","last_peered":"2023-09-23T22:34:44.055591+0000","last_clean":"2023-09-23T22:34:44.055591+0000","last_became_active":"2023-09-23T22:33:49.195659+0000","last_became_peered":"2023-09-23T22:33:49.195659+0000","last_unstale":"2023-09-23T22:34:44.055591+0000","last_undegraded":"2023-09-23T22:34:44.055591+0000","last_fullsized":"2023-09-23T22:34:44.055591+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:16.241442+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":132,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053463+0000","last_change":"2023-09-23T22:34:19.322203+0000","last_active":"2023-09-23T22:34:44.053463+0000","last_peered":"2023-09-23T22:34:44.053463+0000","last_clean":"2023-09-23T22:34:44.053463+0000","last_became_active":"2023-09-23T22:34:19.321604+0000","last_became_peered":"2023-09-23T22:34:19.321604+0000","last_unstale":"2023-09-23T22:34:44.053463+0000","last_undegraded":"2023-09-23T22:34:44.053463+0000","last_fullsized":"2023-09-23T22:34:44.053463+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:49.087103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":939,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.052544+0000","last_change":"2023-09-23T22:29:04.058838+0000","last_active":"2023-09-23T22:34:43.052544+0000","last_peered":"2023-09-23T22:34:43.052544+0000","last_clean":"2023-09-23T22:34:43.052544+0000","last_became_active":"2023-09-23T22:29:04.058360+0000","last_became_peered":"2023-09-23T22:29:04.058360+0000","last_unstale":"2023-09-23T22:34:43.052544+0000","last_undegraded":"2023-09-23T22:34:43.052544+0000","last_fullsized":"2023-09-23T22:34:43.052544+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:31.380722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":22,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053428+0000","last_change":"2023-09-23T22:34:32.969775+0000","last_active":"2023-09-23T22:34:44.053428+0000","last_peered":"2023-09-23T22:34:44.053428+0000","last_clean":"2023-09-23T22:34:44.053428+0000","last_became_active":"2023-09-23T22:34:32.969468+0000","last_became_peered":"2023-09-23T22:34:32.969468+0000","last_unstale":"2023-09-23T22:34:44.053428+0000","last_undegraded":"2023-09-23T22:34:44.053428+0000","last_fullsized":"2023-09-23T22:34:44.053428+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:59.587600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.d","version":"440'11679","reported_seq":17849,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T22:34:42.332483+0000","last_change":"2023-09-23T22:34:38.116580+0000","last_active":"2023-09-23T22:34:42.332483+0000","last_peered":"2023-09-23T22:34:42.332483+0000","last_clean":"2023-09-23T22:34:42.332483+0000","last_became_active":"2023-09-23T22:34:38.069066+0000","last_became_peered":"2023-09-23T22:34:38.069066+0000","last_unstale":"2023-09-23T22:34:42.332483+0000","last_undegraded":"2023-09-23T22:34:42.332483+0000","last_fullsized":"2023-09-23T22:34:42.332483+0000","mapping_epoch":435,"log_start":"400'5420","ondisk_log_start":"400'5420","created":373,"last_epoch_clean":436,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:31:07.200270+0000","last_clean_scrub_stamp":"2023-09-23T22:31:07.200270+0000","objects_scrubbed":0,"log_size":6259,"log_dups_size":0,"ondisk_log_size":6259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:55.059221+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38010904,"num_objects":581,"num_object_clones":0,"num_object_copies":1743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":581,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11679,"num_write_kb":392257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3016,"num_bytes_recovered":197591064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":600,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054432+0000","last_change":"2023-09-23T22:34:42.461411+0000","last_active":"2023-09-23T22:34:44.054432+0000","last_peered":"2023-09-23T22:34:44.054432+0000","last_clean":"2023-09-23T22:34:44.054432+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:34:36.002999+0000","last_unstale":"2023-09-23T22:34:44.054432+0000","last_undegraded":"2023-09-23T22:34:44.054432+0000","last_fullsized":"2023-09-23T22:34:44.054432+0000","mapping_epoch":433,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":434,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.591643+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.591643+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1144050,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":174,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055431+0000","last_change":"2023-09-23T22:33:46.230339+0000","last_active":"2023-09-23T22:34:44.055431+0000","last_peered":"2023-09-23T22:34:44.055431+0000","last_clean":"2023-09-23T22:34:44.055431+0000","last_became_active":"2023-09-23T22:33:46.208097+0000","last_became_peered":"2023-09-23T22:33:46.208097+0000","last_unstale":"2023-09-23T22:34:44.055431+0000","last_undegraded":"2023-09-23T22:34:44.055431+0000","last_fullsized":"2023-09-23T22:34:44.055431+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:39.668780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":183,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.052484+0000","last_change":"2023-09-23T22:34:19.322776+0000","last_active":"2023-09-23T22:34:43.052484+0000","last_peered":"2023-09-23T22:34:43.052484+0000","last_clean":"2023-09-23T22:34:43.052484+0000","last_became_active":"2023-09-23T22:34:19.322454+0000","last_became_peered":"2023-09-23T22:34:19.322454+0000","last_unstale":"2023-09-23T22:34:43.052484+0000","last_undegraded":"2023-09-23T22:34:43.052484+0000","last_fullsized":"2023-09-23T22:34:43.052484+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:24.708890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":322,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053812+0000","last_change":"2023-09-23T22:34:32.964257+0000","last_active":"2023-09-23T22:34:44.053812+0000","last_peered":"2023-09-23T22:34:44.053812+0000","last_clean":"2023-09-23T22:34:44.053812+0000","last_became_active":"2023-09-23T22:34:32.963898+0000","last_became_peered":"2023-09-23T22:34:32.963898+0000","last_unstale":"2023-09-23T22:34:44.053812+0000","last_undegraded":"2023-09-23T22:34:44.053812+0000","last_fullsized":"2023-09-23T22:34:44.053812+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:57.657853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":60,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.051609+0000","last_change":"2023-09-23T22:34:19.321469+0000","last_active":"2023-09-23T22:34:43.051609+0000","last_peered":"2023-09-23T22:34:43.051609+0000","last_clean":"2023-09-23T22:34:43.051609+0000","last_became_active":"2023-09-23T22:34:19.320651+0000","last_became_peered":"2023-09-23T22:34:19.320651+0000","last_unstale":"2023-09-23T22:34:43.051609+0000","last_undegraded":"2023-09-23T22:34:43.051609+0000","last_fullsized":"2023-09-23T22:34:43.051609+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:06.082385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":35,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.048963+0000","last_change":"2023-09-23T22:33:46.210878+0000","last_active":"2023-09-23T22:34:41.048963+0000","last_peered":"2023-09-23T22:34:41.048963+0000","last_clean":"2023-09-23T22:34:41.048963+0000","last_became_active":"2023-09-23T22:33:46.210512+0000","last_became_peered":"2023-09-23T22:33:46.210512+0000","last_unstale":"2023-09-23T22:34:41.048963+0000","last_undegraded":"2023-09-23T22:34:41.048963+0000","last_fullsized":"2023-09-23T22:34:41.048963+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:35.605270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":433,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.054249+0000","last_change":"2023-09-23T22:27:52.181622+0000","last_active":"2023-09-23T22:34:44.054249+0000","last_peered":"2023-09-23T22:34:44.054249+0000","last_clean":"2023-09-23T22:34:44.054249+0000","last_became_active":"2023-09-23T22:27:52.091251+0000","last_became_peered":"2023-09-23T22:27:52.091251+0000","last_unstale":"2023-09-23T22:34:44.054249+0000","last_undegraded":"2023-09-23T22:34:44.054249+0000","last_fullsized":"2023-09-23T22:34:44.054249+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:42.222131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":448,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T22:34:43.052023+0000","last_change":"2023-09-23T22:33:46.213002+0000","last_active":"2023-09-23T22:34:43.052023+0000","last_peered":"2023-09-23T22:34:43.052023+0000","last_clean":"2023-09-23T22:34:43.052023+0000","last_became_active":"2023-09-23T22:33:46.212600+0000","last_became_peered":"2023-09-23T22:33:46.212600+0000","last_unstale":"2023-09-23T22:34:43.052023+0000","last_undegraded":"2023-09-23T22:34:43.052023+0000","last_fullsized":"2023-09-23T22:34:43.052023+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:09.953315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":458,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.053340+0000","last_change":"2023-09-23T22:29:01.675775+0000","last_active":"2023-09-23T22:34:44.053340+0000","last_peered":"2023-09-23T22:34:44.053340+0000","last_clean":"2023-09-23T22:34:44.053340+0000","last_became_active":"2023-09-23T22:29:01.597224+0000","last_became_peered":"2023-09-23T22:29:01.597224+0000","last_unstale":"2023-09-23T22:34:44.053340+0000","last_undegraded":"2023-09-23T22:34:44.053340+0000","last_fullsized":"2023-09-23T22:34:44.053340+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:47.781269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":227,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.047581+0000","last_change":"2023-09-23T22:34:19.426852+0000","last_active":"2023-09-23T22:34:41.047581+0000","last_peered":"2023-09-23T22:34:41.047581+0000","last_clean":"2023-09-23T22:34:41.047581+0000","last_became_active":"2023-09-23T22:34:19.426452+0000","last_became_peered":"2023-09-23T22:34:19.426452+0000","last_unstale":"2023-09-23T22:34:41.047581+0000","last_undegraded":"2023-09-23T22:34:41.047581+0000","last_fullsized":"2023-09-23T22:34:41.047581+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:50.108144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":144,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055372+0000","last_change":"2023-09-23T22:33:46.209205+0000","last_active":"2023-09-23T22:34:44.055372+0000","last_peered":"2023-09-23T22:34:44.055372+0000","last_clean":"2023-09-23T22:34:44.055372+0000","last_became_active":"2023-09-23T22:33:46.208800+0000","last_became_peered":"2023-09-23T22:33:46.208800+0000","last_unstale":"2023-09-23T22:34:44.055372+0000","last_undegraded":"2023-09-23T22:34:44.055372+0000","last_fullsized":"2023-09-23T22:34:44.055372+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:58.881258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":44,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.047857+0000","last_change":"2023-09-23T22:34:32.975238+0000","last_active":"2023-09-23T22:34:41.047857+0000","last_peered":"2023-09-23T22:34:41.047857+0000","last_clean":"2023-09-23T22:34:41.047857+0000","last_became_active":"2023-09-23T22:34:32.974927+0000","last_became_peered":"2023-09-23T22:34:32.974927+0000","last_unstale":"2023-09-23T22:34:41.047857+0000","last_undegraded":"2023-09-23T22:34:41.047857+0000","last_fullsized":"2023-09-23T22:34:41.047857+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:48.150567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":382,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T22:34:41.049563+0000","last_change":"2023-09-23T22:34:32.982953+0000","last_active":"2023-09-23T22:34:41.049563+0000","last_peered":"2023-09-23T22:34:41.049563+0000","last_clean":"2023-09-23T22:34:41.049563+0000","last_became_active":"2023-09-23T22:34:32.974430+0000","last_became_peered":"2023-09-23T22:34:32.974430+0000","last_unstale":"2023-09-23T22:34:41.049563+0000","last_undegraded":"2023-09-23T22:34:41.049563+0000","last_fullsized":"2023-09-23T22:34:41.049563+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:10.333737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":501,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T22:34:44.055086+0000","last_change":"2023-09-23T22:29:01.591795+0000","last_active":"2023-09-23T22:34:44.055086+0000","last_peered":"2023-09-23T22:34:44.055086+0000","last_clean":"2023-09-23T22:34:44.055086+0000","last_became_active":"2023-09-23T22:29:01.591381+0000","last_became_peered":"2023-09-23T22:29:01.591381+0000","last_unstale":"2023-09-23T22:34:44.055086+0000","last_undegraded":"2023-09-23T22:34:44.055086+0000","last_fullsized":"2023-09-23T22:34:44.055086+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:00.036227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":96,"num_bytes_recovered":27064,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31314,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":597557272,"num_objects":9119,"num_object_clones":0,"num_object_copies":27357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":947,"num_objects_unfound":0,"num_objects_dirty":9119,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":188661,"num_write_kb":6328897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24088,"num_bytes_recovered":1558380568,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2217820160,"data_stored":2217803872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82514,"ondisk_log_size":82514,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":316358,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2194802,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":265,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234364,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":46,"seq":197568496000,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":559972,"kb_used_data":344732,"kb_used_omap":0,"kb_used_meta":215232,"kb_avail":93185180,"statfs":{"total":95995035648,"available":95421624320,"internally_reserved":0,"allocated":353005568,"data_stored":351442761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":220397568},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,3],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:34:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":7.4400000000000004,"15min":3.4279999999999999},"min":{"1min":0.23100000000000001,"5min":0.22700000000000001,"15min":0.19600000000000001},"max":{"1min":13.723000000000001,"5min":213.23699999999999,"15min":213.23699999999999},"last":0.37},{"interface":"front","average":{"1min":3.1589999999999998,"5min":7.9119999999999999,"15min":4.4720000000000004},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":6.9500000000000002,"5min":213.304,"15min":213.304},"last":0.313}]},{"osd":1,"last update":"Sat Sep 23 22:33:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":2.0819999999999999,"15min":3.3969999999999998},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":6.2880000000000003,"5min":58.143999999999998,"15min":129.964},"last":0.29399999999999998},{"interface":"front","average":{"1min":2.0960000000000001,"5min":2.2040000000000002,"15min":3.7029999999999998},"min":{"1min":0.39000000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.0209999999999999,"5min":63.073999999999998,"15min":131.18600000000001},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:34:02 2023","interfaces":[{"interface":"back","average":{"1min":6.8780000000000001,"5min":2.9220000000000002,"15min":2.2429999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":103.548,"5min":103.548,"15min":103.548},"last":0.40799999999999997},{"interface":"front","average":{"1min":6.8620000000000001,"5min":2.9580000000000002,"15min":2.419},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":105.99299999999999,"5min":105.99299999999999,"15min":105.99299999999999},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:33:47 2023","interfaces":[{"interface":"back","average":{"1min":2.581,"5min":1.873,"15min":3.1709999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.8179999999999996,"5min":7.8179999999999996,"15min":125.125},"last":0.439},{"interface":"front","average":{"1min":1.8700000000000001,"5min":1.599,"15min":2.8279999999999998},"min":{"1min":0.39400000000000002,"5min":0.33300000000000002,"15min":0.26900000000000002},"max":{"1min":6.351,"5min":9.2569999999999997,"15min":76.406999999999996},"last":0.51300000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:33:57 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":1.4239999999999999,"15min":1.5249999999999999},"min":{"1min":0.27200000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.221,"5min":11.221,"15min":67.483999999999995},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.484,"5min":2.3889999999999998,"15min":1.845},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.255},"max":{"1min":10.385,"5min":11.561,"15min":67.814999999999998},"last":0.45800000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:33:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4329999999999998,"5min":3.3170000000000002,"15min":1.845},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":11.053000000000001,"5min":88.194999999999993,"15min":88.194999999999993},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.8169999999999999,"5min":2.5070000000000001,"15min":1.6120000000000001},"min":{"1min":0.32600000000000001,"5min":0.28499999999999998,"15min":0.26900000000000002},"max":{"1min":6.4820000000000002,"5min":88.025000000000006,"15min":88.025000000000006},"last":0.55900000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:34:24 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":1.556,"15min":1.2450000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.314},"max":{"1min":7.5670000000000002,"5min":9.2270000000000003,"15min":82.801000000000002},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.2810000000000001,"5min":2.0379999999999998,"15min":1.4219999999999999},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.29699999999999999},"max":{"1min":7.6929999999999996,"5min":10.324999999999999,"15min":83.052999999999997},"last":0.498}]}]},{"osd":1,"up_from":98,"seq":420906795272,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1050060,"kb_used_data":333892,"kb_used_omap":7,"kb_used_meta":716152,"kb_avail":92695092,"statfs":{"total":95995035648,"available":94919774208,"internally_reserved":0,"allocated":341905408,"data_stored":339595176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":733340020},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":6.5700000000000003,"15min":5.2850000000000001},"min":{"1min":0.20599999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":17.547000000000001,"5min":202.73500000000001,"15min":202.73500000000001},"last":5.4320000000000004},{"interface":"front","average":{"1min":1.649,"5min":7.0419999999999998,"15min":5.8499999999999996},"min":{"1min":0.20100000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.9649999999999999,"5min":202.80799999999999,"15min":202.80799999999999},"last":3.8180000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:33:52 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.88300000000000001,"15min":0.999},"min":{"1min":0.27000000000000002,"5min":0.216,"15min":0.216},"max":{"1min":2.8860000000000001,"5min":5.8250000000000002,"15min":5.8250000000000002},"last":5.4560000000000004},{"interface":"front","average":{"1min":1.248,"5min":1.6759999999999999,"15min":2.1389999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":8.4610000000000003,"5min":9.2349999999999994,"15min":9.2349999999999994},"last":3.847}]},{"osd":3,"last update":"Sat Sep 23 22:34:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9299999999999999,"5min":1.887,"15min":4.1669999999999998},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":5.9530000000000003,"5min":10.587999999999999,"15min":160.499},"last":3.7290000000000001},{"interface":"front","average":{"1min":3.0129999999999999,"5min":1.9350000000000001,"15min":4.0899999999999999},"min":{"1min":0.27500000000000002,"5min":0.255,"15min":0.223},"max":{"1min":11.757,"5min":11.757,"15min":160.465},"last":2.5329999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.5630000000000002,"5min":1.4990000000000001,"15min":2.3839999999999999},"min":{"1min":0.372,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":13.138,"5min":13.138,"15min":234.79900000000001},"last":5.3449999999999998},{"interface":"front","average":{"1min":2.6339999999999999,"5min":2.1859999999999999,"15min":2.9060000000000001},"min":{"1min":0.38400000000000001,"5min":0.32700000000000001,"15min":0.29499999999999998},"max":{"1min":12.246,"5min":12.246,"15min":234.81700000000001},"last":0.79000000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":2.173,"15min":2.4140000000000001},"min":{"1min":0.38700000000000001,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":7.0810000000000004,"5min":9.2759999999999998,"15min":196.79900000000001},"last":0.81999999999999995},{"interface":"front","average":{"1min":3.1760000000000002,"5min":1.8089999999999999,"15min":2.407},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.29099999999999998},"max":{"1min":12.006,"5min":12.006,"15min":196.68299999999999},"last":2.5939999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:34:39 2023","interfaces":[{"interface":"back","average":{"1min":4.6719999999999997,"5min":3.0760000000000001,"15min":2.3010000000000002},"min":{"1min":0.33700000000000002,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":17.757000000000001,"5min":77.033000000000001,"15min":77.033000000000001},"last":2.569},{"interface":"front","average":{"1min":3.7730000000000001,"5min":2.7930000000000001,"15min":1.776},"min":{"1min":0.39600000000000002,"5min":0.34100000000000003,"15min":0.32700000000000001},"max":{"1min":17.789000000000001,"5min":76.965999999999994,"15min":76.965999999999994},"last":5.407}]},{"osd":7,"last update":"Sat Sep 23 22:34:39 2023","interfaces":[{"interface":"back","average":{"1min":3.3029999999999999,"5min":2.2210000000000001,"15min":2.0419999999999998},"min":{"1min":0.39100000000000001,"5min":0.371,"15min":0.33800000000000002},"max":{"1min":11.849,"5min":11.849,"15min":16.265000000000001},"last":2.6259999999999999},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.603,"15min":1.3300000000000001},"min":{"1min":0.48199999999999998,"5min":0.318,"15min":0.318},"max":{"1min":13.173,"5min":13.173,"15min":13.173},"last":5.3810000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958801,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1105764,"kb_used_data":330396,"kb_used_omap":8,"kb_used_meta":775351,"kb_avail":92639388,"statfs":{"total":95995035648,"available":94862733312,"internally_reserved":0,"allocated":338325504,"data_stored":335933639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":793960298},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:34:41 2023","interfaces":[{"interface":"back","average":{"1min":20.896000000000001,"5min":10.239000000000001,"15min":6.0529999999999999},"min":{"1min":0.19900000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":234.55600000000001,"5min":288.96699999999998,"15min":288.96699999999998},"last":4.3220000000000001},{"interface":"front","average":{"1min":20.318000000000001,"5min":10.375999999999999,"15min":6},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":234.18600000000001,"5min":290.38099999999997,"15min":290.38099999999997},"last":4.6580000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:34:02 2023","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.8440000000000001,"15min":2.7450000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.468,"5min":22.263000000000002,"15min":22.263000000000002},"last":4.6909999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.1779999999999999,"15min":1.615},"min":{"1min":0.26200000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.7320000000000002,"5min":7.1920000000000002,"15min":7.1920000000000002},"last":4.0410000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:34:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":1.601,"15min":3.2309999999999999},"min":{"1min":0.29599999999999999,"5min":0.23200000000000001,"15min":0.22500000000000001},"max":{"1min":4.3470000000000004,"5min":10.292999999999999,"15min":146.87299999999999},"last":4.0960000000000001},{"interface":"front","average":{"1min":1.3089999999999999,"5min":1.8320000000000001,"15min":3.2549999999999999},"min":{"1min":0.27200000000000002,"5min":0.251,"15min":0.23799999999999999},"max":{"1min":4.6200000000000001,"5min":22.303000000000001,"15min":145.48099999999999},"last":4.6200000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:34:17 2023","interfaces":[{"interface":"back","average":{"1min":5.5700000000000003,"5min":3.169,"15min":2.5750000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":65.640000000000001,"5min":65.640000000000001,"15min":82.825000000000003},"last":0.30399999999999999},{"interface":"front","average":{"1min":5.2460000000000004,"5min":2.363,"15min":1.919},"min":{"1min":0.34200000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":65.349999999999994,"5min":65.349999999999994,"15min":75.930999999999997},"last":4.21}]},{"osd":5,"last update":"Sat Sep 23 22:34:17 2023","interfaces":[{"interface":"back","average":{"1min":2.2730000000000001,"5min":2.6890000000000001,"15min":2.827},"min":{"1min":0.32700000000000001,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":9.0239999999999991,"5min":13.564,"15min":93.685000000000002},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.7589999999999999,"5min":1.931,"15min":2.226},"min":{"1min":0.376,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.7590000000000003,"5min":13.486000000000001,"15min":93.546000000000006},"last":4.2510000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:34:17 2023","interfaces":[{"interface":"back","average":{"1min":5.7400000000000002,"5min":3.5670000000000002,"15min":2.4470000000000001},"min":{"1min":0.34499999999999997,"5min":0.308,"15min":0.308},"max":{"1min":74.355000000000004,"5min":74.355000000000004,"15min":74.355000000000004},"last":0.371},{"interface":"front","average":{"1min":5.327,"5min":2.7469999999999999,"15min":1.895},"min":{"1min":0.38400000000000001,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":73.150000000000006,"5min":73.150000000000006,"15min":73.150000000000006},"last":4.2839999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:34:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":2.1120000000000001,"15min":1.837},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.29999999999999999},"max":{"1min":5.4370000000000003,"5min":22.338999999999999,"15min":32.773000000000003},"last":4.577},{"interface":"front","average":{"1min":1.9870000000000001,"5min":2.48,"15min":2.1120000000000001},"min":{"1min":0.42599999999999999,"5min":0.34200000000000003,"15min":0.307},"max":{"1min":9.0879999999999992,"5min":10.666,"15min":16.381},"last":0.40200000000000002}]}]},{"osd":2,"up_from":349,"seq":1498943586364,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":532660,"kb_used_data":347372,"kb_used_omap":15,"kb_used_meta":185264,"kb_avail":93212492,"statfs":{"total":95995035648,"available":95449591808,"internally_reserved":0,"allocated":355708928,"data_stored":353462600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":189711094},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:34:00 2023","interfaces":[{"interface":"back","average":{"1min":5.1669999999999998,"5min":9.4760000000000009,"15min":15.913},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":80.793999999999997,"5min":194.06299999999999,"15min":194.06299999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":5.1820000000000004,"5min":10.583,"15min":17.853999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":80.894000000000005,"5min":196.947,"15min":196.947},"last":0.27200000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:34:00 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":5.1900000000000004,"15min":7.1559999999999997},"min":{"1min":0.23899999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.1349999999999998,"5min":130.63800000000001,"15min":130.63800000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.1459999999999999,"5min":5.8250000000000002,"15min":8.125},"min":{"1min":0.29399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.1970000000000001,"5min":130.66399999999999,"15min":130.66399999999999},"last":0.307}]},{"osd":3,"last update":"Sat Sep 23 22:34:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1639999999999999,"5min":1.179,"15min":1.5429999999999999},"min":{"1min":0.26000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.7720000000000002,"5min":7.915,"15min":7.915},"last":0.47499999999999998},{"interface":"front","average":{"1min":1.254,"5min":0.876,"15min":0.92000000000000004},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.1779999999999999,"5min":6.5549999999999997,"15min":6.5549999999999997},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:34:00 2023","interfaces":[{"interface":"back","average":{"1min":7.5279999999999996,"5min":2.3919999999999999,"15min":1.47},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":120.645,"5min":120.645,"15min":120.645},"last":0.67800000000000005},{"interface":"front","average":{"1min":8.0600000000000005,"5min":2.7829999999999999,"15min":1.851},"min":{"1min":0.371,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":120.19499999999999,"5min":120.19499999999999,"15min":120.19499999999999},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:34:00 2023","interfaces":[{"interface":"back","average":{"1min":15.797000000000001,"5min":4.3710000000000004,"15min":2.4609999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":144.16900000000001,"5min":144.16900000000001,"15min":144.16900000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":15.664999999999999,"5min":4.0149999999999997,"15min":1.992},"min":{"1min":0.40899999999999997,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":143.815,"5min":143.815,"15min":143.815},"last":0.63900000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:34:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":1.712,"15min":1.494},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":4.2249999999999996,"5min":8.3100000000000005,"15min":8.3100000000000005},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.5549999999999999,"5min":1.7549999999999999,"15min":1.708},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.2030000000000003,"5min":8.3330000000000002,"15min":8.3330000000000002},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:34:00 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":1.419,"15min":1.345},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.6059999999999999,"5min":6.4980000000000002,"15min":6.4980000000000002},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.204,"15min":1.3440000000000001},"min":{"1min":0.42499999999999999,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":3.29,"5min":4.8239999999999998,"15min":4.8239999999999998},"last":0.65300000000000002}]}]},{"osd":3,"up_from":0,"seq":463856468219,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":30,"seq":128849019269,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":957532,"kb_used_data":501396,"kb_used_omap":0,"kb_used_meta":456128,"kb_avail":92787620,"statfs":{"total":95995035648,"available":95014522880,"internally_reserved":0,"allocated":513429504,"data_stored":511874314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":467075072},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:34:36 2023","interfaces":[{"interface":"back","average":{"1min":8.5730000000000004,"5min":6.4320000000000004,"15min":4.9720000000000004},"min":{"1min":0.20799999999999999,"5min":0.20499999999999999,"15min":0.19700000000000001},"max":{"1min":128.43600000000001,"5min":128.43600000000001,"15min":150.63499999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":8.4239999999999995,"5min":6.5599999999999996,"15min":5.0410000000000004},"min":{"1min":0.28899999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":128.42099999999999,"5min":128.42099999999999,"15min":150.672},"last":0.38500000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:34:36 2023","interfaces":[{"interface":"back","average":{"1min":20.606000000000002,"5min":10.266999999999999,"15min":5.6520000000000001},"min":{"1min":0.23799999999999999,"5min":0.224,"15min":0.224},"max":{"1min":168.92099999999999,"5min":168.92099999999999,"15min":168.92099999999999},"last":0.504},{"interface":"front","average":{"1min":21.209,"5min":10.337,"15min":5.7919999999999998},"min":{"1min":0.32800000000000001,"5min":0.26100000000000001,"15min":0.22900000000000001},"max":{"1min":169.18899999999999,"5min":169.18899999999999,"15min":169.18899999999999},"last":0.318}]},{"osd":2,"last update":"Sat Sep 23 22:33:56 2023","interfaces":[{"interface":"back","average":{"1min":12.071,"5min":3.8639999999999999,"15min":2.734},"min":{"1min":0.313,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":165.667,"5min":165.667,"15min":165.667},"last":0.35299999999999998},{"interface":"front","average":{"1min":11.707000000000001,"5min":3.5139999999999998,"15min":2.198},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":165.685,"5min":165.685,"15min":165.685},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:33:46 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":1.363,"15min":2.4489999999999998},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.29699999999999999},"max":{"1min":9.1560000000000006,"5min":9.1560000000000006,"15min":102.896},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.4870000000000001,"5min":1.4970000000000001,"15min":2.516},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.27000000000000002},"max":{"1min":6.3099999999999996,"5min":14.318,"15min":112.667},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:33:46 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":1.6659999999999999,"15min":1.157},"min":{"1min":0.28299999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.8849999999999998,"5min":13.407,"15min":28.399000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.7040000000000002,"5min":1.7390000000000001,"15min":1.25},"min":{"1min":0.35799999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":14.016999999999999,"5min":14.212,"15min":28.350999999999999},"last":0.436}]},{"osd":6,"last update":"Sat Sep 23 22:34:02 2023","interfaces":[{"interface":"back","average":{"1min":3.27,"5min":1.601,"15min":1.0509999999999999},"min":{"1min":0.33300000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":22.193000000000001,"5min":22.193000000000001,"15min":22.193000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":3.681,"5min":1.865,"15min":1.099},"min":{"1min":0.47599999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":22.231999999999999,"5min":22.231999999999999,"15min":22.231999999999999},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:34:36 2023","interfaces":[{"interface":"back","average":{"1min":1.266,"5min":1.482,"15min":0.98199999999999998},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.27500000000000002},"max":{"1min":4.9649999999999999,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.903,"5min":1.542,"15min":1.0489999999999999},"min":{"1min":0.35099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.2770000000000001,"5min":9.1110000000000007,"15min":9.1110000000000007},"last":0.40200000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757637,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":813316,"kb_used_data":364604,"kb_used_omap":0,"kb_used_meta":448704,"kb_avail":92931836,"statfs":{"total":95995035648,"available":95162200064,"internally_reserved":0,"allocated":373354496,"data_stored":371818431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":459472896},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:34:27 2023","interfaces":[{"interface":"back","average":{"1min":8,"5min":6.1269999999999998,"15min":5.2009999999999996},"min":{"1min":0.35499999999999998,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":103.51000000000001,"5min":124.892,"15min":163.54499999999999},"last":2.6030000000000002},{"interface":"front","average":{"1min":6.9130000000000003,"5min":5.2930000000000001,"15min":4.7599999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.22900000000000001},"max":{"1min":103.80800000000001,"5min":123.807,"15min":162.81800000000001},"last":0.25800000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:34:38 2023","interfaces":[{"interface":"back","average":{"1min":2.0859999999999999,"5min":2.7109999999999999,"15min":3.1120000000000001},"min":{"1min":0.254,"5min":0.254,"15min":0.23799999999999999},"max":{"1min":12.055999999999999,"5min":61.621000000000002,"15min":113.514},"last":2.6440000000000001},{"interface":"front","average":{"1min":1.895,"5min":2.8679999999999999,"15min":3.1339999999999999},"min":{"1min":0.42599999999999999,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":6.0389999999999997,"5min":61.645000000000003,"15min":113.42700000000001},"last":2.657}]},{"osd":2,"last update":"Sat Sep 23 22:33:57 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.8999999999999999,"15min":2.2400000000000002},"min":{"1min":0.318,"5min":0.316,"15min":0.316},"max":{"1min":3.5470000000000002,"5min":10.451000000000001,"15min":10.451000000000001},"last":3.1059999999999999},{"interface":"front","average":{"1min":2.0470000000000002,"5min":2.0489999999999999,"15min":2.3690000000000002},"min":{"1min":0.375,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.3390000000000004,"5min":9.3390000000000004,"15min":9.3390000000000004},"last":2.4510000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:34:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7130000000000001,"5min":1.9850000000000001,"15min":3.7480000000000002},"min":{"1min":0.379,"5min":0.30199999999999999,"15min":0.28299999999999997},"max":{"1min":9.4120000000000008,"5min":9.4120000000000008,"15min":132.59},"last":4.1029999999999998},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.845,"15min":3.6030000000000002},"min":{"1min":0.40899999999999997,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.0119999999999996,"5min":7.8390000000000004,"15min":129.381},"last":1.1699999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:34:38 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":2.012,"15min":1.615},"min":{"1min":0.246,"5min":0.246,"15min":0.24099999999999999},"max":{"1min":5.8879999999999999,"5min":36.343000000000004,"15min":126.679},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.4590000000000001,"5min":2.423,"15min":1.7789999999999999},"min":{"1min":0.34100000000000003,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":9.5199999999999996,"5min":36.421999999999997,"15min":126.78100000000001},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:33:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7570000000000001,"5min":2.3610000000000002,"15min":1.2470000000000001},"min":{"1min":0.36299999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.4309999999999992,"5min":70.581999999999994,"15min":70.581999999999994},"last":0.312},{"interface":"front","average":{"1min":1.722,"5min":2.3079999999999998,"15min":1.214},"min":{"1min":0.33300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":4.6230000000000002,"5min":71.093000000000004,"15min":71.093000000000004},"last":0.39200000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:34:27 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.7689999999999999,"15min":1.093},"min":{"1min":0.35199999999999998,"5min":0.28000000000000003,"15min":0.27100000000000002},"max":{"1min":12.013999999999999,"5min":40.548000000000002,"15min":40.548000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.1360000000000001,"5min":2.157,"15min":1.1539999999999999},"min":{"1min":0.38600000000000001,"5min":0.34499999999999997,"15min":0.308},"max":{"1min":8.3089999999999993,"5min":41.073,"15min":41.073},"last":0.45000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":331022336,"data_stored":331022336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327946240,"data_stored":327942168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":336990208,"data_stored":336986136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":498466816,"data_stored":498466816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":350687232,"data_stored":350683160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":359927808,"data_stored":359923736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":12779520,"data_stored":12779520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:34:48.966 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 16 2023-09-23T22:34:48.966 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:49.032 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:34:49.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:49.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:34:49.141 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:49.178 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:49.231 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:49.234 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:34:49.234 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:34:49.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:34:49.234 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:34:49.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.235 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83452 7.e 7:7212fb35:::benchmark_data_smithi026_410749_object83451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:34:49.235 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.976183+0000", 2023-09-23T22:34:49.235 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 521.25302081400002, 2023-09-23T22:34:49.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.748405781, 2023-09-23T22:34:49.235 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.235 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.236 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.236 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.236 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.236 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83452 2023-09-23T22:34:49.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.236 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976183+0000", 2023-09-23T22:34:49.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976183+0000", 2023-09-23T22:34:49.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976181+0000", 2023-09-23T22:34:49.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:34:49.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976213+0000", 2023-09-23T22:34:49.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2251e-05 2023-09-23T22:34:49.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976230+0000", 2023-09-23T22:34:49.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6982000000000001e-05 2023-09-23T22:34:49.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976242+0000", 2023-09-23T22:34:49.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.219e-05 2023-09-23T22:34:49.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976670+0000", 2023-09-23T22:34:49.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042763099999999999 2023-09-23T22:34:49.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976680+0000", 2023-09-23T22:34:49.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9739999999999997e-06 2023-09-23T22:34:49.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.703132+0000", 2023-09-23T22:34:49.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7264517640000001 2023-09-23T22:34:49.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705750+0000", 2023-09-23T22:34:49.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026186209999999998 2023-09-23T22:34:49.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:34:49.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.705939+0000", 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018901399999999999 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.712672+0000", 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067324810000000002 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.716114+0000", 2023-09-23T22:34:49.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034423779999999998 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720906+0000", 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047918270000000002 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.720954+0000", 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8099000000000001e-05 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.724589+0000", 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036344390000000002 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:83457 7.5 7:ae7c1643:::benchmark_data_smithi026_410749_object83456:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e272)", 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:07.976262+0000", 2023-09-23T22:34:49.247 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 521.25294186500003, 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.709729018, 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 83457 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976262+0000", 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976262+0000", 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976261+0000", 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976329+0000", 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7922999999999993e-05 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976339+0000", 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0276e-05 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976347+0000", 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5920000000000003e-06 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976407+0000", 2023-09-23T22:34:49.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0486000000000003e-05 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:07.976419+0000", 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1871e-05 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.662197+0000", 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6857776609999999 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.662351+0000", 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153938 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.662471+0000", 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012044099999999999 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.669804+0000", 2023-09-23T22:34:49.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073327050000000001 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.670014+0000", 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000209817 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.681794+0000", 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011780163 2023-09-23T22:34:49.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.681867+0000", 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3206000000000003e-05 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:09.685991+0000", 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041235569999999999 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.255 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.256 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.256 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115950 7.c 7:34d3d495:::benchmark_data_smithi026_410749_object25613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T22:34:49.256 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:27.660282+0000", 2023-09-23T22:34:49.256 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 381.56892180300002, 2023-09-23T22:34:49.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0168458550000001, 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115950 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660282+0000", 2023-09-23T22:34:49.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660282+0000", 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660280+0000", 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:34:49.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660288+0000", 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9459999999999998e-06 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660301+0000", 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2959000000000001e-05 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.660309+0000", 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3289999999999994e-06 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.510116+0000", 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.849806879 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.525207+0000", 2023-09-23T22:34:49.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015090694999999999 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.592500+0000", 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.067292850000000001 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.597705+0000", 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052054579999999996 2023-09-23T22:34:49.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:34:49.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.608095+0000", 2023-09-23T22:34:49.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010389557000000001 2023-09-23T22:34:49.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.640854+0000", 2023-09-23T22:34:49.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.032759282000000001 2023-09-23T22:34:49.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.642265+0000", 2023-09-23T22:34:49.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014111219999999999 2023-09-23T22:34:49.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.645324+0000", 2023-09-23T22:34:49.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030586519999999998 2023-09-23T22:34:49.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.647991+0000", 2023-09-23T22:34:49.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026672029999999999 2023-09-23T22:34:49.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.652086+0000", 2023-09-23T22:34:49.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040946309999999996 2023-09-23T22:34:49.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.677128+0000", 2023-09-23T22:34:49.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025042180000000001 2023-09-23T22:34:49.267 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.267 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.267 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.267 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:115953 7.4 7:2a290ef6:::benchmark_data_smithi026_410749_object25616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T22:34:49.268 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:27.887950+0000", 2023-09-23T22:34:49.268 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 381.34125393699998, 2023-09-23T22:34:49.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7040569749999999, 2023-09-23T22:34:49.268 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.268 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.268 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.268 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.269 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.269 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 115953 2023-09-23T22:34:49.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.269 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887950+0000", 2023-09-23T22:34:49.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887950+0000", 2023-09-23T22:34:49.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887949+0000", 2023-09-23T22:34:49.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:34:49.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887955+0000", 2023-09-23T22:34:49.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6150000000000002e-06 2023-09-23T22:34:49.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887969+0000", 2023-09-23T22:34:49.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3961000000000001e-05 2023-09-23T22:34:49.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:27.887977+0000", 2023-09-23T22:34:49.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.571e-06 2023-09-23T22:34:49.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.560215+0000", 2023-09-23T22:34:49.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6722379759999999 2023-09-23T22:34:49.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.560336+0000", 2023-09-23T22:34:49.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121458 2023-09-23T22:34:49.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:34:49.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.560445+0000", 2023-09-23T22:34:49.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108448 2023-09-23T22:34:49.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.565997+0000", 2023-09-23T22:34:49.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055525820000000004 2023-09-23T22:34:49.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.566005+0000", 2023-09-23T22:34:49.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5989999999999996e-06 2023-09-23T22:34:49.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.591899+0000", 2023-09-23T22:34:49.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025894090000000002 2023-09-23T22:34:49.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.591964+0000", 2023-09-23T22:34:49.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4912000000000005e-05 2023-09-23T22:34:49.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:30.592007+0000", 2023-09-23T22:34:49.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3003999999999997e-05 2023-09-23T22:34:49.279 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.279 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.280 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.280 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117528 7.c 7:3d671bb3:::benchmark_data_smithi026_410749_object27191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:34:49.280 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.558906+0000", 2023-09-23T22:34:49.280 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 365.67029783200002, 2023-09-23T22:34:49.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.678631703, 2023-09-23T22:34:49.281 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.281 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.281 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117528 2023-09-23T22:34:49.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.282 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558906+0000", 2023-09-23T22:34:49.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558906+0000", 2023-09-23T22:34:49.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558905+0000", 2023-09-23T22:34:49.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:34:49.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558910+0000", 2023-09-23T22:34:49.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6879999999999996e-06 2023-09-23T22:34:49.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558922+0000", 2023-09-23T22:34:49.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1355000000000001e-05 2023-09-23T22:34:49.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.558928+0000", 2023-09-23T22:34:49.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0589999999999999e-06 2023-09-23T22:34:49.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.947133+0000", 2023-09-23T22:34:49.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.388205039 2023-09-23T22:34:49.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:34:49.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.970820+0000", 2023-09-23T22:34:49.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023686631999999999 2023-09-23T22:34:49.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.055724+0000", 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.084904269000000004 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.063411+0000", 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076874899999999999 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.076919+0000", 2023-09-23T22:34:49.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013507277 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.143918+0000", 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.066999283000000007 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.215628+0000", 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.071710228000000001 2023-09-23T22:34:49.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.217535+0000", 2023-09-23T22:34:49.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00190687 2023-09-23T22:34:49.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.219406+0000", 2023-09-23T22:34:49.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018707789999999999 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.222250+0000", 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028438980000000001 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.237538+0000", 2023-09-23T22:34:49.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015287945000000001 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117532 7.c 7:34bdbea4:::benchmark_data_smithi026_410749_object27195:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.692044+0000", 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 365.53715925500001, 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.585611748, 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.292 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117532 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692044+0000", 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692044+0000", 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692043+0000", 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:34:49.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692052+0000", 2023-09-23T22:34:49.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8149999999999995e-06 2023-09-23T22:34:49.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692064+0000", 2023-09-23T22:34:49.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.287e-05 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.692073+0000", 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5059999999999995e-06 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.972002+0000", 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2799290349999999 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.988907+0000", 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016904634000000002 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.117641+0000", 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12873475000000001 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.122617+0000", 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049758129999999999 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.134973+0000", 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012355414 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.212825+0000", 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.077852370000000004 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.240596+0000", 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027771139 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.242949+0000", 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023527230000000001 2023-09-23T22:34:49.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.245243+0000", 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022938670000000002 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.249823+0000", 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045802660000000004 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.277656+0000", 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027833238 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117533 7.c 7:3256a181:::benchmark_data_smithi026_410749_object27196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:43.722390+0000", 2023-09-23T22:34:49.301 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 365.50681326199998, 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5963978089999999, 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117533 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722390+0000", 2023-09-23T22:34:49.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722390+0000", 2023-09-23T22:34:49.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722389+0000", 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722399+0000", 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0902e-05 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722415+0000", 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5068999999999999e-05 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:43.722425+0000", 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0562e-05 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.989730+0000", 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.267304719 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.013955+0000", 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024225126999999999 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.176096+0000", 2023-09-23T22:34:49.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16214060799999999 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.186383+0000", 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010287566999999999 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.198704+0000", 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01232137 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.238136+0000", 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.039431352000000003 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.282727+0000", 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.044590888000000002 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.287402+0000", 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0046750940000000003 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.292487+0000", 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050851259999999997 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.297353+0000", 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004866006 2023-09-23T22:34:49.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.318788+0000", 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021435279000000002 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:117550 7.c 7:387185b4:::benchmark_data_smithi026_410749_object27213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e324)", 2023-09-23T22:34:49.310 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:28:44.045345+0000", 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 365.183859047, 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.502615335, 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 117550 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045345+0000", 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045345+0000", 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045342+0000", 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045353+0000", 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1263e-05 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045371+0000", 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7277000000000001e-05 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:44.045382+0000", 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1154e-05 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.319779+0000", 2023-09-23T22:34:49.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.274396912 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.351357+0000", 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.031578531999999999 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.409179+0000", 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.057821600000000001 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.414769+0000", 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055900719999999997 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,7", 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.425662+0000", 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010892808 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.525001+0000", 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.099338953999999993 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.525613+0000", 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00061288300000000005 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.527861+0000", 2023-09-23T22:34:49.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022476280000000002 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.529884+0000", 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002022739 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.533315+0000", 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034314139999999998 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:28:45.547960+0000", 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014644714 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119469 7.c 7:37fe5ea5:::benchmark_data_smithi026_410749_object29132:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.322397+0000", 2023-09-23T22:34:49.319 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.90680698900002, 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9204321049999997, 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119469 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322397+0000", 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322397+0000", 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322396+0000", 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322399+0000", 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.548e-06 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322408+0000", 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2410000000000001e-06 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322413+0000", 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.135e-06 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322457+0000", 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3455999999999999e-05 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322472+0000", 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5319999999999999e-05 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053967+0000", 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.73149523100000002 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053971+0000", 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.737e-06 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.223011+0000", 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1690396930000002 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.224476+0000", 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001465273 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.226493+0000", 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020173280000000001 2023-09-23T22:34:49.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.238434+0000", 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011941267 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.238582+0000", 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147921 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.238990+0000", 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040729999999999998 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.239462+0000", 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047265700000000003 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.240135+0000", 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00067237199999999997 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.242829+0000", 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026940850000000001 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.328 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119490 7.c 7:3eaefa36:::benchmark_data_smithi026_410749_object29153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.356832+0000", 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.87237171999999, 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8905167999999999, 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119490 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356832+0000", 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356832+0000", 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356831+0000", 2023-09-23T22:34:49.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356836+0000", 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1410000000000004e-06 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356845+0000", 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8470000000000008e-06 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356852+0000", 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8990000000000004e-06 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356902+0000", 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0081999999999999e-05 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356914+0000", 2023-09-23T22:34:49.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1416e-05 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053982+0000", 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.69706834200000001 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053986+0000", 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3809999999999999e-06 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.230378+0000", 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1763928799999999 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.231993+0000", 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001614376 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:34:49.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.234257+0000", 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022642140000000001 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.242966+0000", 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087088479999999999 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.243295+0000", 2023-09-23T22:34:49.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032858300000000002 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.243696+0000", 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040189499999999999 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.244112+0000", 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041517399999999998 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.244703+0000", 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059169699999999997 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.247349+0000", 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002645484 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119497 7.c 7:362bd5ae:::benchmark_data_smithi026_410749_object29160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.362138+0000", 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.86706574900001, 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9043871029999999, 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119497 2023-09-23T22:34:49.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362138+0000", 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362138+0000", 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362136+0000", 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362143+0000", 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1210000000000001e-06 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362156+0000", 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2310000000000001e-05 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362165+0000", 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7140000000000005e-06 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362228+0000", 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2415999999999993e-05 2023-09-23T22:34:49.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362236+0000", 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8429999999999999e-06 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053996+0000", 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.69175968499999996 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053999+0000", 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3210000000000001e-06 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.247536+0000", 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1935366379999999 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.248998+0000", 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001461681 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.251332+0000", 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023344030000000001 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.262650+0000", 2023-09-23T22:34:49.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011318004 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.262974+0000", 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032428099999999999 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.263074+0000", 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9837e-05 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.263467+0000", 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039298099999999998 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.264022+0000", 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055437699999999999 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.266525+0000", 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025033540000000002 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119502 7.c 7:39b9d062:::benchmark_data_smithi026_410749_object29165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.368183+0000", 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.86102108900002, 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9164363140000003, 2023-09-23T22:34:49.347 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119502 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368183+0000", 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368183+0000", 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368182+0000", 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368187+0000", 2023-09-23T22:34:49.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6529999999999999e-06 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368199+0000", 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1875000000000001e-05 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368207+0000", 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0499999999999992e-06 2023-09-23T22:34:49.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368265+0000", 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8074e-05 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368277+0000", 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1640999999999999e-05 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054010+0000", 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68573267699999996 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054013+0000", 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3349999999999998e-06 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.254782+0000", 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2007688540000001 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.256184+0000", 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014024669999999999 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.258340+0000", 2023-09-23T22:34:49.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021561169999999999 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.266945+0000", 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086043339999999999 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.267354+0000", 2023-09-23T22:34:49.354 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040936299999999999 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.267803+0000", 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044911400000000002 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.282079+0000", 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014275512000000001 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.282581+0000", 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050250700000000002 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.284619+0000", 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002037681 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119523 7.c 7:3440ea60:::benchmark_data_smithi026_410749_object29186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.498288+0000", 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.73091611500001, 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7907858110000001, 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119523 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498288+0000", 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498288+0000", 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.358 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498287+0000", 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498292+0000", 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4890000000000003e-06 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498301+0000", 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9789999999999999e-06 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498315+0000", 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3635000000000001e-05 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498356+0000", 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1279999999999998e-05 2023-09-23T22:34:49.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498362+0000", 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1199999999999999e-06 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054024+0000", 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.555662137 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054028+0000", 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.343e-06 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.267979+0000", 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2139512649999999 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.269550+0000", 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001571645 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.271850+0000", 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022990599999999999 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285119+0000", 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013269311000000001 2023-09-23T22:34:49.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285481+0000", 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036202899999999999 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285593+0000", 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111935 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.286135+0000", 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054262699999999995 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.286669+0000", 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00053313299999999998 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.289073+0000", 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002404822 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119530 7.c 7:36830f67:::benchmark_data_smithi026_410749_object29193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.567375+0000", 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.66182825599998, 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7261192759999999, 2023-09-23T22:34:49.366 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119530 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567375+0000", 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567375+0000", 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567374+0000", 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567380+0000", 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1850000000000003e-06 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567389+0000", 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5410000000000006e-06 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567394+0000", 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1220000000000003e-06 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567454+0000", 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9857e-05 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567465+0000", 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1219e-05 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054039+0000", 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.48657352799999998 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054042+0000", 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5599999999999998e-06 2023-09-23T22:34:49.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.274794+0000", 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2207516300000001 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.276186+0000", 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001392162 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.278086+0000", 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001899502 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285725+0000", 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076398430000000003 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.289503+0000", 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037773630000000002 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.289936+0000", 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043365299999999999 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.290271+0000", 2023-09-23T22:34:49.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033435799999999999 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.290826+0000", 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000554742 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.293495+0000", 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002669121 2023-09-23T22:34:49.375 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119565 7.c 7:366e6103:::benchmark_data_smithi026_410749_object29228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.701580+0000", 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.52762410000003, 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6102326480000002, 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.376 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119565 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701580+0000", 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701580+0000", 2023-09-23T22:34:49.377 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701579+0000", 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701583+0000", 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4100000000000001e-06 2023-09-23T22:34:49.378 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701596+0000", 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3318e-05 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701602+0000", 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2179999999999998e-06 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.379 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701649+0000", 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7822999999999999e-05 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701661+0000", 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1739000000000001e-05 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.380 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054055+0000", 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.35239350400000002 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054059+0000", 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.605e-06 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.381 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.293665+0000", 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2396055229999998 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.295090+0000", 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001425078 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.297158+0000", 2023-09-23T22:34:49.382 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002068318 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.307433+0000", 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010274817 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.307802+0000", 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036851599999999999 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.383 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.308198+0000", 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039608500000000001 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.308594+0000", 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039657100000000002 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.384 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.309219+0000", 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00062512600000000005 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.311812+0000", 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025928660000000001 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.385 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151873 8.6 8:675a56d2:::benchmark_data_smithi026_497658_object52981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 64.180786589999997, 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6652286869999999, 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151873 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.386 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.387 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048419+0000", 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7810000000000001e-06 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048428+0000", 2023-09-23T22:34:49.388 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2649999999999996e-06 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048434+0000", 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2419999999999997e-06 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048492+0000", 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7899999999999998e-05 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.389 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048608+0000", 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116111 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048734+0000", 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012566199999999999 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.390 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.057680+0000", 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008946173 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202205+0000", 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.144524878 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202215+0000", 2023-09-23T22:34:49.391 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0217000000000001e-05 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198539+0000", 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99632416099999999 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198553+0000", 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4236999999999999e-05 2023-09-23T22:34:49.392 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713565+0000", 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5150114879999999 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713646+0000", 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1019000000000005e-05 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.393 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151874 8.e 8:70ae8a39:::benchmark_data_smithi026_497658_object52982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 64.174431193999993, 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6486478249999998, 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:34:49.394 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151874 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.395 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054771+0000", 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.397 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054778+0000", 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7599999999999997e-06 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054794+0000", 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6138999999999999e-05 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.398 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054803+0000", 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8610000000000004e-06 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054857+0000", 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3900000000000002e-05 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.055039+0000", 2023-09-23T22:34:49.399 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018265100000000001 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.055175+0000", 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013610799999999999 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.057826+0000", 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00265037 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.400 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.060056+0000", 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022297520000000002 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202153+0000", 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.142097219 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.401 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202169+0000", 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5707000000000001e-05 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198586+0000", 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99641781500000004 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198596+0000", 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.662e-06 2023-09-23T22:34:49.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703316+0000", 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.504720442 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703420+0000", 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010388800000000001 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.403 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151883 8.e 8:73a3339c:::benchmark_data_smithi026_497658_object52991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.930636481999997, 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4185350429999999, 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:34:49.404 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151883 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:34:49.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298566+0000", 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298574+0000", 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6419999999999996e-06 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298592+0000", 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.749e-05 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298603+0000", 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1551e-05 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298672+0000", 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8804000000000001e-05 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298688+0000", 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6004000000000001e-05 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198619+0000", 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89993140699999996 2023-09-23T22:34:49.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198625+0000", 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0419999999999999e-06 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703671+0000", 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.505045454 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703821+0000", 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014994300000000001 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703937+0000", 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116397 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713345+0000", 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094075259999999994 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713414+0000", 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9202999999999995e-05 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713603+0000", 2023-09-23T22:34:49.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000189373 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716935+0000", 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033319270000000002 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716988+0000", 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3121000000000002e-05 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.717102+0000", 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011384599999999999 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151885 8.e 8:7d6f345f:::benchmark_data_smithi026_497658_object52993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.929645720000003, 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4256615969999999, 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.413 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151885 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:34:49.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:34:49.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299557+0000", 2023-09-23T22:34:49.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:34:49.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299563+0000", 2023-09-23T22:34:49.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1279999999999996e-06 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299579+0000", 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5736e-05 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299589+0000", 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9720000000000004e-06 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.417 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299662+0000", 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3312000000000006e-05 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299675+0000", 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3414999999999999e-05 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198647+0000", 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89897187599999995 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198655+0000", 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1219999999999995e-06 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706694+0000", 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5080385459999999 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706833+0000", 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013966099999999999 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706985+0000", 2023-09-23T22:34:49.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151915 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713514+0000", 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065285760000000003 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713609+0000", 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5025000000000006e-05 2023-09-23T22:34:49.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716876+0000", 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032670960000000001 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725068+0000", 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0081923059999999999 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725134+0000", 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5544999999999997e-05 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725220+0000", 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5604000000000002e-05 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.423 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151886 8.e 8:74052c51:::benchmark_data_smithi026_497658_object52994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 63.927364160000003, 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4236035519999999, 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:34:49.424 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151886 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:34:49.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301838+0000", 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301844+0000", 2023-09-23T22:34:49.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2099999999999998e-06 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301857+0000", 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2943e-05 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301867+0000", 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3810000000000005e-06 2023-09-23T22:34:49.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301924+0000", 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7370000000000001e-05 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301936+0000", 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2551999999999999e-05 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198678+0000", 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89674120599999996 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198686+0000", 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0330000000000008e-06 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:34:49.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.709941+0000", 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5112555969999999 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.710081+0000", 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013977400000000001 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.710232+0000", 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015061799999999999 2023-09-23T22:34:49.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713594+0000", 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033623500000000001 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716839+0000", 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032453930000000001 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716906+0000", 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7041999999999999e-05 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725274+0000", 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083672850000000003 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:34:49.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725329+0000", 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5031000000000002e-05 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725443+0000", 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114305 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:34:49.433 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:34:49.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:49.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:49.434 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:49 smithi070 ceph-mon[192652]: from='client.36177 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:34:49.434 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:49 smithi070 ceph-mon[192652]: pgmap v1143: 99 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 96 active+clean; 545 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 652/26034 objects misplaced (2.504%); 37 MiB/s, 594 objects/s recovering 2023-09-23T22:34:49.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:49 smithi026 ceph-mon[223014]: from='client.36177 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:34:49.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:49 smithi026 ceph-mon[223014]: pgmap v1143: 99 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 96 active+clean; 545 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 652/26034 objects misplaced (2.504%); 37 MiB/s, 594 objects/s recovering 2023-09-23T22:34:49.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:49 smithi026 ceph-mon[215775]: from='client.36177 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:34:49.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:49 smithi026 ceph-mon[215775]: pgmap v1143: 99 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 96 active+clean; 545 MiB data, 4.8 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 586 op/s; 652/26034 objects misplaced (2.504%); 37 MiB/s, 594 objects/s recovering 2023-09-23T22:34:49.535 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:49.584 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:49.638 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:49.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:49.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:49.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:49.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:34:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:34:49] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.43.0" 2023-09-23T22:34:50.326 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:50.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:50.352 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:34:50.441 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:50.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:50 smithi026 ceph-mon[223014]: 1.4 scrub starts 2023-09-23T22:34:50.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:50 smithi026 ceph-mon[223014]: 1.4 scrub ok 2023-09-23T22:34:50.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:50 smithi026 ceph-mon[215775]: 1.4 scrub starts 2023-09-23T22:34:50.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:50 smithi026 ceph-mon[215775]: 1.4 scrub ok 2023-09-23T22:34:50.532 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:50.620 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:50.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:50 smithi070 ceph-mon[192652]: 1.4 scrub starts 2023-09-23T22:34:50.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:50 smithi070 ceph-mon[192652]: 1.4 scrub ok 2023-09-23T22:34:50.668 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:50.710 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:50.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:50.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:50.817 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:51.013 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:51.257 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:34:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:51 smithi070 ceph-mon[192652]: pgmap v1144: 99 pgs: 99 active+clean; 504 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 538 op/s; 34 MiB/s, 542 objects/s recovering 2023-09-23T22:34:51.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:51 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2332708460' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:51.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:51 smithi026 ceph-mon[223014]: pgmap v1144: 99 pgs: 99 active+clean; 504 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 538 op/s; 34 MiB/s, 542 objects/s recovering 2023-09-23T22:34:51.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:51 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2332708460' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:51.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:51 smithi026 ceph-mon[215775]: pgmap v1144: 99 pgs: 99 active+clean; 504 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 538 op/s; 34 MiB/s, 542 objects/s recovering 2023-09-23T22:34:51.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:51 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2332708460' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:34:52.264 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:34:52.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:34:52.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[215775]: failed to encode map e446 with expected crc 2023-09-23T22:34:52.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2332708460' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:34:52.265 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[215775]: osdmap e446: 8 total, 8 up, 6 in 2023-09-23T22:34:52.537 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:34:52.537 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:34:52.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[223014]: failed to encode map e446 with expected crc 2023-09-23T22:34:52.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2332708460' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:34:52.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:52 smithi026 ceph-mon[223014]: osdmap e446: 8 total, 8 up, 6 in 2023-09-23T22:34:52.563 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:52.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:52.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:52 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:34:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:52 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:34:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:52 smithi070 ceph-mon[192652]: failed to encode map e446 with expected crc 2023-09-23T22:34:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2332708460' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:34:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:52 smithi070 ceph-mon[192652]: osdmap e446: 8 total, 8 up, 6 in 2023-09-23T22:34:52.671 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:52.739 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:53.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:53 smithi026 ceph-mon[215775]: pgmap v1146: 99 pgs: 99 active+clean; 402 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 455 op/s; 9.0 MiB/s, 145 objects/s recovering 2023-09-23T22:34:53.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:53 smithi026 ceph-mon[223014]: pgmap v1146: 99 pgs: 99 active+clean; 402 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 455 op/s; 9.0 MiB/s, 145 objects/s recovering 2023-09-23T22:34:53.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:53 smithi070 ceph-mon[192652]: pgmap v1146: 99 pgs: 99 active+clean; 402 MiB data, 4.3 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 455 op/s; 9.0 MiB/s, 145 objects/s recovering 2023-09-23T22:34:53.825 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:53.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:53.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:53.932 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:53.959 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:54.001 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:54.196 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:54.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:54.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:54.300 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:54.403 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:54.695 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:54.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:54.734 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:34:54.776 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:34:54.804 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:54.808 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:34:54.852 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:34:54.853 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:54.889 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:34:54.913 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:54.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:54.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:54.929 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:34:54.965 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:34:55.009 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:34:55.018 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:55.030 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:34:55.030 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:34:55.030 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:34:55.030 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:34:55.030 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:34:55.030 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:34:55.031 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:34:55.031 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:34:55.054 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:55.118 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:55.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:55.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:55 smithi070 ceph-mon[192652]: pgmap v1147: 99 pgs: 99 active+clean; 335 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 7.1 MiB/s, 115 objects/s recovering 2023-09-23T22:34:55.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:55.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:55.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:55.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2865532135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:55.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:55.228 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:55.267 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:55.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[215775]: pgmap v1147: 99 pgs: 99 active+clean; 335 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 7.1 MiB/s, 115 objects/s recovering 2023-09-23T22:34:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2865532135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:55.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[223014]: pgmap v1147: 99 pgs: 99 active+clean; 335 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 7.1 MiB/s, 115 objects/s recovering 2023-09-23T22:34:55.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:55.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:34:55.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:34:55.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2865532135' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:34:55.910 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:34:56.111 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:56.120 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 8 pgp_num to 16 2023-09-23T22:34:56.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:56.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:56.225 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:34:56.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/481496233' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:34:56.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:56 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:34:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/481496233' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:34:56.238 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:56 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:34:56.267 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/481496233' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:34:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:56 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T22:34:56.503 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 98889 objects 2023-09-23T22:34:56.504 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :131.384 2023-09-23T22:34:56.789 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:34:56.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:56.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:34:56.893 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:56.920 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:56.961 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:56.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:56.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:57.068 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:34:57.093 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:57.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:34:57.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:57.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:34:57.241 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:57.271 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:57.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:57 smithi070 ceph-mon[192652]: pgmap v1148: 99 pgs: 99 active+clean; 120 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 844 op/s; 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:34:57.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:34:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:57 smithi070 ceph-mon[192652]: osdmap e447: 8 total, 8 up, 6 in 2023-09-23T22:34:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:57 smithi070 ceph-mon[192652]: failed to encode map e447 with expected crc 2023-09-23T22:34:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:57 smithi070 ceph-mon[192652]: failed to encode map e447 with expected crc 2023-09-23T22:34:57.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[223014]: pgmap v1148: 99 pgs: 99 active+clean; 120 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 844 op/s; 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:34:57.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:34:57.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[223014]: osdmap e447: 8 total, 8 up, 6 in 2023-09-23T22:34:57.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[223014]: failed to encode map e447 with expected crc 2023-09-23T22:34:57.375 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[223014]: failed to encode map e447 with expected crc 2023-09-23T22:34:57.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[215775]: pgmap v1148: 99 pgs: 99 active+clean; 120 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 844 op/s; 6.3 MiB/s, 101 objects/s recovering 2023-09-23T22:34:57.375 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T22:34:57.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[215775]: osdmap e447: 8 total, 8 up, 6 in 2023-09-23T22:34:57.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[215775]: failed to encode map e447 with expected crc 2023-09-23T22:34:57.376 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:57 smithi026 ceph-mon[215775]: failed to encode map e447 with expected crc 2023-09-23T22:34:57.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:57.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:57.477 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:57.507 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:57.606 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:57.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:57.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:57.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:34:57.738 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:57.742 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:57.781 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:34:57.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:57.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:34:57.826 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:34:57.826 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:34:57.888 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:34:57.914 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:57.955 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:34:57.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:57.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:34:58.060 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:34:58.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:59.192 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:34:59.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:59.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:34:59.216 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:34:59.299 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:34:59.328 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:59.335 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:34:59.435 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:34:59.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:34:59 smithi026 ceph-mon[215775]: pgmap v1150: 99 pgs: 99 active+clean; 63 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 878 op/s 2023-09-23T22:34:59.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:34:59 smithi026 ceph-mon[223014]: pgmap v1150: 99 pgs: 99 active+clean; 63 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 878 op/s 2023-09-23T22:34:59.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:59.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:34:59.542 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:59.578 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:59.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:34:59 smithi070 ceph-mon[192652]: pgmap v1150: 99 pgs: 99 active+clean; 63 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 878 op/s 2023-09-23T22:34:59.686 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:59.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:59.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:59.699 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:34:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:34:59] "GET /metrics HTTP/1.1" 200 36390 "" "Prometheus/2.43.0" 2023-09-23T22:34:59.790 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:34:59.821 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:34:59.876 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:34:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:34:59.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:34:59.890 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:59.908 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:34:59.981 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:00.010 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:00.121 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:00.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:00.201 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:35:00.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:00.255 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:00.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3094053855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:35:00.284 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:00 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:35:00.284 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3094053855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:35:00.285 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:00 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:35:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3094053855' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:35:00.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:00 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:35:00.901 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T22:35:00.901 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T22:35:01.065 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:01.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:01.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:01.169 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:01.198 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:01.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[223014]: pgmap v1151: 99 pgs: 99 active+clean; 36 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 696 op/s 2023-09-23T22:35:01.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:35:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[223014]: osdmap e448: 8 total, 8 up, 6 in 2023-09-23T22:35:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[223014]: failed to encode map e448 with expected crc 2023-09-23T22:35:01.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[223014]: failed to encode map e448 with expected crc 2023-09-23T22:35:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[215775]: pgmap v1151: 99 pgs: 99 active+clean; 36 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 696 op/s 2023-09-23T22:35:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:35:01.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[215775]: osdmap e448: 8 total, 8 up, 6 in 2023-09-23T22:35:01.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[215775]: failed to encode map e448 with expected crc 2023-09-23T22:35:01.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:01 smithi026 ceph-mon[215775]: failed to encode map e448 with expected crc 2023-09-23T22:35:01.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:01 smithi070 ceph-mon[192652]: pgmap v1151: 99 pgs: 99 active+clean; 36 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 696 op/s 2023-09-23T22:35:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:01 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:01 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:35:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:01 smithi070 ceph-mon[192652]: osdmap e448: 8 total, 8 up, 6 in 2023-09-23T22:35:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:01 smithi070 ceph-mon[192652]: failed to encode map e448 with expected crc 2023-09-23T22:35:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:01 smithi070 ceph-mon[192652]: failed to encode map e448 with expected crc 2023-09-23T22:35:01.751 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:01.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:01.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:01.855 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:01.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:02.221 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:02.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:02.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:02.292 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:35:02.292 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:35:02.292 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T22:35:02.292 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:35:02.292 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:35:02.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T22:35:02.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:02 smithi026 ceph-mon[215775]: failed to encode map e448 with expected crc 2023-09-23T22:35:02.318 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:02 smithi026 ceph-mon[215775]: failed to encode map e448 with expected crc 2023-09-23T22:35:02.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:02 smithi026 ceph-mon[223014]: failed to encode map e448 with expected crc 2023-09-23T22:35:02.319 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:02 smithi026 ceph-mon[223014]: failed to encode map e448 with expected crc 2023-09-23T22:35:02.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:02.343 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:02.356 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:02.404 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:02.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:02.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:02.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:02.534 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:02 smithi070 ceph-mon[192652]: failed to encode map e448 with expected crc 2023-09-23T22:35:02.535 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:02 smithi070 ceph-mon[192652]: failed to encode map e448 with expected crc 2023-09-23T22:35:02.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:02.580 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:02.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:02.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:02.687 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:02.689 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:35:02.708 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:35:02.714 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:02.728 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:35:02.748 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:35:02.755 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:02.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:02.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:02.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:35:02.787 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:35:02.806 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:35:02.838 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:35:02.839 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:35:02.839 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:35:02.839 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:35:02.839 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:35:02.840 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:35:02.840 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:35:02.840 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:35:02.840 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:35:02.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:02.869 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:35:02.869 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:35:02.869 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:35:02.869 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:35:02.869 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:35:02.888 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:02.929 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:02.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:02.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:03.038 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:03.064 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:03.108 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:03.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:03.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:03.213 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:03.238 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:03.280 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:03.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:03.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:03.385 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:35:03.410 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:03.452 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:35:03.457 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:03 smithi070 ceph-mon[192652]: pgmap v1153: 99 pgs: 99 active+clean; 4.6 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T22:35:03.457 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1078383987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:03.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:03.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:35:03.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:03 smithi026 ceph-mon[215775]: pgmap v1153: 99 pgs: 99 active+clean; 4.6 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T22:35:03.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1078383987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:03.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:03 smithi026 ceph-mon[223014]: pgmap v1153: 99 pgs: 99 active+clean; 4.6 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 661 op/s 2023-09-23T22:35:03.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1078383987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:03.558 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:03.589 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:03.722 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:35:04.058 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:04.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:04.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:04.118 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:35:04.163 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:04.191 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:04.202 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:04.232 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:04.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:04.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:04.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:35:04.364 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:04.405 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:35:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:04.411 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:35:04.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:04.542 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:04.868 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:04.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:04.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:04.972 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:05.001 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:05.042 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:05.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:05.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:05.145 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:05.179 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:05.210 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:05.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:05 smithi070 ceph-mon[192652]: pgmap v1154: 99 pgs: 99 active+clean; 4.6 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 231 op/s 2023-09-23T22:35:05.436 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:05.438 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:05 smithi026 ceph-mon[223014]: pgmap v1154: 99 pgs: 99 active+clean; 4.6 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 231 op/s 2023-09-23T22:35:05.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:05.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:05.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:05 smithi026 ceph-mon[215775]: pgmap v1154: 99 pgs: 99 active+clean; 4.6 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 231 op/s 2023-09-23T22:35:05.541 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:05.569 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:05.615 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:05.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:05.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:05.719 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:05.748 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:05.792 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:05.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:05.797 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:05.898 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:05.925 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:06.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:35:06.100 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:06.122 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:06.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:06.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:06.212 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/485097288' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:35:06.213 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/485097288' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:35:06.231 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:06.239 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_3' removed 2023-09-23T22:35:06.289 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:06.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/485097288' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:35:07.456 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:07.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:07.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:07.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:07 smithi026 ceph-mon[215775]: pgmap v1155: 99 pgs: 99 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 193 op/s 2023-09-23T22:35:07.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/485097288' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:35:07.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:07 smithi026 ceph-mon[215775]: osdmap e449: 8 total, 8 up, 6 in 2023-09-23T22:35:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:07 smithi026 ceph-mon[223014]: pgmap v1155: 99 pgs: 99 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 193 op/s 2023-09-23T22:35:07.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/485097288' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:35:07.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:07 smithi026 ceph-mon[223014]: osdmap e449: 8 total, 8 up, 6 in 2023-09-23T22:35:07.560 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:35:07.584 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:07 smithi070 ceph-mon[192652]: pgmap v1155: 99 pgs: 99 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 193 op/s 2023-09-23T22:35:07.584 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/485097288' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:35:07.584 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:07 smithi070 ceph-mon[192652]: osdmap e449: 8 total, 8 up, 6 in 2023-09-23T22:35:07.586 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:07.628 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:35:07.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:07.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:35:07.733 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:07.753 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:35:07.754 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:35:07.754 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T22:35:07.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T22:35:07.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:07.800 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:07.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:07.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:07.932 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:07.973 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:07.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:07.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:08.032 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:35:08.032 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "" 2023-09-23T22:35:08.032 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:35:08.079 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:08.112 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:08.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:08 smithi026 ceph-mon[215775]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:08 smithi026 ceph-mon[215775]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:08 smithi026 ceph-mon[215775]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:08 smithi026 ceph-mon[215775]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:08 smithi026 ceph-mon[223014]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:08 smithi026 ceph-mon[223014]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:08 smithi026 ceph-mon[223014]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:08 smithi026 ceph-mon[223014]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:08 smithi070 ceph-mon[192652]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:08 smithi070 ceph-mon[192652]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:08 smithi070 ceph-mon[192652]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:08 smithi070 ceph-mon[192652]: failed to encode map e449 with expected crc 2023-09-23T22:35:08.698 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:08.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:08.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:08.802 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:35:08.834 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:08.883 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:35:08.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:08.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:35:08.989 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:09.017 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:09.059 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:09.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:09.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:09.167 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:09.201 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:09.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:09 smithi026 ceph-mon[215775]: pgmap v1157: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:09.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:09.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:09.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:09 smithi026 ceph-mon[223014]: pgmap v1157: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:09.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:09.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:09.507 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:09.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:09.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:09.517 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:09.612 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:35:09.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:09 smithi070 ceph-mon[192652]: pgmap v1157: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:09.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:09.642 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:09.689 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:35:09.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:09.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:35:09.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:35:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:35:09] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2023-09-23T22:35:09.796 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:09.831 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:09.832 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:09.938 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:09.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:09.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:09.951 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:35:09.951 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:35:09.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:35:09.951 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96581 7.b 7:da8af227:::benchmark_data_smithi026_410749_object6244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.121529+0000", 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.82510239200002, 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0460254070000001, 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:09.952 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96581 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121529+0000", 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121529+0000", 2023-09-23T22:35:09.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:09.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121527+0000", 2023-09-23T22:35:09.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:09.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121535+0000", 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2330000000000006e-06 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121557+0000", 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2014e-05 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:09.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121568+0000", 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0974e-05 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121632+0000", 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.347e-05 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.121645+0000", 2023-09-23T22:35:09.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2959000000000001e-05 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061244+0000", 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93959893500000002 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061260+0000", 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.613e-05 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.159051+0000", 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.097790874 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.159184+0000", 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133869 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.159271+0000", 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6310999999999994e-05 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.161846+0000", 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025755970000000002 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167032+0000", 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005186052 2023-09-23T22:35:09.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167199+0000", 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016674600000000001 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167355+0000", 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000155494 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167418+0000", 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3830000000000004e-05 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167554+0000", 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013549800000000001 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96604 7.b 7:dba0fa8d:::benchmark_data_smithi026_410749_object6267:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.174896+0000", 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.77173473900001, 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.992895646, 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:09.962 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96604 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174896+0000", 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174896+0000", 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174894+0000", 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174904+0000", 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8190000000000003e-06 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174927+0000", 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2821000000000001e-05 2023-09-23T22:35:09.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.174937+0000", 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0549e-05 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.175009+0000", 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1465999999999996e-05 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.175023+0000", 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4574e-05 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061288+0000", 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.88626441600000005 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061298+0000", 2023-09-23T22:35:09.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0083e-05 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.161879+0000", 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1005808749999999 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.161966+0000", 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6909000000000004e-05 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:09.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.162040+0000", 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4089000000000002e-05 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167210+0000", 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051705910000000004 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167270+0000", 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0257000000000003e-05 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167296+0000", 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5072e-05 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167628+0000", 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033241999999999999 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:09.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167683+0000", 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4676999999999998e-05 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167792+0000", 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001092 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.970 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96657 7.b 7:d2054ff9:::benchmark_data_smithi026_410749_object6320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.297353+0000", 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.649278294, 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8706818570000001, 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96657 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:09.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297353+0000", 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297353+0000", 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:09.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297351+0000", 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297359+0000", 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4349999999999997e-06 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297379+0000", 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9244000000000001e-05 2023-09-23T22:35:09.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297390+0000", 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1504e-05 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297505+0000", 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115328 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.297519+0000", 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3552e-05 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061322+0000", 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.76380318800000002 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061332+0000", 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0074999999999999e-05 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.164234+0000", 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.102901989 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.164356+0000", 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121854 2023-09-23T22:35:09.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.164444+0000", 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.81e-05 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167572+0000", 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031280320000000002 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167853+0000", 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00028055799999999998 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167883+0000", 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0664000000000003e-05 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167932+0000", 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8953999999999999e-05 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.167966+0000", 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4050000000000001e-05 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.168034+0000", 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8077999999999994e-05 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.979 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96659 7.b 7:d16d3249:::benchmark_data_smithi026_410749_object6322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.305815+0000", 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.640815868, 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8766973179999999, 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:09.980 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96659 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305815+0000", 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305815+0000", 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305813+0000", 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305821+0000", 2023-09-23T22:35:09.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1159999999999999e-06 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305834+0000", 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3122000000000001e-05 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305842+0000", 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7489999999999998e-06 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305913+0000", 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0431000000000004e-05 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.305925+0000", 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2017e-05 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061357+0000", 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.75543235500000006 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061367+0000", 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0026e-05 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.168087+0000", 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1067203699999999 2023-09-23T22:35:09.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.168241+0000", 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000153883 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.168393+0000", 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015202299999999999 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175249+0000", 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068552550000000002 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175306+0000", 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7188000000000002e-05 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182294+0000", 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069879249999999999 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182309+0000", 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5254e-05 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182372+0000", 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3219000000000006e-05 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182512+0000", 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140285 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96680 7.b 7:d07ca5d2:::benchmark_data_smithi026_410749_object6343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.352046+0000", 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.59458524899998, 2023-09-23T22:35:09.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8377739719999999, 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96680 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:09.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352046+0000", 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352046+0000", 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352044+0000", 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352052+0000", 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1629999999999996e-06 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352071+0000", 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8278e-05 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:09.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352080+0000", 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4709999999999996e-06 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352153+0000", 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3120000000000005e-05 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.352166+0000", 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2813000000000001e-05 2023-09-23T22:35:09.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061392+0000", 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.70922612900000004 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061402+0000", 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0190999999999999e-05 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.171558+0000", 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1101559830000001 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.171689+0000", 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130395 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.171813+0000", 2023-09-23T22:35:09.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124401 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182245+0000", 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010431573 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182523+0000", 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027883900000000002 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182578+0000", 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4280000000000002e-05 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189651+0000", 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007072811 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189703+0000", 2023-09-23T22:35:09.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2033999999999997e-05 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189820+0000", 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000117064 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.998 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96686 7.b 7:de804cf2:::benchmark_data_smithi026_410749_object6349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.359794+0000", 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.58683686699999, 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8301854309999999, 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96686 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:09.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359794+0000", 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359794+0000", 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359792+0000", 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:35:10.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359803+0000", 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0549e-05 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359828+0000", 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5647e-05 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359839+0000", 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0575e-05 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359912+0000", 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3368000000000006e-05 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.359926+0000", 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3757000000000001e-05 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061427+0000", 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.70150106899999998 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061437+0000", 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9420000000000007e-06 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175332+0000", 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1138945499999999 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175468+0000", 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136501 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.175599+0000", 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130784 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182530+0000", 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069310630000000003 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182624+0000", 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4302999999999995e-05 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182699+0000", 2023-09-23T22:35:10.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5062999999999996e-05 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189860+0000", 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071608649999999998 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189907+0000", 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6369000000000003e-05 2023-09-23T22:35:10.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189979+0000", 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2970000000000001e-05 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96695 7.b 7:dbf81365:::benchmark_data_smithi026_410749_object6358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.382326+0000", 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.56430473699999, 2023-09-23T22:35:10.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8078309880000001, 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96695 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382326+0000", 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382326+0000", 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382325+0000", 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382333+0000", 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7649999999999993e-06 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382351+0000", 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8131e-05 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382361+0000", 2023-09-23T22:35:10.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9890000000000004e-06 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382441+0000", 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0789000000000002e-05 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.382465+0000", 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3802999999999999e-05 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061462+0000", 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.67899700699999999 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061472+0000", 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0189e-05 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.178943+0000", 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1174706839999999 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.179093+0000", 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149708 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.179222+0000", 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012925999999999999 2023-09-23T22:35:10.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182586+0000", 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033642749999999999 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182661+0000", 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4276000000000005e-05 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182742+0000", 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0872999999999996e-05 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190018+0000", 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072768939999999999 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190065+0000", 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6468000000000003e-05 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190157+0000", 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2299000000000005e-05 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.016 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96696 7.b 7:d9fb316f:::benchmark_data_smithi026_410749_object6359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.384266+0000", 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.56236529900002, 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8062237539999999, 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96696 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384266+0000", 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384266+0000", 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384264+0000", 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384271+0000", 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0369999999999997e-06 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384290+0000", 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8411e-05 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.384298+0000", 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8689999999999993e-06 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.388227+0000", 2023-09-23T22:35:10.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039287009999999997 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.388243+0000", 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5744e-05 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061496+0000", 2023-09-23T22:35:10.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.67325356400000003 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061505+0000", 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3189999999999995e-06 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182772+0000", 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.121266777 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.182923+0000", 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015133400000000001 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.183082+0000", 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159405 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.189616+0000", 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065341460000000002 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190195+0000", 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057832200000000004 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190320+0000", 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125647 2023-09-23T22:35:10.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190329+0000", 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4719999999999995e-06 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190377+0000", 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8112000000000003e-05 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190489+0000", 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011237499999999999 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96701 7.b 7:d8abcfde:::benchmark_data_smithi026_410749_object6364:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.393752+0000", 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.55287862699998, 2023-09-23T22:35:10.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8036669359999999, 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96701 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393752+0000", 2023-09-23T22:35:10.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393752+0000", 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393751+0000", 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393759+0000", 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6139999999999992e-06 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393776+0000", 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6566e-05 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393783+0000", 2023-09-23T22:35:10.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4250000000000001e-06 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393852+0000", 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8328e-05 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.393863+0000", 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1618e-05 2023-09-23T22:35:10.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061537+0000", 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.66767378899999996 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061546+0000", 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3919999999999992e-06 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.186152+0000", 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1246053899999999 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.186265+0000", 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113108 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.186366+0000", 2023-09-23T22:35:10.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100787 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190277+0000", 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039108930000000004 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190498+0000", 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00022148799999999999 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190539+0000", 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1427000000000002e-05 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197233+0000", 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066931229999999996 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197292+0000", 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8946000000000001e-05 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197419+0000", 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127704 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.035 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96708 7.b 7:dc997113:::benchmark_data_smithi026_410749_object6371:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.411022+0000", 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.53560869699999, 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.793953546, 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96708 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411022+0000", 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411022+0000", 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411020+0000", 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411032+0000", 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2551000000000001e-05 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411053+0000", 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0922000000000001e-05 2023-09-23T22:35:10.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411067+0000", 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3495e-05 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411157+0000", 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0869e-05 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.411172+0000", 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4972e-05 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061570+0000", 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.65039764600000005 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061580+0000", 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5040000000000008e-06 2023-09-23T22:35:10.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190561+0000", 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1289812429999999 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190672+0000", 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011156500000000001 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.190807+0000", 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134771 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197194+0000", 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063872620000000003 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204752+0000", 2023-09-23T22:35:10.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007557355 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204761+0000", 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9029999999999993e-06 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204808+0000", 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7187000000000001e-05 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204863+0000", 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5390999999999999e-05 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204976+0000", 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112508 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96711 7.b 7:d9a0e45e:::benchmark_data_smithi026_410749_object6374:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.419858+0000", 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.52677272599999, 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7984610889999999, 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96711 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419858+0000", 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419858+0000", 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419856+0000", 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419865+0000", 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0950000000000002e-06 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419882+0000", 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6135000000000001e-05 2023-09-23T22:35:10.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419891+0000", 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0839999999999998e-06 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419955+0000", 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4425999999999996e-05 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.419965+0000", 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0171e-05 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061599+0000", 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.64163331400000001 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061604+0000", 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0599999999999998e-06 2023-09-23T22:35:10.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.193594+0000", 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1319901510000001 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.193776+0000", 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018225899999999999 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.193924+0000", 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014823899999999999 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197465+0000", 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003541237 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204767+0000", 2023-09-23T22:35:10.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073014100000000004 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218008+0000", 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013241404 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218125+0000", 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011712199999999999 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218183+0000", 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7797999999999998e-05 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218319+0000", 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013606100000000001 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96742 7.b 7:d66e3e8b:::benchmark_data_smithi026_410749_object6405:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.501784+0000", 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.44484654500002, 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.716760217, 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96742 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.054 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501784+0000", 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501784+0000", 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501782+0000", 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501794+0000", 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1527e-05 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501819+0000", 2023-09-23T22:35:10.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5318000000000001e-05 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501831+0000", 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1579e-05 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501934+0000", 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010391399999999999 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.501954+0000", 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9068000000000001e-05 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061616+0000", 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.55966211899999996 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061624+0000", 2023-09-23T22:35:10.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4319999999999999e-06 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197491+0000", 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.135866592 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197627+0000", 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013658 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.197765+0000", 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013777400000000001 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204695+0000", 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069303510000000004 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.204773+0000", 2023-09-23T22:35:10.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7848000000000004e-05 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218048+0000", 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013274555 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218390+0000", 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000342295 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218440+0000", 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9842000000000002e-05 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218545+0000", 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104691 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96747 7.b 7:d5f72434:::benchmark_data_smithi026_410749_object6410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.510418+0000", 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.436212721, 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.711354499, 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96747 2023-09-23T22:35:10.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510418+0000", 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510418+0000", 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510417+0000", 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510424+0000", 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7739999999999999e-06 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510436+0000", 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2357999999999999e-05 2023-09-23T22:35:10.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.510445+0000", 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9479999999999997e-06 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.514417+0000", 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039724570000000004 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.514438+0000", 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0241000000000002e-05 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061645+0000", 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.54720765500000002 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061651+0000", 2023-09-23T22:35:10.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3360000000000002e-06 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.201041+0000", 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.139390049 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.201189+0000", 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014889699999999999 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.201334+0000", 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000144089 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212102+0000", 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010767995000000001 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212149+0000", 2023-09-23T22:35:10.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7225000000000003e-05 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218091+0000", 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059417910000000001 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.221580+0000", 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003489771 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.221637+0000", 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6943e-05 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.221773+0000", 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135439 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.071 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96750 7.b 7:d0c71cb9:::benchmark_data_smithi026_410749_object6413:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.520260+0000", 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.42637085799998, 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7056295430000001, 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96750 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520260+0000", 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520260+0000", 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520258+0000", 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:10.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520267+0000", 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0909999999999993e-06 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520282+0000", 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5747e-05 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520293+0000", 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0292000000000001e-05 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520369+0000", 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6327000000000003e-05 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.520386+0000", 2023-09-23T22:35:10.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7005999999999999e-05 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061666+0000", 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.54127963700000004 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061674+0000", 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5380000000000002e-06 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.205004+0000", 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.143329743 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.205127+0000", 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012354600000000001 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.205248+0000", 2023-09-23T22:35:10.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012061 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212157+0000", 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069090289999999997 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212199+0000", 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2474e-05 2023-09-23T22:35:10.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218595+0000", 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063951340000000002 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.225719+0000", 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071245279999999998 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.225774+0000", 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5337000000000002e-05 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.225890+0000", 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000115142 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.080 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96754 7.b 7:d80856d8:::benchmark_data_smithi026_410749_object6417:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.533130+0000", 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.41350139500003, 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7019917420000001, 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96754 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533130+0000", 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533130+0000", 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533127+0000", 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533138+0000", 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.044e-05 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533172+0000", 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3939000000000002e-05 2023-09-23T22:35:10.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533183+0000", 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1053000000000001e-05 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533258+0000", 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5605000000000005e-05 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.533271+0000", 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2901000000000001e-05 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061697+0000", 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.52842535099999999 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061705+0000", 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1850000000000004e-06 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.208491+0000", 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1467860519999999 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.208647+0000", 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015611999999999999 2023-09-23T22:35:10.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.208815+0000", 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016839099999999999 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.212236+0000", 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034210099999999999 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218561+0000", 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006324311 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.218756+0000", 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000195515 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.234972+0000", 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016215947000000001 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235011+0000", 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8461999999999999e-05 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235121+0000", 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011069399999999999 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:96780 7.b 7:da651c72:::benchmark_data_smithi026_410749_object6443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:26:39.699047+0000", 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 510.24758411699997, 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5362475739999999, 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 96780 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699047+0000", 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699047+0000", 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699045+0000", 2023-09-23T22:35:10.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699054+0000", 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.405e-06 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699072+0000", 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8097999999999999e-05 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699079+0000", 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2919999999999998e-06 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699132+0000", 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3053999999999997e-05 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:39.699143+0000", 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0866000000000001e-05 2023-09-23T22:35:10.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061734+0000", 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.36259154900000001 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:40.061742+0000", 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9130000000000003e-06 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.214041+0000", 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1522982420000001 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.214184+0000", 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000143299 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.214321+0000", 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000137145 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.219748+0000", 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054267309999999997 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.220699+0000", 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00095117499999999996 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.229541+0000", 2023-09-23T22:35:10.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088419870000000008 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235160+0000", 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056194740000000002 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235208+0000", 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7886999999999997e-05 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:26:41.235294+0000", 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6030000000000001e-05 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151889 8.a 8:5329de88:::benchmark_data_smithi026_497658_object52997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.305137+0000", 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.641493772000004, 2023-09-23T22:35:10.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3767564390000002, 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151889 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.305137+0000", 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.305137+0000", 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.305135+0000", 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.305145+0000", 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.577e-06 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.305164+0000", 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9429e-05 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.305174+0000", 2023-09-23T22:35:10.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0073e-05 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.305257+0000", 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3125000000000001e-05 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.305274+0000", 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6299000000000001e-05 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.195482+0000", 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89020808200000001 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.195497+0000", 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5582000000000001e-05 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.674121+0000", 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4786231380000001 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.674253+0000", 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013211000000000001 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.674382+0000", 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012951999999999999 2023-09-23T22:35:10.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.679978+0000", 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005595872 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.679990+0000", 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.166e-05 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.680026+0000", 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6384999999999998e-05 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.681791+0000", 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001765036 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.681826+0000", 2023-09-23T22:35:10.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5216999999999998e-05 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.681894+0000", 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7218000000000006e-05 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151868 8.f 8:fb3fe1ac:::benchmark_data_smithi026_497658_object52976:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T22:35:10.107 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:46.199307+0000", 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.747323796000003, 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.505988629, 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151868 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.199307+0000", 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.199307+0000", 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.199306+0000", 2023-09-23T22:35:10.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.199312+0000", 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4779999999999996e-06 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.199324+0000", 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2196e-05 2023-09-23T22:35:10.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.199335+0000", 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0560000000000001e-05 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.199522+0000", 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018746100000000001 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.199531+0000", 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6500000000000002e-06 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.705214+0000", 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5056826160000001 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.705296+0000", 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2150999999999993e-05 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188841 8.0 8:0a0c08c7:::benchmark_data_smithi026_497658_object89949:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:45.093412+0000", 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.853218697999999, 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99677500299999999, 2023-09-23T22:35:10.113 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188841 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.093412+0000", 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.093412+0000", 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.093411+0000", 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.093417+0000", 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4819999999999999e-06 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.093430+0000", 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2391e-05 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.093439+0000", 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2450000000000004e-06 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.093494+0000", 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5104e-05 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.093503+0000", 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7660000000000004e-06 2023-09-23T22:35:10.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.083997+0000", 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99049394099999999 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.084162+0000", 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016538300000000001 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.084324+0000", 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016163500000000001 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.086690+0000", 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023665399999999999 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.086752+0000", 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2084999999999999e-05 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.090092+0000", 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033395289999999999 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.090147+0000", 2023-09-23T22:35:10.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5299e-05 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.090187+0000", 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0095999999999997e-05 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188868 8.0 8:03d7e495:::benchmark_data_smithi026_497658_object89976:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e443)", 2023-09-23T22:35:10.121 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:45.132294+0000", 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.814336926999999, 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95819184199999996, 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188868 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:35:10.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.132294+0000", 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.132294+0000", 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.132292+0000", 2023-09-23T22:35:10.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.132299+0000", 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8349999999999998e-06 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.132316+0000", 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7456e-05 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.132323+0000", 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7220000000000002e-06 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.132376+0000", 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3020999999999999e-05 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:45.132389+0000", 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2447e-05 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.086811+0000", 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95442235799999997 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.086943+0000", 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013175900000000001 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.087072+0000", 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012916800000000001 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.090221+0000", 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031493480000000002 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.090346+0000", 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012482300000000001 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.090407+0000", 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1309000000000005e-05 2023-09-23T22:35:10.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.090451+0000", 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3278000000000001e-05 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:46.090486+0000", 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5259999999999998e-05 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.129 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:35:10.130 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:35:10.130 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:35:10.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:10.162 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:10.210 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:10.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:10.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:10.235 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:35:10.316 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:10.343 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:10.386 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:10.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:10.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:10.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3333408303' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:10.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3333408303' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:10.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:10.493 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:10.524 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:10.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3333408303' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:10.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:10.973 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:10.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:10.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:11.077 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:11.104 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:11.287 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:35:11.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:11 smithi026 ceph-mon[223014]: pgmap v1158: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:11.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:11 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:11.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:35:11.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:11 smithi026 ceph-mon[223014]: osdmap e450: 8 total, 8 up, 6 in 2023-09-23T22:35:11.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:11 smithi026 ceph-mon[215775]: pgmap v1158: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:11.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:11 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:11.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:35:11.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:11 smithi026 ceph-mon[215775]: osdmap e450: 8 total, 8 up, 6 in 2023-09-23T22:35:11.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:11 smithi070 ceph-mon[192652]: pgmap v1158: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:11.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:11 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:35:11.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:11 smithi070 ceph-mon[192652]: osdmap e450: 8 total, 8 up, 6 in 2023-09-23T22:35:12.065 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:12.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:12.169 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:12.376 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:12 smithi026 ceph-mon[223014]: failed to encode map e450 with expected crc 2023-09-23T22:35:12.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:12 smithi026 ceph-mon[223014]: failed to encode map e450 with expected crc 2023-09-23T22:35:12.377 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:12 smithi026 ceph-mon[223014]: failed to encode map e450 with expected crc 2023-09-23T22:35:12.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:12 smithi026 ceph-mon[215775]: failed to encode map e450 with expected crc 2023-09-23T22:35:12.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:12 smithi026 ceph-mon[215775]: failed to encode map e450 with expected crc 2023-09-23T22:35:12.377 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:12 smithi026 ceph-mon[215775]: failed to encode map e450 with expected crc 2023-09-23T22:35:12.380 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:12.609 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:12.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:12.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:12.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:12 smithi070 ceph-mon[192652]: failed to encode map e450 with expected crc 2023-09-23T22:35:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:12 smithi070 ceph-mon[192652]: failed to encode map e450 with expected crc 2023-09-23T22:35:12.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:12 smithi070 ceph-mon[192652]: failed to encode map e450 with expected crc 2023-09-23T22:35:12.714 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:12.749 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:12.798 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:12.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:12.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:12.903 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:12.935 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:13.043 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:13.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:13.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:13.057 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:35:13.077 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:35:13.096 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:35:13.116 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:35:13.135 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:35:13.142 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:13.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:13.172 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:35:13.179 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:13.208 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:35:13.227 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:13.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:13.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:13.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[215775]: pgmap v1160: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:13.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[215775]: failed to encode map e450 with expected crc 2023-09-23T22:35:13.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[215775]: failed to encode map e450 with expected crc 2023-09-23T22:35:13.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[215775]: failed to encode map e450 with expected crc 2023-09-23T22:35:13.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1226602833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:13.237 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:35:13.239 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:35:13.239 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:35:13.239 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:35:13.239 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:35:13.239 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:35:13.239 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:35:13.239 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:35:13.240 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:35:13.335 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:35:13.364 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:13.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:35:13.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:13.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:35:13.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[223014]: pgmap v1160: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:13.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[223014]: failed to encode map e450 with expected crc 2023-09-23T22:35:13.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[223014]: failed to encode map e450 with expected crc 2023-09-23T22:35:13.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[223014]: failed to encode map e450 with expected crc 2023-09-23T22:35:13.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1226602833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:13.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:13.552 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:13 smithi070 ceph-mon[192652]: pgmap v1160: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:13 smithi070 ceph-mon[192652]: failed to encode map e450 with expected crc 2023-09-23T22:35:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:13 smithi070 ceph-mon[192652]: failed to encode map e450 with expected crc 2023-09-23T22:35:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:13 smithi070 ceph-mon[192652]: failed to encode map e450 with expected crc 2023-09-23T22:35:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1226602833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:14.152 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:14.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:14.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:14.248 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_4' created 2023-09-23T22:35:14.258 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:14.289 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:14.335 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:14.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:14.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:14.441 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:14.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:14 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/618348929' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T22:35:14.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:14 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T22:35:14.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:14 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/618348929' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T22:35:14.467 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:14 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T22:35:14.470 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:14.485 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2] out_osds: [3, 7] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:35:14.485 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:35:14.485 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T22:35:14.486 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T22:35:14.519 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:14.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:14.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:14.623 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:14 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/618348929' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T22:35:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:14 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T22:35:14.650 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:14.690 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:35:15.371 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:15.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:15.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:15.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:15.506 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[223014]: pgmap v1161: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:15.506 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[223014]: failed to encode map e451 with expected crc 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[223014]: osdmap e451: 8 total, 8 up, 6 in 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[223014]: failed to encode map e451 with expected crc 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[223014]: failed to encode map e451 with expected crc 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[215775]: pgmap v1161: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[215775]: failed to encode map e451 with expected crc 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[215775]: osdmap e451: 8 total, 8 up, 6 in 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[215775]: failed to encode map e451 with expected crc 2023-09-23T22:35:15.507 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:15 smithi026 ceph-mon[215775]: failed to encode map e451 with expected crc 2023-09-23T22:35:15.508 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:15.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T22:35:15.606 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:15.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:15.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:15 smithi070 ceph-mon[192652]: pgmap v1161: 83 pgs: 83 active+clean; 4.6 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:15 smithi070 ceph-mon[192652]: failed to encode map e451 with expected crc 2023-09-23T22:35:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:15 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T22:35:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:15 smithi070 ceph-mon[192652]: osdmap e451: 8 total, 8 up, 6 in 2023-09-23T22:35:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:15 smithi070 ceph-mon[192652]: failed to encode map e451 with expected crc 2023-09-23T22:35:15.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:15 smithi070 ceph-mon[192652]: failed to encode map e451 with expected crc 2023-09-23T22:35:15.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:35:15.742 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:15.791 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:35:15.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:15.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:35:15.857 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:15.899 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:15.902 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:15.927 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:15.969 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:35:15.969 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:35:15.973 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:15.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:15.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:16.081 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:35:16.120 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:16.161 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:35:16.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:16.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:35:16.228 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:35:16.268 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:16.295 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:16.337 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:16.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:16.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:16.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:16 smithi070 ceph-mon[192652]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:16 smithi070 ceph-mon[192652]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:16 smithi070 ceph-mon[192652]: osdmap e452: 8 total, 8 up, 6 in 2023-09-23T22:35:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:16 smithi070 ceph-mon[192652]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:16 smithi070 ceph-mon[192652]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:16 smithi070 ceph-mon[192652]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.445 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:16.470 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:16.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:16.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:16.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:16.620 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:16.645 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[223014]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[223014]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[223014]: osdmap e452: 8 total, 8 up, 6 in 2023-09-23T22:35:16.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[223014]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[223014]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[223014]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[215775]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[215775]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[215775]: osdmap e452: 8 total, 8 up, 6 in 2023-09-23T22:35:16.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[215775]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[215775]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.647 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:16 smithi026 ceph-mon[215775]: failed to encode map e452 with expected crc 2023-09-23T22:35:16.648 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:17.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:35:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:17 smithi070 ceph-mon[192652]: pgmap v1164: 99 pgs: 5 creating+activating, 6 creating+peering, 2 unknown, 86 active+clean; 4.6 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:17 smithi070 ceph-mon[192652]: failed to encode map e453 with expected crc 2023-09-23T22:35:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:17 smithi070 ceph-mon[192652]: failed to encode map e453 with expected crc 2023-09-23T22:35:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:17 smithi070 ceph-mon[192652]: osdmap e453: 8 total, 8 up, 6 in 2023-09-23T22:35:17.674 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:17 smithi026 ceph-mon[215775]: pgmap v1164: 99 pgs: 5 creating+activating, 6 creating+peering, 2 unknown, 86 active+clean; 4.6 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:17.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:17 smithi026 ceph-mon[215775]: failed to encode map e453 with expected crc 2023-09-23T22:35:17.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:17 smithi026 ceph-mon[215775]: failed to encode map e453 with expected crc 2023-09-23T22:35:17.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:17 smithi026 ceph-mon[215775]: osdmap e453: 8 total, 8 up, 6 in 2023-09-23T22:35:17.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:17 smithi026 ceph-mon[223014]: pgmap v1164: 99 pgs: 5 creating+activating, 6 creating+peering, 2 unknown, 86 active+clean; 4.6 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:17.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:17 smithi026 ceph-mon[223014]: failed to encode map e453 with expected crc 2023-09-23T22:35:17.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:17 smithi026 ceph-mon[223014]: failed to encode map e453 with expected crc 2023-09-23T22:35:17.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:17 smithi026 ceph-mon[223014]: osdmap e453: 8 total, 8 up, 6 in 2023-09-23T22:35:18.316 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:18.350 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:18.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:18.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:18.458 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:18.509 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:18.567 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:18.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:18.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:18.671 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:18.700 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:19.222 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:19.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:19.262 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.7. 2023-09-23T22:35:19.326 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:19.356 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:19.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:19 smithi026 ceph-mon[223014]: pgmap v1166: 99 pgs: 5 creating+activating, 6 creating+peering, 88 active+clean; 4.6 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:19.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1504614481' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:35:19.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:35:19.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:19 smithi026 conmon[215751]: 2023-09-23T22:35:19.127+0000 7f4580f04700 -1 mon.a@0(leader).osd e453 definitely_dead 0 2023-09-23T22:35:19.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:19 smithi026 ceph-mon[215775]: pgmap v1166: 99 pgs: 5 creating+activating, 6 creating+peering, 88 active+clean; 4.6 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:19.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1504614481' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:35:19.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:35:19.576 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:19.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:19.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:19.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:19 smithi070 ceph-mon[192652]: pgmap v1166: 99 pgs: 5 creating+activating, 6 creating+peering, 88 active+clean; 4.6 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:19.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1504614481' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:35:19.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:35:19.681 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:19.713 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:19.722 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:19.762 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:19.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:19.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:19.867 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:19.893 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:35:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:35:19] "GET /metrics HTTP/1.1" 200 34020 "" "Prometheus/2.43.0" 2023-09-23T22:35:19.896 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:20.268 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:35:20.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:20 smithi070 ceph-mon[192652]: failed to encode map e454 with expected crc 2023-09-23T22:35:20.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:20 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T22:35:20.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:20 smithi070 ceph-mon[192652]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T22:35:20.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3204550009' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:20.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:20 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:20.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[223014]: failed to encode map e454 with expected crc 2023-09-23T22:35:20.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T22:35:20.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[223014]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T22:35:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3204550009' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:20.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[215775]: failed to encode map e454 with expected crc 2023-09-23T22:35:20.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T22:35:20.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[215775]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T22:35:20.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3204550009' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:20.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:20 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:21.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: pgmap v1168: 99 pgs: 10 peering, 5 creating+activating, 1 unknown, 2 creating+peering, 81 active+clean; 4.6 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:21.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:21.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:35:21.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.267 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[215775]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: pgmap v1168: 99 pgs: 10 peering, 5 creating+activating, 1 unknown, 2 creating+peering, 81 active+clean; 4.6 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.268 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T22:35:21.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.269 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:21 smithi026 ceph-mon[223014]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.596 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:21.621 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:21.622 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T22:35:21.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:21.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: pgmap v1168: 99 pgs: 10 peering, 5 creating+activating, 1 unknown, 2 creating+peering, 81 active+clean; 4.6 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:35:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:35:21.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T22:35:21.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:21 smithi070 ceph-mon[192652]: failed to encode map e455 with expected crc 2023-09-23T22:35:21.725 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:21.757 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:21.805 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:21.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:21.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:21.911 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:21.942 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:22.270 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:22.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:22.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:22.289 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T22:35:22.375 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:22.406 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:22.452 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:22.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:22.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:22.559 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:22.587 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[215775]: failed to encode map e456 with expected crc 2023-09-23T22:35:22.587 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[215775]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T22:35:22.587 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[215775]: failed to encode map e456 with expected crc 2023-09-23T22:35:22.587 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[215775]: failed to encode map e456 with expected crc 2023-09-23T22:35:22.587 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3119051178' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:35:22.587 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:35:22.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[223014]: failed to encode map e456 with expected crc 2023-09-23T22:35:22.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[223014]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T22:35:22.587 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[223014]: failed to encode map e456 with expected crc 2023-09-23T22:35:22.588 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[223014]: failed to encode map e456 with expected crc 2023-09-23T22:35:22.588 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3119051178' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:35:22.588 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:22 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:35:22.591 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:22.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:22 smithi070 ceph-mon[192652]: failed to encode map e456 with expected crc 2023-09-23T22:35:22.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:22 smithi070 ceph-mon[192652]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T22:35:22.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:22 smithi070 ceph-mon[192652]: failed to encode map e456 with expected crc 2023-09-23T22:35:22.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:22 smithi070 ceph-mon[192652]: failed to encode map e456 with expected crc 2023-09-23T22:35:22.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3119051178' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:35:22.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:22 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:35:23.372 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:35:23.390 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:35:23.395 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:23.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[223014]: pgmap v1171: 99 pgs: 4 activating, 1 activating+degraded, 3 remapped+peering, 17 peering, 1 active+recovery_wait+degraded, 1 unknown, 72 active+clean; 4.6 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 5/78 objects degraded (6.410%); 874 B/s, 0 objects/s recovering 2023-09-23T22:35:23.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[223014]: failed to encode map e456 with expected crc 2023-09-23T22:35:23.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 5/78 objects degraded (6.410%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:23.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[223014]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:35:23.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[223014]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T22:35:23.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[223014]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[223014]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.398 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[223014]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[215775]: pgmap v1171: 99 pgs: 4 activating, 1 activating+degraded, 3 remapped+peering, 17 peering, 1 active+recovery_wait+degraded, 1 unknown, 72 active+clean; 4.6 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 5/78 objects degraded (6.410%); 874 B/s, 0 objects/s recovering 2023-09-23T22:35:23.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[215775]: failed to encode map e456 with expected crc 2023-09-23T22:35:23.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 5/78 objects degraded (6.410%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:23.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[215775]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:35:23.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[215775]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T22:35:23.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[215775]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[215775]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:23 smithi026 ceph-mon[215775]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:23.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:23.410 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:35:23.429 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:35:23.448 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:35:23.468 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:35:23.488 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:35:23.501 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:23.507 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:35:23.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:35:23.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:35:23.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:35:23.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:35:23.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:35:23.510 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:35:23.510 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:35:23.510 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:35:23.541 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:23.589 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:23.591 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:35:23.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:23.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:23 smithi070 ceph-mon[192652]: pgmap v1171: 99 pgs: 4 activating, 1 activating+degraded, 3 remapped+peering, 17 peering, 1 active+recovery_wait+degraded, 1 unknown, 72 active+clean; 4.6 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 5/78 objects degraded (6.410%); 874 B/s, 0 objects/s recovering 2023-09-23T22:35:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:23 smithi070 ceph-mon[192652]: failed to encode map e456 with expected crc 2023-09-23T22:35:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:23 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 5/78 objects degraded (6.410%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:23 smithi070 ceph-mon[192652]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:23 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:35:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:23 smithi070 ceph-mon[192652]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T22:35:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:23 smithi070 ceph-mon[192652]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:23 smithi070 ceph-mon[192652]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:23 smithi070 ceph-mon[192652]: failed to encode map e457 with expected crc 2023-09-23T22:35:23.693 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:23.722 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:24.300 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T22:35:24.302 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:35:24.302 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:35:24.351 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T22:35:24.351 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:35:24.351 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_555954 2023-09-23T22:35:24.351 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:35:24.351 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:35:24.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[215775]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[215775]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[215775]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T22:35:24.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[215775]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[215775]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[215775]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4044073434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:24.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:24.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:24.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[223014]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[223014]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[223014]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T22:35:24.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[223014]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[223014]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[223014]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4044073434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:24.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:24.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:24.524 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:24.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:24.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:24 smithi070 ceph-mon[192652]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:24 smithi070 ceph-mon[192652]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:24 smithi070 ceph-mon[192652]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T22:35:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:24 smithi070 ceph-mon[192652]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:24 smithi070 ceph-mon[192652]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:24 smithi070 ceph-mon[192652]: failed to encode map e458 with expected crc 2023-09-23T22:35:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4044073434' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:24.640 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:24.682 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:24.736 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:24.757 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:24.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:24.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:24.866 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:24.920 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:25.020 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:25.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:25.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:25.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:25.170 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:25.253 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:25.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:25.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:25.352 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 3476 3460 216.116 216.25 0.00423986 0.00460883 2023-09-23T22:35:25.360 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:25.361 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:35:25.361 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:35:25.361 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:35:25.361 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:35:25.361 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:35:25.424 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:25 smithi070 ceph-mon[192652]: pgmap v1174: 99 pgs: 4 activating, 1 activating+degraded, 3 remapped+peering, 19 peering, 1 active+recovery_wait+degraded, 1 unknown, 70 active+clean; 4.6 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 5/78 objects degraded (6.410%); 1.2 KiB/s, 2 keys/s, 0 objects/s recovering 2023-09-23T22:35:25.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:25 smithi026 ceph-mon[215775]: pgmap v1174: 99 pgs: 4 activating, 1 activating+degraded, 3 remapped+peering, 19 peering, 1 active+recovery_wait+degraded, 1 unknown, 70 active+clean; 4.6 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 5/78 objects degraded (6.410%); 1.2 KiB/s, 2 keys/s, 0 objects/s recovering 2023-09-23T22:35:25.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:25 smithi026 ceph-mon[223014]: pgmap v1174: 99 pgs: 4 activating, 1 activating+degraded, 3 remapped+peering, 19 peering, 1 active+recovery_wait+degraded, 1 unknown, 70 active+clean; 4.6 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 5/78 objects degraded (6.410%); 1.2 KiB/s, 2 keys/s, 0 objects/s recovering 2023-09-23T22:35:26.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:26.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:26.188 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:26.235 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:26.327 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:26.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:26.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:26.352 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 7029 7013 219.07 222.062 0.0027435 0.00455692 2023-09-23T22:35:26.436 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:26.483 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:26 smithi070 ceph-mon[192652]: pgmap v1175: 99 pgs: 2 peering, 97 active+clean; 141 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 212 KiB/s rd, 25 MiB/s wr, 572 op/s; 60 KiB/s, 1 keys/s, 14 objects/s recovering 2023-09-23T22:35:26.483 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:26 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/78 objects degraded (6.410%), 2 pgs degraded) 2023-09-23T22:35:26.488 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:26.543 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:35:26.581 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:26.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:26.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:26.593 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:26 smithi026 ceph-mon[215775]: pgmap v1175: 99 pgs: 2 peering, 97 active+clean; 141 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 212 KiB/s rd, 25 MiB/s wr, 572 op/s; 60 KiB/s, 1 keys/s, 14 objects/s recovering 2023-09-23T22:35:26.594 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:26 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/78 objects degraded (6.410%), 2 pgs degraded) 2023-09-23T22:35:26.594 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:26 smithi026 ceph-mon[223014]: pgmap v1175: 99 pgs: 2 peering, 97 active+clean; 141 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 212 KiB/s rd, 25 MiB/s wr, 572 op/s; 60 KiB/s, 1 keys/s, 14 objects/s recovering 2023-09-23T22:35:26.594 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:26 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/78 objects degraded (6.410%), 2 pgs degraded) 2023-09-23T22:35:26.623 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:35:26.623 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:35:26.623 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [0, 1, 3, 4, 5, 6, 7, 2] 2023-09-23T22:35:26.624 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T22:35:26.624 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T22:35:26.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:26.736 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:26 smithi070 systemd[1]: Stopping Ceph osd.4 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:35:26.739 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:26.842 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:26.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:26.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:26.948 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:26.997 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:27.152 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:27.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:27.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:27.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:27.284 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:27.342 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:27 smithi070 conmon[113214]: 2023-09-23T22:35:27.130+0000 7f749a44a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:35:27.342 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:27 smithi070 conmon[113214]: 2023-09-23T22:35:27.130+0000 7f749a44a700 -1 osd.4 458 *** Got signal Terminated *** 2023-09-23T22:35:27.342 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:27 smithi070 conmon[113214]: 2023-09-23T22:35:27.130+0000 7f749a44a700 -1 osd.4 458 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:35:27.352 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 9499 9483 197.507 154.375 0.00465377 0.00505457 2023-09-23T22:35:27.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:27 smithi026 ceph-mon[215775]: osd.4 marked itself down and dead 2023-09-23T22:35:27.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:27 smithi026 ceph-mon[223014]: osd.4 marked itself down and dead 2023-09-23T22:35:27.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:27 smithi070 ceph-mon[192652]: osd.4 marked itself down and dead 2023-09-23T22:35:27.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:27.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:27.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:27.787 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:27.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:27.858 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:27.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:27.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:27.961 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:27.990 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:28.315 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:35:28.341 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:28.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:28.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:28.352 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 9551 9535 148.949 3.25 0.00130211 0.00518123 2023-09-23T22:35:28.374 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:28 smithi070 bash[338311]: 92889bd7f5eec3b99b8f22018354d23df2c45f496753ca138dd3b443526d4b0f 2023-09-23T22:35:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:28 smithi070 ceph-mon[192652]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:35:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:28 smithi070 ceph-mon[192652]: failed to encode map e459 with expected crc 2023-09-23T22:35:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:28 smithi070 ceph-mon[192652]: osdmap e459: 8 total, 7 up, 7 in 2023-09-23T22:35:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:28 smithi070 ceph-mon[192652]: pgmap v1177: 99 pgs: 7 stale+active+clean, 9 peering, 83 active+clean; 215 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 190 KiB/s rd, 35 MiB/s wr, 711 op/s; 53 KiB/s, 1 keys/s, 11 objects/s recovering 2023-09-23T22:35:28.385 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:28.449 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:28.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:28 smithi026 ceph-mon[223014]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:35:28.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:28 smithi026 ceph-mon[223014]: failed to encode map e459 with expected crc 2023-09-23T22:35:28.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:28 smithi026 ceph-mon[223014]: osdmap e459: 8 total, 7 up, 7 in 2023-09-23T22:35:28.515 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:28 smithi026 ceph-mon[223014]: pgmap v1177: 99 pgs: 7 stale+active+clean, 9 peering, 83 active+clean; 215 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 190 KiB/s rd, 35 MiB/s wr, 711 op/s; 53 KiB/s, 1 keys/s, 11 objects/s recovering 2023-09-23T22:35:28.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:28 smithi026 ceph-mon[215775]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:35:28.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:28 smithi026 ceph-mon[215775]: failed to encode map e459 with expected crc 2023-09-23T22:35:28.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:28 smithi026 ceph-mon[215775]: osdmap e459: 8 total, 7 up, 7 in 2023-09-23T22:35:28.516 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:28 smithi026 ceph-mon[215775]: pgmap v1177: 99 pgs: 7 stale+active+clean, 9 peering, 83 active+clean; 215 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 190 KiB/s rd, 35 MiB/s wr, 711 op/s; 53 KiB/s, 1 keys/s, 11 objects/s recovering 2023-09-23T22:35:28.526 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:28.730 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:28.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:28.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:28.838 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:28.881 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:28.951 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:28.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:29.056 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:29.099 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:29.162 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:29.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:29.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:29.267 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:29.338 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:29.352 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 12044 12028 150.32 155.812 0.00943796 0.00664442 2023-09-23T22:35:29.445 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:29.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:29.552 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:29.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:29 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:35:29.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:29 smithi070 ceph-mon[192652]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:29 smithi070 ceph-mon[192652]: osdmap e460: 8 total, 7 up, 7 in 2023-09-23T22:35:29.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:29 smithi070 ceph-mon[192652]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:29 smithi070 ceph-mon[192652]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.576 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:29 smithi070 ceph-mon[192652]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.579 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:29.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:35:29.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[215775]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[215775]: osdmap e460: 8 total, 7 up, 7 in 2023-09-23T22:35:29.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[215775]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[215775]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[215775]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:35:29.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[223014]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[223014]: osdmap e460: 8 total, 7 up, 7 in 2023-09-23T22:35:29.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[223014]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[223014]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:29 smithi026 ceph-mon[223014]: failed to encode map e460 with expected crc 2023-09-23T22:35:29.621 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:29.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:29.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:29.724 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:29.752 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:29.795 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:29.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:29.798 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:29.898 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:29.924 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:29.964 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:29.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:29.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:29.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:35:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:35:29] "GET /metrics HTTP/1.1" 200 36313 "" "Prometheus/2.43.0" 2023-09-23T22:35:30.068 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:30.111 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:30.183 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:30.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:30.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:30.289 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:30.308 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:30.321 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:30.352 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 13514 13498 140.578 91.875 0.00703882 0.00709933 2023-09-23T22:35:30.385 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:35:30.610 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:30.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:30.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:30.615 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2371515236' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:30.615 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:30 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:30.615 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:30 smithi070 ceph-mon[192652]: pgmap v1179: 99 pgs: 4 active+undersized+degraded, 9 active+undersized, 6 stale+active+clean, 9 peering, 71 active+clean; 436 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 179 KiB/s rd, 68 MiB/s wr, 1.23k op/s; 1570/20778 objects degraded (7.556%); 50 KiB/s, 9 objects/s recovering 2023-09-23T22:35:30.644 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2371515236' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:30.644 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:30 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:30.644 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:30 smithi026 ceph-mon[215775]: pgmap v1179: 99 pgs: 4 active+undersized+degraded, 9 active+undersized, 6 stale+active+clean, 9 peering, 71 active+clean; 436 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 179 KiB/s rd, 68 MiB/s wr, 1.23k op/s; 1570/20778 objects degraded (7.556%); 50 KiB/s, 9 objects/s recovering 2023-09-23T22:35:30.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2371515236' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:30.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:30 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:30.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:30 smithi026 ceph-mon[223014]: pgmap v1179: 99 pgs: 4 active+undersized+degraded, 9 active+undersized, 6 stale+active+clean, 9 peering, 71 active+clean; 436 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 179 KiB/s rd, 68 MiB/s wr, 1.23k op/s; 1570/20778 objects degraded (7.556%); 50 KiB/s, 9 objects/s recovering 2023-09-23T22:35:30.715 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:30.771 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:31.355 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 15620 15604 139.244 131.625 0.00494869 0.00717542 2023-09-23T22:35:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:31 smithi070 ceph-mon[192652]: Health check failed: Degraded data redundancy: 1570/20778 objects degraded (7.556%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:31 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:35:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:31 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:35:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:31 smithi070 ceph-mon[192652]: osdmap e461: 8 total, 7 up, 7 in 2023-09-23T22:35:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:31 smithi070 ceph-mon[192652]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:31 smithi070 ceph-mon[192652]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:31 smithi070 ceph-mon[192652]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:31 smithi070 ceph-mon[192652]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.649 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:31.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[223014]: Health check failed: Degraded data redundancy: 1570/20778 objects degraded (7.556%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:31.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:35:31.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:35:31.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[223014]: osdmap e461: 8 total, 7 up, 7 in 2023-09-23T22:35:31.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[223014]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[223014]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.654 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[223014]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.655 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[223014]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:31.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:31.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[215775]: Health check failed: Degraded data redundancy: 1570/20778 objects degraded (7.556%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:31.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:35:31.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:35:31.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[215775]: osdmap e461: 8 total, 7 up, 7 in 2023-09-23T22:35:31.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[215775]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[215775]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.656 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[215775]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:31 smithi026 ceph-mon[215775]: failed to encode map e461 with expected crc 2023-09-23T22:35:31.756 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:31.830 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:32.244 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:32.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:32.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:32.356 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:32.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 18220 18204 142.079 162.5 0.00423738 0.00703321 2023-09-23T22:35:32.418 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:35:32.429 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:32.457 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:35:32.496 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:35:32.515 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:32.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:32 smithi026 ceph-mon[215775]: pgmap v1181: 99 pgs: 15 active+undersized+degraded, 27 active+undersized, 9 peering, 48 active+clean; 787 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s; 4063/37632 objects degraded (10.797%) 2023-09-23T22:35:32.518 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:32 smithi026 ceph-mon[223014]: pgmap v1181: 99 pgs: 15 active+undersized+degraded, 27 active+undersized, 9 peering, 48 active+clean; 787 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s; 4063/37632 objects degraded (10.797%) 2023-09-23T22:35:32.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:32.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:32.535 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:35:32.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:32.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:32 smithi070 ceph-mon[192652]: pgmap v1181: 99 pgs: 15 active+undersized+degraded, 27 active+undersized, 9 peering, 48 active+clean; 787 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 108 MiB/s wr, 1.72k op/s; 4063/37632 objects degraded (10.797%) 2023-09-23T22:35:32.649 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:32.693 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:32.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:32.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:32.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:32.828 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:32.876 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:32.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:32.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:32.970 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:35:32.970 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:35:32.970 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:35:32.970 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:35:32.970 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:35:32.970 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:35:32.971 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:35:32.971 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T22:35:32.971 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:35:32.971 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:35:32.971 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:35:32.981 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:33.034 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:33.359 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T22:35:33.360 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 19699 19683 136.56 92.4375 0.00407503 0.00731718 2023-09-23T22:35:33.435 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:35:33.435 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T22:35:33.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:33 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1339148835' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:33 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1339148835' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:33 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1339148835' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:33.908 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:35:34.264 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:34.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:34.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:34.360 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 21949 21933 136.965 140.625 0.00826855 0.00729534 2023-09-23T22:35:34.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:34.405 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:35:34.405 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:35:34.439 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:34.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:34 smithi026 ceph-mon[215775]: pgmap v1182: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 852 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s; 4066/40752 objects degraded (9.977%) 2023-09-23T22:35:34.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:34 smithi026 ceph-mon[223014]: pgmap v1182: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 852 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s; 4066/40752 objects degraded (9.977%) 2023-09-23T22:35:34.605 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:34.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:34.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:34.709 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:34.770 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:34.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:34 smithi070 ceph-mon[192652]: pgmap v1182: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 852 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 113 MiB/s wr, 1.81k op/s; 4066/40752 objects degraded (9.977%) 2023-09-23T22:35:34.881 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:34.900 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:34.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:34.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:34.948 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:35:34.949 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:35:35.008 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:35.057 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:35.140 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:35.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:35.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:35.248 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:35.300 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:35.360 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 24129 24113 136.898 136.25 0.00609802 0.00729989 2023-09-23T22:35:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:35 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:35:35.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:35 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:35:35.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:35 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:35:36.360 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 26145 26129 135.99 126 0.0099129 0.00734906 2023-09-23T22:35:36.407 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:35:36.431 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:36.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:36.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:36.536 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:36.609 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:35:36.614 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:36.641 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:36 smithi070 ceph-mon[192652]: pgmap v1183: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 1.4 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 151 MiB/s wr, 2.41k op/s; 7296/68166 objects degraded (10.703%) 2023-09-23T22:35:36.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:36 smithi026 ceph-mon[215775]: pgmap v1183: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 1.4 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 151 MiB/s wr, 2.41k op/s; 7296/68166 objects degraded (10.703%) 2023-09-23T22:35:36.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:36 smithi026 ceph-mon[223014]: pgmap v1183: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 1.4 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 151 MiB/s wr, 2.41k op/s; 7296/68166 objects degraded (10.703%) 2023-09-23T22:35:36.767 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:36.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:36.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:36.874 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:36.938 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:37.147 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:37.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:37.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:37.204 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:35:37.253 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:37.317 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:37.360 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 28160 28144 135.213 125.938 0.00310812 0.00738929 2023-09-23T22:35:37.399 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:37.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:37.503 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:37.530 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:37.574 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:37.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:37 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7296/68166 objects degraded (10.703%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:37.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:37 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7296/68166 objects degraded (10.703%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:37.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:37.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:37.678 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:37.731 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:37.804 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:37 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7296/68166 objects degraded (10.703%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:38.056 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:38.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:38.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:38.161 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:38.229 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:38.233 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:38.329 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:38.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:38.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:38.360 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 29713 29697 132.489 97.0625 0.00977728 0.00754237 2023-09-23T22:35:38.436 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:38.436 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7] out_osds: [3] dead_osds: [4] live_osds: [0, 1, 3, 5, 6, 7, 2] 2023-09-23T22:35:38.436 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:35:38.436 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2023-09-23T22:35:38.436 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2023-09-23T22:35:38.437 DEBUG:teuthology.orchestra.run.smithi070:osd.4> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T22:35:38.440 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl start ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T22:35:38.483 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:38.486 INFO:journalctl@ceph.osd.4.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:35:38.563 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:38.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:38.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:38.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:38.712 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:38.874 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:38 smithi070 systemd[1]: Starting Ceph osd.4 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:35:39.363 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 31546 31530 131.27 114.562 0.00612182 0.00761134 2023-09-23T22:35:39.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:39 smithi026 ceph-mon[223014]: pgmap v1184: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s; 7296/71892 objects degraded (10.149%) 2023-09-23T22:35:39.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:39.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:39.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:39.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:39 smithi026 ceph-mon[215775]: pgmap v1184: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s; 7296/71892 objects degraded (10.149%) 2023-09-23T22:35:39.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:39.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:39.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:39.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:39 smithi070 ceph-mon[192652]: pgmap v1184: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s; 7296/71892 objects degraded (10.149%) 2023-09-23T22:35:39.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:35:39.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:39.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:39.987 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:35:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:35:39] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.43.0" 2023-09-23T22:35:40.152 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:40.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:40.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:40.240 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:35:40.259 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:40.319 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:40.366 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 33093 33077 129.11 96.6875 0.00655536 0.00774011 2023-09-23T22:35:40.427 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:40.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:40.433 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/905308004' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:35:40.478 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/905308004' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:35:40.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/905308004' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:35:40.534 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:40.598 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:40.689 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:40.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:40.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:40.796 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:40.827 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:41.012 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:35:41.012 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:35:41.013 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:41.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:41.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:41.122 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:41.201 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:41.345 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:41 smithi070 ceph-mon[192652]: pgmap v1185: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 1.6 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s; 8486/80952 objects degraded (10.483%) 2023-09-23T22:35:41.345 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:41 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:41.345 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/905308004' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:35:41.345 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:41 smithi070 ceph-mon[192652]: osdmap e462: 8 total, 7 up, 7 in 2023-09-23T22:35:41.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:41 smithi070 ceph-mon[192652]: failed to encode map e462 with expected crc 2023-09-23T22:35:41.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:41 smithi070 ceph-mon[192652]: failed to encode map e462 with expected crc 2023-09-23T22:35:41.346 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:41 smithi070 ceph-mon[192652]: failed to encode map e462 with expected crc 2023-09-23T22:35:41.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 34634 34618 127.179 96.3125 0.00841725 0.00785759 2023-09-23T22:35:41.385 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:41.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:41.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:41.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[223014]: pgmap v1185: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 1.6 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s; 8486/80952 objects degraded (10.483%) 2023-09-23T22:35:41.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:41.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/905308004' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:35:41.479 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[223014]: osdmap e462: 8 total, 7 up, 7 in 2023-09-23T22:35:41.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[223014]: failed to encode map e462 with expected crc 2023-09-23T22:35:41.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[223014]: failed to encode map e462 with expected crc 2023-09-23T22:35:41.480 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[223014]: failed to encode map e462 with expected crc 2023-09-23T22:35:41.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[215775]: pgmap v1185: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 1.6 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s; 8486/80952 objects degraded (10.483%) 2023-09-23T22:35:41.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:41.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/905308004' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:35:41.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[215775]: osdmap e462: 8 total, 7 up, 7 in 2023-09-23T22:35:41.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[215775]: failed to encode map e462 with expected crc 2023-09-23T22:35:41.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[215775]: failed to encode map e462 with expected crc 2023-09-23T22:35:41.481 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:41 smithi026 ceph-mon[215775]: failed to encode map e462 with expected crc 2023-09-23T22:35:41.491 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:41.554 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:41.662 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:41.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:41.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:41.769 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:41.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:35:41.814 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:35:41.841 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:41.854 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:35:41.890 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:35:41.930 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:35:41.943 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:41.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:41.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:41.972 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:35:42.007 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:35:42.007 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:35:42.007 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:35:42.007 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:35:42.007 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:35:42.007 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T22:35:42.007 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:35:42.007 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:35:42.008 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:35:42.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:42.096 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:41 smithi070 bash[340230]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:35:42.096 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:41 smithi070 bash[340230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-09-23T22:35:42.096 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:41 smithi070 bash[340230]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-4 --no-mon-config 2023-09-23T22:35:42.096 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:41 smithi070 bash[340230]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-4/block 2023-09-23T22:35:42.096 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:41 smithi070 bash[340230]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-4/block 2023-09-23T22:35:42.096 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:41 smithi070 bash[340230]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T22:35:42.096 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:41 smithi070 bash[340230]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2023-09-23T22:35:42.096 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:41 smithi070 bash[340230]: --> ceph-volume lvm activate successful for osd ID: 4 2023-09-23T22:35:42.104 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:42.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 36211 36195 125.589 98.5625 0.00866505 0.00795677 2023-09-23T22:35:42.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3200518599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3200518599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:42.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3200518599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:42.732 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:42.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:42.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:42.839 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:42.866 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:43.307 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:35:43.314 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:35:43.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 37757 37741 124.065 96.625 0.00861266 0.00805605 2023-09-23T22:35:43.432 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:43.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:43.440 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:43.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:43 smithi026 ceph-mon[215775]: pgmap v1187: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s; 10211/96117 objects degraded (10.624%) 2023-09-23T22:35:43.480 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10211/96117 objects degraded (10.624%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:43.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:43 smithi026 ceph-mon[223014]: pgmap v1187: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s; 10211/96117 objects degraded (10.624%) 2023-09-23T22:35:43.483 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10211/96117 objects degraded (10.624%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:43.513 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:35:43.513 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:35:43.541 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:43.609 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:43 smithi070 ceph-mon[192652]: pgmap v1187: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.0 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s; 10211/96117 objects degraded (10.624%) 2023-09-23T22:35:43.609 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10211/96117 objects degraded (10.624%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:43.615 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:43.710 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:43.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:43.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:43.815 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:43.878 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:43.978 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:35:43.982 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:44.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:44.011 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:44.104 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:44.169 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:44.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:35:44.364690+0000 min lat: 0.00117176 max lat: 1.02666 avg lat: 0.00816323 2023-09-23T22:35:44.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:35:44.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 39217 39201 122.425 91.25 0.0158853 0.00816323 2023-09-23T22:35:44.391 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:44.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:44.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:44.502 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:44.569 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:44.764 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:44 smithi070 ceph-osd[341576]: -- 172.21.15.70:0/646870531 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dae6c70b60 con 0x55dae6c36c00 2023-09-23T22:35:44.766 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:44.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:44.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:44.872 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:44.930 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:45.112 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:44 smithi070 conmon[341485]: 2023-09-23T22:35:44.864+0000 7f50c5984540 -1 Falling back to public interface 2023-09-23T22:35:45.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 40885 40869 121.559 104.25 0.00226188 0.00819879 2023-09-23T22:35:45.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:45 smithi070 ceph-mon[192652]: pgmap v1188: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s; 10211/99132 objects degraded (10.300%) 2023-09-23T22:35:45.374 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:45 smithi070 bash[341304]: 3e1513b12ef20402fede1fdd37cd0dcd0fa7a89202d4e6332293cc37fb6159ce 2023-09-23T22:35:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:45 smithi026 ceph-mon[215775]: pgmap v1188: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s; 10211/99132 objects degraded (10.300%) 2023-09-23T22:35:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:45 smithi026 ceph-mon[223014]: pgmap v1188: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.0 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 122 MiB/s wr, 1.95k op/s; 10211/99132 objects degraded (10.300%) 2023-09-23T22:35:45.514 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:35:45.747 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:35:45.964 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:45.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:46.042 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:45 smithi070 systemd[1]: Started Ceph osd.4 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:35:46.071 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:46.146 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:46.322 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:46.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:46.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:46.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 42022 42006 119.264 71.0625 0.00760995 0.00837861 2023-09-23T22:35:46.425 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:46.438 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:46.514 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:46.575 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:35:46.706 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:35:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:35:46.706 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:35:46.707 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:35:46.707 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:35:46.726 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:46.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:46.846 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:46.908 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:47.143 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:47.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:47.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:47.247 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:47.288 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:47.340 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:47 smithi070 ceph-mon[192652]: pgmap v1189: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s; 12613/119514 objects degraded (10.554%) 2023-09-23T22:35:47.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 43465 43449 118 90.1875 0.00846849 0.00846824 2023-09-23T22:35:47.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:47 smithi026 ceph-mon[223014]: pgmap v1189: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s; 12613/119514 objects degraded (10.554%) 2023-09-23T22:35:47.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:47 smithi026 ceph-mon[215775]: pgmap v1189: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.4 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 107 MiB/s wr, 1.71k op/s; 12613/119514 objects degraded (10.554%) 2023-09-23T22:35:47.920 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:47.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:47.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:48.025 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:48.089 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:48.253 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:35:48.294 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1163392574' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:48.294 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:48 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:48.295 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12613/119514 objects degraded (10.554%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:48.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 44679 44663 116.246 75.875 0.00545197 0.00859717 2023-09-23T22:35:48.724 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:35:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1163392574' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:48 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12613/119514 objects degraded (10.554%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:48.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1163392574' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:48 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:35:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12613/119514 objects degraded (10.554%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:48.837 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:48.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:48.950 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:48.994 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:49.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:49 smithi070 ceph-mon[192652]: pgmap v1190: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s; 12613/122175 objects degraded (10.324%) 2023-09-23T22:35:49.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:49 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:49.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:49 smithi070 ceph-mon[192652]: failed to encode map e463 with expected crc 2023-09-23T22:35:49.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:49 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:35:49.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:49 smithi070 ceph-mon[192652]: osdmap e463: 8 total, 7 up, 7 in 2023-09-23T22:35:49.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:49 smithi070 ceph-mon[192652]: failed to encode map e463 with expected crc 2023-09-23T22:35:49.260 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:49 smithi070 ceph-mon[192652]: failed to encode map e463 with expected crc 2023-09-23T22:35:49.365 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 45756 45740 114.289 67.3125 0.00934356 0.00874521 2023-09-23T22:35:49.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[215775]: pgmap v1190: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s; 12613/122175 objects degraded (10.324%) 2023-09-23T22:35:49.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:49.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[215775]: failed to encode map e463 with expected crc 2023-09-23T22:35:49.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:35:49.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[215775]: osdmap e463: 8 total, 7 up, 7 in 2023-09-23T22:35:49.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[215775]: failed to encode map e463 with expected crc 2023-09-23T22:35:49.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[215775]: failed to encode map e463 with expected crc 2023-09-23T22:35:49.618 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:35:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:35:49] "GET /metrics HTTP/1.1" 200 36372 "" "Prometheus/2.43.0" 2023-09-23T22:35:49.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[223014]: pgmap v1190: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.5 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 105 MiB/s wr, 1.68k op/s; 12613/122175 objects degraded (10.324%) 2023-09-23T22:35:49.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:49.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[223014]: failed to encode map e463 with expected crc 2023-09-23T22:35:49.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:35:49.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[223014]: osdmap e463: 8 total, 7 up, 7 in 2023-09-23T22:35:49.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[223014]: failed to encode map e463 with expected crc 2023-09-23T22:35:49.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:49 smithi026 ceph-mon[223014]: failed to encode map e463 with expected crc 2023-09-23T22:35:49.722 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:49.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:49.812 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:35:49.835 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:49.899 INFO:teuthology.orchestra.run.smithi070.stderr:no valid command found; 10 closest matches: 2023-09-23T22:35:49.899 INFO:teuthology.orchestra.run.smithi070.stderr:0 2023-09-23T22:35:49.899 INFO:teuthology.orchestra.run.smithi070.stderr:1 2023-09-23T22:35:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:2 2023-09-23T22:35:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:abort 2023-09-23T22:35:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:assert 2023-09-23T22:35:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:bluefs debug_inject_read_zeros 2023-09-23T22:35:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:bluefs files list 2023-09-23T22:35:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:bluefs stats 2023-09-23T22:35:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:bluestore allocator dump block 2023-09-23T22:35:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:bluestore allocator fragmentation block 2023-09-23T22:35:49.900 INFO:teuthology.orchestra.run.smithi070.stderr:admin_socket: invalid command 2023-09-23T22:35:49.940 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:50.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:50.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:50.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:50.185 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:50.239 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:50.286 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:35:50.318 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:35:50.365 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 47087 47071 113.093 83.1875 0.00984807 0.00883769 2023-09-23T22:35:50.371 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:35:50.402 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:35:50.441 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:35:50.480 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:35:50.517 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:35:50.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:35:50.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:35:50.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:35:50.517 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:35:50.517 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.4 2023-09-23T22:35:50.518 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:35:50.518 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:35:50.518 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:35:50.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:50 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:50.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:50 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:50.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:50 smithi070 ceph-mon[192652]: pgmap v1192: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 13445/128310 objects degraded (10.479%) 2023-09-23T22:35:50.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4086120984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:50.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:50 smithi070 ceph-mon[192652]: from='osd.4 [v2:172.21.15.70:6800/3366415869,v1:172.21.15.70:6801/3366415869]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:35:50.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:50 smithi070 ceph-mon[192652]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:35:50.875 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:50 smithi070 conmon[341485]: 2023-09-23T22:35:50.533+0000 7f50c5984540 -1 osd.4 458 log_to_monitors true 2023-09-23T22:35:50.876 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:35:50 smithi070 conmon[341485]: 2023-09-23T22:35:50.709+0000 7f50b7766700 -1 osd.4 458 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:35:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:50.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[223014]: pgmap v1192: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 13445/128310 objects degraded (10.479%) 2023-09-23T22:35:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4086120984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[223014]: from='osd.4 [v2:172.21.15.70:6800/3366415869,v1:172.21.15.70:6801/3366415869]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:35:50.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[223014]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:35:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[215775]: pgmap v1192: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 93 MiB/s wr, 1.49k op/s; 13445/128310 objects degraded (10.479%) 2023-09-23T22:35:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4086120984' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[215775]: from='osd.4 [v2:172.21.15.70:6800/3366415869,v1:172.21.15.70:6801/3366415869]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:35:50.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:50 smithi026 ceph-mon[215775]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T22:35:51.135 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:51.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:51.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:51.214 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T22:35:51.214 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2023-09-23T22:35:51.240 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:51.277 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:51.355 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:51.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:51.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:51.364 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 48470 48454 112.106 86.4375 0.00536825 0.00891617 2023-09-23T22:35:51.465 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:51.525 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:51.738 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:51.738 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:35:51.739 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:35:51.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:51.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:51.769 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:35:51.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[215775]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:35:51.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[215775]: osdmap e464: 8 total, 7 up, 7 in 2023-09-23T22:35:51.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[215775]: from='osd.4 [v2:172.21.15.70:6800/3366415869,v1:172.21.15.70:6801/3366415869]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:35:51.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[215775]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:35:51.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[215775]: failed to encode map e464 with expected crc 2023-09-23T22:35:51.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[215775]: failed to encode map e464 with expected crc 2023-09-23T22:35:51.807 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[215775]: failed to encode map e464 with expected crc 2023-09-23T22:35:51.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[223014]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:35:51.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[223014]: osdmap e464: 8 total, 7 up, 7 in 2023-09-23T22:35:51.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[223014]: from='osd.4 [v2:172.21.15.70:6800/3366415869,v1:172.21.15.70:6801/3366415869]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:35:51.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[223014]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:35:51.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[223014]: failed to encode map e464 with expected crc 2023-09-23T22:35:51.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[223014]: failed to encode map e464 with expected crc 2023-09-23T22:35:51.808 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:51 smithi026 ceph-mon[223014]: failed to encode map e464 with expected crc 2023-09-23T22:35:51.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:51.871 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:51 smithi070 ceph-mon[192652]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T22:35:51.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:51 smithi070 ceph-mon[192652]: osdmap e464: 8 total, 7 up, 7 in 2023-09-23T22:35:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:51 smithi070 ceph-mon[192652]: from='osd.4 [v2:172.21.15.70:6800/3366415869,v1:172.21.15.70:6801/3366415869]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:35:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:51 smithi070 ceph-mon[192652]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:35:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:51 smithi070 ceph-mon[192652]: failed to encode map e464 with expected crc 2023-09-23T22:35:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:51 smithi070 ceph-mon[192652]: failed to encode map e464 with expected crc 2023-09-23T22:35:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:51 smithi070 ceph-mon[192652]: failed to encode map e464 with expected crc 2023-09-23T22:35:51.917 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:51.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:51.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:52.023 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:52.052 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:52.104 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:52.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:52.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:52.154 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:52.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:52.238 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:52.365 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 48990 48974 109.264 32.5 0.00319403 0.00892999 2023-09-23T22:35:52.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: pgmap v1194: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 14665/138798 objects degraded (10.566%) 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: osd.4 [v2:172.21.15.70:6800/3366415869,v1:172.21.15.70:6801/3366415869] boot 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: osdmap e465: 8 total, 8 up, 7 in 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: failed to encode map e465 with expected crc 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: failed to encode map e465 with expected crc 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: failed to encode map e465 with expected crc 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: failed to encode map e465 with expected crc 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:52.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[215775]: osdmap e466: 8 total, 8 up, 7 in 2023-09-23T22:35:52.688 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:52.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:52.692 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:52.793 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:52.823 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:52.870 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:52.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:52.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:52.973 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: pgmap v1194: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 14665/138798 objects degraded (10.566%) 2023-09-23T22:35:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:35:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: osd.4 [v2:172.21.15.70:6800/3366415869,v1:172.21.15.70:6801/3366415869] boot 2023-09-23T22:35:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: osdmap e465: 8 total, 8 up, 7 in 2023-09-23T22:35:52.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: failed to encode map e465 with expected crc 2023-09-23T22:35:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:35:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: failed to encode map e465 with expected crc 2023-09-23T22:35:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: failed to encode map e465 with expected crc 2023-09-23T22:35:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: failed to encode map e465 with expected crc 2023-09-23T22:35:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:52.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:52 smithi026 ceph-mon[223014]: osdmap e466: 8 total, 8 up, 7 in 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: pgmap v1194: 99 pgs: 17 active+undersized+degraded, 34 active+undersized, 48 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 14665/138798 objects degraded (10.566%) 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: osd.4 [v2:172.21.15.70:6800/3366415869,v1:172.21.15.70:6801/3366415869] boot 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: osdmap e465: 8 total, 8 up, 7 in 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: failed to encode map e465 with expected crc 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: failed to encode map e465 with expected crc 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: failed to encode map e465 with expected crc 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: failed to encode map e465 with expected crc 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:53.000 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:53.001 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:52 smithi070 ceph-mon[192652]: osdmap e466: 8 total, 8 up, 7 in 2023-09-23T22:35:53.002 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:53.097 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:53.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:53.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:53.201 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:53.229 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:53.278 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:53.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:53.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:53.307 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:35:53.307 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:35:53.365 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 48990 48974 105.497 0 - 0.00892999 2023-09-23T22:35:53.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:53.411 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:53.509 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:53.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:53.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:53.615 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:53.644 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:53.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e459 with expected crc 2023-09-23T22:35:53.739 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:35:53.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:53.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:53.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:53.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e465 with expected crc 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e465 with expected crc 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.956 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:53.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:53.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: pgmap v1197: 99 pgs: 5 peering, 15 active+undersized+degraded, 31 active+undersized, 48 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s; 14662/140829 objects degraded (10.411%) 2023-09-23T22:35:53.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:53.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 14665/138798 objects degraded (10.566%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:53.957 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[215775]: osdmap e467: 8 total, 8 up, 7 in 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e459 with expected crc 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e465 with expected crc 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e465 with expected crc 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.958 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: failed to encode map e466 with expected crc 2023-09-23T22:35:53.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:53.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:53.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: pgmap v1197: 99 pgs: 5 peering, 15 active+undersized+degraded, 31 active+undersized, 48 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s; 14662/140829 objects degraded (10.411%) 2023-09-23T22:35:53.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:53.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 14665/138798 objects degraded (10.566%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:53.959 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:53 smithi026 ceph-mon[223014]: osdmap e467: 8 total, 8 up, 7 in 2023-09-23T22:35:53.972 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:54.104 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:35:54.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:54.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:35:54.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e459 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e465 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e465 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: failed to encode map e466 with expected crc 2023-09-23T22:35:54.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:35:54.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:35:54.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: pgmap v1197: 99 pgs: 5 peering, 15 active+undersized+degraded, 31 active+undersized, 48 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s; 14662/140829 objects degraded (10.411%) 2023-09-23T22:35:54.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:54.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 14665/138798 objects degraded (10.566%), 17 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:54.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:53 smithi070 ceph-mon[192652]: osdmap e467: 8 total, 8 up, 7 in 2023-09-23T22:35:54.159 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:35:54.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:54.301 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:54.365 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 49528 49512 103.102 16.8125 0.0180565 0.00969361 2023-09-23T22:35:54.390 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:35:54.413 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:54.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:54.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:54.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:54.572 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:54.658 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:54.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:54.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:54.702 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:54 smithi070 ceph-mon[192652]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.702 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:54 smithi070 ceph-mon[192652]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.702 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:54 smithi070 ceph-mon[192652]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.702 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:54 smithi070 ceph-mon[192652]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.702 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:54 smithi070 ceph-mon[192652]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-09-23T22:35:54.702 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:54.702 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:54.702 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2636685039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:54.703 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:54 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:54.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:54.823 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:54.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[215775]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[215775]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[215775]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[215775]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[215775]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-09-23T22:35:54.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:54.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:54.824 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2636685039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:54.825 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:54.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[223014]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[223014]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[223014]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[223014]: failed to encode map e467 with expected crc 2023-09-23T22:35:54.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[223014]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-09-23T22:35:54.826 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:35:54.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:35:54.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2636685039' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:54.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:54 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:35:54.848 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:35:54.947 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:54.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:54.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:55.061 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:55.114 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:35:55.120 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:55.204 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:55.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:55.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:55.308 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:35:55.365 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 50723 50707 102.186 74.6875 0.0192747 0.00977999 2023-09-23T22:35:55.376 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:55.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:35:55.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:55.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:35:55.587 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:55.649 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:55.737 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:55.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:55 smithi070 ceph-mon[192652]: pgmap v1199: 99 pgs: 5 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 22 active+undersized, 59 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 29 MiB/s wr, 456 op/s; 13789/144315 objects degraded (9.555%); 588 KiB/s, 9 objects/s recovering 2023-09-23T22:35:55.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:55 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:55.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:55 smithi070 ceph-mon[192652]: failed to encode map e468 with expected crc 2023-09-23T22:35:55.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:55 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:35:55.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:55 smithi070 ceph-mon[192652]: osdmap e468: 8 total, 8 up, 7 in 2023-09-23T22:35:55.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:55 smithi070 ceph-mon[192652]: failed to encode map e468 with expected crc 2023-09-23T22:35:55.740 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:55 smithi070 ceph-mon[192652]: failed to encode map e468 with expected crc 2023-09-23T22:35:55.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:55.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:55.842 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:55.890 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:55.933 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:55.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:55.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[215775]: pgmap v1199: 99 pgs: 5 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 22 active+undersized, 59 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 29 MiB/s wr, 456 op/s; 13789/144315 objects degraded (9.555%); 588 KiB/s, 9 objects/s recovering 2023-09-23T22:35:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[215775]: failed to encode map e468 with expected crc 2023-09-23T22:35:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:35:55.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[215775]: osdmap e468: 8 total, 8 up, 7 in 2023-09-23T22:35:55.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[215775]: failed to encode map e468 with expected crc 2023-09-23T22:35:55.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[215775]: failed to encode map e468 with expected crc 2023-09-23T22:35:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[223014]: pgmap v1199: 99 pgs: 5 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 22 active+undersized, 59 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 29 MiB/s wr, 456 op/s; 13789/144315 objects degraded (9.555%); 588 KiB/s, 9 objects/s recovering 2023-09-23T22:35:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:35:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[223014]: failed to encode map e468 with expected crc 2023-09-23T22:35:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:35:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[223014]: osdmap e468: 8 total, 8 up, 7 in 2023-09-23T22:35:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[223014]: failed to encode map e468 with expected crc 2023-09-23T22:35:55.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:55 smithi026 ceph-mon[223014]: failed to encode map e468 with expected crc 2023-09-23T22:35:56.036 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:56.080 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:56.122 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:56.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:56.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:56.215 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:35:56.225 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:35:56.318 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:56.365 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 51469 51453 100.45 46.625 0.00290408 0.00993336 2023-09-23T22:35:56.596 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:35:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:56.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:35:56.710 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:56.770 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:56.844 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:35:57.108 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:57.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:57.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:57.214 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:57.259 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:35:57.259 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:35:57.279 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:57.365 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 52355 52339 99.0843 55.375 0.0114065 0.0100873 2023-09-23T22:35:57.408 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:57.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:57.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:57.519 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:57.572 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:57.695 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:57.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:57.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:57.803 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:35:57.836 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:57.923 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:35:57.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:57.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:35:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:57 smithi026 ceph-mon[215775]: pgmap v1201: 99 pgs: 5 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s; 12403/150417 objects degraded (8.246%); 471 KiB/s, 6 objects/s recovering 2023-09-23T22:35:57.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:57 smithi026 ceph-mon[223014]: pgmap v1201: 99 pgs: 5 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s; 12403/150417 objects degraded (8.246%); 471 KiB/s, 6 objects/s recovering 2023-09-23T22:35:57.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:35:58.028 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:35:58.029 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:35:58.069 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:35:58.101 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:58.117 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:35:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:57 smithi070 ceph-mon[192652]: pgmap v1201: 99 pgs: 5 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 89 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 48 MiB/s wr, 770 op/s; 12403/150417 objects degraded (8.246%); 471 KiB/s, 6 objects/s recovering 2023-09-23T22:35:58.155 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:35:58.198 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:35:58.204 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:35:58.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:58.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:35:58.232 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:35:58.267 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:35:58.280 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:35:58.280 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:35:58.280 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:35:58.280 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:35:58.280 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:35:58.280 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:35:58.280 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:35:58.280 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:35:58.313 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:35:58.365 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:58.365 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 53504 53488 98.2817 71.8125 0.00609263 0.0101689 2023-09-23T22:35:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3854210484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:35:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12403/150417 objects degraded (8.246%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:58.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3854210484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:58.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:35:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12403/150417 objects degraded (8.246%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:58.816 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:35:59.013 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3854210484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:35:59.013 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:35:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12403/150417 objects degraded (8.246%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:35:59.048 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:35:59.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:59.053 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:35:59.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:59.188 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:59.293 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:59.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:59.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:59.304 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:35:59.366 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 54243 54227 96.7937 46.1875 0.00828466 0.0103252 2023-09-23T22:35:59.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:35:59.463 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:35:59.587 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:54510 9.4 e468/466 9:20856175:::benchmark_data_smithi026_555954_object54509:head v 468'3392, mlcod=458'609)", 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:59.621225+0000", 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0093856140000000005, 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094176399999999997, 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:35:59.631 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.621225+0000", 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.621225+0000", 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.621225+0000", 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.621262+0000", 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7518999999999998e-05 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.621276+0000", 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3553999999999999e-05 2023-09-23T22:35:59.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.621282+0000", 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1349999999999997e-06 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.623924+0000", 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026422759999999998 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:35:59.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.623958+0000", 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3552e-05 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:54509 9.e e468/466 9:7d115eda:::benchmark_data_smithi026_555954_object54508:head v 468'3489, mlcod=461'1024)", 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:59.623985+0000", 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0066257210000000002, 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067338149999999998, 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:35:59.635 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.623985+0000", 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.623985+0000", 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:35:59.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.623984+0000", 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624027+0000", 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2876000000000001e-05 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624039+0000", 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2291e-05 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624047+0000", 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6899999999999992e-06 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624086+0000", 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8674000000000003e-05 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:35:59.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624103+0000", 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7271000000000001e-05 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:54517 9.4 e468/466 9:2613f06b:::benchmark_data_smithi026_555954_object54516:head v 468'3393, mlcod=458'609)", 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:59.624060+0000", 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.006550602, 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067203799999999998, 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:35:59.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624060+0000", 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624060+0000", 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:35:59.640 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624060+0000", 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624093+0000", 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2515999999999998e-05 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624100+0000", 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5569999999999998e-06 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.624105+0000", 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.583e-06 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.627140+0000", 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030351150000000001 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.627163+0000", 2023-09-23T22:35:59.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2758000000000001e-05 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:54518 9.1 e468/466 9:8e3809d2:::benchmark_data_smithi026_555954_object54517:head v 468'3320, mlcod=458'557)", 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:59.627849+0000", 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0027617150000000001, 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029746659999999999, 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:35:59.643 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.627849+0000", 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.627849+0000", 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:35:59.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.627848+0000", 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.627874+0000", 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6024e-05 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.627894+0000", 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9324999999999999e-05 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.627901+0000", 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0500000000000003e-06 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.628406+0000", 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00050566900000000004 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:59.628428+0000", 2023-09-23T22:35:59.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1395000000000001e-05 2023-09-23T22:35:59.647 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:35:59.647 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:35:59.647 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:35:59.647 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:35:59.647 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T22:35:59.647 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 4 2023-09-23T22:35:59.647 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:35:59.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:35:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:35:59] "GET /metrics HTTP/1.1" 200 36305 "" "Prometheus/2.43.0" 2023-09-23T22:35:59.763 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:35:59.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:35:59.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:35:59.877 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:35:59.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:00.039 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:00.046 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:00.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:00.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:00.156 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:00.202 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:00.258 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:00.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:00.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:00.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:00 smithi026 ceph-mon[223014]: pgmap v1202: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 12403/151584 objects degraded (8.182%); 393 KiB/s, 5 objects/s recovering 2023-09-23T22:36:00.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:00 smithi026 ceph-mon[215775]: pgmap v1202: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 12403/151584 objects degraded (8.182%); 393 KiB/s, 5 objects/s recovering 2023-09-23T22:36:00.362 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:00.366 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 55168 55152 95.7111 57.8125 0.0518999 0.0104184 2023-09-23T22:36:00.392 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:00.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:00 smithi070 ceph-mon[192652]: pgmap v1202: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 37 MiB/s wr, 594 op/s; 12403/151584 objects degraded (8.182%); 393 KiB/s, 5 objects/s recovering 2023-09-23T22:36:00.692 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:00.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:00.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:00.796 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:00.864 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:01.261 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:01.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:01.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:01.366 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 56347 56331 95.1159 73.6875 0.0151082 0.0105082 2023-09-23T22:36:01.379 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:01.413 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:01.457 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:01.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:01.561 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:01.607 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:01.763 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:36:01.897 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:01.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:01.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:02.005 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:02.049 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:02.187 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:02.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:02.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:02.293 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:02.344 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:02.366 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 57326 57310 94.2231 61.1875 0.0175469 0.0106065 2023-09-23T22:36:02.450 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:02.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:02.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:02.458 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:02 smithi070 ceph-mon[192652]: pgmap v1203: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 60 MiB/s wr, 954 op/s; 12014/163752 objects degraded (7.337%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T22:36:02.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:02 smithi026 ceph-mon[215775]: pgmap v1203: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 60 MiB/s wr, 954 op/s; 12014/163752 objects degraded (7.337%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T22:36:02.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:02 smithi026 ceph-mon[223014]: pgmap v1203: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 60 MiB/s wr, 954 op/s; 12014/163752 objects degraded (7.337%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T22:36:02.524 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:36:02.525 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:36:02.525 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:36:02.525 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:36:02.525 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:36:02.559 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:02.637 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:02.707 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:02.727 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:02.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:02.739 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:02.840 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:02.914 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:03.039 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:03.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:03.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:03.146 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:03.186 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:03.278 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:36:03.366 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 58177 58161 93.17 53.1875 0.00955696 0.0107273 2023-09-23T22:36:03.472 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3224239654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:03.472 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:03 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:03.472 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12014/163752 objects degraded (7.337%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:03.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3224239654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:03.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:03 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:03.472 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12014/163752 objects degraded (7.337%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:03.514 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3224239654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:03.514 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:03 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:03.514 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12014/163752 objects degraded (7.337%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:03.521 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:03.526 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:36:03.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:03.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:03.633 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:03.711 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:03.825 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:03.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:03.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:03.934 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:03.997 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:04.231 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:36:04.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[215775]: pgmap v1204: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 57 MiB/s wr, 916 op/s; 12014/165654 objects degraded (7.252%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:36:04.300 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:04.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:04.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:04.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:36:04.367392+0000 min lat: 0.00117176 max lat: 2.19936 avg lat: 0.0107874 2023-09-23T22:36:04.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:36:04.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 59300 59284 92.5953 70.1875 0.00570766 0.0107874 2023-09-23T22:36:04.408 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:04.454 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:04.501 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:04.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[223014]: pgmap v1204: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 57 MiB/s wr, 916 op/s; 12014/165654 objects degraded (7.252%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:36:04.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:36:04.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[223014]: failed to encode map e469 with expected crc 2023-09-23T22:36:04.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[223014]: failed to encode map e469 with expected crc 2023-09-23T22:36:04.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:04.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[223014]: osdmap e469: 8 total, 8 up, 7 in 2023-09-23T22:36:04.503 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[223014]: failed to encode map e469 with expected crc 2023-09-23T22:36:04.503 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:36:04.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[215775]: failed to encode map e469 with expected crc 2023-09-23T22:36:04.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[215775]: failed to encode map e469 with expected crc 2023-09-23T22:36:04.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:04.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[215775]: osdmap e469: 8 total, 8 up, 7 in 2023-09-23T22:36:04.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:04 smithi026 ceph-mon[215775]: failed to encode map e469 with expected crc 2023-09-23T22:36:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:04.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:04.519 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:04 smithi070 ceph-mon[192652]: pgmap v1204: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 57 MiB/s wr, 916 op/s; 12014/165654 objects degraded (7.252%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:36:04.519 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:04 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:36:04.519 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:04 smithi070 ceph-mon[192652]: failed to encode map e469 with expected crc 2023-09-23T22:36:04.519 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:04 smithi070 ceph-mon[192652]: failed to encode map e469 with expected crc 2023-09-23T22:36:04.520 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:04 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:04.520 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:04 smithi070 ceph-mon[192652]: osdmap e469: 8 total, 8 up, 7 in 2023-09-23T22:36:04.520 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:04 smithi070 ceph-mon[192652]: failed to encode map e469 with expected crc 2023-09-23T22:36:04.605 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:04.672 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:04.980 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:04.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:05.085 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:05.165 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:05.238 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:05.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:05.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:05.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:05.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 60124 60108 91.5931 51.5 0.0032252 0.0108746 2023-09-23T22:36:05.369 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:05.833 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-09-23T22:36:05.834 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:36:05.834 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T22:36:05.834 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:36:05.887 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:05.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:05.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:05.996 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:06.083 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:06.191 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:06.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:06.313 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:06.365 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:06 smithi070 ceph-mon[192652]: pgmap v1206: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s; 11639/170307 objects degraded (6.834%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:36:06.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 61207 61191 91.024 67.6875 0.0123393 0.0109812 2023-09-23T22:36:06.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:06.435 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:36:06.435 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:36:06.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.35406.0:119346 7.2 7.d96857c2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:29:00.616173+0000", 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 425.80774980699999, 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95561031100000005, 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.35406", 2023-09-23T22:36:06.436 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119346 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616173+0000", 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616173+0000", 2023-09-23T22:36:06.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616172+0000", 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616176+0000", 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4700000000000004e-06 2023-09-23T22:36:06.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616185+0000", 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5620000000000004e-06 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616191+0000", 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4189999999999999e-06 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616252+0000", 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0748999999999997e-05 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:00.616261+0000", 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3499999999999997e-06 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:01.571770+0000", 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95550986299999996 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:29:01.571783+0000", 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2985e-05 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165180 8.7 8:e62e0041:::benchmark_data_smithi026_497658_object66288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.441 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.194288+0000", 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.229634921, 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.03319117, 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165180 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.194288+0000", 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.194288+0000", 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.194287+0000", 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.194292+0000", 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.631e-06 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.194304+0000", 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1738e-05 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.194311+0000", 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.63e-06 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.194350+0000", 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9617999999999999e-05 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.194360+0000", 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7880000000000001e-06 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.203842+0000", 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.009481842 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.203974+0000", 2023-09-23T22:36:06.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131586 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.204057+0000", 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3149e-05 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.210633+0000", 2023-09-23T22:36:06.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0065763130000000003 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227397+0000", 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016764112000000001 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227426+0000", 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8721000000000002e-05 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227454+0000", 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8382e-05 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227479+0000", 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4878000000000001e-05 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165194 8.7 8:ee3a7b94:::benchmark_data_smithi026_497658_object66302:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.213860+0000", 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.210063027, 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0176969410000001, 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165194 2023-09-23T22:36:06.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.213860+0000", 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.213860+0000", 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.213858+0000", 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.213865+0000", 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8029999999999999e-06 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.213881+0000", 2023-09-23T22:36:06.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5315000000000002e-05 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.213887+0000", 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7399999999999998e-06 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.215741+0000", 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018534319999999999 2023-09-23T22:36:06.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.215751+0000", 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0348e-05 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.206125+0000", 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99037375599999999 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.206222+0000", 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7582000000000002e-05 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.206296+0000", 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3527000000000003e-05 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.217202+0000", 2023-09-23T22:36:06.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010905632 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227521+0000", 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010319403 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231476+0000", 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039547269999999999 2023-09-23T22:36:06.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231518+0000", 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2250999999999999e-05 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231557+0000", 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8797000000000001e-05 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.457 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165208 8.7 8:ea24af71:::benchmark_data_smithi026_497658_object66316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.223713+0000", 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.20020981499999, 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.007924475, 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.458 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165208 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.223713+0000", 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.223713+0000", 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.223712+0000", 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.223717+0000", 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9110000000000001e-06 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.223726+0000", 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6600000000000001e-06 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.223731+0000", 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.994e-06 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.223772+0000", 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1770999999999997e-05 2023-09-23T22:36:06.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.223778+0000", 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6840000000000001e-06 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.207746+0000", 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98396795599999998 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.207852+0000", 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000106328 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.207924+0000", 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1527999999999997e-05 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.217206+0000", 2023-09-23T22:36:06.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0092818009999999992 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227551+0000", 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010344826 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231585+0000", 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040342770000000002 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231615+0000", 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0269000000000001e-05 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231638+0000", 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2405e-05 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165226 8.7 8:e6b6030b:::benchmark_data_smithi026_497658_object66334:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.248353+0000", 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.17556960899999, 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98340321799999997, 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165226 2023-09-23T22:36:06.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.248353+0000", 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.248353+0000", 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.248352+0000", 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.248362+0000", 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0293e-05 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.248376+0000", 2023-09-23T22:36:06.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4266e-05 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.248383+0000", 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6109999999999999e-06 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.248433+0000", 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9821000000000003e-05 2023-09-23T22:36:06.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.248441+0000", 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0979999999999994e-06 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.209219+0000", 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96077865900000003 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.209300+0000", 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0433999999999996e-05 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.209367+0000", 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6960999999999998e-05 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.220120+0000", 2023-09-23T22:36:06.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010753248999999999 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227567+0000", 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0074469410000000003 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231677+0000", 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041104760000000001 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231715+0000", 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7177e-05 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231756+0000", 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1801000000000002e-05 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165231 8.7 8:ee3b6d40:::benchmark_data_smithi026_497658_object66339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.254323+0000", 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.169599995, 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97751349799999998, 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165231 2023-09-23T22:36:06.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.254323+0000", 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.254323+0000", 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.254322+0000", 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.254327+0000", 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6470000000000003e-06 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.254339+0000", 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1219e-05 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.254343+0000", 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.724e-06 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.254389+0000", 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5485000000000002e-05 2023-09-23T22:36:06.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.254399+0000", 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9310000000000003e-06 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.210669+0000", 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95626990899999997 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.210754+0000", 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5669999999999998e-05 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.210823+0000", 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9123999999999998e-05 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.220124+0000", 2023-09-23T22:36:06.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0093005410000000007 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227660+0000", 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075364710000000003 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231791+0000", 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041308969999999997 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231816+0000", 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4456000000000001e-05 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231836+0000", 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0530000000000002e-05 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165238 8.7 8:e2fa30c0:::benchmark_data_smithi026_497658_object66346:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.259618+0000", 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.16430472099999, 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97234068699999998, 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165238 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.482 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.259618+0000", 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.259618+0000", 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.259617+0000", 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.259622+0000", 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4279999999999998e-06 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.259634+0000", 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2481e-05 2023-09-23T22:36:06.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.259640+0000", 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1440000000000002e-06 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.259675+0000", 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5515000000000001e-05 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.259681+0000", 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6670000000000001e-06 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.211954+0000", 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95227331800000004 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.212044+0000", 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0173999999999994e-05 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.212110+0000", 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5837e-05 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.220127+0000", 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0080174600000000006 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227698+0000", 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075704530000000004 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231878+0000", 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041795570000000004 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231915+0000", 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7261000000000001e-05 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231959+0000", 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.405e-05 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.489 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165241 8.7 8:e4204f52:::benchmark_data_smithi026_497658_object66349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.261427+0000", 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.162496132, 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97063850200000001, 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165241 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261427+0000", 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261427+0000", 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261426+0000", 2023-09-23T22:36:06.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261432+0000", 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.733e-06 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261449+0000", 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7971000000000001e-05 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261456+0000", 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3740000000000003e-06 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261501+0000", 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5117999999999997e-05 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261509+0000", 2023-09-23T22:36:06.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7559999999999999e-06 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.213290+0000", 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95178085599999995 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.213381+0000", 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1747000000000001e-05 2023-09-23T22:36:06.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.213453+0000", 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1796000000000002e-05 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.220131+0000", 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066777360000000001 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227719+0000", 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075880699999999997 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.232001+0000", 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042817949999999997 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.232032+0000", 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1455999999999998e-05 2023-09-23T22:36:06.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.232065+0000", 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3036e-05 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165242 8.7 8:ea26c154:::benchmark_data_smithi026_497658_object66350:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.261619+0000", 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.162304127, 2023-09-23T22:36:06.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97054259700000001, 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165242 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261619+0000", 2023-09-23T22:36:06.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261619+0000", 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261618+0000", 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:06.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261622+0000", 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.386e-06 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261633+0000", 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0975000000000001e-05 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261638+0000", 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1399999999999999e-06 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261677+0000", 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8875000000000003e-05 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.261684+0000", 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6510000000000004e-06 2023-09-23T22:36:06.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.214594+0000", 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95291034500000005 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.214684+0000", 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9806e-05 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.214755+0000", 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0390999999999997e-05 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.220134+0000", 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0053797259999999996 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227734+0000", 2023-09-23T22:36:06.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007599532 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.232093+0000", 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043595070000000003 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.232132+0000", 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8671000000000003e-05 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.232161+0000", 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9331000000000001e-05 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165243 8.7 8:eb828167:::benchmark_data_smithi026_497658_object66351:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.262800+0000", 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.161123199, 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.992423893, 2023-09-23T22:36:06.505 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165243 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.262800+0000", 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.262800+0000", 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.262799+0000", 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.262804+0000", 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6229999999999996e-06 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.262814+0000", 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0631e-05 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.262820+0000", 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5589999999999996e-06 2023-09-23T22:36:06.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.262874+0000", 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4387e-05 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.262880+0000", 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9619999999999998e-06 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.215880+0000", 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95299973100000002 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.215982+0000", 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010173199999999999 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.216056+0000", 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4153999999999995e-05 2023-09-23T22:36:06.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.222887+0000", 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0068308539999999999 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227754+0000", 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048672790000000004 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.255132+0000", 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027378454999999999 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.255183+0000", 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0253000000000001e-05 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.255224+0000", 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0914999999999997e-05 2023-09-23T22:36:06.512 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165251 8.7 8:ea7a1135:::benchmark_data_smithi026_497658_object66359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.270673+0000", 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.15324973200001, 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98465710299999998, 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.513 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165251 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.270673+0000", 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.270673+0000", 2023-09-23T22:36:06.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.270672+0000", 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.270680+0000", 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8679999999999999e-06 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.270696+0000", 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6898999999999999e-05 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.270703+0000", 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.494e-06 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.275076+0000", 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043728370000000001 2023-09-23T22:36:06.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.275083+0000", 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2599999999999999e-06 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.217245+0000", 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.942162045 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.217333+0000", 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7859e-05 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.217400+0000", 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6679000000000003e-05 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.222892+0000", 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0054923359999999996 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227795+0000", 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0049032620000000002 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.255256+0000", 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027460902999999998 2023-09-23T22:36:06.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.255296+0000", 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0093999999999999e-05 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.255330+0000", 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3995000000000002e-05 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165260 8.7 8:ed552c53:::benchmark_data_smithi026_497658_object66368:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.284070+0000", 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.139853321, 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97697189200000001, 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165260 2023-09-23T22:36:06.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.284070+0000", 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.284070+0000", 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.284068+0000", 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.284075+0000", 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7789999999999996e-06 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.284093+0000", 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7479e-05 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.284100+0000", 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7499999999999997e-06 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.284155+0000", 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5257000000000003e-05 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.284166+0000", 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0567999999999999e-05 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.218676+0000", 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93451006199999997 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.218759+0000", 2023-09-23T22:36:06.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3264999999999996e-05 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.218827+0000", 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7960999999999995e-05 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.225660+0000", 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.006833375 2023-09-23T22:36:06.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227842+0000", 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021820979999999999 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.260963+0000", 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033120910000000003 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.527 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.260996+0000", 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2570999999999997e-05 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.261041+0000", 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5646000000000003e-05 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.528 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165267 8.7 8:e20bf70e:::benchmark_data_smithi026_497658_object66375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.292986+0000", 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 121.13093662199999, 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96819027800000002, 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165267 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.292986+0000", 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.292986+0000", 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.292985+0000", 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.292992+0000", 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1509999999999998e-06 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.293009+0000", 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7147999999999999e-05 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.293016+0000", 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0049999999999998e-06 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.293069+0000", 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2513999999999999e-05 2023-09-23T22:36:06.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.293077+0000", 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8189999999999998e-06 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.220172+0000", 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92709428999999999 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.220257+0000", 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5677e-05 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.220324+0000", 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6458000000000006e-05 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.225685+0000", 2023-09-23T22:36:06.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0053615310000000001 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227866+0000", 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002180918 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.261090+0000", 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033223381000000003 2023-09-23T22:36:06.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.261131+0000", 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1239000000000003e-05 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.261177+0000", 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5549e-05 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.536 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:165295 8.7 8:eed8ec9d:::benchmark_data_smithi026_497658_object66403:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:05.464610+0000", 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 120.959312888, 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79667475600000004, 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.537 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 165295 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.464610+0000", 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.464610+0000", 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.464609+0000", 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.464615+0000", 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6490000000000002e-06 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.464625+0000", 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8679999999999994e-06 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.464631+0000", 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2100000000000001e-06 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.464665+0000", 2023-09-23T22:36:06.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4507e-05 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:05.464671+0000", 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.032e-06 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.221488+0000", 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75681736799999999 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.221567+0000", 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8559000000000005e-05 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2", 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.221635+0000", 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8133999999999995e-05 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.227142+0000", 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005506431 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.231111+0000", 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039689249999999999 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.261232+0000", 2023-09-23T22:36:06.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030121782999999999 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.261263+0000", 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0942999999999998e-05 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:06.261285+0000", 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1481999999999999e-05 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.544 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:188716 8.c 8:3d7276a0:::benchmark_data_smithi026_497658_object89824:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:44.310727+0000", 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 82.113196219000002, 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.80145883900000003, 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.545 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 188716 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.310727+0000", 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.310727+0000", 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.310726+0000", 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.310728+0000", 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7659999999999999e-06 2023-09-23T22:36:06.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.310733+0000", 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9439999999999996e-06 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.310736+0000", 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6129999999999998e-06 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.310779+0000", 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3637999999999998e-05 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.310788+0000", 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2330000000000006e-06 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.094814+0000", 2023-09-23T22:36:06.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.78402623800000004 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.094974+0000", 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016014999999999999 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.095317+0000", 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034339499999999997 2023-09-23T22:36:06.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.105762+0000", 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01044486 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.107310+0000", 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001548118 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.112135+0000", 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048247619999999998 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.112163+0000", 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7433999999999999e-05 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.112185+0000", 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2962000000000001e-05 2023-09-23T22:36:06.552 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:188793 8.c 8:3248d742:::benchmark_data_smithi026_497658_object89901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e442)", 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:44.396523+0000", 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 82.02740034, 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.71580229200000001, 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.553 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 188793 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.396523+0000", 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.396523+0000", 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.396522+0000", 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.396528+0000", 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3239999999999999e-06 2023-09-23T22:36:06.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.396540+0000", 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1959e-05 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.396545+0000", 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.95e-06 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.396592+0000", 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6743999999999998e-05 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:44.396599+0000", 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.221e-06 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.102309+0000", 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.70571035100000001 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.102440+0000", 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013042500000000001 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.102570+0000", 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000130283 2023-09-23T22:36:06.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.105768+0000", 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031975509999999999 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.109963+0000", 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041950420000000004 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.112235+0000", 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022724210000000002 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.112288+0000", 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2843000000000002e-05 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:45.112325+0000", 2023-09-23T22:36:06.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6987999999999999e-05 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9515 9.1 9:81f3cd39:::benchmark_data_smithi026_555954_object9514:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.451619+0000", 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 38.972303349999997, 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91870674600000002, 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.561 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.562 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:06.562 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:06.562 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9515 2023-09-23T22:36:06.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.562 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.451619+0000", 2023-09-23T22:36:06.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.451619+0000", 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.451619+0000", 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.567 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.451647+0000", 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7742999999999998e-05 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.451654+0000", 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4000000000000003e-06 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.451666+0000", 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1995e-05 2023-09-23T22:36:06.568 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.451708+0000", 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1505999999999999e-05 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.451712+0000", 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.583e-06 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.569 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.364320+0000", 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91260790899999999 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.364458+0000", 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000138119 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 7", 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.364614+0000", 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015602199999999999 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.570 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.367628+0000", 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030135510000000002 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370199+0000", 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025709019999999999 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.571 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370242+0000", 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3566000000000003e-05 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370326+0000", 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3959e-05 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.572 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9526 9.1 9:8ab01e48:::benchmark_data_smithi026_555954_object9525:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.454360+0000", 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 38.969563016000002, 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91609184600000004, 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9526 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.573 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454360+0000", 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454360+0000", 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.574 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454359+0000", 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454469+0000", 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109598 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454479+0000", 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0346999999999999e-05 2023-09-23T22:36:06.575 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454485+0000", 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7849999999999997e-06 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454535+0000", 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0046999999999997e-05 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454542+0000", 2023-09-23T22:36:06.576 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5900000000000002e-06 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.365137+0000", 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91059469599999998 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.365245+0000", 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000108107 2023-09-23T22:36:06.577 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 7", 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.365389+0000", 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014343 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.367636+0000", 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022467920000000001 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.578 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370364+0000", 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027286459999999999 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370393+0000", 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9057e-05 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370452+0000", 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8458999999999997e-05 2023-09-23T22:36:06.579 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9533 9.1 9:847e3a93:::benchmark_data_smithi026_555954_object9532:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.454961+0000", 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 38.968961368000002, 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91567604000000002, 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.580 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9533 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454961+0000", 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.581 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454961+0000", 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.454961+0000", 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455009+0000", 2023-09-23T22:36:06.582 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7769000000000003e-05 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455014+0000", 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.848e-06 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455017+0000", 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1889999999999999e-06 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.583 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455054+0000", 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7543000000000003e-05 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455060+0000", 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9490000000000001e-06 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.584 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.365771+0000", 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91071068099999997 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.365892+0000", 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012108300000000001 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 7", 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.366041+0000", 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014940099999999999 2023-09-23T22:36:06.585 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.367641+0000", 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001599975 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370502+0000", 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028609999999999998 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.586 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370558+0000", 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5939000000000001e-05 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370638+0000", 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9153999999999995e-05 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.587 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9548 9.1 9:83d73095:::benchmark_data_smithi026_555954_object9547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.461768+0000", 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 38.962154884999997, 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90903977499999999, 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9548 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.588 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.461768+0000", 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.461768+0000", 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:06.589 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.461758+0000", 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.99999 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.461806+0000", 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8124999999999997e-05 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.461818+0000", 2023-09-23T22:36:06.590 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2439000000000001e-05 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.461823+0000", 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7450000000000003e-06 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.461869+0000", 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6217000000000002e-05 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.591 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.461879+0000", 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1139999999999995e-06 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.366770+0000", 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90489162899999998 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.366882+0000", 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0001115 2023-09-23T22:36:06.592 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 7", 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.367039+0000", 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015698099999999999 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.368094+0000", 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001055554 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.593 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370689+0000", 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002594861 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370730+0000", 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1165999999999999e-05 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:06.594 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.370808+0000", 2023-09-23T22:36:06.595 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7527999999999994e-05 2023-09-23T22:36:06.595 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.595 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.595 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.595 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:06.595 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:06.595 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:36:06.596 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:06 smithi026 ceph-mon[215775]: pgmap v1206: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s; 11639/170307 objects degraded (6.834%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:36:06.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:06 smithi026 ceph-mon[223014]: pgmap v1206: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 859 op/s; 11639/170307 objects degraded (6.834%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T22:36:06.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:06.597 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:06.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:06.667 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:06.698 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:06.711 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:36:06.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:36:06.784 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:06.821 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:36:06.858 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:36:06.878 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:36:06.883 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:06.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:06.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:06.899 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:36:06.987 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:07.024 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:36:07.034 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:07.065 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:36:07.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:36:07.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:36:07.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:36:07.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:36:07.069 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:36:07.070 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:36:07.070 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:36:07.070 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:36:07.268 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:07.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:07.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:07.312 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2433620654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:07.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2433620654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:07.324 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2433620654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:07.363 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:36:07.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 62116 62100 90.2284 56.8125 0.0125767 0.0110764 2023-09-23T22:36:07.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:07.436 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:08.001 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:36:08.019 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:36:08.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 63066 63050 89.5273 59.375 0.00475897 0.0111215 2023-09-23T22:36:08.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:08 smithi026 ceph-mon[223014]: pgmap v1207: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 937 op/s; 11639/178539 objects degraded (6.519%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:36:08.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 11639/178539 objects degraded (6.519%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:08.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:08 smithi026 ceph-mon[215775]: pgmap v1207: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 937 op/s; 11639/178539 objects degraded (6.519%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:36:08.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 11639/178539 objects degraded (6.519%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:08.500 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:08.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:08.611 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:08.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:08 smithi070 ceph-mon[192652]: pgmap v1207: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.6 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 59 MiB/s wr, 937 op/s; 11639/178539 objects degraded (6.519%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:36:08.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 11639/178539 objects degraded (6.519%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:08.678 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:09.056 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:09.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:09.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:09.160 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:09.217 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:09.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 63915 63899 88.7171 53.0625 0.00812741 0.0112657 2023-09-23T22:36:09.508 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:09.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:09.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:09.616 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:09.619 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:09.674 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:09.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:36:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:36:09] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2023-09-23T22:36:09.829 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:09.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:09.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:09.937 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:09.998 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:10.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:10 smithi070 ceph-mon[192652]: pgmap v1208: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 11639/180432 objects degraded (6.451%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:36:10.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:36:10.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:10.189 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:36:10.189 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:36:10.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:10 smithi026 ceph-mon[223014]: pgmap v1208: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 11639/180432 objects degraded (6.451%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:36:10.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:36:10.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:10 smithi026 ceph-mon[215775]: pgmap v1208: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 11639/180432 objects degraded (6.451%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:36:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:36:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:10.279 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:10.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:10.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:10.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 64936 64920 88.1757 63.8125 0.0282142 0.0113362 2023-09-23T22:36:10.393 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:10.455 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:10.547 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:10.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:10.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:10.654 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:36:10.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:10.657 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:36:10.688 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:10.734 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:10.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:10.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:10.838 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:10.897 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:10.981 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:10.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:10.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:11.086 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:11.116 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:11.166 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:36:11.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 65772 65756 87.4115 52.25 0.0105212 0.0114357 2023-09-23T22:36:11.432 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:36:11.750 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:36:12.157 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:12.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:12.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:12.269 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:12.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:12 smithi070 ceph-mon[192652]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:36:12.317 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:12 smithi070 ceph-mon[192652]: pgmap v1209: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s; 11254/192978 objects degraded (5.832%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:36:12.321 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:12.333 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1208,"stamp":"2023-09-23T22:36:08.688824+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3944552371,"num_objects":60144,"num_object_clones":0,"num_object_copies":180432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11639,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60144,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":61023,"num_write_kb":3860149,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1202,"num_bytes_recovered":74876197,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62918,"ondisk_log_size":62918,"up":297,"acting":292,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13763184,"kb_used_data":10798008,"kb_used_omap":43,"kb_used_meta":2965076,"kb_avail":642452880,"statfs":{"total":671965249536,"available":657871749120,"internally_reserved":0,"allocated":11057160192,"data_stored":11043696237,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44475,"internal_metadata":3036238405},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,3,2,1,5,0,0,2,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":630194176,"num_objects":9616,"num_object_clones":0,"num_object_copies":28848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-764,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9616,"num_write_kb":615424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":764,"num_bytes_recovered":50069504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004772"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":512,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.103816+0000","last_change":"2023-09-23T22:35:52.683364+0000","last_active":"2023-09-23T22:35:55.103816+0000","last_peered":"2023-09-23T22:35:55.103816+0000","last_clean":"2023-09-23T22:35:55.103816+0000","last_became_active":"2023-09-23T22:35:52.677978+0000","last_became_peered":"2023-09-23T22:35:52.677978+0000","last_unstale":"2023-09-23T22:35:55.103816+0000","last_undegraded":"2023-09-23T22:35:55.103816+0000","last_fullsized":"2023-09-23T22:35:55.103816+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:37.770380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":202,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.105839+0000","last_change":"2023-09-23T22:35:52.679312+0000","last_active":"2023-09-23T22:35:55.105839+0000","last_peered":"2023-09-23T22:35:55.105839+0000","last_clean":"2023-09-23T22:35:55.105839+0000","last_became_active":"2023-09-23T22:35:52.679056+0000","last_became_peered":"2023-09-23T22:35:52.679056+0000","last_unstale":"2023-09-23T22:35:55.105839+0000","last_undegraded":"2023-09-23T22:35:55.105839+0000","last_fullsized":"2023-09-23T22:35:55.105839+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:32.907649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":202,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.105378+0000","last_change":"2023-09-23T22:35:52.683740+0000","last_active":"2023-09-23T22:35:55.105378+0000","last_peered":"2023-09-23T22:35:55.105378+0000","last_clean":"2023-09-23T22:35:55.105378+0000","last_became_active":"2023-09-23T22:35:52.683474+0000","last_became_peered":"2023-09-23T22:35:52.683474+0000","last_unstale":"2023-09-23T22:35:55.105378+0000","last_undegraded":"2023-09-23T22:35:55.105378+0000","last_fullsized":"2023-09-23T22:35:55.105378+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:51.547183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":334,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.106991+0000","last_change":"2023-09-23T22:35:20.336179+0000","last_active":"2023-09-23T22:35:55.106991+0000","last_peered":"2023-09-23T22:35:55.106991+0000","last_clean":"2023-09-23T22:35:55.106991+0000","last_became_active":"2023-09-23T22:35:20.335246+0000","last_became_peered":"2023-09-23T22:35:20.335246+0000","last_unstale":"2023-09-23T22:35:55.106991+0000","last_undegraded":"2023-09-23T22:35:55.106991+0000","last_fullsized":"2023-09-23T22:35:55.106991+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:56.822233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":527,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.105861+0000","last_change":"2023-09-23T22:34:19.426816+0000","last_active":"2023-09-23T22:35:55.105861+0000","last_peered":"2023-09-23T22:35:55.105861+0000","last_clean":"2023-09-23T22:35:55.105861+0000","last_became_active":"2023-09-23T22:34:19.426413+0000","last_became_peered":"2023-09-23T22:34:19.426413+0000","last_unstale":"2023-09-23T22:35:55.105861+0000","last_undegraded":"2023-09-23T22:35:55.105861+0000","last_fullsized":"2023-09-23T22:35:55.105861+0000","mapping_epoch":424,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":425,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:58.025095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":150,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110889+0000","last_change":"2023-09-23T22:29:01.593733+0000","last_active":"2023-09-23T22:35:55.110889+0000","last_peered":"2023-09-23T22:35:55.110889+0000","last_clean":"2023-09-23T22:35:55.110889+0000","last_became_active":"2023-09-23T22:29:01.587059+0000","last_became_peered":"2023-09-23T22:29:01.587059+0000","last_unstale":"2023-09-23T22:35:55.110889+0000","last_undegraded":"2023-09-23T22:35:55.110889+0000","last_fullsized":"2023-09-23T22:35:55.110889+0000","mapping_epoch":328,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":329,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:54.018001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":290,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110690+0000","last_change":"2023-09-23T22:34:32.965370+0000","last_active":"2023-09-23T22:35:55.110690+0000","last_peered":"2023-09-23T22:35:55.110690+0000","last_clean":"2023-09-23T22:35:55.110690+0000","last_became_active":"2023-09-23T22:34:32.965095+0000","last_became_peered":"2023-09-23T22:34:32.965095+0000","last_unstale":"2023-09-23T22:35:55.110690+0000","last_undegraded":"2023-09-23T22:35:55.110690+0000","last_fullsized":"2023-09-23T22:35:55.110690+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:15.925188+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":167,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.109855+0000","last_change":"2023-09-23T22:35:52.677713+0000","last_active":"2023-09-23T22:35:55.109855+0000","last_peered":"2023-09-23T22:35:55.109855+0000","last_clean":"2023-09-23T22:35:55.109855+0000","last_became_active":"2023-09-23T22:35:52.677484+0000","last_became_peered":"2023-09-23T22:35:52.677484+0000","last_unstale":"2023-09-23T22:35:55.109855+0000","last_undegraded":"2023-09-23T22:35:55.109855+0000","last_fullsized":"2023-09-23T22:35:55.109855+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:58.459027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":476,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107215+0000","last_change":"2023-09-23T22:35:20.336418+0000","last_active":"2023-09-23T22:35:55.107215+0000","last_peered":"2023-09-23T22:35:55.107215+0000","last_clean":"2023-09-23T22:35:55.107215+0000","last_became_active":"2023-09-23T22:35:20.335798+0000","last_became_peered":"2023-09-23T22:35:20.335798+0000","last_unstale":"2023-09-23T22:35:55.107215+0000","last_undegraded":"2023-09-23T22:35:55.107215+0000","last_fullsized":"2023-09-23T22:35:55.107215+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:43.391862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":170,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.109705+0000","last_change":"2023-09-23T22:35:20.282336+0000","last_active":"2023-09-23T22:35:55.109705+0000","last_peered":"2023-09-23T22:35:55.109705+0000","last_clean":"2023-09-23T22:35:55.109705+0000","last_became_active":"2023-09-23T22:35:20.281709+0000","last_became_peered":"2023-09-23T22:35:20.281709+0000","last_unstale":"2023-09-23T22:35:55.109705+0000","last_undegraded":"2023-09-23T22:35:55.109705+0000","last_fullsized":"2023-09-23T22:35:55.109705+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:06.657258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":47,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.104716+0000","last_change":"2023-09-23T22:35:52.688239+0000","last_active":"2023-09-23T22:35:55.104716+0000","last_peered":"2023-09-23T22:35:55.104716+0000","last_clean":"2023-09-23T22:35:55.104716+0000","last_became_active":"2023-09-23T22:35:52.688016+0000","last_became_peered":"2023-09-23T22:35:52.688016+0000","last_unstale":"2023-09-23T22:35:55.104716+0000","last_undegraded":"2023-09-23T22:35:55.104716+0000","last_fullsized":"2023-09-23T22:35:55.104716+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:10.687941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":128,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107996+0000","last_change":"2023-09-23T22:33:46.207512+0000","last_active":"2023-09-23T22:35:55.107996+0000","last_peered":"2023-09-23T22:35:55.107996+0000","last_clean":"2023-09-23T22:35:55.107996+0000","last_became_active":"2023-09-23T22:33:46.207193+0000","last_became_peered":"2023-09-23T22:33:46.207193+0000","last_unstale":"2023-09-23T22:35:55.107996+0000","last_undegraded":"2023-09-23T22:35:55.107996+0000","last_fullsized":"2023-09-23T22:35:55.107996+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:59:23.755903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":501,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110070+0000","last_change":"2023-09-23T22:35:52.679574+0000","last_active":"2023-09-23T22:35:55.110070+0000","last_peered":"2023-09-23T22:35:55.110070+0000","last_clean":"2023-09-23T22:35:55.110070+0000","last_became_active":"2023-09-23T22:35:52.679134+0000","last_became_peered":"2023-09-23T22:35:52.679134+0000","last_unstale":"2023-09-23T22:35:55.110070+0000","last_undegraded":"2023-09-23T22:35:55.110070+0000","last_fullsized":"2023-09-23T22:35:55.110070+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:45.005432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":468,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.103902+0000","last_change":"2023-09-23T22:35:52.680639+0000","last_active":"2023-09-23T22:35:55.103902+0000","last_peered":"2023-09-23T22:35:55.103902+0000","last_clean":"2023-09-23T22:35:55.103902+0000","last_became_active":"2023-09-23T22:35:52.680372+0000","last_became_peered":"2023-09-23T22:35:52.680372+0000","last_unstale":"2023-09-23T22:35:55.103902+0000","last_undegraded":"2023-09-23T22:35:55.103902+0000","last_fullsized":"2023-09-23T22:35:55.103902+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:57.196483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":101,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:03.271730+0000","last_change":"2023-09-23T22:35:52.674513+0000","last_active":"2023-09-23T22:36:03.271730+0000","last_peered":"2023-09-23T22:36:03.271730+0000","last_clean":"2023-09-23T22:36:03.271730+0000","last_became_active":"2023-09-23T22:35:52.674194+0000","last_became_peered":"2023-09-23T22:35:52.674194+0000","last_unstale":"2023-09-23T22:36:03.271730+0000","last_undegraded":"2023-09-23T22:36:03.271730+0000","last_fullsized":"2023-09-23T22:36:03.271730+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:54.383524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":74,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:03.271213+0000","last_change":"2023-09-23T22:35:20.335024+0000","last_active":"2023-09-23T22:36:03.271213+0000","last_peered":"2023-09-23T22:36:03.271213+0000","last_clean":"2023-09-23T22:36:03.271213+0000","last_became_active":"2023-09-23T22:35:20.334700+0000","last_became_peered":"2023-09-23T22:35:20.334700+0000","last_unstale":"2023-09-23T22:36:03.271213+0000","last_undegraded":"2023-09-23T22:36:03.271213+0000","last_fullsized":"2023-09-23T22:36:03.271213+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:02.807961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":363,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107009+0000","last_change":"2023-09-23T22:35:20.331190+0000","last_active":"2023-09-23T22:35:55.107009+0000","last_peered":"2023-09-23T22:35:55.107009+0000","last_clean":"2023-09-23T22:35:55.107009+0000","last_became_active":"2023-09-23T22:35:20.330863+0000","last_became_peered":"2023-09-23T22:35:20.330863+0000","last_unstale":"2023-09-23T22:35:55.107009+0000","last_undegraded":"2023-09-23T22:35:55.107009+0000","last_fullsized":"2023-09-23T22:35:55.107009+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:38.402197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":442,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110347+0000","last_change":"2023-09-23T22:35:20.331543+0000","last_active":"2023-09-23T22:35:55.110347+0000","last_peered":"2023-09-23T22:35:55.110347+0000","last_clean":"2023-09-23T22:35:55.110347+0000","last_became_active":"2023-09-23T22:35:20.331175+0000","last_became_peered":"2023-09-23T22:35:20.331175+0000","last_unstale":"2023-09-23T22:35:55.110347+0000","last_undegraded":"2023-09-23T22:35:55.110347+0000","last_fullsized":"2023-09-23T22:35:55.110347+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:43.419355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":425,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.105642+0000","last_change":"2023-09-23T22:35:52.691011+0000","last_active":"2023-09-23T22:35:55.105642+0000","last_peered":"2023-09-23T22:35:55.105642+0000","last_clean":"2023-09-23T22:35:55.105642+0000","last_became_active":"2023-09-23T22:35:52.690675+0000","last_became_peered":"2023-09-23T22:35:52.690675+0000","last_unstale":"2023-09-23T22:35:55.105642+0000","last_undegraded":"2023-09-23T22:35:55.105642+0000","last_fullsized":"2023-09-23T22:35:55.105642+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:13.148948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":378,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.098332+0000","last_change":"2023-09-23T22:35:20.325523+0000","last_active":"2023-09-23T22:35:55.098332+0000","last_peered":"2023-09-23T22:35:55.098332+0000","last_clean":"2023-09-23T22:35:55.098332+0000","last_became_active":"2023-09-23T22:35:20.325152+0000","last_became_peered":"2023-09-23T22:35:20.325152+0000","last_unstale":"2023-09-23T22:35:55.098332+0000","last_undegraded":"2023-09-23T22:35:55.098332+0000","last_fullsized":"2023-09-23T22:35:55.098332+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:56.569011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":180,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.109683+0000","last_change":"2023-09-23T22:34:33.228716+0000","last_active":"2023-09-23T22:35:55.109683+0000","last_peered":"2023-09-23T22:35:55.109683+0000","last_clean":"2023-09-23T22:35:55.109683+0000","last_became_active":"2023-09-23T22:34:33.017756+0000","last_became_peered":"2023-09-23T22:34:33.017756+0000","last_unstale":"2023-09-23T22:35:55.109683+0000","last_undegraded":"2023-09-23T22:35:55.109683+0000","last_fullsized":"2023-09-23T22:35:55.109683+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:47.604622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.098476+0000","last_change":"2023-09-23T22:35:20.335092+0000","last_active":"2023-09-23T22:35:55.098476+0000","last_peered":"2023-09-23T22:35:55.098476+0000","last_clean":"2023-09-23T22:35:55.098476+0000","last_became_active":"2023-09-23T22:35:20.334784+0000","last_became_peered":"2023-09-23T22:35:20.334784+0000","last_unstale":"2023-09-23T22:35:55.098476+0000","last_undegraded":"2023-09-23T22:35:55.098476+0000","last_fullsized":"2023-09-23T22:35:55.098476+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:39.022681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.5","version":"469'3649","reported_seq":3675,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:05.039672+0000","last_change":"2023-09-23T22:35:20.331588+0000","last_active":"2023-09-23T22:36:05.039672+0000","last_peered":"2023-09-23T22:36:05.039672+0000","last_clean":"2023-09-23T22:36:05.039672+0000","last_became_active":"2023-09-23T22:35:20.331099+0000","last_became_peered":"2023-09-23T22:35:20.331099+0000","last_unstale":"2023-09-23T22:36:05.039672+0000","last_undegraded":"2023-09-23T22:36:05.039672+0000","last_fullsized":"2023-09-23T22:36:05.039672+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3649,"log_dups_size":0,"ondisk_log_size":3649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:00.949156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239140864,"num_objects":3649,"num_object_clones":0,"num_object_copies":10947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3649,"num_write_kb":233536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":495,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.103821+0000","last_change":"2023-09-23T22:35:52.675241+0000","last_active":"2023-09-23T22:35:55.103821+0000","last_peered":"2023-09-23T22:35:55.103821+0000","last_clean":"2023-09-23T22:35:55.103821+0000","last_became_active":"2023-09-23T22:35:52.674769+0000","last_became_peered":"2023-09-23T22:35:52.674769+0000","last_unstale":"2023-09-23T22:35:55.103821+0000","last_undegraded":"2023-09-23T22:35:55.103821+0000","last_fullsized":"2023-09-23T22:35:55.103821+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:43.611334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":452,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.109420+0000","last_change":"2023-09-23T22:35:52.677872+0000","last_active":"2023-09-23T22:35:55.109420+0000","last_peered":"2023-09-23T22:35:55.109420+0000","last_clean":"2023-09-23T22:35:55.109420+0000","last_became_active":"2023-09-23T22:35:52.677579+0000","last_became_peered":"2023-09-23T22:35:52.677579+0000","last_unstale":"2023-09-23T22:35:55.109420+0000","last_undegraded":"2023-09-23T22:35:55.109420+0000","last_fullsized":"2023-09-23T22:35:55.109420+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:35.388219+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.4","version":"469'3695","reported_seq":3737,"reported_epoch":469,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:36:04.416217+0000","last_change":"2023-09-23T22:35:53.848580+0000","last_active":"2023-09-23T22:36:04.416217+0000","last_peered":"2023-09-23T22:36:04.416217+0000","last_clean":"2023-09-23T22:35:27.345185+0000","last_became_active":"2023-09-23T22:35:53.841324+0000","last_became_peered":"2023-09-23T22:35:53.841324+0000","last_unstale":"2023-09-23T22:36:04.416217+0000","last_undegraded":"2023-09-23T22:35:53.682235+0000","last_fullsized":"2023-09-23T22:35:53.662538+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3695,"log_dups_size":0,"ondisk_log_size":3695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:30.970344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242155520,"num_objects":3695,"num_object_clones":0,"num_object_copies":11085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2423,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3695,"num_write_kb":236480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1272}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":327,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107343+0000","last_change":"2023-09-23T22:35:52.680981+0000","last_active":"2023-09-23T22:35:55.107343+0000","last_peered":"2023-09-23T22:35:55.107343+0000","last_clean":"2023-09-23T22:35:55.107343+0000","last_became_active":"2023-09-23T22:35:52.680178+0000","last_became_peered":"2023-09-23T22:35:52.680178+0000","last_unstale":"2023-09-23T22:35:55.107343+0000","last_undegraded":"2023-09-23T22:35:55.107343+0000","last_fullsized":"2023-09-23T22:35:55.107343+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:28.477224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":432,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.105280+0000","last_change":"2023-09-23T22:35:52.683687+0000","last_active":"2023-09-23T22:35:55.105280+0000","last_peered":"2023-09-23T22:35:55.105280+0000","last_clean":"2023-09-23T22:35:55.105280+0000","last_became_active":"2023-09-23T22:35:52.683415+0000","last_became_peered":"2023-09-23T22:35:52.683415+0000","last_unstale":"2023-09-23T22:35:55.105280+0000","last_undegraded":"2023-09-23T22:35:55.105280+0000","last_fullsized":"2023-09-23T22:35:55.105280+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:28.081728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"469'3853","reported_seq":3889,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:07.374453+0000","last_change":"2023-09-23T22:35:20.330288+0000","last_active":"2023-09-23T22:36:07.374453+0000","last_peered":"2023-09-23T22:36:07.374453+0000","last_clean":"2023-09-23T22:36:07.374453+0000","last_became_active":"2023-09-23T22:35:20.329884+0000","last_became_peered":"2023-09-23T22:35:20.329884+0000","last_unstale":"2023-09-23T22:36:07.374453+0000","last_undegraded":"2023-09-23T22:36:07.374453+0000","last_fullsized":"2023-09-23T22:36:07.374453+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3853,"log_dups_size":0,"ondisk_log_size":3853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:17.172674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252510208,"num_objects":3853,"num_object_clones":0,"num_object_copies":11559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3853,"num_write_kb":246592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":355,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110431+0000","last_change":"2023-09-23T22:35:52.680078+0000","last_active":"2023-09-23T22:35:55.110431+0000","last_peered":"2023-09-23T22:35:55.110431+0000","last_clean":"2023-09-23T22:35:55.110431+0000","last_became_active":"2023-09-23T22:35:52.679111+0000","last_became_peered":"2023-09-23T22:35:52.679111+0000","last_unstale":"2023-09-23T22:35:55.110431+0000","last_undegraded":"2023-09-23T22:35:55.110431+0000","last_fullsized":"2023-09-23T22:35:55.110431+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:56.382083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":170,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110131+0000","last_change":"2023-09-23T22:35:20.333357+0000","last_active":"2023-09-23T22:35:55.110131+0000","last_peered":"2023-09-23T22:35:55.110131+0000","last_clean":"2023-09-23T22:35:55.110131+0000","last_became_active":"2023-09-23T22:35:20.332971+0000","last_became_peered":"2023-09-23T22:35:20.332971+0000","last_unstale":"2023-09-23T22:35:55.110131+0000","last_undegraded":"2023-09-23T22:35:55.110131+0000","last_fullsized":"2023-09-23T22:35:55.110131+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:39.765709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"469'3841","reported_seq":3868,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:07.381620+0000","last_change":"2023-09-23T22:35:15.255601+0000","last_active":"2023-09-23T22:36:07.381620+0000","last_peered":"2023-09-23T22:36:07.381620+0000","last_clean":"2023-09-23T22:36:07.381620+0000","last_became_active":"2023-09-23T22:35:15.255306+0000","last_became_peered":"2023-09-23T22:35:15.255306+0000","last_unstale":"2023-09-23T22:36:07.381620+0000","last_undegraded":"2023-09-23T22:36:07.381620+0000","last_fullsized":"2023-09-23T22:36:07.381620+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3841,"log_dups_size":0,"ondisk_log_size":3841,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:12:45.966276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251723776,"num_objects":3841,"num_object_clones":0,"num_object_copies":11523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3841,"num_write_kb":245824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":401,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.109898+0000","last_change":"2023-09-23T22:34:32.968460+0000","last_active":"2023-09-23T22:35:55.109898+0000","last_peered":"2023-09-23T22:35:55.109898+0000","last_clean":"2023-09-23T22:35:55.109898+0000","last_became_active":"2023-09-23T22:34:32.968169+0000","last_became_peered":"2023-09-23T22:34:32.968169+0000","last_unstale":"2023-09-23T22:35:55.109898+0000","last_undegraded":"2023-09-23T22:35:55.109898+0000","last_fullsized":"2023-09-23T22:35:55.109898+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:02.842773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":250,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.105350+0000","last_change":"2023-09-23T22:35:52.676300+0000","last_active":"2023-09-23T22:35:55.105350+0000","last_peered":"2023-09-23T22:35:55.105350+0000","last_clean":"2023-09-23T22:35:55.105350+0000","last_became_active":"2023-09-23T22:35:52.675995+0000","last_became_peered":"2023-09-23T22:35:52.675995+0000","last_unstale":"2023-09-23T22:35:55.105350+0000","last_undegraded":"2023-09-23T22:35:55.105350+0000","last_fullsized":"2023-09-23T22:35:55.105350+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:05.634878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"469'3664","reported_seq":3718,"reported_epoch":469,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:36:05.221838+0000","last_change":"2023-09-23T22:35:53.838418+0000","last_active":"2023-09-23T22:36:05.221838+0000","last_peered":"2023-09-23T22:36:05.221838+0000","last_clean":"2023-09-23T22:35:27.126327+0000","last_became_active":"2023-09-23T22:35:53.830914+0000","last_became_peered":"2023-09-23T22:35:53.830914+0000","last_unstale":"2023-09-23T22:36:05.221838+0000","last_undegraded":"2023-09-23T22:35:53.678488+0000","last_fullsized":"2023-09-23T22:35:53.663350+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3664,"log_dups_size":0,"ondisk_log_size":3664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:04:24.274799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240123904,"num_objects":3664,"num_object_clones":0,"num_object_copies":10992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2408,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3664,"num_write_kb":234496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[{"shards":"6,7","objects":1256}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":795,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:36:02.387272+0000","last_change":"2023-09-23T22:35:23.289779+0000","last_active":"2023-09-23T22:36:02.387272+0000","last_peered":"2023-09-23T22:36:02.387272+0000","last_clean":"2023-09-23T22:36:02.387272+0000","last_became_active":"2023-09-23T22:35:23.289304+0000","last_became_peered":"2023-09-23T22:35:23.289304+0000","last_unstale":"2023-09-23T22:36:02.387272+0000","last_undegraded":"2023-09-23T22:36:02.387272+0000","last_fullsized":"2023-09-23T22:36:02.387272+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:55.971566+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":317,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107466+0000","last_change":"2023-09-23T22:35:52.676196+0000","last_active":"2023-09-23T22:35:55.107466+0000","last_peered":"2023-09-23T22:35:55.107466+0000","last_clean":"2023-09-23T22:35:55.107466+0000","last_became_active":"2023-09-23T22:35:52.675873+0000","last_became_peered":"2023-09-23T22:35:52.675873+0000","last_unstale":"2023-09-23T22:35:55.107466+0000","last_undegraded":"2023-09-23T22:35:55.107466+0000","last_fullsized":"2023-09-23T22:35:55.107466+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:57.694228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":415,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.109990+0000","last_change":"2023-09-23T22:35:52.679361+0000","last_active":"2023-09-23T22:35:55.109990+0000","last_peered":"2023-09-23T22:35:55.109990+0000","last_clean":"2023-09-23T22:35:55.109990+0000","last_became_active":"2023-09-23T22:35:52.678590+0000","last_became_peered":"2023-09-23T22:35:52.678590+0000","last_unstale":"2023-09-23T22:35:55.109990+0000","last_undegraded":"2023-09-23T22:35:55.109990+0000","last_fullsized":"2023-09-23T22:35:55.109990+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:08.755380+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.0","version":"469'3676","reported_seq":3703,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:04.371646+0000","last_change":"2023-09-23T22:35:15.256179+0000","last_active":"2023-09-23T22:36:04.371646+0000","last_peered":"2023-09-23T22:36:04.371646+0000","last_clean":"2023-09-23T22:36:04.371646+0000","last_became_active":"2023-09-23T22:35:15.255873+0000","last_became_peered":"2023-09-23T22:35:15.255873+0000","last_unstale":"2023-09-23T22:36:04.371646+0000","last_undegraded":"2023-09-23T22:36:04.371646+0000","last_fullsized":"2023-09-23T22:36:04.371646+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3676,"log_dups_size":0,"ondisk_log_size":3676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:53.523937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240910336,"num_objects":3676,"num_object_clones":0,"num_object_copies":11028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3676,"num_write_kb":235264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":878,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.102806+0000","last_change":"2023-09-23T22:35:52.674511+0000","last_active":"2023-09-23T22:35:55.102806+0000","last_peered":"2023-09-23T22:35:55.102806+0000","last_clean":"2023-09-23T22:35:55.102806+0000","last_became_active":"2023-09-23T22:35:52.674078+0000","last_became_peered":"2023-09-23T22:35:52.674078+0000","last_unstale":"2023-09-23T22:35:55.102806+0000","last_undegraded":"2023-09-23T22:35:55.102806+0000","last_fullsized":"2023-09-23T22:35:55.102806+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:17.325528+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":167,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110219+0000","last_change":"2023-09-23T22:35:52.685819+0000","last_active":"2023-09-23T22:35:55.110219+0000","last_peered":"2023-09-23T22:35:55.110219+0000","last_clean":"2023-09-23T22:35:55.110219+0000","last_became_active":"2023-09-23T22:35:52.685516+0000","last_became_peered":"2023-09-23T22:35:52.685516+0000","last_unstale":"2023-09-23T22:35:55.110219+0000","last_undegraded":"2023-09-23T22:35:55.110219+0000","last_fullsized":"2023-09-23T22:35:55.110219+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:56.714065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":80,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:36:02.387371+0000","last_change":"2023-09-23T22:35:52.677753+0000","last_active":"2023-09-23T22:36:02.387371+0000","last_peered":"2023-09-23T22:36:02.387371+0000","last_clean":"2023-09-23T22:36:02.387371+0000","last_became_active":"2023-09-23T22:35:52.677432+0000","last_became_peered":"2023-09-23T22:35:52.677432+0000","last_unstale":"2023-09-23T22:36:02.387371+0000","last_undegraded":"2023-09-23T22:36:02.387371+0000","last_fullsized":"2023-09-23T22:36:02.387371+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:38.847793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"469'3766","reported_seq":3802,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:05.193566+0000","last_change":"2023-09-23T22:35:20.333357+0000","last_active":"2023-09-23T22:36:05.193566+0000","last_peered":"2023-09-23T22:36:05.193566+0000","last_clean":"2023-09-23T22:36:05.193566+0000","last_became_active":"2023-09-23T22:35:20.333002+0000","last_became_peered":"2023-09-23T22:35:20.333002+0000","last_unstale":"2023-09-23T22:36:05.193566+0000","last_undegraded":"2023-09-23T22:36:05.193566+0000","last_fullsized":"2023-09-23T22:36:05.193566+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3766,"log_dups_size":0,"ondisk_log_size":3766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:30:44.027165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246808576,"num_objects":3766,"num_object_clones":0,"num_object_copies":11298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3766,"num_write_kb":241024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":483,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.104032+0000","last_change":"2023-09-23T22:35:52.681932+0000","last_active":"2023-09-23T22:35:55.104032+0000","last_peered":"2023-09-23T22:35:55.104032+0000","last_clean":"2023-09-23T22:35:55.104032+0000","last_became_active":"2023-09-23T22:35:52.681575+0000","last_became_peered":"2023-09-23T22:35:52.681575+0000","last_unstale":"2023-09-23T22:35:55.104032+0000","last_undegraded":"2023-09-23T22:35:55.104032+0000","last_fullsized":"2023-09-23T22:35:55.104032+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:25.183658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":457,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110983+0000","last_change":"2023-09-23T22:35:52.678032+0000","last_active":"2023-09-23T22:35:55.110983+0000","last_peered":"2023-09-23T22:35:55.110983+0000","last_clean":"2023-09-23T22:35:55.110983+0000","last_became_active":"2023-09-23T22:35:52.677659+0000","last_became_peered":"2023-09-23T22:35:52.677659+0000","last_unstale":"2023-09-23T22:35:55.110983+0000","last_undegraded":"2023-09-23T22:35:55.110983+0000","last_fullsized":"2023-09-23T22:35:55.110983+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:52.220155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.2","version":"469'3693","reported_seq":3729,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:04.574574+0000","last_change":"2023-09-23T22:35:20.335388+0000","last_active":"2023-09-23T22:36:04.574574+0000","last_peered":"2023-09-23T22:36:04.574574+0000","last_clean":"2023-09-23T22:36:04.574574+0000","last_became_active":"2023-09-23T22:35:20.335040+0000","last_became_peered":"2023-09-23T22:35:20.335040+0000","last_unstale":"2023-09-23T22:36:04.574574+0000","last_undegraded":"2023-09-23T22:36:04.574574+0000","last_fullsized":"2023-09-23T22:36:04.574574+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3693,"log_dups_size":0,"ondisk_log_size":3693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:23:20.824994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242024448,"num_objects":3693,"num_object_clones":0,"num_object_copies":11079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3693,"num_write_kb":236352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":843,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.108436+0000","last_change":"2023-09-23T22:33:49.199701+0000","last_active":"2023-09-23T22:35:55.108436+0000","last_peered":"2023-09-23T22:35:55.108436+0000","last_clean":"2023-09-23T22:35:55.108436+0000","last_became_active":"2023-09-23T22:33:49.198936+0000","last_became_peered":"2023-09-23T22:33:49.198936+0000","last_unstale":"2023-09-23T22:35:55.108436+0000","last_undegraded":"2023-09-23T22:35:55.108436+0000","last_fullsized":"2023-09-23T22:35:55.108436+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.214906+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":220,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.105989+0000","last_change":"2023-09-23T22:35:52.675194+0000","last_active":"2023-09-23T22:35:55.105989+0000","last_peered":"2023-09-23T22:35:55.105989+0000","last_clean":"2023-09-23T22:35:55.105989+0000","last_became_active":"2023-09-23T22:35:52.674899+0000","last_became_peered":"2023-09-23T22:35:52.674899+0000","last_unstale":"2023-09-23T22:35:55.105989+0000","last_undegraded":"2023-09-23T22:35:55.105989+0000","last_fullsized":"2023-09-23T22:35:55.105989+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:26:42.485550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":220,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.106359+0000","last_change":"2023-09-23T22:35:52.675104+0000","last_active":"2023-09-23T22:35:55.106359+0000","last_peered":"2023-09-23T22:35:55.106359+0000","last_clean":"2023-09-23T22:35:55.106359+0000","last_became_active":"2023-09-23T22:35:52.674798+0000","last_became_peered":"2023-09-23T22:35:52.674798+0000","last_unstale":"2023-09-23T22:35:55.106359+0000","last_undegraded":"2023-09-23T22:35:55.106359+0000","last_fullsized":"2023-09-23T22:35:55.106359+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:06.087851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"469'3683","reported_seq":3709,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:05.202472+0000","last_change":"2023-09-23T22:35:15.253793+0000","last_active":"2023-09-23T22:36:05.202472+0000","last_peered":"2023-09-23T22:36:05.202472+0000","last_clean":"2023-09-23T22:36:05.202472+0000","last_became_active":"2023-09-23T22:35:15.253386+0000","last_became_peered":"2023-09-23T22:35:15.253386+0000","last_unstale":"2023-09-23T22:36:05.202472+0000","last_undegraded":"2023-09-23T22:36:05.202472+0000","last_fullsized":"2023-09-23T22:36:05.202472+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3683,"log_dups_size":0,"ondisk_log_size":3683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:28.130011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241369088,"num_objects":3683,"num_object_clones":0,"num_object_copies":11049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3683,"num_write_kb":235712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":804,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110818+0000","last_change":"2023-09-23T22:35:23.295959+0000","last_active":"2023-09-23T22:35:55.110818+0000","last_peered":"2023-09-23T22:35:55.110818+0000","last_clean":"2023-09-23T22:35:55.110818+0000","last_became_active":"2023-09-23T22:35:23.295558+0000","last_became_peered":"2023-09-23T22:35:23.295558+0000","last_unstale":"2023-09-23T22:35:55.110818+0000","last_undegraded":"2023-09-23T22:35:55.110818+0000","last_fullsized":"2023-09-23T22:35:55.110818+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:04.908380+0000","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":933,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107287+0000","last_change":"2023-09-23T22:35:23.293368+0000","last_active":"2023-09-23T22:35:55.107287+0000","last_peered":"2023-09-23T22:35:55.107287+0000","last_clean":"2023-09-23T22:35:55.107287+0000","last_became_active":"2023-09-23T22:35:23.292601+0000","last_became_peered":"2023-09-23T22:35:23.292601+0000","last_unstale":"2023-09-23T22:35:55.107287+0000","last_undegraded":"2023-09-23T22:35:55.107287+0000","last_fullsized":"2023-09-23T22:35:55.107287+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:32.168018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":543,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.099385+0000","last_change":"2023-09-23T22:35:20.326348+0000","last_active":"2023-09-23T22:35:55.099385+0000","last_peered":"2023-09-23T22:35:55.099385+0000","last_clean":"2023-09-23T22:35:55.099385+0000","last_became_active":"2023-09-23T22:35:20.326029+0000","last_became_peered":"2023-09-23T22:35:20.326029+0000","last_unstale":"2023-09-23T22:35:55.099385+0000","last_undegraded":"2023-09-23T22:35:55.099385+0000","last_fullsized":"2023-09-23T22:35:55.099385+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:44:34.048067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,7],"acting":[2,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":202,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107866+0000","last_change":"2023-09-23T22:35:52.684793+0000","last_active":"2023-09-23T22:35:55.107866+0000","last_peered":"2023-09-23T22:35:55.107866+0000","last_clean":"2023-09-23T22:35:55.107866+0000","last_became_active":"2023-09-23T22:35:52.684519+0000","last_became_peered":"2023-09-23T22:35:52.684519+0000","last_unstale":"2023-09-23T22:35:55.107866+0000","last_undegraded":"2023-09-23T22:35:55.107866+0000","last_fullsized":"2023-09-23T22:35:55.107866+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:44.680776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"469'3714","reported_seq":3750,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:05.220267+0000","last_change":"2023-09-23T22:35:20.330224+0000","last_active":"2023-09-23T22:36:05.220267+0000","last_peered":"2023-09-23T22:36:05.220267+0000","last_clean":"2023-09-23T22:36:05.220267+0000","last_became_active":"2023-09-23T22:35:20.329853+0000","last_became_peered":"2023-09-23T22:35:20.329853+0000","last_unstale":"2023-09-23T22:36:05.220267+0000","last_undegraded":"2023-09-23T22:36:05.220267+0000","last_fullsized":"2023-09-23T22:36:05.220267+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3714,"log_dups_size":0,"ondisk_log_size":3714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.075006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243400704,"num_objects":3714,"num_object_clones":0,"num_object_copies":11142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3714,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3714,"num_write_kb":237696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":837,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.098333+0000","last_change":"2023-09-23T22:35:52.683800+0000","last_active":"2023-09-23T22:35:55.098333+0000","last_peered":"2023-09-23T22:35:55.098333+0000","last_clean":"2023-09-23T22:35:55.098333+0000","last_became_active":"2023-09-23T22:35:52.683466+0000","last_became_peered":"2023-09-23T22:35:52.683466+0000","last_unstale":"2023-09-23T22:35:55.098333+0000","last_undegraded":"2023-09-23T22:35:55.098333+0000","last_fullsized":"2023-09-23T22:35:55.098333+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:18.772312+0000","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1347,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.108528+0000","last_change":"2023-09-23T22:29:51.201252+0000","last_active":"2023-09-23T22:35:55.108528+0000","last_peered":"2023-09-23T22:35:55.108528+0000","last_clean":"2023-09-23T22:35:55.108528+0000","last_became_active":"2023-09-23T22:29:51.200799+0000","last_became_peered":"2023-09-23T22:29:51.200799+0000","last_unstale":"2023-09-23T22:35:55.108528+0000","last_undegraded":"2023-09-23T22:35:55.108528+0000","last_fullsized":"2023-09-23T22:35:55.108528+0000","mapping_epoch":351,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":352,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:55.014595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":435,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.111350+0000","last_change":"2023-09-23T22:35:52.680092+0000","last_active":"2023-09-23T22:35:55.111350+0000","last_peered":"2023-09-23T22:35:55.111350+0000","last_clean":"2023-09-23T22:35:55.111350+0000","last_became_active":"2023-09-23T22:35:52.679779+0000","last_became_peered":"2023-09-23T22:35:52.679779+0000","last_unstale":"2023-09-23T22:35:55.111350+0000","last_undegraded":"2023-09-23T22:35:55.111350+0000","last_fullsized":"2023-09-23T22:35:55.111350+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:29.344281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":65,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.099033+0000","last_change":"2023-09-23T22:33:46.211031+0000","last_active":"2023-09-23T22:35:55.099033+0000","last_peered":"2023-09-23T22:35:55.099033+0000","last_clean":"2023-09-23T22:35:55.099033+0000","last_became_active":"2023-09-23T22:33:46.210699+0000","last_became_peered":"2023-09-23T22:33:46.210699+0000","last_unstale":"2023-09-23T22:35:55.099033+0000","last_undegraded":"2023-09-23T22:35:55.099033+0000","last_fullsized":"2023-09-23T22:35:55.099033+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:06:26.338584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"469'3786","reported_seq":3822,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:05.187481+0000","last_change":"2023-09-23T22:35:20.332854+0000","last_active":"2023-09-23T22:36:05.187481+0000","last_peered":"2023-09-23T22:36:05.187481+0000","last_clean":"2023-09-23T22:36:05.187481+0000","last_became_active":"2023-09-23T22:35:20.332499+0000","last_became_peered":"2023-09-23T22:35:20.332499+0000","last_unstale":"2023-09-23T22:36:05.187481+0000","last_undegraded":"2023-09-23T22:36:05.187481+0000","last_fullsized":"2023-09-23T22:36:05.187481+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3786,"log_dups_size":0,"ondisk_log_size":3786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:35.031807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248119296,"num_objects":3786,"num_object_clones":0,"num_object_copies":11358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3786,"num_write_kb":242304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":834,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.100716+0000","last_change":"2023-09-23T22:35:52.688068+0000","last_active":"2023-09-23T22:35:55.100716+0000","last_peered":"2023-09-23T22:35:55.100716+0000","last_clean":"2023-09-23T22:35:55.100716+0000","last_became_active":"2023-09-23T22:35:52.687742+0000","last_became_peered":"2023-09-23T22:35:52.687742+0000","last_unstale":"2023-09-23T22:35:55.100716+0000","last_undegraded":"2023-09-23T22:35:55.100716+0000","last_fullsized":"2023-09-23T22:35:55.100716+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:09.825230+0000","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":91,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.098499+0000","last_change":"2023-09-23T22:33:46.222482+0000","last_active":"2023-09-23T22:35:55.098499+0000","last_peered":"2023-09-23T22:35:55.098499+0000","last_clean":"2023-09-23T22:35:55.098499+0000","last_became_active":"2023-09-23T22:33:46.205871+0000","last_became_peered":"2023-09-23T22:33:46.205871+0000","last_unstale":"2023-09-23T22:35:55.098499+0000","last_undegraded":"2023-09-23T22:35:55.098499+0000","last_fullsized":"2023-09-23T22:35:55.098499+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:54.426538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":65,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.102971+0000","last_change":"2023-09-23T22:33:46.213597+0000","last_active":"2023-09-23T22:35:55.102971+0000","last_peered":"2023-09-23T22:35:55.102971+0000","last_clean":"2023-09-23T22:35:55.102971+0000","last_became_active":"2023-09-23T22:33:46.213294+0000","last_became_peered":"2023-09-23T22:33:46.213294+0000","last_unstale":"2023-09-23T22:35:55.102971+0000","last_undegraded":"2023-09-23T22:35:55.102971+0000","last_fullsized":"2023-09-23T22:35:55.102971+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:29.121139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":137,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.106271+0000","last_change":"2023-09-23T22:35:20.335842+0000","last_active":"2023-09-23T22:35:55.106271+0000","last_peered":"2023-09-23T22:35:55.106271+0000","last_clean":"2023-09-23T22:35:55.106271+0000","last_became_active":"2023-09-23T22:35:20.335452+0000","last_became_peered":"2023-09-23T22:35:20.335452+0000","last_unstale":"2023-09-23T22:35:55.106271+0000","last_undegraded":"2023-09-23T22:35:55.106271+0000","last_fullsized":"2023-09-23T22:35:55.106271+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:17.101594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"469'3868","reported_seq":3904,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:04.441277+0000","last_change":"2023-09-23T22:35:20.339882+0000","last_active":"2023-09-23T22:36:04.441277+0000","last_peered":"2023-09-23T22:36:04.441277+0000","last_clean":"2023-09-23T22:36:04.441277+0000","last_became_active":"2023-09-23T22:35:20.339404+0000","last_became_peered":"2023-09-23T22:35:20.339404+0000","last_unstale":"2023-09-23T22:36:04.441277+0000","last_undegraded":"2023-09-23T22:36:04.441277+0000","last_fullsized":"2023-09-23T22:36:04.441277+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3868,"log_dups_size":0,"ondisk_log_size":3868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:11.755000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253493248,"num_objects":3868,"num_object_clones":0,"num_object_copies":11604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3868,"num_write_kb":247552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1301,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.098776+0000","last_change":"2023-09-23T22:35:23.295043+0000","last_active":"2023-09-23T22:35:55.098776+0000","last_peered":"2023-09-23T22:35:55.098776+0000","last_clean":"2023-09-23T22:35:55.098776+0000","last_became_active":"2023-09-23T22:35:23.294354+0000","last_became_peered":"2023-09-23T22:35:23.294354+0000","last_unstale":"2023-09-23T22:35:55.098776+0000","last_undegraded":"2023-09-23T22:35:55.098776+0000","last_fullsized":"2023-09-23T22:35:55.098776+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:55.184477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":756,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.109768+0000","last_change":"2023-09-23T22:35:23.294541+0000","last_active":"2023-09-23T22:35:55.109768+0000","last_peered":"2023-09-23T22:35:55.109768+0000","last_clean":"2023-09-23T22:35:55.109768+0000","last_became_active":"2023-09-23T22:35:23.294096+0000","last_became_peered":"2023-09-23T22:35:23.294096+0000","last_unstale":"2023-09-23T22:35:55.109768+0000","last_undegraded":"2023-09-23T22:35:55.109768+0000","last_fullsized":"2023-09-23T22:35:55.109768+0000","mapping_epoch":457,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":458,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:51:32.124694+0000","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":269,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110052+0000","last_change":"2023-09-23T22:35:52.685855+0000","last_active":"2023-09-23T22:35:55.110052+0000","last_peered":"2023-09-23T22:35:55.110052+0000","last_clean":"2023-09-23T22:35:55.110052+0000","last_became_active":"2023-09-23T22:35:52.685522+0000","last_became_peered":"2023-09-23T22:35:52.685522+0000","last_unstale":"2023-09-23T22:35:55.110052+0000","last_undegraded":"2023-09-23T22:35:55.110052+0000","last_fullsized":"2023-09-23T22:35:55.110052+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:01.954753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":187,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110555+0000","last_change":"2023-09-23T22:35:52.687955+0000","last_active":"2023-09-23T22:35:55.110555+0000","last_peered":"2023-09-23T22:35:55.110555+0000","last_clean":"2023-09-23T22:35:55.110555+0000","last_became_active":"2023-09-23T22:35:52.687663+0000","last_became_peered":"2023-09-23T22:35:52.687663+0000","last_unstale":"2023-09-23T22:35:55.110555+0000","last_undegraded":"2023-09-23T22:35:55.110555+0000","last_fullsized":"2023-09-23T22:35:55.110555+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:54.076621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"469'3779","reported_seq":3805,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:05.217200+0000","last_change":"2023-09-23T22:35:15.479260+0000","last_active":"2023-09-23T22:36:05.217200+0000","last_peered":"2023-09-23T22:36:05.217200+0000","last_clean":"2023-09-23T22:36:05.217200+0000","last_became_active":"2023-09-23T22:35:15.478813+0000","last_became_peered":"2023-09-23T22:35:15.478813+0000","last_unstale":"2023-09-23T22:36:05.217200+0000","last_undegraded":"2023-09-23T22:36:05.217200+0000","last_fullsized":"2023-09-23T22:36:05.217200+0000","mapping_epoch":451,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3779,"log_dups_size":0,"ondisk_log_size":3779,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:33.079825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247660544,"num_objects":3779,"num_object_clones":0,"num_object_copies":11337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3779,"num_write_kb":241856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":718,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.111571+0000","last_change":"2023-09-23T22:35:52.680487+0000","last_active":"2023-09-23T22:35:55.111571+0000","last_peered":"2023-09-23T22:35:55.111571+0000","last_clean":"2023-09-23T22:35:55.111571+0000","last_became_active":"2023-09-23T22:35:52.680256+0000","last_became_peered":"2023-09-23T22:35:52.680256+0000","last_unstale":"2023-09-23T22:35:55.111571+0000","last_undegraded":"2023-09-23T22:35:55.111571+0000","last_fullsized":"2023-09-23T22:35:55.111571+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:45.503016+0000","scrub_duration":0.0034076229999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":903,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.106009+0000","last_change":"2023-09-23T22:35:20.365482+0000","last_active":"2023-09-23T22:35:55.106009+0000","last_peered":"2023-09-23T22:35:55.106009+0000","last_clean":"2023-09-23T22:35:55.106009+0000","last_became_active":"2023-09-23T22:35:20.339867+0000","last_became_peered":"2023-09-23T22:35:20.339867+0000","last_unstale":"2023-09-23T22:35:55.106009+0000","last_undegraded":"2023-09-23T22:35:55.106009+0000","last_fullsized":"2023-09-23T22:35:55.106009+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:58.856960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":211,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.104102+0000","last_change":"2023-09-23T22:35:52.686882+0000","last_active":"2023-09-23T22:35:55.104102+0000","last_peered":"2023-09-23T22:35:55.104102+0000","last_clean":"2023-09-23T22:35:55.104102+0000","last_became_active":"2023-09-23T22:35:52.685147+0000","last_became_peered":"2023-09-23T22:35:52.685147+0000","last_unstale":"2023-09-23T22:35:55.104102+0000","last_undegraded":"2023-09-23T22:35:55.104102+0000","last_fullsized":"2023-09-23T22:35:55.104102+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:52.173682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":343,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.104446+0000","last_change":"2023-09-23T22:35:52.675187+0000","last_active":"2023-09-23T22:35:55.104446+0000","last_peered":"2023-09-23T22:35:55.104446+0000","last_clean":"2023-09-23T22:35:55.104446+0000","last_became_active":"2023-09-23T22:35:52.674444+0000","last_became_peered":"2023-09-23T22:35:52.674444+0000","last_unstale":"2023-09-23T22:35:55.104446+0000","last_undegraded":"2023-09-23T22:35:55.104446+0000","last_fullsized":"2023-09-23T22:35:55.104446+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:17.802244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.e","version":"469'3791","reported_seq":5441,"reported_epoch":469,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:36:04.568268+0000","last_change":"2023-09-23T22:35:53.835859+0000","last_active":"2023-09-23T22:36:04.568268+0000","last_peered":"2023-09-23T22:36:04.568268+0000","last_clean":"2023-09-23T22:35:27.119300+0000","last_became_active":"2023-09-23T22:35:53.820174+0000","last_became_peered":"2023-09-23T22:35:53.820174+0000","last_unstale":"2023-09-23T22:36:04.568268+0000","last_undegraded":"2023-09-23T22:35:53.681301+0000","last_fullsized":"2023-09-23T22:35:53.662783+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3791,"log_dups_size":0,"ondisk_log_size":3791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:34.258475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248446976,"num_objects":3791,"num_object_clones":0,"num_object_copies":11373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1732,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3791,"num_write_kb":242624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":801,"num_bytes_recovered":52494336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":2059}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":836,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.111248+0000","last_change":"2023-09-23T22:35:52.678639+0000","last_active":"2023-09-23T22:35:55.111248+0000","last_peered":"2023-09-23T22:35:55.111248+0000","last_clean":"2023-09-23T22:35:55.111248+0000","last_became_active":"2023-09-23T22:35:52.678321+0000","last_became_peered":"2023-09-23T22:35:52.678321+0000","last_unstale":"2023-09-23T22:35:55.111248+0000","last_undegraded":"2023-09-23T22:35:55.111248+0000","last_fullsized":"2023-09-23T22:35:55.111248+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:07.924822+0000","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":61,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.104802+0000","last_change":"2023-09-23T22:35:52.688113+0000","last_active":"2023-09-23T22:35:55.104802+0000","last_peered":"2023-09-23T22:35:55.104802+0000","last_clean":"2023-09-23T22:35:55.104802+0000","last_became_active":"2023-09-23T22:35:52.687708+0000","last_became_peered":"2023-09-23T22:35:52.687708+0000","last_unstale":"2023-09-23T22:35:55.104802+0000","last_undegraded":"2023-09-23T22:35:55.104802+0000","last_fullsized":"2023-09-23T22:35:55.104802+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:07.029149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":74,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.099115+0000","last_change":"2023-09-23T22:34:32.974362+0000","last_active":"2023-09-23T22:35:55.099115+0000","last_peered":"2023-09-23T22:35:55.099115+0000","last_clean":"2023-09-23T22:35:55.099115+0000","last_became_active":"2023-09-23T22:34:32.973858+0000","last_became_peered":"2023-09-23T22:34:32.973858+0000","last_unstale":"2023-09-23T22:35:55.099115+0000","last_undegraded":"2023-09-23T22:35:55.099115+0000","last_fullsized":"2023-09-23T22:35:55.099115+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:15.264858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":302,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.105606+0000","last_change":"2023-09-23T22:35:20.335737+0000","last_active":"2023-09-23T22:35:55.105606+0000","last_peered":"2023-09-23T22:35:55.105606+0000","last_clean":"2023-09-23T22:35:55.105606+0000","last_became_active":"2023-09-23T22:35:20.335363+0000","last_became_peered":"2023-09-23T22:35:20.335363+0000","last_unstale":"2023-09-23T22:35:55.105606+0000","last_undegraded":"2023-09-23T22:35:55.105606+0000","last_fullsized":"2023-09-23T22:35:55.105606+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:38.482168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.d","version":"469'3886","reported_seq":3922,"reported_epoch":469,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:36:05.207708+0000","last_change":"2023-09-23T22:35:53.846084+0000","last_active":"2023-09-23T22:36:05.207708+0000","last_peered":"2023-09-23T22:36:05.207708+0000","last_clean":"2023-09-23T22:35:27.351559+0000","last_became_active":"2023-09-23T22:35:53.840426+0000","last_became_peered":"2023-09-23T22:35:53.840426+0000","last_unstale":"2023-09-23T22:36:05.207708+0000","last_undegraded":"2023-09-23T22:35:53.677830+0000","last_fullsized":"2023-09-23T22:35:53.663431+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3886,"log_dups_size":0,"ondisk_log_size":3886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:33.813139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254672896,"num_objects":3886,"num_object_clones":0,"num_object_copies":11658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3886,"num_write_kb":248704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[{"shards":"1,7","objects":1310}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":810,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.104678+0000","last_change":"2023-09-23T22:35:52.682635+0000","last_active":"2023-09-23T22:35:55.104678+0000","last_peered":"2023-09-23T22:35:55.104678+0000","last_clean":"2023-09-23T22:35:55.104678+0000","last_became_active":"2023-09-23T22:35:52.677026+0000","last_became_peered":"2023-09-23T22:35:52.677026+0000","last_unstale":"2023-09-23T22:35:55.104678+0000","last_undegraded":"2023-09-23T22:35:55.104678+0000","last_fullsized":"2023-09-23T22:35:55.104678+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:22:57.263576+0000","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":172,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110231+0000","last_change":"2023-09-23T22:35:20.346473+0000","last_active":"2023-09-23T22:35:55.110231+0000","last_peered":"2023-09-23T22:35:55.110231+0000","last_clean":"2023-09-23T22:35:55.110231+0000","last_became_active":"2023-09-23T22:35:20.327239+0000","last_became_peered":"2023-09-23T22:35:20.327239+0000","last_unstale":"2023-09-23T22:35:55.110231+0000","last_undegraded":"2023-09-23T22:35:55.110231+0000","last_fullsized":"2023-09-23T22:35:55.110231+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:40.623854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,0],"acting":[5,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":981,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107140+0000","last_change":"2023-09-23T22:35:52.680755+0000","last_active":"2023-09-23T22:35:55.107140+0000","last_peered":"2023-09-23T22:35:55.107140+0000","last_clean":"2023-09-23T22:35:55.107140+0000","last_became_active":"2023-09-23T22:35:52.679534+0000","last_became_peered":"2023-09-23T22:35:52.679534+0000","last_unstale":"2023-09-23T22:35:55.107140+0000","last_undegraded":"2023-09-23T22:35:55.107140+0000","last_fullsized":"2023-09-23T22:35:55.107140+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:56.134807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":24,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.101147+0000","last_change":"2023-09-23T22:35:20.335440+0000","last_active":"2023-09-23T22:35:55.101147+0000","last_peered":"2023-09-23T22:35:55.101147+0000","last_clean":"2023-09-23T22:35:55.101147+0000","last_became_active":"2023-09-23T22:35:20.335130+0000","last_became_peered":"2023-09-23T22:35:20.335130+0000","last_unstale":"2023-09-23T22:35:55.101147+0000","last_undegraded":"2023-09-23T22:35:55.101147+0000","last_fullsized":"2023-09-23T22:35:55.101147+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:05.380227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"469'3772","reported_seq":3819,"reported_epoch":469,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:36:05.058729+0000","last_change":"2023-09-23T22:35:53.839344+0000","last_active":"2023-09-23T22:36:05.058729+0000","last_peered":"2023-09-23T22:36:05.058729+0000","last_clean":"2023-09-23T22:35:27.348348+0000","last_became_active":"2023-09-23T22:35:53.834288+0000","last_became_peered":"2023-09-23T22:35:53.834288+0000","last_unstale":"2023-09-23T22:36:05.058729+0000","last_undegraded":"2023-09-23T22:35:53.679898+0000","last_fullsized":"2023-09-23T22:35:53.662675+0000","mapping_epoch":466,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":3772,"log_dups_size":0,"ondisk_log_size":3772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:13:09.892850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247201792,"num_objects":3772,"num_object_clones":0,"num_object_copies":11316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2500,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3772,"num_write_kb":241408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1272}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":677,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.109801+0000","last_change":"2023-09-23T22:35:52.679470+0000","last_active":"2023-09-23T22:35:55.109801+0000","last_peered":"2023-09-23T22:35:55.109801+0000","last_clean":"2023-09-23T22:35:55.109801+0000","last_became_active":"2023-09-23T22:35:52.679019+0000","last_became_peered":"2023-09-23T22:35:52.679019+0000","last_unstale":"2023-09-23T22:35:55.109801+0000","last_undegraded":"2023-09-23T22:35:55.109801+0000","last_fullsized":"2023-09-23T22:35:55.109801+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:11.504416+0000","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":223,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.110765+0000","last_change":"2023-09-23T22:35:20.357456+0000","last_active":"2023-09-23T22:35:55.110765+0000","last_peered":"2023-09-23T22:35:55.110765+0000","last_clean":"2023-09-23T22:35:55.110765+0000","last_became_active":"2023-09-23T22:35:20.330309+0000","last_became_peered":"2023-09-23T22:35:20.330309+0000","last_unstale":"2023-09-23T22:35:55.110765+0000","last_undegraded":"2023-09-23T22:35:55.110765+0000","last_fullsized":"2023-09-23T22:35:55.110765+0000","mapping_epoch":454,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:19.537165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":225,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107069+0000","last_change":"2023-09-23T22:35:52.680525+0000","last_active":"2023-09-23T22:35:55.107069+0000","last_peered":"2023-09-23T22:35:55.107069+0000","last_clean":"2023-09-23T22:35:55.107069+0000","last_became_active":"2023-09-23T22:35:52.679179+0000","last_became_peered":"2023-09-23T22:35:52.679179+0000","last_unstale":"2023-09-23T22:35:55.107069+0000","last_undegraded":"2023-09-23T22:35:55.107069+0000","last_fullsized":"2023-09-23T22:35:55.107069+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:00.095043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":365,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.109496+0000","last_change":"2023-09-23T22:35:52.685857+0000","last_active":"2023-09-23T22:35:55.109496+0000","last_peered":"2023-09-23T22:35:55.109496+0000","last_clean":"2023-09-23T22:35:55.109496+0000","last_became_active":"2023-09-23T22:35:52.685235+0000","last_became_peered":"2023-09-23T22:35:52.685235+0000","last_unstale":"2023-09-23T22:35:55.109496+0000","last_undegraded":"2023-09-23T22:35:55.109496+0000","last_fullsized":"2023-09-23T22:35:55.109496+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:17.995934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":123,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.107267+0000","last_change":"2023-09-23T22:35:52.675772+0000","last_active":"2023-09-23T22:35:55.107267+0000","last_peered":"2023-09-23T22:35:55.107267+0000","last_clean":"2023-09-23T22:35:55.107267+0000","last_became_active":"2023-09-23T22:35:52.675279+0000","last_became_peered":"2023-09-23T22:35:52.675279+0000","last_unstale":"2023-09-23T22:35:55.107267+0000","last_undegraded":"2023-09-23T22:35:55.107267+0000","last_fullsized":"2023-09-23T22:35:55.107267+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.513213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":64,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.099197+0000","last_change":"2023-09-23T22:33:46.210878+0000","last_active":"2023-09-23T22:35:55.099197+0000","last_peered":"2023-09-23T22:35:55.099197+0000","last_clean":"2023-09-23T22:35:55.099197+0000","last_became_active":"2023-09-23T22:33:46.210512+0000","last_became_peered":"2023-09-23T22:33:46.210512+0000","last_unstale":"2023-09-23T22:35:55.099197+0000","last_undegraded":"2023-09-23T22:35:55.099197+0000","last_fullsized":"2023-09-23T22:35:55.099197+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:35.605270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":476,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.108530+0000","last_change":"2023-09-23T22:35:52.682631+0000","last_active":"2023-09-23T22:35:55.108530+0000","last_peered":"2023-09-23T22:35:55.108530+0000","last_clean":"2023-09-23T22:35:55.108530+0000","last_became_active":"2023-09-23T22:35:52.682292+0000","last_became_peered":"2023-09-23T22:35:52.682292+0000","last_unstale":"2023-09-23T22:35:55.108530+0000","last_undegraded":"2023-09-23T22:35:55.108530+0000","last_fullsized":"2023-09-23T22:35:55.108530+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:46.851957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":491,"reported_epoch":469,"state":"active+clean","last_fresh":"2023-09-23T22:36:03.259868+0000","last_change":"2023-09-23T22:35:52.676188+0000","last_active":"2023-09-23T22:36:03.259868+0000","last_peered":"2023-09-23T22:36:03.259868+0000","last_clean":"2023-09-23T22:36:03.259868+0000","last_became_active":"2023-09-23T22:35:52.675880+0000","last_became_peered":"2023-09-23T22:35:52.675880+0000","last_unstale":"2023-09-23T22:36:03.259868+0000","last_undegraded":"2023-09-23T22:36:03.259868+0000","last_fullsized":"2023-09-23T22:36:03.259868+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:16.794100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":501,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.108638+0000","last_change":"2023-09-23T22:35:52.687609+0000","last_active":"2023-09-23T22:35:55.108638+0000","last_peered":"2023-09-23T22:35:55.108638+0000","last_clean":"2023-09-23T22:35:55.108638+0000","last_became_active":"2023-09-23T22:35:52.687247+0000","last_became_peered":"2023-09-23T22:35:52.687247+0000","last_unstale":"2023-09-23T22:35:55.108638+0000","last_undegraded":"2023-09-23T22:35:55.108638+0000","last_fullsized":"2023-09-23T22:35:55.108638+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:11.284798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":271,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.099320+0000","last_change":"2023-09-23T22:35:52.685015+0000","last_active":"2023-09-23T22:35:55.099320+0000","last_peered":"2023-09-23T22:35:55.099320+0000","last_clean":"2023-09-23T22:35:55.099320+0000","last_became_active":"2023-09-23T22:35:52.684685+0000","last_became_peered":"2023-09-23T22:35:52.684685+0000","last_unstale":"2023-09-23T22:35:55.099320+0000","last_undegraded":"2023-09-23T22:35:55.099320+0000","last_fullsized":"2023-09-23T22:35:55.099320+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:17.431203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":170,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.111157+0000","last_change":"2023-09-23T22:33:46.209205+0000","last_active":"2023-09-23T22:35:55.111157+0000","last_peered":"2023-09-23T22:35:55.111157+0000","last_clean":"2023-09-23T22:35:55.111157+0000","last_became_active":"2023-09-23T22:33:46.208800+0000","last_became_peered":"2023-09-23T22:33:46.208800+0000","last_unstale":"2023-09-23T22:35:55.111157+0000","last_undegraded":"2023-09-23T22:35:55.111157+0000","last_fullsized":"2023-09-23T22:35:55.111157+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:58.881258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":73,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.098666+0000","last_change":"2023-09-23T22:34:32.975238+0000","last_active":"2023-09-23T22:35:55.098666+0000","last_peered":"2023-09-23T22:35:55.098666+0000","last_clean":"2023-09-23T22:35:55.098666+0000","last_became_active":"2023-09-23T22:34:32.974927+0000","last_became_peered":"2023-09-23T22:34:32.974927+0000","last_unstale":"2023-09-23T22:35:55.098666+0000","last_undegraded":"2023-09-23T22:35:55.098666+0000","last_fullsized":"2023-09-23T22:35:55.098666+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:48.150567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":411,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.098599+0000","last_change":"2023-09-23T22:34:32.982953+0000","last_active":"2023-09-23T22:35:55.098599+0000","last_peered":"2023-09-23T22:35:55.098599+0000","last_clean":"2023-09-23T22:35:55.098599+0000","last_became_active":"2023-09-23T22:34:32.974430+0000","last_became_peered":"2023-09-23T22:34:32.974430+0000","last_unstale":"2023-09-23T22:35:55.098599+0000","last_undegraded":"2023-09-23T22:35:55.098599+0000","last_fullsized":"2023-09-23T22:35:55.098599+0000","mapping_epoch":430,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:10.333737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,6],"acting":[2,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":528,"reported_epoch":468,"state":"active+clean","last_fresh":"2023-09-23T22:35:55.103341+0000","last_change":"2023-09-23T22:35:52.676866+0000","last_active":"2023-09-23T22:35:55.103341+0000","last_peered":"2023-09-23T22:35:55.103341+0000","last_clean":"2023-09-23T22:35:55.103341+0000","last_became_active":"2023-09-23T22:35:52.676551+0000","last_became_peered":"2023-09-23T22:35:52.676551+0000","last_unstale":"2023-09-23T22:35:55.103341+0000","last_undegraded":"2023-09-23T22:35:55.103341+0000","last_fullsized":"2023-09-23T22:35:55.103341+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:51.546191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":31814,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44001,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1378432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3939762176,"num_objects":60116,"num_object_clones":0,"num_object_copies":180348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11639,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":60116,"num_write_kb":3847424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":801,"num_bytes_recovered":52494336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11071913984,"data_stored":11071913984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":60116,"ondisk_log_size":60116,"up":48,"acting":43,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":295,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234380,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3043640,"kb_used_data":2646896,"kb_used_omap":0,"kb_used_meta":396736,"kb_avail":90701512,"statfs":{"total":95995035648,"available":92878348288,"internally_reserved":0,"allocated":2710421504,"data_stored":2708857951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":406257664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:35:25 2023","interfaces":[{"interface":"back","average":{"1min":1.012,"5min":3.8330000000000002,"15min":4.6379999999999999},"min":{"1min":0.29099999999999998,"5min":0.215,"15min":0.20300000000000001},"max":{"1min":3.9079999999999999,"5min":124.34699999999999,"15min":220.66},"last":0.25800000000000001},{"interface":"front","average":{"1min":0.88900000000000001,"5min":4.3079999999999998,"15min":5.0679999999999996},"min":{"1min":0.254,"5min":0.23699999999999999,"15min":0.23499999999999999},"max":{"1min":3.9750000000000001,"5min":124.73099999999999,"15min":220.59899999999999},"last":2.452}]},{"osd":1,"last update":"Sat Sep 23 22:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":4.2859999999999996,"15min":4.0099999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.23799999999999999},"max":{"1min":4.8769999999999998,"5min":168.18700000000001,"15min":168.18700000000001},"last":0.88400000000000001},{"interface":"front","average":{"1min":1.089,"5min":4.2869999999999999,"15min":4.1589999999999998},"min":{"1min":0.30099999999999999,"5min":0.28599999999999998,"15min":0.26100000000000001},"max":{"1min":4.8090000000000002,"5min":165.053,"15min":165.053},"last":2.0880000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:36:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1000000000000001,"5min":2.3039999999999998,"15min":1.5820000000000001},"min":{"1min":0.32100000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":4.5270000000000001,"5min":70.183999999999997,"15min":70.183999999999997},"last":2.2879999999999998},{"interface":"front","average":{"1min":1.044,"5min":2.589,"15min":1.7270000000000001},"min":{"1min":0.41999999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.4449999999999998,"5min":70.140000000000001,"15min":70.140000000000001},"last":3.4449999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.403,"15min":3.2989999999999999},"min":{"1min":0.34999999999999998,"5min":0.309,"15min":0.26900000000000002},"max":{"1min":1.919,"5min":8.407,"15min":219.90700000000001},"last":4.3010000000000002},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.5409999999999999,"15min":3.4809999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.28299999999999997},"max":{"1min":9.7919999999999998,"5min":9.7919999999999998,"15min":219.976},"last":2.1520000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:35:13 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":2.4340000000000002,"15min":1.341},"min":{"1min":0.32200000000000001,"5min":0.27100000000000002,"15min":0.23499999999999999},"max":{"1min":3.9449999999999998,"5min":106.13800000000001,"15min":106.13800000000001},"last":1.53},{"interface":"front","average":{"1min":1.046,"5min":2.4950000000000001,"15min":1.3779999999999999},"min":{"1min":0.30599999999999999,"5min":0.27700000000000002,"15min":0.25900000000000001},"max":{"1min":3.8759999999999999,"5min":106.179,"15min":106.179},"last":1.9950000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:35:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":2.9319999999999999,"15min":1.4950000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":5.7530000000000001,"5min":125.083,"15min":125.083},"last":1.635},{"interface":"front","average":{"1min":1.2130000000000001,"5min":2.7229999999999999,"15min":1.5269999999999999},"min":{"1min":0.33100000000000002,"5min":0.317,"15min":0.27300000000000002},"max":{"1min":4.8760000000000003,"5min":122.351,"15min":122.351},"last":2.0499999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:35:13 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":2.2839999999999998,"15min":1.1859999999999999},"min":{"1min":0.35699999999999998,"5min":0.33400000000000002,"15min":0.28999999999999998},"max":{"1min":5.7060000000000004,"5min":88.284999999999997,"15min":88.284999999999997},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.85999999999999999,"5min":2.089,"15min":1.2210000000000001},"min":{"1min":0.38800000000000001,"5min":0.34000000000000002,"15min":0.33700000000000002},"max":{"1min":4.0419999999999998,"5min":88.355000000000004,"15min":88.355000000000004},"last":2.117}]}]},{"osd":6,"up_from":46,"seq":197568496016,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1670268,"kb_used_data":1428596,"kb_used_omap":0,"kb_used_meta":241664,"kb_avail":92074884,"statfs":{"total":95995035648,"available":94284681216,"internally_reserved":0,"allocated":1462882304,"data_stored":1461315949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":247463936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:35:26 2023","interfaces":[{"interface":"back","average":{"1min":2.7040000000000002,"5min":6.9210000000000003,"15min":3.4430000000000001},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.19600000000000001},"max":{"1min":44.158000000000001,"5min":213.23699999999999,"15min":213.23699999999999},"last":0.442},{"interface":"front","average":{"1min":3.4359999999999999,"5min":7.3979999999999997,"15min":4.4870000000000001},"min":{"1min":0.28299999999999997,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":44.220999999999997,"5min":213.304,"15min":213.304},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:35:50 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":3.9180000000000001,"15min":3.8340000000000001},"min":{"1min":0.27600000000000002,"5min":0.25700000000000001,"15min":0.22800000000000001},"max":{"1min":2.4950000000000001,"5min":116.06399999999999,"15min":129.964},"last":2.4740000000000002},{"interface":"front","average":{"1min":0.99399999999999999,"5min":3.9489999999999998,"15min":4.1020000000000003},"min":{"1min":0.314,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.8820000000000001,"5min":116.233,"15min":131.18600000000001},"last":1.9219999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:35:07 2023","interfaces":[{"interface":"back","average":{"1min":7.3499999999999996,"5min":4.0110000000000001,"15min":2.6059999999999999},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":52.249000000000002,"5min":103.548,"15min":103.548},"last":0.41899999999999998},{"interface":"front","average":{"1min":7.3449999999999998,"5min":3.9969999999999999,"15min":2.7650000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":51.887,"5min":105.99299999999999,"15min":105.99299999999999},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 22:35:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89900000000000002,"5min":1.8879999999999999,"15min":3.073},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.734,"5min":7.8179999999999996,"15min":125.125},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.151,"5min":1.5669999999999999,"15min":2.7290000000000001},"min":{"1min":0.41699999999999998,"5min":0.313,"15min":0.26900000000000002},"max":{"1min":2.9569999999999999,"5min":9.2569999999999997,"15min":76.406999999999996},"last":0.47199999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:35:00 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.544,"15min":1.54},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":5.133,"5min":11.221,"15min":67.483999999999995},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.8500000000000001,"5min":2.6379999999999999,"15min":1.897},"min":{"1min":0.309,"5min":0.29099999999999998,"15min":0.255},"max":{"1min":13.407999999999999,"5min":13.407999999999999,"15min":67.814999999999998},"last":0.39300000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":3.4649999999999999,"15min":1.728},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.167,"5min":88.194999999999993,"15min":88.194999999999993},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.0309999999999999,"5min":2.5680000000000001,"15min":1.4319999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.4370000000000003,"5min":88.025000000000006,"15min":88.025000000000006},"last":0.36699999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:35:26 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":1.536,"15min":1.2490000000000001},"min":{"1min":0.34499999999999997,"5min":0.32600000000000001,"15min":0.314},"max":{"1min":3.7480000000000002,"5min":9.2270000000000003,"15min":82.801000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.982,"15min":1.4039999999999999},"min":{"1min":0.35199999999999998,"5min":0.315,"15min":0.29699999999999999},"max":{"1min":1.589,"5min":10.324999999999999,"15min":83.052999999999997},"last":0.38400000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795288,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3653952,"kb_used_data":2861176,"kb_used_omap":7,"kb_used_meta":792760,"kb_avail":90091200,"statfs":{"total":95995035648,"available":92253388800,"internally_reserved":0,"allocated":2929844224,"data_stored":2927446668,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":811786612},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,0,0,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:35:41 2023","interfaces":[{"interface":"back","average":{"1min":8.7840000000000007,"5min":7.6219999999999999,"15min":5.8029999999999999},"min":{"1min":0.21299999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":190.715,"5min":202.73500000000001,"15min":202.73500000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":8.9429999999999996,"5min":7.9210000000000003,"15min":6.2919999999999998},"min":{"1min":0.247,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":191.41499999999999,"5min":202.80799999999999,"15min":202.80799999999999},"last":0.26200000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:35:59 2023","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":1.224,"15min":1.1120000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.216},"max":{"1min":3.548,"5min":17.594000000000001,"15min":17.594000000000001},"last":1.6950000000000001},{"interface":"front","average":{"1min":0.81499999999999995,"5min":1.2709999999999999,"15min":2.004},"min":{"1min":0.29899999999999999,"5min":0.222,"15min":0.222},"max":{"1min":3.3740000000000001,"5min":11.978999999999999,"15min":11.978999999999999},"last":1.1930000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.429,"15min":4.109},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.246},"max":{"1min":3.7290000000000001,"5min":10.587999999999999,"15min":160.499},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.0549999999999999,"5min":1.643,"15min":4.0119999999999996},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.223},"max":{"1min":5.2779999999999996,"5min":11.757,"15min":160.465},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 22:34:39 2023","interfaces":[{"interface":"back","average":{"1min":2.5630000000000002,"5min":1.4990000000000001,"15min":2.3839999999999999},"min":{"1min":0.372,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":13.138,"5min":13.138,"15min":234.79900000000001},"last":0.71999999999999997},{"interface":"front","average":{"1min":2.6339999999999999,"5min":2.1859999999999999,"15min":2.9060000000000001},"min":{"1min":0.38400000000000001,"5min":0.32700000000000001,"15min":0.29499999999999998},"max":{"1min":12.246,"5min":12.246,"15min":234.81700000000001},"last":0.55000000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:35:41 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.593,"15min":2.347},"min":{"1min":0.34799999999999998,"5min":0.28799999999999998,"15min":0.25900000000000001},"max":{"1min":1.179,"5min":8.1600000000000001,"15min":196.79900000000001},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.0149999999999999,"5min":1.734,"15min":2.3319999999999999},"min":{"1min":0.38300000000000001,"5min":0.30299999999999999,"15min":0.29099999999999998},"max":{"1min":5.3499999999999996,"5min":12.006,"15min":196.68299999999999},"last":0.59999999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":2.9710000000000001,"15min":2.2269999999999999},"min":{"1min":0.33700000000000002,"5min":0.32200000000000001,"15min":0.30399999999999999},"max":{"1min":5.3179999999999996,"5min":77.033000000000001,"15min":77.033000000000001},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.95999999999999996,"5min":2.7200000000000002,"15min":1.758},"min":{"1min":0.36099999999999999,"5min":0.34699999999999998,"15min":0.32700000000000001},"max":{"1min":5.407,"5min":76.965999999999994,"15min":76.965999999999994},"last":0.76900000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0269999999999999,"5min":1.929,"15min":1.9770000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.35699999999999998},"max":{"1min":5.3739999999999997,"5min":11.849,"15min":16.265000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.468,"15min":1.3340000000000001},"min":{"1min":0.48899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":5.3810000000000002,"5min":13.173,"15min":13.173},"last":0.80300000000000005}]}]},{"osd":0,"up_from":93,"seq":399431958817,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2263636,"kb_used_data":1443980,"kb_used_omap":8,"kb_used_meta":819639,"kb_avail":91481516,"statfs":{"total":95995035648,"available":93677072384,"internally_reserved":0,"allocated":1478635520,"data_stored":1476159480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":839311210},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:35:43 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":9.1890000000000001,"15min":6.056},"min":{"1min":0.23999999999999999,"5min":0.19900000000000001,"15min":0.19400000000000001},"max":{"1min":9.0600000000000005,"5min":288.96699999999998,"15min":288.96699999999998},"last":2.383},{"interface":"front","average":{"1min":0.77100000000000002,"5min":8.8810000000000002,"15min":5.883},"min":{"1min":0.222,"5min":0.217,"15min":0.217},"max":{"1min":2.6150000000000002,"5min":290.38099999999997,"15min":290.38099999999997},"last":1.143}]},{"osd":2,"last update":"Sat Sep 23 22:35:03 2023","interfaces":[{"interface":"back","average":{"1min":16.059999999999999,"5min":4.4160000000000004,"15min":3.6019999999999999},"min":{"1min":0.23899999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":175.89400000000001,"5min":175.89400000000001,"15min":175.89400000000001},"last":1.534},{"interface":"front","average":{"1min":16.169,"5min":4.0449999999999999,"15min":2.5710000000000002},"min":{"1min":0.26400000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":175.66300000000001,"5min":175.66300000000001,"15min":175.66300000000001},"last":0.23799999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:35:43 2023","interfaces":[{"interface":"back","average":{"1min":0.77300000000000002,"5min":1.343,"15min":3.1989999999999998},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.22500000000000001},"max":{"1min":2.0750000000000002,"5min":10.292999999999999,"15min":146.87299999999999},"last":2.323},{"interface":"front","average":{"1min":0.75900000000000001,"5min":1.1950000000000001,"15min":3.1909999999999998},"min":{"1min":0.34399999999999997,"5min":0.27200000000000002,"15min":0.247},"max":{"1min":3.012,"5min":10.43,"15min":145.48099999999999},"last":1.173}]},{"osd":4,"last update":"Sat Sep 23 22:35:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":2.8879999999999999,"15min":2.4609999999999999},"min":{"1min":0.30399999999999999,"5min":0.27400000000000002,"15min":0.27000000000000002},"max":{"1min":6.8250000000000002,"5min":65.640000000000001,"15min":82.825000000000003},"last":1.3999999999999999},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.2530000000000001,"15min":1.9279999999999999},"min":{"1min":0.30399999999999999,"5min":0.29599999999999999,"15min":0.24399999999999999},"max":{"1min":4.21,"5min":65.349999999999994,"15min":75.930999999999997},"last":1.4399999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:35:18 2023","interfaces":[{"interface":"back","average":{"1min":6.3179999999999996,"5min":3.2469999999999999,"15min":3.052},"min":{"1min":0.33900000000000002,"5min":0.32400000000000001,"15min":0.29299999999999998},"max":{"1min":106.98699999999999,"5min":106.98699999999999,"15min":106.98699999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.778,"15min":2.2429999999999999},"min":{"1min":0.36599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.2510000000000003,"5min":9.8190000000000008,"15min":93.546000000000006},"last":2.4009999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:35:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":3.2429999999999999,"15min":2.3159999999999998},"min":{"1min":0.371,"5min":0.32900000000000001,"15min":0.308},"max":{"1min":3.5110000000000001,"5min":74.355000000000004,"15min":74.355000000000004},"last":0.627},{"interface":"front","average":{"1min":1.135,"5min":2.6640000000000001,"15min":1.895},"min":{"1min":0.39500000000000002,"5min":0.35099999999999998,"15min":0.34999999999999998},"max":{"1min":4.2839999999999998,"5min":73.150000000000006,"15min":73.150000000000006},"last":2.363}]},{"osd":7,"last update":"Sat Sep 23 22:35:18 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.764,"15min":1.7949999999999999},"min":{"1min":0.47299999999999998,"5min":0.41199999999999998,"15min":0.29999999999999999},"max":{"1min":4.577,"5min":22.338999999999999,"15min":32.773000000000003},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.2889999999999999,"5min":2.2160000000000002,"15min":1.9970000000000001},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.307},"max":{"1min":6.8710000000000004,"5min":10.666,"15min":16.381},"last":1.0349999999999999}]}]},{"osd":2,"up_from":349,"seq":1498943586381,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1425412,"kb_used_data":1222076,"kb_used_omap":15,"kb_used_meta":203312,"kb_avail":92319740,"statfs":{"total":95995035648,"available":94535413760,"internally_reserved":0,"allocated":1251405824,"data_stored":1249063932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":208192246},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":3.2530000000000001,"15min":13.839},"min":{"1min":0.222,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.6840000000000002,"5min":86.620000000000005,"15min":194.06299999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.3129999999999999,"5min":3.5339999999999998,"15min":15.504},"min":{"1min":0.223,"5min":0.215,"15min":0.215},"max":{"1min":4.5330000000000004,"5min":86.355000000000004,"15min":196.947},"last":2.0070000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:36:06 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":2.75,"15min":6.343},"min":{"1min":0.30599999999999999,"5min":0.23899999999999999,"15min":0.23200000000000001},"max":{"1min":3.4990000000000001,"5min":81.984999999999999,"15min":130.63800000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.175,"5min":2.847,"15min":7.133},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.27600000000000002},"max":{"1min":4.2050000000000001,"5min":80.349000000000004,"15min":130.66399999999999},"last":0.67400000000000004}]},{"osd":3,"last update":"Sat Sep 23 22:36:06 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.0069999999999999,"15min":1.486},"min":{"1min":0.28100000000000003,"5min":0.25,"15min":0.25},"max":{"1min":1.962,"5min":12.193,"15min":12.193},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.80000000000000004,"5min":1.0649999999999999,"15min":0.98299999999999998},"min":{"1min":0.313,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.968,"5min":14.128,"15min":14.128},"last":0.94999999999999996}]},{"osd":4,"last update":"Sat Sep 23 22:35:03 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":2.5169999999999999,"15min":1.512},"min":{"1min":0.32600000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.742000000000001,"5min":120.645,"15min":120.645},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.323,"5min":2.7709999999999999,"15min":1.847},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.5800000000000001,"5min":120.19499999999999,"15min":120.19499999999999},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:36:06 2023","interfaces":[{"interface":"back","average":{"1min":7.3019999999999996,"5min":5.7240000000000002,"15min":2.9119999999999999},"min":{"1min":0.39000000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":78.510000000000005,"5min":144.16900000000001,"15min":144.16900000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":7.2270000000000003,"5min":5.4359999999999999,"15min":2.4660000000000002},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":78.530000000000001,"5min":143.815,"15min":143.815},"last":0.66300000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:36:06 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":1.7030000000000001,"15min":1.4910000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.3499999999999996,"5min":8.3100000000000005,"15min":8.3100000000000005},"last":4.3499999999999996},{"interface":"front","average":{"1min":1.0589999999999999,"5min":2.4900000000000002,"15min":1.9530000000000001},"min":{"1min":0.432,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.3949999999999996,"5min":75.275999999999996,"15min":75.275999999999996},"last":4.3949999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:36:06 2023","interfaces":[{"interface":"back","average":{"1min":6.3070000000000004,"5min":2.4020000000000001,"15min":1.673},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":108.113,"5min":108.113,"15min":108.113},"last":1.407},{"interface":"front","average":{"1min":6.2220000000000004,"5min":3.0880000000000001,"15min":1.972},"min":{"1min":0.46400000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":108.093,"5min":108.093,"15min":108.093},"last":3.1840000000000002}]}]},{"osd":3,"up_from":0,"seq":463856468235,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":465,"seq":1997159792644,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":525376,"kb_used_data":477688,"kb_used_omap":12,"kb_used_meta":47667,"kb_avail":93219776,"statfs":{"total":95995035648,"available":95457050624,"internally_reserved":0,"allocated":489152512,"data_stored":487581634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":48811633},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44600000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.495}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35099999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47099999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":104.521}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.27500000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757653,"num_pgs":12,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1180900,"kb_used_data":717596,"kb_used_omap":0,"kb_used_meta":463296,"kb_avail":92564252,"statfs":{"total":95995035648,"available":94785794048,"internally_reserved":0,"allocated":734818304,"data_stored":733270623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":474415104},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:35:27 2023","interfaces":[{"interface":"back","average":{"1min":9.109,"5min":7.2610000000000001,"15min":5.5519999999999996},"min":{"1min":0.28199999999999997,"5min":0.22700000000000001,"15min":0.20799999999999999},"max":{"1min":145.46600000000001,"5min":145.46600000000001,"15min":163.54499999999999},"last":1.76},{"interface":"front","average":{"1min":8.9280000000000008,"5min":6.7249999999999996,"15min":5.1299999999999999},"min":{"1min":0.25800000000000001,"5min":0.24099999999999999,"15min":0.22900000000000001},"max":{"1min":144.429,"5min":144.429,"15min":162.81800000000001},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:35:39 2023","interfaces":[{"interface":"back","average":{"1min":4.0650000000000004,"5min":2.9980000000000002,"15min":3.266},"min":{"1min":0.312,"5min":0.254,"15min":0.23799999999999999},"max":{"1min":58.087000000000003,"5min":61.621000000000002,"15min":113.514},"last":0.40899999999999997},{"interface":"front","average":{"1min":4.032,"5min":3.0499999999999998,"15min":3.3119999999999998},"min":{"1min":0.28699999999999998,"5min":0.27600000000000002,"15min":0.255},"max":{"1min":58.100999999999999,"5min":61.645000000000003,"15min":113.42700000000001},"last":4.8120000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:35:59 2023","interfaces":[{"interface":"back","average":{"1min":5.2690000000000001,"5min":2.5110000000000001,"15min":2.444},"min":{"1min":0.33700000000000002,"5min":0.318,"15min":0.316},"max":{"1min":70.153000000000006,"5min":70.153000000000006,"15min":70.153000000000006},"last":1.2509999999999999},{"interface":"front","average":{"1min":5.0650000000000004,"5min":2.508,"15min":2.5219999999999998},"min":{"1min":0.37,"5min":0.33000000000000002,"15min":0.30499999999999999},"max":{"1min":70.078999999999994,"5min":70.078999999999994,"15min":70.078999999999994},"last":1.212}]},{"osd":3,"last update":"Sat Sep 23 22:35:39 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.631,"15min":3.6600000000000001},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.28299999999999997},"max":{"1min":4.1029999999999998,"5min":9.4120000000000008,"15min":132.59},"last":1.1699999999999999},{"interface":"front","average":{"1min":0.67300000000000004,"5min":1.375,"15min":3.5049999999999999},"min":{"1min":0.40999999999999998,"5min":0.34000000000000002,"15min":0.29299999999999998},"max":{"1min":1.452,"5min":7.8390000000000004,"15min":129.381},"last":2.3799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:34:38 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":2.012,"15min":1.615},"min":{"1min":0.246,"5min":0.246,"15min":0.24099999999999999},"max":{"1min":5.8879999999999999,"5min":36.343000000000004,"15min":126.679},"last":1.119},{"interface":"front","average":{"1min":2.4590000000000001,"5min":2.423,"15min":1.7789999999999999},"min":{"1min":0.34100000000000003,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":9.5199999999999996,"5min":36.421999999999997,"15min":126.78100000000001},"last":1.1890000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.6060000000000001,"15min":1.278},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":3.6880000000000002,"5min":9.4309999999999992,"15min":70.581999999999994},"last":1.05},{"interface":"front","average":{"1min":0.94899999999999995,"5min":1.3500000000000001,"15min":1.2430000000000001},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.248},"max":{"1min":4.806,"5min":7.25,"15min":71.093000000000004},"last":0.443}]},{"osd":7,"last update":"Sat Sep 23 22:35:27 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":1.7130000000000001,"15min":1.0549999999999999},"min":{"1min":0.38500000000000001,"5min":0.34300000000000003,"15min":0.27100000000000002},"max":{"1min":1.115,"5min":40.548000000000002,"15min":40.548000000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.0880000000000001,"15min":1.151},"min":{"1min":0.37,"5min":0.34499999999999997,"15min":0.308},"max":{"1min":0.76500000000000001,"5min":41.073,"15min":41.073},"last":1.155}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7820,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1476853760,"data_stored":1476853760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2925133824,"data_stored":2925133824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1259012096,"data_stored":1259012096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":489881600,"data_stored":489881600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":735838208,"data_stored":735838208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1467351040,"data_stored":1467351040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2717843456,"data_stored":2717843456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:36:12.341 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T22:36:12.341 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-09-23T22:36:12.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 66716 66700 86.8196 59 0.020064 0.0115116 2023-09-23T22:36:12.403 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:12.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:12.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:12.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:12 smithi026 ceph-mon[223014]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:36:12.422 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:12 smithi026 ceph-mon[223014]: pgmap v1209: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s; 11254/192978 objects degraded (5.832%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:36:12.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:12 smithi026 ceph-mon[215775]: from='client.46220 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:36:12.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:12 smithi026 ceph-mon[215775]: pgmap v1209: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 61 MiB/s wr, 973 op/s; 11254/192978 objects degraded (5.832%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:36:12.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:12.522 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:12.648 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:12.837 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:12.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:12.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:12.941 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:12.986 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:13.341 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:13.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:13.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:13.367 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 67507 67491 86.0568 49.4375 0.0202966 0.0116142 2023-09-23T22:36:13.444 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:13.503 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:13.708 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:13.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:13.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:13.816 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:13.890 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:13.945 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:13.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:13.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:13.984 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:36:13.984 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:36:14.048 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:14.079 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:14.180 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:14.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:14.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:14.242 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:36:14.288 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:14.343 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:14 smithi026 ceph-mon[223014]: pgmap v1210: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 60 MiB/s wr, 965 op/s; 11254/194643 objects degraded (5.782%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:36:14.343 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:14 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 11254/194643 objects degraded (5.782%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:14.343 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:14 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/91339545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:14.343 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:14 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:14.344 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:14 smithi026 ceph-mon[215775]: pgmap v1210: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 60 MiB/s wr, 965 op/s; 11254/194643 objects degraded (5.782%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:36:14.344 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:14 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 11254/194643 objects degraded (5.782%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:14.344 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:14 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/91339545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:14.345 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:14 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:14.351 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:14.368 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 68380 68364 85.4269 54.5625 0.0101364 0.0117 2023-09-23T22:36:14.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:14 smithi070 ceph-mon[192652]: pgmap v1210: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 60 MiB/s wr, 965 op/s; 11254/194643 objects degraded (5.782%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T22:36:14.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:14 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 11254/194643 objects degraded (5.782%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:14.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:14 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/91339545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:14.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:14 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:14.715 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:14.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:14.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:14.829 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:14.884 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:14.887 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:36:15.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:15 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:15.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:15 smithi070 ceph-mon[192652]: failed to encode map e470 with expected crc 2023-09-23T22:36:15.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:15 smithi070 ceph-mon[192652]: failed to encode map e470 with expected crc 2023-09-23T22:36:15.128 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:15 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:36:15.129 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:15 smithi070 ceph-mon[192652]: osdmap e470: 8 total, 8 up, 7 in 2023-09-23T22:36:15.370 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 69199 69183 84.7514 51.1875 0.00853178 0.0117942 2023-09-23T22:36:15.419 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:15.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:15.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:15.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[215775]: failed to encode map e470 with expected crc 2023-09-23T22:36:15.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[215775]: failed to encode map e470 with expected crc 2023-09-23T22:36:15.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:36:15.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[215775]: osdmap e470: 8 total, 8 up, 7 in 2023-09-23T22:36:15.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:15.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[223014]: failed to encode map e470 with expected crc 2023-09-23T22:36:15.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[223014]: failed to encode map e470 with expected crc 2023-09-23T22:36:15.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:36:15.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:15 smithi026 ceph-mon[223014]: osdmap e470: 8 total, 8 up, 7 in 2023-09-23T22:36:15.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:15.529 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:15.547 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:36:15.560 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:15.614 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:15.699 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:15.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:15.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:15.805 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:15.876 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:16.041 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:16.041 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:36:16.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:16.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:16.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:36:16.130 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:36:16.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:16.164 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:36:16.192 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:16.215 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:36:16.257 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:36:16.296 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:36:16.327 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:16.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:16.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:16.342 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:36:16.349 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:36:16.349 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:36:16.349 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:36:16.349 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:36:16.349 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:36:16.349 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:36:16.349 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:36:16.350 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:36:16.370 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 70177 70161 84.297 61.125 0.0234931 0.011858 2023-09-23T22:36:16.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:16 smithi026 ceph-mon[223014]: pgmap v1212: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 59 MiB/s wr, 945 op/s; 10830/198711 objects degraded (5.450%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:36:16.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3748092401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:16.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:16 smithi026 ceph-mon[215775]: pgmap v1212: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 59 MiB/s wr, 945 op/s; 10830/198711 objects degraded (5.450%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:36:16.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3748092401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:16.431 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:16.488 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:16 smithi070 ceph-mon[192652]: pgmap v1212: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 59 MiB/s wr, 945 op/s; 10830/198711 objects degraded (5.450%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:36:16.488 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3748092401' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:16.495 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:16.538 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:16.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:16.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:16.642 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:16.692 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:17.371 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 70991 70975 83.6665 50.875 0.018593 0.0119462 2023-09-23T22:36:17.604 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:36:17.743 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9495 9.d 9:b6f72c49:::benchmark_data_smithi026_555954_object9494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.350545+0000", 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 50.382808353999998, 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.022504189, 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:36:17.744 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9495 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350545+0000", 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350545+0000", 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350543+0000", 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350613+0000", 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0388000000000004e-05 2023-09-23T22:36:17.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350628+0000", 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4489e-05 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350635+0000", 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6150000000000002e-06 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350683+0000", 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8446000000000003e-05 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350811+0000", 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128374 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350977+0000", 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016547600000000001 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.352085+0000", 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00110797 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.352551+0000", 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00046640899999999999 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370196+0000", 2023-09-23T22:36:17.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017644389999999999 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370205+0000", 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4939999999999992e-06 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372976+0000", 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0027705790000001 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373049+0000", 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3143000000000001e-05 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9496 9.c 9:3bf3d114:::benchmark_data_smithi026_555954_object9495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.370618+0000", 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 50.362735116000003, 2023-09-23T22:36:17.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.001084182, 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9496 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370618+0000", 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370618+0000", 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370617+0000", 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370666+0000", 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9101999999999998e-05 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370677+0000", 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1586999999999999e-05 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370684+0000", 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2400000000000004e-06 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370725+0000", 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1288000000000002e-05 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370731+0000", 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8649999999999998e-06 2023-09-23T22:36:17.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371493+0000", 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.000761907 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371702+0000", 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020950600000000001 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.756 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9509 9.c 9:31b495ae:::benchmark_data_smithi026_555954_object9508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.442632+0000", 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 50.290721163000001, 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93168581900000003, 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.757 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9509 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442632+0000", 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442632+0000", 2023-09-23T22:36:17.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442630+0000", 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442697+0000", 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6773000000000006e-05 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442707+0000", 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5789999999999993e-06 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442714+0000", 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0979999999999996e-06 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442752+0000", 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8105000000000002e-05 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442757+0000", 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5249999999999996e-06 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371728+0000", 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92897071200000003 2023-09-23T22:36:17.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371868+0000", 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000139462 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372052+0000", 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000184446 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373673+0000", 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001620511 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374175+0000", 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000502203 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374228+0000", 2023-09-23T22:36:17.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2825999999999999e-05 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374318+0000", 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0082999999999998e-05 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9535 9.d 9:b4eb0ec8:::benchmark_data_smithi026_555954_object9534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:17.764 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.455847+0000", 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 50.277506093, 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92259973200000001, 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9535 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455847+0000", 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455847+0000", 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455846+0000", 2023-09-23T22:36:17.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455906+0000", 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0608e-05 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455923+0000", 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6758000000000001e-05 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455930+0000", 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7780000000000002e-06 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455990+0000", 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0136000000000002e-05 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456000+0000", 2023-09-23T22:36:17.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8220000000000002e-06 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373303+0000", 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91730366500000005 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373384+0000", 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0984999999999998e-05 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373486+0000", 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000101313 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.375253+0000", 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001767565 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378309+0000", 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030558080000000001 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378360+0000", 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0806000000000001e-05 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378447+0000", 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6707999999999997e-05 2023-09-23T22:36:17.771 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9536 9.c 9:382e37e1:::benchmark_data_smithi026_555954_object9535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.456300+0000", 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 50.277052756000003, 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92362326400000005, 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.772 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9536 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456300+0000", 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456300+0000", 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456299+0000", 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456394+0000", 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4780999999999999e-05 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456405+0000", 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0876999999999999e-05 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456411+0000", 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7259999999999999e-06 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456453+0000", 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2178000000000002e-05 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456466+0000", 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3247e-05 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372550+0000", 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91608398700000004 2023-09-23T22:36:17.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372617+0000", 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7168000000000005e-05 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372714+0000", 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6972000000000006e-05 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373682+0000", 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00096728799999999996 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379826+0000", 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061445110000000001 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379868+0000", 2023-09-23T22:36:17.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1446999999999998e-05 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379924+0000", 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5908000000000001e-05 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9540 9.d 9:bd7479be:::benchmark_data_smithi026_555954_object9539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:36:17.779 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 50.275444702999998, 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92071582799999996, 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9540 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457972+0000", 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3987999999999998e-05 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457986+0000", 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3869999999999999e-05 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457992+0000", 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6499999999999999e-06 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.458039+0000", 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6647000000000002e-05 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.458044+0000", 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0610000000000003e-06 2023-09-23T22:36:17.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373970+0000", 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91592621799999996 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374074+0000", 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103479 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:36:17.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374200+0000", 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000126317 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.375260+0000", 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010597790000000001 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378498+0000", 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032380619999999999 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378543+0000", 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5624999999999998e-05 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378624+0000", 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0729999999999994e-05 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9544 9.d 9:b6a337e4:::benchmark_data_smithi026_555954_object9543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.459597+0000", 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 50.273755967, 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91919865700000003, 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9544 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.787 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459597+0000", 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459597+0000", 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459596+0000", 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459657+0000", 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0798999999999998e-05 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459668+0000", 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0868e-05 2023-09-23T22:36:17.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459673+0000", 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4889999999999997e-06 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459722+0000", 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8953999999999999e-05 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459728+0000", 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7449999999999998e-06 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374739+0000", 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91501085900000001 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374843+0000", 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104354 2023-09-23T22:36:17.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374936+0000", 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2578999999999996e-05 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.375383+0000", 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044722000000000002 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378674+0000", 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032907280000000001 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378716+0000", 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2168e-05 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378796+0000", 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9506999999999996e-05 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.793 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9551 9.c 9:30ade42e:::benchmark_data_smithi026_555954_object9550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.465719+0000", 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 50.267634456000003, 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.914369392, 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.794 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9551 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465719+0000", 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465719+0000", 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465718+0000", 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465753+0000", 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5376999999999997e-05 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465780+0000", 2023-09-23T22:36:17.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7169e-05 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465785+0000", 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5059999999999997e-06 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465841+0000", 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6444999999999999e-05 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465846+0000", 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9490000000000004e-06 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373010+0000", 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90716359499999999 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373077+0000", 2023-09-23T22:36:17.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7021999999999996e-05 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373205+0000", 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012822300000000001 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374744+0000", 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015386899999999999 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379977+0000", 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052328569999999996 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.380030+0000", 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3511000000000003e-05 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.380088+0000", 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7877000000000002e-05 2023-09-23T22:36:17.800 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:15537 9.c 9:38a0d8b2:::benchmark_data_smithi026_555954_object15536:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:31.082775+0000", 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 46.650578328000002, 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.256626197, 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.801 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 15537 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.082775+0000", 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.082775+0000", 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.082774+0000", 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.082825+0000", 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0455999999999999e-05 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.082832+0000", 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6490000000000008e-06 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.082837+0000", 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0509999999999996e-06 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.082878+0000", 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0852000000000001e-05 2023-09-23T22:36:17.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.083003+0000", 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012456500000000001 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.083144+0000", 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000141397 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.084209+0000", 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010646200000000001 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.339242+0000", 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.25503280499999997 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.339304+0000", 2023-09-23T22:36:17.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2281999999999998e-05 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.339401+0000", 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7058000000000005e-05 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:15566 9.c 9:3c21e70e:::benchmark_data_smithi026_555954_object15565:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T22:36:17.807 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:31.100543+0000", 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 46.632809795, 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.23900254000000001, 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 15566 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.100543+0000", 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.100543+0000", 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.100542+0000", 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.100602+0000", 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0372999999999999e-05 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.100615+0000", 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2848e-05 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.100621+0000", 2023-09-23T22:36:17.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2870000000000004e-06 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.100659+0000", 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8503e-05 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.100751+0000", 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2251999999999997e-05 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.100839+0000", 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8083000000000004e-05 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.101521+0000", 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068117600000000003 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.339457+0000", 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.237936646 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.339499+0000", 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1492000000000002e-05 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:31.339546+0000", 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7088e-05 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:40917 9.d 9:b83b0e03:::benchmark_data_smithi026_555954_object40916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e462)", 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:45.425371+0000", 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.307981685999998, 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20836825, 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 40917 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.814 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425371+0000", 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425371+0000", 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425364+0000", 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999933 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425605+0000", 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024092100000000001 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425618+0000", 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2824e-05 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425625+0000", 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1879999999999996e-06 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425676+0000", 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0278999999999998e-05 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425789+0000", 2023-09-23T22:36:17.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011309 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425942+0000", 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000153284 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427977+0000", 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020352959999999998 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633614+0000", 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20563620899999999 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633659+0000", 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5707000000000001e-05 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633740+0000", 2023-09-23T22:36:17.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0296000000000005e-05 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:40924 9.e e462/459 9:7886a519:::benchmark_data_smithi026_555954_object40923:head v 462'2589, mlcod=462'2587)", 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:45.425840+0000", 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.307512541999998, 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.207578865, 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425840+0000", 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425840+0000", 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.425833+0000", 2023-09-23T22:36:17.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999928 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.426086+0000", 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025266600000000002 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.426100+0000", 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3839000000000001e-05 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.426105+0000", 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3850000000000001e-06 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427633+0000", 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015283230000000001 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427662+0000", 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8493e-05 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.429994+0000", 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023319230000000001 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633419+0000", 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20342542499999999 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.824 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:40922 9.9 e462/454 9:97bbbbe5:::benchmark_data_smithi026_555954_object40921:head v 462'2527, mlcod=462'2525)", 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:45.427525+0000", 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.305828177000002, 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.205902106, 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427525+0000", 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427525+0000", 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427524+0000", 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:17.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427552+0000", 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8309e-05 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427564+0000", 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1336e-05 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427585+0000", 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1489e-05 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427651+0000", 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6303000000000005e-05 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.427669+0000", 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7606000000000001e-05 2023-09-23T22:36:17.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.430023+0000", 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0023536299999999998 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633427+0000", 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20340436200000001 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:40926 9.7 e462/454 9:e066147c:::benchmark_data_smithi026_555954_object40925:head v 462'2542, mlcod=462'2540)", 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:45.428101+0000", 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.305251886000001, 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.205303236, 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.428101+0000", 2023-09-23T22:36:17.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.428101+0000", 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.428093+0000", 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999919 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.428132+0000", 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9153999999999998e-05 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.428143+0000", 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1280999999999999e-05 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.428149+0000", 2023-09-23T22:36:17.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2179999999999998e-06 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.428219+0000", 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0895999999999993e-05 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.428236+0000", 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7e-05 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.431428+0000", 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031920339999999998 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633404+0000", 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20197585400000001 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:40912 9.8 e462/454 9:1a468da5:::benchmark_data_smithi026_555954_object40911:head v 462'2653, mlcod=462'2650)", 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:45.429728+0000", 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.303625056999998, 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20712230500000001, 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.429728+0000", 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.429728+0000", 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.429727+0000", 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.429772+0000", 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5198999999999999e-05 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.429783+0000", 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1270000000000001e-05 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.429789+0000", 2023-09-23T22:36:17.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7289999999999997e-06 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.429840+0000", 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0701999999999997e-05 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.429859+0000", 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9587000000000001e-05 2023-09-23T22:36:17.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.636783+0000", 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20692317199999999 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.636850+0000", 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7737000000000006e-05 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.839 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:40927 9.3 e462/454 9:c74f530f:::benchmark_data_smithi026_555954_object40926:head v 462'2576, mlcod=462'2573)", 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:45.430251+0000", 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.303102000000003, 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20758642199999999, 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.430251+0000", 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.430251+0000", 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.430250+0000", 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:17.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.430272+0000", 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1277999999999999e-05 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633350+0000", 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20307794500000001 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633361+0000", 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1351e-05 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633407+0000", 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5702999999999999e-05 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633430+0000", 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3699000000000001e-05 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.637812+0000", 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043814650000000002 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.637837+0000", 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5590999999999999e-05 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.844 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:40929 9.1 e462/459 9:8aa581b6:::benchmark_data_smithi026_555954_object40928:head v 462'2473, mlcod=462'2471)", 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:45.431943+0000", 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 32.301409589999999, 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20490281900000001, 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.431943+0000", 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.431943+0000", 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.431943+0000", 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.431971+0000", 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8347000000000002e-05 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633373+0000", 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20140202300000001 2023-09-23T22:36:17.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633381+0000", 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5769999999999996e-06 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633418+0000", 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7793999999999998e-05 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.633442+0000", 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3589999999999999e-05 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.636826+0000", 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033845450000000001 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:45.636846+0000", 2023-09-23T22:36:17.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9752e-05 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:60106 9.d 9:babbd1d2:::benchmark_data_smithi026_555954_object60105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e469)", 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:05.208227+0000", 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 12.52512649, 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17700128100000001, 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.850 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 60106 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.208227+0000", 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.208227+0000", 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.208226+0000", 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.208249+0000", 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2748999999999999e-05 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.208254+0000", 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5230000000000003e-06 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.208258+0000", 2023-09-23T22:36:17.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5360000000000001e-06 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.214546+0000", 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0062879839999999999 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.214701+0000", 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015553499999999999 2023-09-23T22:36:17.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.214878+0000", 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017620600000000001 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.219233+0000", 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043550009999999998 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.225963+0000", 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067303199999999997 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.383996+0000", 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.158032748 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.384048+0000", 2023-09-23T22:36:17.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2667999999999998e-05 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:05.385228+0000", 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001179395 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:70296 9.d 9:b549d026:::benchmark_data_smithi026_555954_object70295:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T22:36:17.857 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:16.475225+0000", 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.2581284779999999, 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.187788659, 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 70296 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.475225+0000", 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.475225+0000", 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.475223+0000", 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.475262+0000", 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9162999999999998e-05 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.475275+0000", 2023-09-23T22:36:17.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2644e-05 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.475281+0000", 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8940000000000004e-06 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.475327+0000", 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5290999999999997e-05 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.475493+0000", 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000165909 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.475662+0000", 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000169581 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.478381+0000", 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027183839999999999 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.488450+0000", 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010069666 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.661237+0000", 2023-09-23T22:36:17.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.172786361 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.661296+0000", 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9147999999999997e-05 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.663013+0000", 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017173959999999999 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.864 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:70304 9.d 9:b90199ce:::benchmark_data_smithi026_555954_object70303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e470)", 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:16.485044+0000", 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 1.2483085949999999, 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17938812100000001, 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:36:17.865 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 70304 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.485044+0000", 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:36:17.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.485044+0000", 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.485043+0000", 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.485083+0000", 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0178000000000001e-05 2023-09-23T22:36:17.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.485096+0000", 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3135e-05 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.485103+0000", 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5529999999999998e-06 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.485200+0000", 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7717e-05 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.485356+0000", 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015546000000000001 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:36:17.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.485518+0000", 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000162385 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.660884+0000", 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.17536539500000001 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.660906+0000", 2023-09-23T22:36:17.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2016000000000001e-05 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.663059+0000", 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021532510000000001 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.663109+0000", 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0612000000000003e-05 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:36:17.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:36:17.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:36:17.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:16.664433+0000", 2023-09-23T22:36:17.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013231149999999999 2023-09-23T22:36:17.872 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.872 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.872 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.872 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:36:17.872 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:36:17.872 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:36:18.079 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:18.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:18.189 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:18.226 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:18.276 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:18.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:18.294 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:18.371 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 71818 71802 83.0731 51.6875 0.0139768 0.0120319 2023-09-23T22:36:18.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:18 smithi026 ceph-mon[215775]: pgmap v1213: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s; 10830/205764 objects degraded (5.263%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:36:18.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10830/205764 objects degraded (5.263%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:18.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:18 smithi026 ceph-mon[223014]: pgmap v1213: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s; 10830/205764 objects degraded (5.263%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:36:18.450 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10830/205764 objects degraded (5.263%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:18.453 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:18.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:18 smithi070 ceph-mon[192652]: pgmap v1213: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 57 MiB/s wr, 906 op/s; 10830/205764 objects degraded (5.263%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:36:18.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10830/205764 objects degraded (5.263%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:18.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:36:18.673 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:18.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:18.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:18.780 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:18.845 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:18.921 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:18.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:18.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:19.029 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:19.119 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:19.371 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 72766 72750 82.64 59.25 0.017733 0.012096 2023-09-23T22:36:19.887 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:19.891 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:36:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:36:19] "GET /metrics HTTP/1.1" 200 36468 "" "Prometheus/2.43.0" 2023-09-23T22:36:19.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:19.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:19.993 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:20.067 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:20.169 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:20.213 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:20.217 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:36:20.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:20.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:20.265 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pg_num to 21 2023-09-23T22:36:20.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:20.366 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:20.371 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 16 73518 73502 82.0037 47 0.0137914 0.0121887 2023-09-23T22:36:20.442 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:20.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:20.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:20.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:20 smithi026 ceph-mon[215775]: pgmap v1214: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s; 10830/207507 objects degraded (5.219%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:36:20.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1329896701' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T22:36:20.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:20 smithi026 ceph-mon[223014]: pgmap v1214: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s; 10830/207507 objects degraded (5.219%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:36:20.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1329896701' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T22:36:20.549 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:20.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:20 smithi070 ceph-mon[192652]: pgmap v1214: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 56 MiB/s wr, 901 op/s; 10830/207507 objects degraded (5.219%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:36:20.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1329896701' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T22:36:20.630 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:20.833 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:20.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:20.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:20.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:20.964 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:21.005 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:21.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:21.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:21.111 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:21.172 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:21.251 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:21.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:21.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:21.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:21.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 74299 74283 81.4214 48.8125 0.0102525 0.0122766 2023-09-23T22:36:21.428 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:21.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:21 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1329896701' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T22:36:21.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:21 smithi070 ceph-mon[192652]: osdmap e471: 8 total, 8 up, 7 in 2023-09-23T22:36:21.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:21 smithi070 ceph-mon[192652]: failed to encode map e471 with expected crc 2023-09-23T22:36:21.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:21 smithi070 ceph-mon[192652]: failed to encode map e471 with expected crc 2023-09-23T22:36:21.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:21 smithi070 ceph-mon[192652]: failed to encode map e471 with expected crc 2023-09-23T22:36:21.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T22:36:21.708 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:21.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:21.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:21.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1329896701' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T22:36:21.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[215775]: osdmap e471: 8 total, 8 up, 7 in 2023-09-23T22:36:21.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[215775]: failed to encode map e471 with expected crc 2023-09-23T22:36:21.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[215775]: failed to encode map e471 with expected crc 2023-09-23T22:36:21.729 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[215775]: failed to encode map e471 with expected crc 2023-09-23T22:36:21.729 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T22:36:21.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1329896701' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T22:36:21.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[223014]: osdmap e471: 8 total, 8 up, 7 in 2023-09-23T22:36:21.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[223014]: failed to encode map e471 with expected crc 2023-09-23T22:36:21.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[223014]: failed to encode map e471 with expected crc 2023-09-23T22:36:21.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[223014]: failed to encode map e471 with expected crc 2023-09-23T22:36:21.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T22:36:21.825 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:21.905 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:22.111 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:22.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:22.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:22.219 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:22.281 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:22.301 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:36:22.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[215775]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 10263/218916 objects degraded (4.688%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T22:36:22.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[215775]: failed to encode map e472 with expected crc 2023-09-23T22:36:22.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T22:36:22.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[215775]: osdmap e472: 8 total, 8 up, 7 in 2023-09-23T22:36:22.331 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[215775]: failed to encode map e472 with expected crc 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[215775]: failed to encode map e472 with expected crc 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2759776279' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[223014]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 10263/218916 objects degraded (4.688%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[223014]: failed to encode map e472 with expected crc 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[223014]: osdmap e472: 8 total, 8 up, 7 in 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[223014]: failed to encode map e472 with expected crc 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[223014]: failed to encode map e472 with expected crc 2023-09-23T22:36:22.332 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2759776279' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:22.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:22 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:22.335 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:22.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 75253 75237 81.0456 59.625 0.0206537 0.0123323 2023-09-23T22:36:22.373 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:22.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:22.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:22.479 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:22.533 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:22.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:22 smithi070 ceph-mon[192652]: pgmap v1216: 99 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 63 MiB/s wr, 1.01k op/s; 10263/218916 objects degraded (4.688%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T22:36:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:22 smithi070 ceph-mon[192652]: failed to encode map e472 with expected crc 2023-09-23T22:36:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:22 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T22:36:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:22 smithi070 ceph-mon[192652]: osdmap e472: 8 total, 8 up, 7 in 2023-09-23T22:36:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:22 smithi070 ceph-mon[192652]: failed to encode map e472 with expected crc 2023-09-23T22:36:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:22 smithi070 ceph-mon[192652]: failed to encode map e472 with expected crc 2023-09-23T22:36:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2759776279' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:22.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:22 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:23.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 75960 75944 80.421 44.1875 0.00595359 0.0124255 2023-09-23T22:36:23.587 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: osdmap e473: 8 total, 8 up, 7 in 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10263/218916 objects degraded (4.688%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:23.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[215775]: osdmap e474: 8 total, 8 up, 7 in 2023-09-23T22:36:23.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:23.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:23.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:23.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:23.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: osdmap e473: 8 total, 8 up, 7 in 2023-09-23T22:36:23.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10263/218916 objects degraded (4.688%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:23.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:23 smithi026 ceph-mon[223014]: osdmap e474: 8 total, 8 up, 7 in 2023-09-23T22:36:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: osdmap e473: 8 total, 8 up, 7 in 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: failed to encode map e473 with expected crc 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10263/218916 objects degraded (4.688%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:23 smithi070 ceph-mon[192652]: osdmap e474: 8 total, 8 up, 7 in 2023-09-23T22:36:23.692 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:23.724 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:23.770 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:23.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:23.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:23.875 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:23.902 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:23.947 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:23.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:23.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:24.051 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:24.078 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:24.122 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:24.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:24.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:24.231 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:24.274 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:24.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:36:24.372637+0000 min lat: 0.00117176 max lat: 2.19936 avg lat: 0.0125801 2023-09-23T22:36:24.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:36:24.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 76321 76305 79.4569 22.5625 0.0103463 0.0125801 2023-09-23T22:36:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:24 smithi070 ceph-mon[192652]: pgmap v1219: 109 pgs: 9 unknown, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 10263/220473 objects degraded (4.655%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:36:24.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:24 smithi070 ceph-mon[192652]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:24 smithi070 ceph-mon[192652]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:24 smithi070 ceph-mon[192652]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:24 smithi070 ceph-mon[192652]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:24 smithi070 ceph-mon[192652]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:36:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:24.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:24 smithi070 ceph-mon[192652]: osdmap e475: 8 total, 8 up, 7 in 2023-09-23T22:36:24.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[215775]: pgmap v1219: 109 pgs: 9 unknown, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 10263/220473 objects degraded (4.655%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:36:24.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[215775]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[215775]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[215775]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[215775]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[215775]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:36:24.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:24.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[215775]: osdmap e475: 8 total, 8 up, 7 in 2023-09-23T22:36:24.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[223014]: pgmap v1219: 109 pgs: 9 unknown, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 51 MiB/s wr, 816 op/s; 10263/220473 objects degraded (4.655%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:36:24.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[223014]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[223014]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[223014]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[223014]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[223014]: failed to encode map e474 with expected crc 2023-09-23T22:36:24.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:36:24.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:24.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:24 smithi026 ceph-mon[223014]: osdmap e475: 8 total, 8 up, 7 in 2023-09-23T22:36:24.588 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:24.589 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:36:24.590 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:36:24.590 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:36:24.590 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:36:24.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:24.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:24.696 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:24.745 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:25.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 77277 77261 79.1337 59.75 0.023266 0.0126322 2023-09-23T22:36:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:25 smithi070 ceph-mon[192652]: failed to encode map e475 with expected crc 2023-09-23T22:36:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:25 smithi070 ceph-mon[192652]: failed to encode map e475 with expected crc 2023-09-23T22:36:25.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:25 smithi070 ceph-mon[192652]: failed to encode map e475 with expected crc 2023-09-23T22:36:25.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:25 smithi026 ceph-mon[215775]: failed to encode map e475 with expected crc 2023-09-23T22:36:25.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:25 smithi026 ceph-mon[215775]: failed to encode map e475 with expected crc 2023-09-23T22:36:25.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:25 smithi026 ceph-mon[215775]: failed to encode map e475 with expected crc 2023-09-23T22:36:25.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:25 smithi026 ceph-mon[223014]: failed to encode map e475 with expected crc 2023-09-23T22:36:25.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:25 smithi026 ceph-mon[223014]: failed to encode map e475 with expected crc 2023-09-23T22:36:25.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:25 smithi026 ceph-mon[223014]: failed to encode map e475 with expected crc 2023-09-23T22:36:25.892 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:36:25.893 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:25.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:25.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:26.005 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:26.041 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:36:26.075 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:26.327 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:26.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:26.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:26.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 78165 78149 78.7526 55.5 0.028076 0.0126928 2023-09-23T22:36:26.447 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:26.474 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:26 smithi026 ceph-mon[223014]: pgmap v1222: 109 pgs: 9 unknown, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 23 MiB/s wr, 364 op/s; 10263/223296 objects degraded (4.596%) 2023-09-23T22:36:26.474 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:26 smithi026 ceph-mon[215775]: pgmap v1222: 109 pgs: 9 unknown, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 23 MiB/s wr, 364 op/s; 10263/223296 objects degraded (4.596%) 2023-09-23T22:36:26.493 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:26.596 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:26.605 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:26 smithi070 ceph-mon[192652]: pgmap v1222: 109 pgs: 9 unknown, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 23 MiB/s wr, 364 op/s; 10263/223296 objects degraded (4.596%) 2023-09-23T22:36:26.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:26.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:26.707 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:26.771 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:36:26.779 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:26.791 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:26.793 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-09-23T22:36:26.793 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:36:26.794 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T22:36:26.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T22:36:26.828 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:36:26.866 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:36:26.870 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:26.884 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:26.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:26.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:26.907 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:36:26.950 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:36:26.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:36:26.995 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:27.049 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:36:27.089 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:36:27.106 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:27.109 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:36:27.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:36:27.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:36:27.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:36:27.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:36:27.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:36:27.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:36:27.110 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:36:27.363 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3797725789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:27.363 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3797725789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:27.373 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 78984 78968 78.3151 51.1875 0.014549 0.0127639 2023-09-23T22:36:27.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3797725789' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:28.348 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:28.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:28.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:28.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:28 smithi026 ceph-mon[215775]: pgmap v1223: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 706 op/s; 9621/230439 objects degraded (4.175%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T22:36:28.362 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 9621/230439 objects degraded (4.175%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:28.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:28 smithi026 ceph-mon[223014]: pgmap v1223: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 706 op/s; 9621/230439 objects degraded (4.175%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T22:36:28.362 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 9621/230439 objects degraded (4.175%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:28.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 79873 79857 77.9595 55.5625 0.0211116 0.0128215 2023-09-23T22:36:28.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:28.519 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:28 smithi070 ceph-mon[192652]: pgmap v1223: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 44 MiB/s wr, 706 op/s; 9621/230439 objects degraded (4.175%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T22:36:28.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 9621/230439 objects degraded (4.175%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:28.627 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:28.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:28.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:28.738 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:28.797 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:29.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 80878 80862 77.7265 62.8125 0.0166724 0.012861 2023-09-23T22:36:29.517 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:29.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:29.622 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:29.677 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:29.755 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:29.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:29.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:29.833 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:36:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:36:29] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-09-23T22:36:29.869 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:29.920 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:30.017 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:30.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:30.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:30.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:30.192 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:30.268 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:30.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:30.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:30.373 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 81802 81786 77.4238 57.75 0.0146183 0.012911 2023-09-23T22:36:30.383 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:30.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:36:30.466 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:30.559 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:30.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:30 smithi070 ceph-mon[192652]: pgmap v1224: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 670 op/s; 9621/231825 objects degraded (4.150%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T22:36:30.631 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:30.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:30.652 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:30 smithi026 ceph-mon[223014]: pgmap v1224: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 670 op/s; 9621/231825 objects degraded (4.150%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T22:36:30.653 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:30 smithi026 ceph-mon[215775]: pgmap v1224: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.7 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 670 op/s; 9621/231825 objects degraded (4.150%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T22:36:30.726 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:36:30.726 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:36:30.753 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:30.828 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:31.217 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:31.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:31.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:31.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:31.347 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:36:31.360 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:31.372 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 82649 82633 77.0583 52.9375 0.00425332 0.0129619 2023-09-23T22:36:31.408 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:31.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:31.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:31.514 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:31.585 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:31.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4260691436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:36:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:31 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:36:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4260691436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:36:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:31 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:36:31.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4260691436' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:36:31.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:31 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:36:31.994 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:31.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:31.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:32.098 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:32.157 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:32.373 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 83430 83414 76.643 48.8125 0.00655917 0.0130174 2023-09-23T22:36:32.568 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:36:32.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[215775]: pgmap v1225: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 928 op/s; 9198/242781 objects degraded (3.789%); 8.3 MiB/s, 133 objects/s recovering 2023-09-23T22:36:32.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:32.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[215775]: failed to encode map e476 with expected crc 2023-09-23T22:36:32.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[215775]: failed to encode map e476 with expected crc 2023-09-23T22:36:32.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:36:32.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[215775]: osdmap e476: 8 total, 8 up, 7 in 2023-09-23T22:36:32.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[215775]: failed to encode map e476 with expected crc 2023-09-23T22:36:32.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:32 smithi070 ceph-mon[192652]: pgmap v1225: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 928 op/s; 9198/242781 objects degraded (3.789%); 8.3 MiB/s, 133 objects/s recovering 2023-09-23T22:36:32.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:32 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:32.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:32 smithi070 ceph-mon[192652]: failed to encode map e476 with expected crc 2023-09-23T22:36:32.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:32 smithi070 ceph-mon[192652]: failed to encode map e476 with expected crc 2023-09-23T22:36:32.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:32 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:36:32.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:32 smithi070 ceph-mon[192652]: osdmap e476: 8 total, 8 up, 7 in 2023-09-23T22:36:32.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:32 smithi070 ceph-mon[192652]: failed to encode map e476 with expected crc 2023-09-23T22:36:32.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[223014]: pgmap v1225: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 58 MiB/s wr, 928 op/s; 9198/242781 objects degraded (3.789%); 8.3 MiB/s, 133 objects/s recovering 2023-09-23T22:36:32.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:32.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[223014]: failed to encode map e476 with expected crc 2023-09-23T22:36:32.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[223014]: failed to encode map e476 with expected crc 2023-09-23T22:36:32.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:36:32.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[223014]: osdmap e476: 8 total, 8 up, 7 in 2023-09-23T22:36:32.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:32 smithi026 ceph-mon[223014]: failed to encode map e476 with expected crc 2023-09-23T22:36:33.011 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:33.020 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:33.120 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:33.173 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:33.242 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:33.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:33.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:33.349 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:33.373 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 84193 84177 76.2233 47.6875 0.0054084 0.0130842 2023-09-23T22:36:33.375 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:33.441 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:33.453 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:33 smithi070 ceph-mon[192652]: 1.1 scrub starts 2023-09-23T22:36:33.453 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 9198/242781 objects degraded (3.789%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:33.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:33.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:33.555 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:33.626 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:33 smithi026 ceph-mon[223014]: 1.1 scrub starts 2023-09-23T22:36:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 9198/242781 objects degraded (3.789%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:33 smithi026 ceph-mon[215775]: 1.1 scrub starts 2023-09-23T22:36:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 9198/242781 objects degraded (3.789%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:33.743 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:33.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:33.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:33.854 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:33.881 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:33.949 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:33.981 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:34.208 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:34.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:34.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:34.315 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:34.374 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 85135 85119 75.9754 58.875 0.0205271 0.0131568 2023-09-23T22:36:34.404 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:34.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:34 smithi026 ceph-mon[215775]: pgmap v1227: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s; 9198/245277 objects degraded (3.750%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T22:36:34.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:34 smithi026 ceph-mon[223014]: pgmap v1227: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s; 9198/245277 objects degraded (3.750%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T22:36:34.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:34 smithi070 ceph-mon[192652]: pgmap v1227: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 61 MiB/s wr, 974 op/s; 9198/245277 objects degraded (3.750%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T22:36:34.710 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:34.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:34.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:34.826 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:34.883 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:34.939 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:36:34.970 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:34.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:34.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:35.082 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:35.136 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:35.137 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:34 smithi026 conmon[215751]: 2023-09-23T22:36:34.960+0000 7f4580f04700 -1 mon.a@0(leader).osd e476 definitely_dead 0 2023-09-23T22:36:35.227 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:35.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:35.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:35.338 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:35.373 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 15 85959 85944 75.6316 51.5625 0.0188827 0.0132165 2023-09-23T22:36:35.382 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.3. 2023-09-23T22:36:35.396 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:35.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/71839751' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:36:35.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:35 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:36:35.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/71839751' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:36:35.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:35 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:36:35.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/71839751' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:36:35.412 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:35 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:36:35.456 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:35.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:35.561 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:35.589 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:36.373 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 85973 85957 74.5926 0.8125 0.00551665 0.0132323 2023-09-23T22:36:36.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:36 smithi070 ceph-mon[192652]: pgmap v1228: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s; 9198/248595 objects degraded (3.700%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T22:36:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:36 smithi070 ceph-mon[192652]: failed to encode map e477 with expected crc 2023-09-23T22:36:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T22:36:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:36 smithi070 ceph-mon[192652]: osdmap e477: 8 total, 8 up, 8 in 2023-09-23T22:36:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:36 smithi070 ceph-mon[192652]: failed to encode map e477 with expected crc 2023-09-23T22:36:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:36 smithi070 ceph-mon[192652]: failed to encode map e477 with expected crc 2023-09-23T22:36:36.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[215775]: pgmap v1228: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s; 9198/248595 objects degraded (3.700%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T22:36:36.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[215775]: failed to encode map e477 with expected crc 2023-09-23T22:36:36.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T22:36:36.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[215775]: osdmap e477: 8 total, 8 up, 8 in 2023-09-23T22:36:36.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[215775]: failed to encode map e477 with expected crc 2023-09-23T22:36:36.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[215775]: failed to encode map e477 with expected crc 2023-09-23T22:36:36.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[223014]: pgmap v1228: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 53 MiB/s wr, 842 op/s; 9198/248595 objects degraded (3.700%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T22:36:36.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[223014]: failed to encode map e477 with expected crc 2023-09-23T22:36:36.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T22:36:36.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[223014]: osdmap e477: 8 total, 8 up, 8 in 2023-09-23T22:36:36.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[223014]: failed to encode map e477 with expected crc 2023-09-23T22:36:36.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:36 smithi026 ceph-mon[223014]: failed to encode map e477 with expected crc 2023-09-23T22:36:37.147 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T22:36:37.158 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:37.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:37.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:37.262 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:37.281 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:37.295 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:37.373 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 85977 85961 73.5745 0.25 0.0166842 0.013258 2023-09-23T22:36:37.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: 1.11 scrub starts 2023-09-23T22:36:37.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: osdmap e478: 8 total, 8 up, 8 in 2023-09-23T22:36:37.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.381 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[223014]: pgmap v1231: 109 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 749 op/s; 8644/256287 objects degraded (3.373%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:36:37.677 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:37.684 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: 1.11 scrub starts 2023-09-23T22:36:37.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: osdmap e478: 8 total, 8 up, 8 in 2023-09-23T22:36:37.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.685 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:37 smithi026 ceph-mon[215775]: pgmap v1231: 109 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 749 op/s; 8644/256287 objects degraded (3.373%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:36:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:37.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:37.754 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:36:37.754 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:36:37.787 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:37.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: 1.11 scrub starts 2023-09-23T22:36:37.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.813 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: osdmap e478: 8 total, 8 up, 8 in 2023-09-23T22:36:37.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: failed to encode map e478 with expected crc 2023-09-23T22:36:37.814 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:37 smithi070 ceph-mon[192652]: pgmap v1231: 109 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 749 op/s; 8644/256287 objects degraded (3.373%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T22:36:37.816 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:37.873 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:37.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:37.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:37.980 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:38.014 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:38.044 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:36:38.073 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:38.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:38.081 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:36:38.121 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:36:38.161 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:36:38.179 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:38.205 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:36:38.225 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:38.254 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:36:38.307 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:36:38.314 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:38.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:38.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:38.349 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:36:38.356 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:36:38.356 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:36:38.356 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:36:38.356 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:36:38.357 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:36:38.357 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:36:38.357 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:36:38.357 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:36:38.373 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 86267 86251 72.8253 18.125 0.0276345 0.0137253 2023-09-23T22:36:38.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:38.456 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:38.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.641 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: osdmap e479: 8 total, 8 up, 8 in 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 476'5487 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: 9.2 starting backfill to osd.3 from (0'0,0'0] MAX to 476'5397 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: 9.4 starting backfill to osd.3 from (0'0,0'0] MAX to 476'5397 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1401941007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 8644/256287 objects degraded (3.373%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:38.642 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[215775]: osdmap e480: 8 total, 8 up, 8 in 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: osdmap e479: 8 total, 8 up, 8 in 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 476'5487 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: 9.2 starting backfill to osd.3 from (0'0,0'0] MAX to 476'5397 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: 9.4 starting backfill to osd.3 from (0'0,0'0] MAX to 476'5397 2023-09-23T22:36:38.643 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1401941007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:38.644 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 8644/256287 objects degraded (3.373%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:38.644 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:38 smithi026 ceph-mon[223014]: osdmap e480: 8 total, 8 up, 8 in 2023-09-23T22:36:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: osdmap e479: 8 total, 8 up, 8 in 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: failed to encode map e479 with expected crc 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: 9.a starting backfill to osd.3 from (0'0,0'0] MAX to 476'5487 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: 9.2 starting backfill to osd.3 from (0'0,0'0] MAX to 476'5397 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: 9.4 starting backfill to osd.3 from (0'0,0'0] MAX to 476'5397 2023-09-23T22:36:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1401941007' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:38.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 8644/256287 objects degraded (3.373%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:38.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:38 smithi070 ceph-mon[192652]: osdmap e480: 8 total, 8 up, 8 in 2023-09-23T22:36:39.375 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 86741 86725 72.2478 29.625 0.013616 0.0138178 2023-09-23T22:36:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[215775]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[215775]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[215775]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[215775]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[215775]: pgmap v1234: 109 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 686 op/s; 8655/256830 objects degraded (3.370%); 8.7 MiB/s, 141 objects/s recovering 2023-09-23T22:36:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[215775]: osdmap e481: 8 total, 8 up, 8 in 2023-09-23T22:36:39.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[223014]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[223014]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[223014]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[223014]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[223014]: pgmap v1234: 109 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 686 op/s; 8655/256830 objects degraded (3.370%); 8.7 MiB/s, 141 objects/s recovering 2023-09-23T22:36:39.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:39.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:39 smithi026 ceph-mon[223014]: osdmap e481: 8 total, 8 up, 8 in 2023-09-23T22:36:39.791 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:36:39.820 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:39.823 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:39 smithi070 ceph-mon[192652]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.824 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:39 smithi070 ceph-mon[192652]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.824 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:39 smithi070 ceph-mon[192652]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.824 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:39 smithi070 ceph-mon[192652]: failed to encode map e480 with expected crc 2023-09-23T22:36:39.824 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:39 smithi070 ceph-mon[192652]: pgmap v1234: 109 pgs: 1 peering, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 43 MiB/s wr, 686 op/s; 8655/256830 objects degraded (3.370%); 8.7 MiB/s, 141 objects/s recovering 2023-09-23T22:36:39.824 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:39.824 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:39 smithi070 ceph-mon[192652]: osdmap e481: 8 total, 8 up, 8 in 2023-09-23T22:36:39.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:39.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:39.931 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:39.933 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:36:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:36:39] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-09-23T22:36:40.027 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:40.141 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:40.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:40.248 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:40.310 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:40.375 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 87382 87366 71.8244 40.0625 0.0136092 0.0139172 2023-09-23T22:36:40.387 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:40.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:40.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:40.415 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:36:40.509 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:40.594 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:40.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:40 smithi070 ceph-mon[192652]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:40 smithi070 ceph-mon[192652]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:40 smithi070 ceph-mon[192652]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:40 smithi070 ceph-mon[192652]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:40 smithi070 ceph-mon[192652]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.726 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:40.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[215775]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[215775]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[215775]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[215775]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[215775]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[223014]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[223014]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[223014]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[223014]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:40 smithi026 ceph-mon[223014]: failed to encode map e481 with expected crc 2023-09-23T22:36:40.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:40.841 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:40.915 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:41.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:41.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:41.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:41.195 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:41.271 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:41.377 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 87944 87928 71.346 35.125 0.0379367 0.0140092 2023-09-23T22:36:41.385 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:41.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:41.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:41.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:41.583 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:41.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:41 smithi026 ceph-mon[223014]: pgmap v1236: 109 pgs: 2 active+recovery_wait+remapped, 1 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22 MiB/s wr, 359 op/s; 23942/260967 objects degraded (9.174%); 31175/260967 objects misplaced (11.946%); 11 MiB/s, 180 objects/s recovering 2023-09-23T22:36:41.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:41 smithi026 ceph-mon[215775]: pgmap v1236: 109 pgs: 2 active+recovery_wait+remapped, 1 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22 MiB/s wr, 359 op/s; 23942/260967 objects degraded (9.174%); 31175/260967 objects misplaced (11.946%); 11 MiB/s, 180 objects/s recovering 2023-09-23T22:36:41.749 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:41.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:41.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:41.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:36:41.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:41 smithi070 ceph-mon[192652]: pgmap v1236: 109 pgs: 2 active+recovery_wait+remapped, 1 peering, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 22 MiB/s wr, 359 op/s; 23942/260967 objects degraded (9.174%); 31175/260967 objects misplaced (11.946%); 11 MiB/s, 180 objects/s recovering 2023-09-23T22:36:41.907 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:42.148 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-09-23T22:36:42.148 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:36:42.148 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T22:36:42.148 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:36:42.148 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:36:42.148 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T22:36:42.386 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 88544 88528 70.9041 37.5 0.0166377 0.0140971 2023-09-23T22:36:42.486 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:36:42.537 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:42.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:42.643 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:42.691 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:42.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/631199832' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:42.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:42 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:42.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/631199832' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:42.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:42 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:42.764 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:42.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/631199832' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:42.767 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:42 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:42.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:42.868 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:42.914 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:43.098 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:43.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:43.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:43.203 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:43.243 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:43.340 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:43.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:43.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:43.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 89189 89173 70.5154 40.3125 0.0236661 0.0141757 2023-09-23T22:36:43.446 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:43.558 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:43.656 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:43.700 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:43.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:43.716 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:36:43.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:43.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T22:36:43.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: pgmap v1238: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 21 MiB/s wr, 339 op/s; 23942/261726 objects degraded (9.148%); 31176/261726 objects misplaced (11.912%); 8.9 MiB/s, 146 objects/s recovering 2023-09-23T22:36:43.717 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 23942/260967 objects degraded (9.174%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:43.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:43.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:36:43.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:43.718 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T22:36:43.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: pgmap v1238: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 21 MiB/s wr, 339 op/s; 23942/261726 objects degraded (9.148%); 31176/261726 objects misplaced (11.912%); 8.9 MiB/s, 146 objects/s recovering 2023-09-23T22:36:43.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 23942/260967 objects degraded (9.174%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:43.819 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:43.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:36:43.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:43.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T22:36:43.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: failed to encode map e482 with expected crc 2023-09-23T22:36:43.861 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: pgmap v1238: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 21 MiB/s wr, 339 op/s; 23942/261726 objects degraded (9.148%); 31176/261726 objects misplaced (11.912%); 8.9 MiB/s, 146 objects/s recovering 2023-09-23T22:36:43.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 23942/260967 objects degraded (9.174%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:43.866 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:43.941 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:43.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:43.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:44.047 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:44.096 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:44.165 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:44.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:44.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:44.272 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:44.355 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:44.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:36:44.388821+0000 min lat: 0.00117176 max lat: 2.611 avg lat: 0.0142571 2023-09-23T22:36:44.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:36:44.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 89791 89775 70.1043 37.625 0.0264854 0.0142571 2023-09-23T22:36:44.523 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:44.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:44.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:44.635 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:44.663 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:44.708 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:44.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:44.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:44.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:44.891 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:45.168 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:45.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:45.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:45.285 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:45.322 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:45.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 90298 90282 69.6302 31.6875 0.0215469 0.0143558 2023-09-23T22:36:45.399 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:45.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:45.503 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:45.594 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:45.703 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:36:45.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:45.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:45.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:45.867 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:45.933 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:45 smithi026 ceph-mon[223014]: pgmap v1239: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 16 MiB/s wr, 252 op/s; 23942/261726 objects degraded (9.148%); 31176/261726 objects misplaced (11.912%); 7.3 MiB/s, 118 objects/s recovering 2023-09-23T22:36:45.940 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:45 smithi026 ceph-mon[215775]: pgmap v1239: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 16 MiB/s wr, 252 op/s; 23942/261726 objects degraded (9.148%); 31176/261726 objects misplaced (11.912%); 7.3 MiB/s, 118 objects/s recovering 2023-09-23T22:36:45.947 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:46.109 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:46.112 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:36:46.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:46.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:46.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:45 smithi070 ceph-mon[192652]: pgmap v1239: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 16 MiB/s wr, 252 op/s; 23942/261726 objects degraded (9.148%); 31176/261726 objects misplaced (11.912%); 7.3 MiB/s, 118 objects/s recovering 2023-09-23T22:36:46.148 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:36:46.195 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:36:46.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:46.231 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:36:46.255 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:46.264 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:36:46.317 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:36:46.326 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:46.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:46.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:46.350 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:36:46.388 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:36:46.388 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 90820 90804 69.179 32.625 0.0287481 0.0144483 2023-09-23T22:36:46.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:46.434 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:36:46.434 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:36:46.434 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:36:46.434 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:36:46.434 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:36:46.434 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:36:46.434 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:36:46.435 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:36:46.546 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:46.756 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:46.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:46.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:46.868 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:46.910 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:46.974 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:46.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:46.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:47.079 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:47.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4124228178' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:47.107 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:47.154 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:47.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:47.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:47.239 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4124228178' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:47.240 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4124228178' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:47.259 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:47.359 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:47.393 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 91300 91284 68.7027 30 0.014189 0.0145497 2023-09-23T22:36:47.527 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:47.566 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:47.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:47.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:47.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:47.718 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:47.795 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:47.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:47.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:47.902 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:47.996 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:48.085 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:47 smithi026 ceph-mon[215775]: pgmap v1240: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s; 23393/270027 objects degraded (8.663%); 31581/270027 objects misplaced (11.695%); 11 MiB/s, 183 objects/s recovering 2023-09-23T22:36:48.086 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:47 smithi026 ceph-mon[223014]: pgmap v1240: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s; 23393/270027 objects degraded (8.663%); 31581/270027 objects misplaced (11.695%); 11 MiB/s, 183 objects/s recovering 2023-09-23T22:36:48.126 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:47 smithi070 ceph-mon[192652]: pgmap v1240: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s; 23393/270027 objects degraded (8.663%); 31581/270027 objects misplaced (11.695%); 11 MiB/s, 183 objects/s recovering 2023-09-23T22:36:48.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:48.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:48.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:48.394 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 91897 91881 68.3292 37.3125 0.020109 0.0146275 2023-09-23T22:36:48.442 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:48.479 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:48.523 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:48.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:48.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:48.627 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:48.674 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:48.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 23393/270027 objects degraded (8.663%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:48.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 23393/270027 objects degraded (8.663%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:49.067 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:49.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:49.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:49.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 23393/270027 objects degraded (8.663%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:49.209 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:49.236 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:49.280 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:49.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:49.383 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:49.394 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 92321 92305 67.8372 26.5 0.0290209 0.0147353 2023-09-23T22:36:49.419 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:49.497 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:49.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:49.567 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:36:49.604 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:49.722 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:36:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:36:49] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-09-23T22:36:49.725 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:49.856 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:36:49.895 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:49.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:49.909 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:50.010 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:50.047 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:50.141 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:50.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:50.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:50.253 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:50.293 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:36:50.352 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:50.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 92820 92804 67.4075 31.1875 0.0120941 0.0148266 2023-09-23T22:36:50.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:50 smithi070 ceph-mon[192652]: pgmap v1241: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 30 MiB/s wr, 485 op/s; 23393/270594 objects degraded (8.645%); 31581/270594 objects misplaced (11.671%); 9.6 MiB/s, 155 objects/s recovering 2023-09-23T22:36:50.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3445005108' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:50.465 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:50 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:50.503 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:50 smithi026 ceph-mon[215775]: pgmap v1241: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 30 MiB/s wr, 485 op/s; 23393/270594 objects degraded (8.645%); 31581/270594 objects misplaced (11.671%); 9.6 MiB/s, 155 objects/s recovering 2023-09-23T22:36:50.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3445005108' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:50.504 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:50 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:50.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:50 smithi026 ceph-mon[223014]: pgmap v1241: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 30 MiB/s wr, 485 op/s; 23393/270594 objects degraded (8.645%); 31581/270594 objects misplaced (11.671%); 9.6 MiB/s, 155 objects/s recovering 2023-09-23T22:36:50.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3445005108' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:50.504 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:50 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:36:50.570 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:50.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:50.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:50.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:50.723 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:50.850 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:50.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:50.855 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:50.916 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:36:50.916 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:36:50.956 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:51.004 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:51.064 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:51.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:51.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:51.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:36:51.258 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:51.392 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:36:51.396 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:51.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 93361 93345 67.0215 33.8125 0.0187196 0.0149146 2023-09-23T22:36:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:51.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:36:51.518 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:51.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:51.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[215775]: failed to encode map e483 with expected crc 2023-09-23T22:36:51.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[215775]: failed to encode map e483 with expected crc 2023-09-23T22:36:51.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:36:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[215775]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T22:36:51.602 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[215775]: failed to encode map e483 with expected crc 2023-09-23T22:36:51.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:51.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[223014]: failed to encode map e483 with expected crc 2023-09-23T22:36:51.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[223014]: failed to encode map e483 with expected crc 2023-09-23T22:36:51.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:36:51.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[223014]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T22:36:51.602 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:51 smithi026 ceph-mon[223014]: failed to encode map e483 with expected crc 2023-09-23T22:36:51.621 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:51 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:51 smithi070 ceph-mon[192652]: failed to encode map e483 with expected crc 2023-09-23T22:36:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:51 smithi070 ceph-mon[192652]: failed to encode map e483 with expected crc 2023-09-23T22:36:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:51 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:36:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:51 smithi070 ceph-mon[192652]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T22:36:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:51 smithi070 ceph-mon[192652]: failed to encode map e483 with expected crc 2023-09-23T22:36:51.760 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:51.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:51.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:51.873 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:51.908 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:52.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 93799 93783 66.5711 27.375 0.0188493 0.0149958 2023-09-23T22:36:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:52 smithi070 ceph-mon[192652]: pgmap v1243: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 39 MiB/s wr, 631 op/s; 22664/277404 objects degraded (8.170%); 31904/277404 objects misplaced (11.501%); 12 MiB/s, 193 objects/s recovering 2023-09-23T22:36:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:52 smithi026 ceph-mon[223014]: pgmap v1243: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 39 MiB/s wr, 631 op/s; 22664/277404 objects degraded (8.170%); 31904/277404 objects misplaced (11.501%); 12 MiB/s, 193 objects/s recovering 2023-09-23T22:36:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:52 smithi026 ceph-mon[215775]: pgmap v1243: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 39 MiB/s wr, 631 op/s; 22664/277404 objects degraded (8.170%); 31904/277404 objects misplaced (11.501%); 12 MiB/s, 193 objects/s recovering 2023-09-23T22:36:53.025 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:53.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:53.110 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:36:53.133 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:36:53.202 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:53.314 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:53.315 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 22664/277404 objects degraded (8.170%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:53.342 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:36:53.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:53.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:36:53.379 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:36:53.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 94307 94291 66.18 31.75 0.00766881 0.0151034 2023-09-23T22:36:53.453 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:36:53.454 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:53.454 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 22664/277404 objects degraded (8.170%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:53.462 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:36:53.462 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 22664/277404 objects degraded (8.170%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:53.565 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:53.690 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:36:53.691 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:36:53.691 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:36:53.697 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:36:53.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:53.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:36:53.776 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:36:53.804 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:53.838 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:36:53.847 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:53.897 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:36:53.939 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:36:53.983 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:36:54.022 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:36:54.049 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:36:54.104 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:36:54.124 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:36:54.124 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:36:54.124 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:36:54.124 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:36:54.124 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:36:54.124 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:36:54.124 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:36:54.125 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:36:54.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 90 4 94726 94722 65.7442 26.9375 0.0471233 0.0151996 2023-09-23T22:36:54.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:54 smithi026 ceph-mon[223014]: pgmap v1244: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 551 op/s; 22664/278286 objects degraded (8.144%); 31904/278286 objects misplaced (11.464%); 10 MiB/s, 160 objects/s recovering 2023-09-23T22:36:54.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/432343717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:54.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:54.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:54 smithi026 ceph-mon[215775]: pgmap v1244: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 551 op/s; 22664/278286 objects degraded (8.144%); 31904/278286 objects misplaced (11.464%); 10 MiB/s, 160 objects/s recovering 2023-09-23T22:36:54.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/432343717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:54.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:54.555 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:54.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:54.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:54 smithi070 ceph-mon[192652]: pgmap v1244: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 34 MiB/s wr, 551 op/s; 22664/278286 objects degraded (8.144%); 31904/278286 objects misplaced (11.464%); 10 MiB/s, 160 objects/s recovering 2023-09-23T22:36:54.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/432343717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:36:54.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:36:54.659 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:36:54.686 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:55.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[215775]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[215775]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T22:36:55.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[223014]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[223014]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T22:36:55.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[223014]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[223014]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.328 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[223014]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.381 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:55.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.0519 2023-09-23T22:36:55.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 94726 2023-09-23T22:36:55.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T22:36:55.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T22:36:55.399 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 65.7441 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 41.6504 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 222.062 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 1051 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 666.406 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 3553 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.015202 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.0478706 2023-09-23T22:36:55.400 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 2.611 2023-09-23T22:36:55.401 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.00117176 2023-09-23T22:36:55.404 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:36:55.443 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:36:55.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:55.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:36:55.483 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:36:55.555 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:55.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:55 smithi070 ceph-mon[192652]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:55 smithi070 ceph-mon[192652]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T22:36:55.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:55 smithi070 ceph-mon[192652]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:55 smithi070 ceph-mon[192652]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:55 smithi070 ceph-mon[192652]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.591 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:55.660 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:55.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:55.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:55.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[215775]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[215775]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:55 smithi026 ceph-mon[215775]: failed to encode map e484 with expected crc 2023-09-23T22:36:55.771 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:55.872 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:56.193 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:56.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:56.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:56.301 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:36:56.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[215775]: pgmap v1246: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 22664/278286 objects degraded (8.144%); 31904/278286 objects misplaced (11.464%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T22:36:56.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[215775]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[215775]: osdmap e485: 8 total, 8 up, 8 in 2023-09-23T22:36:56.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[215775]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.344 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:56.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:56 smithi070 ceph-mon[192652]: pgmap v1246: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 22664/278286 objects degraded (8.144%); 31904/278286 objects misplaced (11.464%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T22:36:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:56 smithi070 ceph-mon[192652]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:56 smithi070 ceph-mon[192652]: osdmap e485: 8 total, 8 up, 8 in 2023-09-23T22:36:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:56 smithi070 ceph-mon[192652]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:56 smithi070 ceph-mon[192652]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:56 smithi070 ceph-mon[192652]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[223014]: pgmap v1246: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 21 MiB/s wr, 343 op/s; 22664/278286 objects degraded (8.144%); 31904/278286 objects misplaced (11.464%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T22:36:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[223014]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[223014]: osdmap e485: 8 total, 8 up, 8 in 2023-09-23T22:36:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[223014]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[223014]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[223014]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[215775]: failed to encode map e485 with expected crc 2023-09-23T22:36:56.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:56 smithi026 ceph-mon[215775]: failed to encode map e485 with expected crc 2023-09-23T22:36:57.412 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:36:57.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:36:57.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:57.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:36:57.821 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:57.873 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:57.924 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:57.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:58.028 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:58.066 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:58.234 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:58.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:58.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:36:58.304 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:36:58.338 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:36:58.433 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:58.461 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:36:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:36:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:36:58.461 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:36:58.461 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:36:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:58 smithi070 ceph-mon[192652]: pgmap v1248: 109 pgs: 1 peering, 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 26 MiB/s wr, 421 op/s; 21982/284070 objects degraded (7.738%); 32097/284070 objects misplaced (11.299%); 17 MiB/s, 278 objects/s recovering 2023-09-23T22:36:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 21982/284070 objects degraded (7.738%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2608928047' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:58 smithi026 ceph-mon[223014]: pgmap v1248: 109 pgs: 1 peering, 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 26 MiB/s wr, 421 op/s; 21982/284070 objects degraded (7.738%); 32097/284070 objects misplaced (11.299%); 17 MiB/s, 278 objects/s recovering 2023-09-23T22:36:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 21982/284070 objects degraded (7.738%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2608928047' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:58.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:58.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:58 smithi026 ceph-mon[215775]: pgmap v1248: 109 pgs: 1 peering, 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 26 MiB/s wr, 421 op/s; 21982/284070 objects degraded (7.738%); 32097/284070 objects misplaced (11.299%); 17 MiB/s, 278 objects/s recovering 2023-09-23T22:36:58.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 21982/284070 objects degraded (7.738%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:36:58.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2608928047' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:58.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:36:58.873 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:36:58.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:58.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:36:58.981 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:36:59.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:59.268 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:36:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:59 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:59 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:59.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:36:59 smithi026 ceph-mon[215775]: osdmap e486: 8 total, 8 up, 8 in 2023-09-23T22:36:59.617 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:36:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:36:59] "GET /metrics HTTP/1.1" 200 36497 "" "Prometheus/2.43.0" 2023-09-23T22:36:59.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:59 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:59.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:59 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:59.617 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:36:59 smithi026 ceph-mon[223014]: osdmap e486: 8 total, 8 up, 8 in 2023-09-23T22:36:59.675 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:36:59.678 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:59 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:36:59.678 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:59 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:36:59.678 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:36:59 smithi070 ceph-mon[192652]: osdmap e486: 8 total, 8 up, 8 in 2023-09-23T22:36:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:59.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:36:59.780 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:36:59.790 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:36:59.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:36:59.896 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:36:59.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:36:59.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:00.005 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:00.037 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:00.392 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:00 smithi026 ceph-mon[215775]: pgmap v1249: 109 pgs: 1 peering, 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 18 MiB/s wr, 296 op/s; 21982/283998 objects degraded (7.740%); 32097/283998 objects misplaced (11.302%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T22:37:00.392 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:00 smithi026 ceph-mon[215775]: failed to encode map e486 with expected crc 2023-09-23T22:37:00.392 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:00 smithi026 ceph-mon[215775]: failed to encode map e486 with expected crc 2023-09-23T22:37:00.392 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:00 smithi026 ceph-mon[215775]: failed to encode map e486 with expected crc 2023-09-23T22:37:00.445 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:00.455 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-09-23T22:37:00.456 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:37:00.456 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T22:37:00.456 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:37:00.456 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:37:00.456 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T22:37:00.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:00.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:00.516 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:37:00.569 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:00.607 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:00.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:00 smithi026 ceph-mon[223014]: pgmap v1249: 109 pgs: 1 peering, 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 18 MiB/s wr, 296 op/s; 21982/283998 objects degraded (7.740%); 32097/283998 objects misplaced (11.302%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T22:37:00.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:00 smithi026 ceph-mon[223014]: failed to encode map e486 with expected crc 2023-09-23T22:37:00.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:00 smithi026 ceph-mon[223014]: failed to encode map e486 with expected crc 2023-09-23T22:37:00.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:00 smithi026 ceph-mon[223014]: failed to encode map e486 with expected crc 2023-09-23T22:37:00.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:00 smithi070 ceph-mon[192652]: pgmap v1249: 109 pgs: 1 peering, 3 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 96 active+clean; 5.8 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 18 MiB/s wr, 296 op/s; 21982/283998 objects degraded (7.740%); 32097/283998 objects misplaced (11.302%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T22:37:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:00 smithi070 ceph-mon[192652]: failed to encode map e486 with expected crc 2023-09-23T22:37:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:00 smithi070 ceph-mon[192652]: failed to encode map e486 with expected crc 2023-09-23T22:37:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:00 smithi070 ceph-mon[192652]: failed to encode map e486 with expected crc 2023-09-23T22:37:01.254 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:01.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:01.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:01.360 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:01.431 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:01.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:01 smithi026 ceph-mon[223014]: pgmap v1251: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 160 B/s rd, 19 MiB/s wr, 682 op/s; 21982/277149 objects degraded (7.931%); 31435/277149 objects misplaced (11.342%); 10 MiB/s, 165 objects/s recovering 2023-09-23T22:37:01.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:01 smithi026 ceph-mon[215775]: pgmap v1251: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 160 B/s rd, 19 MiB/s wr, 682 op/s; 21982/277149 objects degraded (7.931%); 31435/277149 objects misplaced (11.342%); 10 MiB/s, 165 objects/s recovering 2023-09-23T22:37:01.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:01 smithi070 ceph-mon[192652]: pgmap v1251: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 160 B/s rd, 19 MiB/s wr, 682 op/s; 21982/277149 objects degraded (7.931%); 31435/277149 objects misplaced (11.342%); 10 MiB/s, 165 objects/s recovering 2023-09-23T22:37:01.626 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:01.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:01.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:01.735 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:01.777 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:01.993 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:01.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:01.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:02.001 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:37:02.049 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:37:02.076 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:37:02.098 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:02.118 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:37:02.160 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:37:02.174 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:02.177 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:37:02.221 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:02.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:02.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:02.305 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:37:02.325 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:02.347 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:37:02.369 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:37:02.369 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:37:02.369 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:37:02.369 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:37:02.370 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:37:02.370 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:37:02.370 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:37:02.370 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:37:02.424 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1882100934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:02.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:02.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1882100934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:02.666 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:02.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:02.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:02.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1882100934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:02.700 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:37:02.775 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:02.891 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:03.251 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:37:03.332 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:03.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:03.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:03.430 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:37:03.430 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:37:03.448 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:03.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[215775]: pgmap v1252: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 16 MiB/s wr, 589 op/s; 21982/276096 objects degraded (7.962%); 31435/276096 objects misplaced (11.386%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T22:37:03.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 21982/277149 objects degraded (7.931%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:03.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[223014]: pgmap v1252: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 16 MiB/s wr, 589 op/s; 21982/276096 objects degraded (7.962%); 31435/276096 objects misplaced (11.386%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T22:37:03.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 21982/277149 objects degraded (7.931%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:03.608 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:03.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:03 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:03 smithi070 ceph-mon[192652]: pgmap v1252: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 16 MiB/s wr, 589 op/s; 21982/276096 objects degraded (7.962%); 31435/276096 objects misplaced (11.386%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T22:37:03.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 21982/277149 objects degraded (7.931%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:03.753 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:03.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:03.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:03.874 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:03.939 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:04.066 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:04.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:04.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:04.127 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:04.184 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:04.232 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:04.451 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:04.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:04.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:04.556 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:04.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:04 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:04.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:04 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:37:04.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:04 smithi070 ceph-mon[192652]: pgmap v1253: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 124 B/s rd, 2.9 MiB/s wr, 377 op/s; 21982/276096 objects degraded (7.962%); 31435/276096 objects misplaced (11.386%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:37:04.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:04 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:04.590 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:04 smithi070 ceph-mon[192652]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-09-23T22:37:04.593 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:37:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[215775]: pgmap v1253: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 124 B/s rd, 2.9 MiB/s wr, 377 op/s; 21982/276096 objects degraded (7.962%); 31435/276096 objects misplaced (11.386%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:37:04.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:04.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[215775]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-09-23T22:37:04.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:37:04.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:37:04.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[223014]: pgmap v1253: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 124 B/s rd, 2.9 MiB/s wr, 377 op/s; 21982/276096 objects degraded (7.962%); 31435/276096 objects misplaced (11.386%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T22:37:04.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:04.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:04 smithi026 ceph-mon[223014]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2023-09-23T22:37:04.914 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:04.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:04.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:04.997 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:37:05.019 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:05.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:05.124 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:05.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:05.231 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:05.265 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:05.383 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:05.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:05.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:05.488 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:37:05.491 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:05.524 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:05.587 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:05.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:05.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:05.685 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:37:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "" 2023-09-23T22:37:05.685 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:37:05.695 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:05.787 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:05.932 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:05.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:05.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:05.998 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:06.062 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:06.134 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:06.361 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:06.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:06.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:06.466 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:06.591 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:06.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:06 smithi026 ceph-mon[215775]: pgmap v1254: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 115 B/s rd, 341 KiB/s wr, 576 op/s; 21982/269070 objects degraded (8.170%); 30778/269070 objects misplaced (11.439%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:37:06.847 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:06.862 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:06 smithi026 ceph-mon[223014]: pgmap v1254: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 115 B/s rd, 341 KiB/s wr, 576 op/s; 21982/269070 objects degraded (8.170%); 30778/269070 objects misplaced (11.439%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:37:06.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:06.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:06 smithi070 ceph-mon[192652]: pgmap v1254: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 115 B/s rd, 341 KiB/s wr, 576 op/s; 21982/269070 objects degraded (8.170%); 30778/269070 objects misplaced (11.439%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:37:06.965 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:06.966 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:07.044 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:07.146 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:07.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:07.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:07.257 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:07.328 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:07.660 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:37:07.874 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1475129526' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:07 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:07.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1475129526' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:07.886 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:07 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:07.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:07.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:07.887 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1475129526' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:07.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:07 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:07.987 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:08.082 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:08.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[215775]: pgmap v1255: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 115 B/s rd, 115 B/s wr, 603 op/s; 21982/268017 objects degraded (8.202%); 30778/268017 objects misplaced (11.484%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:37:08.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:08.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[215775]: failed to encode map e487 with expected crc 2023-09-23T22:37:08.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[215775]: failed to encode map e487 with expected crc 2023-09-23T22:37:08.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[215775]: failed to encode map e487 with expected crc 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[215775]: osdmap e487: 8 total, 8 up, 8 in 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 21982/268017 objects degraded (8.202%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[223014]: pgmap v1255: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 115 B/s rd, 115 B/s wr, 603 op/s; 21982/268017 objects degraded (8.202%); 30778/268017 objects misplaced (11.484%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[223014]: failed to encode map e487 with expected crc 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[223014]: failed to encode map e487 with expected crc 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[223014]: failed to encode map e487 with expected crc 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[223014]: osdmap e487: 8 total, 8 up, 8 in 2023-09-23T22:37:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 21982/268017 objects degraded (8.202%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:08.811 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:08.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:08.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:08.918 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:08.958 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:08 smithi070 ceph-mon[192652]: pgmap v1255: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 115 B/s rd, 115 B/s wr, 603 op/s; 21982/268017 objects degraded (8.202%); 30778/268017 objects misplaced (11.484%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T22:37:08.958 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:08 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:08.958 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:08 smithi070 ceph-mon[192652]: failed to encode map e487 with expected crc 2023-09-23T22:37:08.958 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:08 smithi070 ceph-mon[192652]: failed to encode map e487 with expected crc 2023-09-23T22:37:08.958 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:08 smithi070 ceph-mon[192652]: failed to encode map e487 with expected crc 2023-09-23T22:37:08.958 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:08 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:37:08.958 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:08 smithi070 ceph-mon[192652]: osdmap e487: 8 total, 8 up, 8 in 2023-09-23T22:37:08.958 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 21982/268017 objects degraded (8.202%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:08.963 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:09.033 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:09.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:09.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:09.140 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:09.169 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:09.225 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:09.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:09.340 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:09.429 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:09.668 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:37:09.668 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:37:09.740 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:37:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:37:09] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2023-09-23T22:37:09.895 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:09.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:10.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:10.128 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:10.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:10.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:10.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:10.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:10.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:10.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:10.428 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:10.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:10.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:10.535 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:10.567 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:10.626 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:10.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:10.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:10.699 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:37:10.733 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:10.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:10.819 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:10.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:10.831 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:37:10.924 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:11.068 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:11.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:11 smithi026 ceph-mon[223014]: pgmap v1257: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 21982/268017 objects degraded (8.202%); 30778/268017 objects misplaced (11.484%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:37:11.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:11 smithi026 ceph-mon[215775]: pgmap v1257: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 21982/268017 objects degraded (8.202%); 30778/268017 objects misplaced (11.484%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:37:11.237 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:11.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:11.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:11.283 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:37:11.338 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:37:11.361 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:11.388 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:37:11.394 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:11 smithi070 ceph-mon[192652]: pgmap v1257: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 344 op/s; 21982/268017 objects degraded (8.202%); 30778/268017 objects misplaced (11.484%); 1.0 MiB/s, 16 objects/s recovering 2023-09-23T22:37:11.397 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:11.441 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:37:11.450 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:11.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:11.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:11.475 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:37:11.509 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:37:11.555 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:37:11.556 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:11.584 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:11.590 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:37:11.618 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:37:11.618 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:37:11.618 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:37:11.618 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:37:11.618 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:37:11.619 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:37:11.619 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:37:11.619 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:37:11.640 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:11.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:11.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:11.654 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:11.706 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 3, 5, 6, 7, 2, 4] 2023-09-23T22:37:11.706 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:37:11.707 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 3, 5, 6, 7, 2, 4] 2023-09-23T22:37:11.707 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T22:37:11.707 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T22:37:11.745 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:11.779 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:11.854 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:11.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:11.962 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:11.989 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:11 smithi026 systemd[1]: Stopping Ceph osd.1 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:37:11.992 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:12.054 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:12.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:12.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:12.161 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:12.188 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:12.233 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:12.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:12.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:12.339 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:12.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4063778228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:12.447 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4063778228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:12.447 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4063778228' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:12.464 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:12.670 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:12.811 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:12.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:12.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:12.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:12.995 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:13.327 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:13 smithi026 ceph-mon[215775]: pgmap v1258: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 21982/260913 objects degraded (8.425%); 30138/260913 objects misplaced (11.551%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T22:37:13.327 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:13 smithi026 ceph-mon[223014]: pgmap v1258: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 21982/260913 objects degraded (8.425%); 30138/260913 objects misplaced (11.551%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T22:37:13.354 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:13.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:13.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:13.467 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:13.531 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:13.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:13 smithi070 ceph-mon[192652]: pgmap v1258: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 21982/260913 objects degraded (8.425%); 30138/260913 objects misplaced (11.551%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T22:37:13.986 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:13 smithi026 conmon[245951]: 2023-09-23T22:37:13.883+0000 7f8bd440c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:37:13.986 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:13 smithi026 conmon[245951]: 2023-09-23T22:37:13.883+0000 7f8bd440c700 -1 osd.1 487 *** Got signal Terminated *** 2023-09-23T22:37:13.986 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:13 smithi026 conmon[245951]: 2023-09-23T22:37:13.883+0000 7f8bd440c700 -1 osd.1 487 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:37:14.328 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:14.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:14.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:14.433 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:14.459 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:14 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 21982/260913 objects degraded (8.425%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:14.459 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:14 smithi070 ceph-mon[192652]: osd.1 marked itself down and dead 2023-09-23T22:37:14.462 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:14.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:14 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 21982/260913 objects degraded (8.425%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:14.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:14 smithi026 ceph-mon[215775]: osd.1 marked itself down and dead 2023-09-23T22:37:14.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:14 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 21982/260913 objects degraded (8.425%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:14.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:14 smithi026 ceph-mon[223014]: osd.1 marked itself down and dead 2023-09-23T22:37:14.509 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:14.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:14.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:14.616 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:14.642 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:14.685 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:14.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:14.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:14.723 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:37:14.793 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:14.824 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:14.964 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:14.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:14.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:15.070 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:15.106 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:15.171 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:15.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:15.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:15.235 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:15 smithi026 bash[578751]: 1fd817f92d4e14c63086e3ebd379fce7ef32e4473c674d7ef0da5d69059283cd 2023-09-23T22:37:15.276 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:15.330 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:15.390 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:15.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:15.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:15.430 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:37:15.430 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:37:15.431 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:37:15.431 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:37:15.431 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:37:15.497 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:15.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:15 smithi070 ceph-mon[192652]: pgmap v1259: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 21982/260472 objects degraded (8.439%); 30138/260472 objects misplaced (11.571%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T22:37:15.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:15 smithi070 ceph-mon[192652]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:37:15.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:15 smithi070 ceph-mon[192652]: failed to encode map e488 with expected crc 2023-09-23T22:37:15.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:15 smithi070 ceph-mon[192652]: osdmap e488: 8 total, 7 up, 8 in 2023-09-23T22:37:15.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:15 smithi070 ceph-mon[192652]: failed to encode map e488 with expected crc 2023-09-23T22:37:15.521 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:15 smithi070 ceph-mon[192652]: failed to encode map e488 with expected crc 2023-09-23T22:37:15.523 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:15.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:15.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:15.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:15.678 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:15.704 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:15.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[223014]: pgmap v1259: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 21982/260472 objects degraded (8.439%); 30138/260472 objects misplaced (11.571%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T22:37:15.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[223014]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:37:15.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[223014]: failed to encode map e488 with expected crc 2023-09-23T22:37:15.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[223014]: osdmap e488: 8 total, 7 up, 8 in 2023-09-23T22:37:15.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[223014]: failed to encode map e488 with expected crc 2023-09-23T22:37:15.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[223014]: failed to encode map e488 with expected crc 2023-09-23T22:37:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[215775]: pgmap v1259: 109 pgs: 3 active+recovery_wait+remapped, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 97 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 520 op/s; 21982/260472 objects degraded (8.439%); 30138/260472 objects misplaced (11.571%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T22:37:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[215775]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:37:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[215775]: failed to encode map e488 with expected crc 2023-09-23T22:37:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[215775]: osdmap e488: 8 total, 7 up, 8 in 2023-09-23T22:37:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[215775]: failed to encode map e488 with expected crc 2023-09-23T22:37:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:15 smithi026 ceph-mon[215775]: failed to encode map e488 with expected crc 2023-09-23T22:37:15.755 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:15.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:15.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:15.860 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:15.889 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:15.935 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:15.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:15.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:16.040 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:16.066 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:16.113 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:16.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:16.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:16.222 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:16.248 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:16.306 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:16.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:16.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:16.416 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:16.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: osdmap e489: 8 total, 7 up, 8 in 2023-09-23T22:37:16.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.455 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: 9.2 continuing backfill to osd.3 from (459'600,487'6540] MIN to 487'6540 2023-09-23T22:37:16.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: 9.4 continuing backfill to osd.3 from (460'700,487'6583] MIN to 487'6583 2023-09-23T22:37:16.456 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[223014]: 9.a continuing backfill to osd.3 from (460'800,487'6684] MIN to 487'6684 2023-09-23T22:37:16.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.456 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: osdmap e489: 8 total, 7 up, 8 in 2023-09-23T22:37:16.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: 9.2 continuing backfill to osd.3 from (459'600,487'6540] MIN to 487'6540 2023-09-23T22:37:16.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: 9.4 continuing backfill to osd.3 from (460'700,487'6583] MIN to 487'6583 2023-09-23T22:37:16.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:16 smithi026 ceph-mon[215775]: 9.a continuing backfill to osd.3 from (460'800,487'6684] MIN to 487'6684 2023-09-23T22:37:16.459 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:16.590 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:16.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:16.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:16.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: osdmap e489: 8 total, 7 up, 8 in 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: failed to encode map e489 with expected crc 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: 9.2 continuing backfill to osd.3 from (459'600,487'6540] MIN to 487'6540 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: 9.4 continuing backfill to osd.3 from (460'700,487'6583] MIN to 487'6583 2023-09-23T22:37:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:16 smithi070 ceph-mon[192652]: 9.a continuing backfill to osd.3 from (460'800,487'6684] MIN to 487'6684 2023-09-23T22:37:16.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:37:16.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:16.721 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:16.766 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:16.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:16.866 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:16.873 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:16.975 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:17.066 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:17.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:17.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:17.174 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:17.201 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:17.250 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:17.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:17.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:17.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[215775]: pgmap v1262: 109 pgs: 1 activating+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 1 remapped+peering, 19 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 2 stale+active+recovery_wait+remapped, 1 stale+active+recovery_wait+degraded+remapped, 3 stale+active+clean, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 32826/256611 objects degraded (12.792%); 29713/256611 objects misplaced (11.579%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:37:17.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:37:17.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[215775]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[215775]: osdmap e490: 8 total, 7 up, 8 in 2023-09-23T22:37:17.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[215775]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[215775]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[215775]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[215775]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[215775]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[223014]: pgmap v1262: 109 pgs: 1 activating+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 1 remapped+peering, 19 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 2 stale+active+recovery_wait+remapped, 1 stale+active+recovery_wait+degraded+remapped, 3 stale+active+clean, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 32826/256611 objects degraded (12.792%); 29713/256611 objects misplaced (11.579%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[223014]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[223014]: osdmap e490: 8 total, 7 up, 8 in 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[223014]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[223014]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[223014]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[223014]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.333 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:17 smithi026 ceph-mon[223014]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.356 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:17.419 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:17.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:17 smithi070 ceph-mon[192652]: pgmap v1262: 109 pgs: 1 activating+undersized, 1 active+undersized+degraded+remapped+backfill_wait, 9 active+undersized, 1 remapped+peering, 19 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 2 stale+active+recovery_wait+remapped, 1 stale+active+recovery_wait+degraded+remapped, 3 stale+active+clean, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 62 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 32826/256611 objects degraded (12.792%); 29713/256611 objects misplaced (11.579%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T22:37:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:17 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T22:37:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:17 smithi070 ceph-mon[192652]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:17 smithi070 ceph-mon[192652]: osdmap e490: 8 total, 7 up, 8 in 2023-09-23T22:37:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:17 smithi070 ceph-mon[192652]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:17 smithi070 ceph-mon[192652]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:17 smithi070 ceph-mon[192652]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:17 smithi070 ceph-mon[192652]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:17 smithi070 ceph-mon[192652]: failed to encode map e490 with expected crc 2023-09-23T22:37:17.658 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:17.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:17.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:17.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:17.789 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:17.838 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:17.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:17.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:17.946 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:17.974 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:18.020 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:18.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:18.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:18.126 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:18.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:18.200 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:18.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:18.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:18.291 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:37:18.305 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:18.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:18.381 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:18.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:18.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:18.486 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:18.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4181384552' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:18.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:18 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:18.510 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 32826/256611 objects degraded (12.792%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:18.513 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:18.560 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:18.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:18.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:18.592 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4181384552' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:18.592 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:18 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:18.592 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 32826/256611 objects degraded (12.792%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:18.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4181384552' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:18.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:18 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:18.592 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 32826/256611 objects degraded (12.792%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:18.667 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:18.705 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:19.175 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:19.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:19.279 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:19.305 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:19.348 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:19.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:19.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:19.384 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:19.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:19.481 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:19.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[215775]: pgmap v1264: 109 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 19 active+undersized, 1 remapped+peering, 19 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 318 op/s; 53951/255174 objects degraded (21.143%); 29426/255174 objects misplaced (11.532%); 3.3 MiB/s, 50 objects/s recovering 2023-09-23T22:37:19.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:37:19.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[215775]: failed to encode map e491 with expected crc 2023-09-23T22:37:19.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[215775]: failed to encode map e491 with expected crc 2023-09-23T22:37:19.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:37:19.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[215775]: osdmap e491: 8 total, 7 up, 8 in 2023-09-23T22:37:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[223014]: pgmap v1264: 109 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 19 active+undersized, 1 remapped+peering, 19 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 318 op/s; 53951/255174 objects degraded (21.143%); 29426/255174 objects misplaced (11.532%); 3.3 MiB/s, 50 objects/s recovering 2023-09-23T22:37:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:37:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[223014]: failed to encode map e491 with expected crc 2023-09-23T22:37:19.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[223014]: failed to encode map e491 with expected crc 2023-09-23T22:37:19.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:37:19.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:19 smithi026 ceph-mon[223014]: osdmap e491: 8 total, 7 up, 8 in 2023-09-23T22:37:19.524 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:19.530 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:19 smithi070 ceph-mon[192652]: pgmap v1264: 109 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 19 active+undersized, 1 remapped+peering, 19 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 318 op/s; 53951/255174 objects degraded (21.143%); 29426/255174 objects misplaced (11.532%); 3.3 MiB/s, 50 objects/s recovering 2023-09-23T22:37:19.530 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:19 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:37:19.530 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:19 smithi070 ceph-mon[192652]: failed to encode map e491 with expected crc 2023-09-23T22:37:19.530 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:19 smithi070 ceph-mon[192652]: failed to encode map e491 with expected crc 2023-09-23T22:37:19.530 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:37:19.530 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:19 smithi070 ceph-mon[192652]: osdmap e491: 8 total, 7 up, 8 in 2023-09-23T22:37:19.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:19.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:19.632 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:19.658 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:19.705 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:19.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:19.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:19.810 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:19.846 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:19.944 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:37:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:37:19] "GET /metrics HTTP/1.1" 200 36496 "" "Prometheus/2.43.0" 2023-09-23T22:37:19.961 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:37:19.980 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:37:20.200 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:20.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:20.304 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:20.331 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:20.380 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:20.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:20.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:20.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1669037425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:20.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1669037425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:20.486 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:20.519 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:20.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1669037425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:20.833 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:20.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:20.840 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:20.941 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:20.971 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:21.187 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T22:37:21.210 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:21.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:21.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:21.256 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:37:21.257 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T22:37:21.315 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:21.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:37:21.341 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:21.390 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:21.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:21.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:21.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:21 smithi026 ceph-mon[223014]: pgmap v1266: 109 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 19 active+undersized, 1 remapped+peering, 19 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 333 op/s; 53951/255174 objects degraded (21.143%); 29426/255174 objects misplaced (11.532%); 3.7 MiB/s, 57 objects/s recovering 2023-09-23T22:37:21.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:21 smithi026 ceph-mon[215775]: pgmap v1266: 109 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 19 active+undersized, 1 remapped+peering, 19 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 333 op/s; 53951/255174 objects degraded (21.143%); 29426/255174 objects misplaced (11.532%); 3.7 MiB/s, 57 objects/s recovering 2023-09-23T22:37:21.495 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:21.530 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:21.592 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:21.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:21.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:21.604 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:37:21.604 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:37:21.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:21 smithi070 ceph-mon[192652]: pgmap v1266: 109 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 1 activating+undersized, 2 active+undersized+degraded+remapped+backfill_wait, 19 active+undersized, 1 remapped+peering, 19 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 333 op/s; 53951/255174 objects degraded (21.143%); 29426/255174 objects misplaced (11.532%); 3.7 MiB/s, 57 objects/s recovering 2023-09-23T22:37:21.697 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:21.729 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:21.779 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:21.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:21.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:21.884 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:21.911 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:21.956 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:21.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:21.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:22.063 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:22.089 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:22.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:22.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:22.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:22.239 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:22.264 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:22.307 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:22.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:22.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:22.412 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:22.445 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:22.454 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:37:22.480 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:22.498 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:22.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:22.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:22.603 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:22.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:22 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:37:22.628 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:22.672 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:22.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:22.681 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:22.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:22 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:37:22.684 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:22 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T22:37:22.782 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:22.817 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:22.976 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:22.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:22.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:23.080 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:23.111 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:23.212 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:23.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:23.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:23.317 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:23.347 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:23 smithi070 ceph-mon[192652]: pgmap v1267: 109 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 12 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 204 op/s; 70674/254670 objects degraded (27.751%); 33093/254670 objects misplaced (12.994%); 7.6 MiB/s, 118 objects/s recovering 2023-09-23T22:37:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 70674/254670 objects degraded (27.751%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:23.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:23 smithi026 ceph-mon[215775]: pgmap v1267: 109 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 12 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 204 op/s; 70674/254670 objects degraded (27.751%); 33093/254670 objects misplaced (12.994%); 7.6 MiB/s, 118 objects/s recovering 2023-09-23T22:37:23.646 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 70674/254670 objects degraded (27.751%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:23.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:23 smithi026 ceph-mon[223014]: pgmap v1267: 109 pgs: 1 activating+undersized+remapped, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 12 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 204 op/s; 70674/254670 objects degraded (27.751%); 33093/254670 objects misplaced (12.994%); 7.6 MiB/s, 118 objects/s recovering 2023-09-23T22:37:23.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 70674/254670 objects degraded (27.751%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:23.668 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:23.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:23.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:23.695 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:23.773 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:23.799 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:23.842 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:23.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:23.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:23.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:23.985 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:24.189 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:24.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:24.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:24.294 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:24.328 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:24.488 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:24.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:24.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:24.592 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:24.621 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:24.668 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:24.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:24.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:24.772 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:24.801 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:25.042 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:25.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:25.046 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:25.147 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:25.178 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:25.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[223014]: pgmap v1268: 109 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 12 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 70601/254511 objects degraded (27.740%); 33065/254511 objects misplaced (12.992%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:37:25.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:25.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:25.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3870905972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:25.287 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:25.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[215775]: pgmap v1268: 109 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 12 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 70601/254511 objects degraded (27.740%); 33065/254511 objects misplaced (12.992%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:37:25.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:25.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:25.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3870905972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:25.288 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:25 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:25.288 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:25.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:25.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:25.342 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:37:25.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:25 smithi070 ceph-mon[192652]: pgmap v1268: 109 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 12 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 87 op/s; 70601/254511 objects degraded (27.740%); 33065/254511 objects misplaced (12.992%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:37:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3870905972' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:25 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:25.394 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:25.420 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:25.465 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:25.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:25.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:25.543 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:37:25.543 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:37:25.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.543 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104826 9.3 9.75090da3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:25.543 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.625968+0000", 2023-09-23T22:37:25.543 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 7.9155037830000001, 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9155264179999998, 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104826 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.625968+0000", 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.625968+0000", 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.625966+0000", 2023-09-23T22:37:25.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:25.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.625978+0000", 2023-09-23T22:37:25.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2099000000000001e-05 2023-09-23T22:37:25.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.625995+0000", 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6722e-05 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.626004+0000", 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9560000000000003e-06 2023-09-23T22:37:25.547 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104959 9.3 9.d7b97ff3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e491)", 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.869975+0000", 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 6.6714968810000004, 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6715591029999999, 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.548 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104959 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.869975+0000", 2023-09-23T22:37:25.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.869975+0000", 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.869973+0000", 2023-09-23T22:37:25.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.869983+0000", 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5910000000000002e-06 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.869994+0000", 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0793e-05 2023-09-23T22:37:25.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.870000+0000", 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7070000000000003e-06 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.552 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104878 9.3 9.9ff22933 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.945372+0000", 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 7.5960992730000001, 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5961933410000002, 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104878 2023-09-23T22:37:25.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.945372+0000", 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.945372+0000", 2023-09-23T22:37:25.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.945371+0000", 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.945378+0000", 2023-09-23T22:37:25.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9369999999999998e-06 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.945393+0000", 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5199e-05 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.945401+0000", 2023-09-23T22:37:25.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2009999999999999e-06 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104803 9.3 9.9cd3aba3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.518175+0000", 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 8.0232967980000005, 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0234226599999996, 2023-09-23T22:37:25.557 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104803 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.518175+0000", 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.518175+0000", 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.518173+0000", 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.518180+0000", 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9460000000000001e-06 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.518193+0000", 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2302e-05 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.518200+0000", 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4610000000000002e-06 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104961 9.3 9.ae5a5a43 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e491)", 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.875807+0000", 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 6.6656647600000003, 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6658212020000001, 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.562 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104961 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.875807+0000", 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.875807+0000", 2023-09-23T22:37:25.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.564 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.875805+0000", 2023-09-23T22:37:25.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:25.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.875813+0000", 2023-09-23T22:37:25.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8299999999999996e-06 2023-09-23T22:37:25.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.875829+0000", 2023-09-23T22:37:25.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6577000000000001e-05 2023-09-23T22:37:25.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.875841+0000", 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1379000000000001e-05 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.566 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104784 9.3 9.5dd0e513 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.363749+0000", 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 8.177722588, 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1779107809999996, 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104784 2023-09-23T22:37:25.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.363749+0000", 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.363749+0000", 2023-09-23T22:37:25.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.363747+0000", 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.363761+0000", 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3823e-05 2023-09-23T22:37:25.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.363777+0000", 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6710000000000001e-05 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.363787+0000", 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2890000000000003e-06 2023-09-23T22:37:25.570 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104962 9.3 9.f1436783 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e491)", 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.880386+0000", 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 6.6610856399999996, 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6613044459999999, 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.571 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104962 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.880386+0000", 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.880386+0000", 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.880384+0000", 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:25.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.880392+0000", 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3459999999999996e-06 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.880408+0000", 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6524e-05 2023-09-23T22:37:25.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.880417+0000", 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6309999999999992e-06 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.575 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104965 9.3 9.aac963c3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e491)", 2023-09-23T22:37:25.576 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.931511+0000", 2023-09-23T22:37:25.576 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 6.6099608950000004, 2023-09-23T22:37:25.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.610217499, 2023-09-23T22:37:25.576 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.576 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.576 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.576 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.576 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.576 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104965 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.931511+0000", 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.931511+0000", 2023-09-23T22:37:25.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.931509+0000", 2023-09-23T22:37:25.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:25.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.931517+0000", 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0180000000000001e-06 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.931532+0000", 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5262000000000001e-05 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.931540+0000", 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7000000000000008e-06 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104691 9.3 9.d6892893 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.279055+0000", 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 10.262416781000001, 2023-09-23T22:37:25.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 10.262705367000001, 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "delayed", 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104691 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279055+0000", 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279055+0000", 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279053+0000", 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279061+0000", 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6020000000000002e-06 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279077+0000", 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6127e-05 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279086+0000", 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4839999999999993e-06 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279152+0000", 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5423e-05 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279163+0000", 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1711e-05 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104914 9.3 9.7e002a93 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.217200+0000", 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 7.3242711099999998, 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.324598763, 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.586 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104914 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.217200+0000", 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.217200+0000", 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.217199+0000", 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.217206+0000", 2023-09-23T22:37:25.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5079999999999999e-06 2023-09-23T22:37:25.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.217219+0000", 2023-09-23T22:37:25.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3287e-05 2023-09-23T22:37:25.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.590 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.217227+0000", 2023-09-23T22:37:25.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5900000000000002e-06 2023-09-23T22:37:25.590 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.590 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.590 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104697 9.3 9.c29b12d3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.279100+0000", 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 10.262372023999999, 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 10.262732106, 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "delayed", 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104697 2023-09-23T22:37:25.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.592 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279100+0000", 2023-09-23T22:37:25.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279100+0000", 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279099+0000", 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279102+0000", 2023-09-23T22:37:25.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7010000000000001e-06 2023-09-23T22:37:25.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.594 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279118+0000", 2023-09-23T22:37:25.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6118999999999999e-05 2023-09-23T22:37:25.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.594 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279123+0000", 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3090000000000003e-06 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279186+0000", 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2349999999999998e-05 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.279192+0000", 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8449999999999997e-06 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104799 9.3 9.75df6223 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.473902+0000", 2023-09-23T22:37:25.596 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 8.0675697240000002, 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0679666660000002, 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104799 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.473902+0000", 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.473902+0000", 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.473900+0000", 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.473908+0000", 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.303e-06 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.473922+0000", 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3665e-05 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.473934+0000", 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2143e-05 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.600 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104800 9.3 9.539d24e3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.477072+0000", 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 8.0643999999999991, 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0648271890000007, 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.601 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.602 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.602 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104800 2023-09-23T22:37:25.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.602 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.602 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.477072+0000", 2023-09-23T22:37:25.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.477072+0000", 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.477070+0000", 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:25.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.477079+0000", 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1309999999999995e-06 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.477092+0000", 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2979999999999999e-05 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.477103+0000", 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1027e-05 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104777 9.3 9.734e16b3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:25.605 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.349251+0000", 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 8.1922207829999998, 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1926811550000007, 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104777 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.606 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.349251+0000", 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.349251+0000", 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.349249+0000", 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.349257+0000", 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2349999999999999e-06 2023-09-23T22:37:25.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.349275+0000", 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7456e-05 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.349284+0000", 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7779999999999996e-06 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.609 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104940 9.3 9.dd736a83 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e491)", 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.674884+0000", 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 6.866587376, 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8670777589999998, 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.610 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.611 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.611 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104940 2023-09-23T22:37:25.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.611 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.611 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.674884+0000", 2023-09-23T22:37:25.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.674884+0000", 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.674882+0000", 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.674893+0000", 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0359e-05 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.674910+0000", 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7136e-05 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.674922+0000", 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2425e-05 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104874 9.3 9.5a3c8ae3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.936689+0000", 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 7.6047823660000002, 2023-09-23T22:37:25.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6053070659999999, 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104874 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:25.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.936689+0000", 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.936689+0000", 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.936687+0000", 2023-09-23T22:37:25.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.936695+0000", 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6359999999999994e-06 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.936731+0000", 2023-09-23T22:37:25.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6365999999999997e-05 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.936741+0000", 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6169999999999996e-06 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:25.618 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:37:25.619 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 16 2023-09-23T22:37:25.619 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:37:25.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:25.653 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:26.258 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [1] live_osds: [0, 3, 5, 6, 7, 2, 4] 2023-09-23T22:37:26.258 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:37:26.258 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-09-23T22:37:26.258 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-09-23T22:37:26.258 DEBUG:teuthology.orchestra.run.smithi026:osd.1> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T22:37:26.261 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl start ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T22:37:26.284 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:37:26.296 INFO:journalctl@ceph.osd.1.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T22:37:26.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:26 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:26.312 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:26 smithi026 ceph-mon[223014]: failed to encode map e492 with expected crc 2023-09-23T22:37:26.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:26 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:37:26.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:26 smithi026 ceph-mon[223014]: osdmap e492: 8 total, 7 up, 8 in 2023-09-23T22:37:26.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:26 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:26.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:26 smithi026 ceph-mon[215775]: failed to encode map e492 with expected crc 2023-09-23T22:37:26.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:26 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:37:26.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:26 smithi026 ceph-mon[215775]: osdmap e492: 8 total, 7 up, 8 in 2023-09-23T22:37:26.313 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:37:26.336 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:37:26.368 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:37:26.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:26 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:26.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:26 smithi070 ceph-mon[192652]: failed to encode map e492 with expected crc 2023-09-23T22:37:26.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:26 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:37:26.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:26 smithi070 ceph-mon[192652]: osdmap e492: 8 total, 7 up, 8 in 2023-09-23T22:37:26.390 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:37:26.390 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:37:26.390 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T22:37:26.390 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:37:26.390 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:37:26.390 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:37:26.390 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:37:26.390 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:37:26.391 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:37:26.597 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:26 smithi026 systemd[1]: Starting Ceph osd.1 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:37:26.623 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:26.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:26.653 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:37:26.728 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:26.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:27.158 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:27.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:27.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:37:27.247 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:37:27.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:27.305 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:27.412 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:27.426 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:27.429 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:27 smithi026 ceph-mon[223014]: pgmap v1270: 109 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 12 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 27 op/s; 69990/254511 objects degraded (27.500%); 33065/254511 objects misplaced (12.992%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:37:27.430 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:27 smithi026 ceph-mon[215775]: pgmap v1270: 109 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 12 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 27 op/s; 69990/254511 objects degraded (27.500%); 33065/254511 objects misplaced (12.992%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:37:27.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:27.431 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:27.533 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:27.574 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:27.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:27 smithi070 ceph-mon[192652]: pgmap v1270: 109 pgs: 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 22 active+undersized, 12 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 1 stale+active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 27 op/s; 69990/254511 objects degraded (27.500%); 33065/254511 objects misplaced (12.992%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:37:27.688 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:27.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:27.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:27.793 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:27.820 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:27.870 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:27.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:27.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:27.976 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:28.015 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:28.136 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:28.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:28.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:28.242 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:28.270 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:28.330 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:28.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:28.344 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:28 smithi026 bash[582766]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:37:28.345 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:28 smithi026 bash[582766]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T22:37:28.345 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:28 smithi026 bash[582766]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T22:37:28.345 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:28 smithi026 bash[582766]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T22:37:28.345 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:28 smithi026 bash[582766]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T22:37:28.345 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:28 smithi026 bash[582766]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T22:37:28.345 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:28 smithi026 bash[582766]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T22:37:28.345 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:28 smithi026 bash[582766]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T22:37:28.348 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 69990/254511 objects degraded (27.500%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:28.437 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:28.476 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:28.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:37:28.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 69990/254511 objects degraded (27.500%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:28.651 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 69990/254511 objects degraded (27.500%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:29.247 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:37:29.279 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:29.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:29.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:29.361 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:29.392 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:29.433 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:29 smithi026 ceph-mon[223014]: pgmap v1271: 109 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 76427/253593 objects degraded (30.138%); 31907/253593 objects misplaced (12.582%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:37:29.433 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:29 smithi026 ceph-mon[215775]: pgmap v1271: 109 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 76427/253593 objects degraded (30.138%); 31907/253593 objects misplaced (12.582%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:37:29.436 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:29.529 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:29.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:29.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:29 smithi070 ceph-mon[192652]: pgmap v1271: 109 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 76427/253593 objects degraded (30.138%); 31907/253593 objects misplaced (12.582%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:37:29.636 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:29.663 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:29.719 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:29.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:29.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:29.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:37:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:37:29] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-09-23T22:37:29.826 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:29.853 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:29.887 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:37:29.887 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:37:29.901 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:29.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:29.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:30.006 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:30.034 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:30.083 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:30.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:30.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:30.191 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:30.240 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:30.406 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:30.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:30.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:30.511 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:30.537 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:30.581 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:30.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:30.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:30.685 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:30.712 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:30.758 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:30.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:30.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:30.865 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:30.893 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:30.940 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:30.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:31.046 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:31.074 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:31.118 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:31.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:31.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:31.184 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:37:31.225 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:31.252 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:31.296 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:31.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:31.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:31.350 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:31 smithi026 ceph-osd[584160]: -- 172.21.15.26:0/3717253029 <== mon.2 v2:172.21.15.26:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55df6f6b44e0 con 0x55df6f687400 2023-09-23T22:37:31.350 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:31 smithi026 bash[583935]: 3877a14b80dfc78d9a2c26a0dbdd4a84406cd87bdcc70479348643e7c189547e 2023-09-23T22:37:31.350 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:31 smithi026 systemd[1]: Started Ceph osd.1 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:37:31.350 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:31 smithi026 ceph-mon[215775]: pgmap v1272: 109 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 76427/253593 objects degraded (30.138%); 31907/253593 objects misplaced (12.582%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T22:37:31.351 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:31 smithi026 ceph-mon[223014]: pgmap v1272: 109 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 76427/253593 objects degraded (30.138%); 31907/253593 objects misplaced (12.582%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T22:37:31.410 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:31.436 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:31.484 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:31.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:31.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:31.519 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:37:31.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:31.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:31 smithi070 ceph-mon[192652]: pgmap v1272: 109 pgs: 1 activating+degraded+remapped, 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 52 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 52 op/s; 76427/253593 objects degraded (30.138%); 31907/253593 objects misplaced (12.582%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T22:37:31.625 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:31.649 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:31 smithi026 conmon[584057]: 2023-09-23T22:37:31.644+0000 7f971cf30640 -1 Falling back to public interface 2023-09-23T22:37:31.663 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:31.877 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:31.890 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:31.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:31.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:31.996 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:32.053 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:32.229 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:32.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:32.339 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:32.365 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:32.414 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:32.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:32.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:32.520 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:32.547 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:32.591 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:32.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:32.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:32.698 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:32.725 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:32.772 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:32.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:32.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:32.878 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:32.905 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:32.952 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:32.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:32.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:33.057 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:33.091 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:33.143 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:33.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:33.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:33.252 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:33.280 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:33.333 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:33.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:33.366 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:37:33.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:33 smithi070 ceph-mon[192652]: pgmap v1273: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 58 op/s; 75990/252918 objects degraded (30.045%); 30667/252918 objects misplaced (12.125%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T22:37:33.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:33 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2442917913' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:33.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:33 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:33.367 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 75990/252918 objects degraded (30.045%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:33.439 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:33.495 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:33.707 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:33 smithi026 ceph-mon[223014]: pgmap v1273: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 58 op/s; 75990/252918 objects degraded (30.045%); 30667/252918 objects misplaced (12.125%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T22:37:33.708 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:33 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2442917913' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:33.708 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:33 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:33.708 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 75990/252918 objects degraded (30.045%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:33.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:33 smithi026 ceph-mon[215775]: pgmap v1273: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 58 op/s; 75990/252918 objects degraded (30.045%); 30667/252918 objects misplaced (12.125%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T22:37:33.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:33 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2442917913' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:33.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:33 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:33.708 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 75990/252918 objects degraded (30.045%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:33.914 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:33.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:33.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:34.018 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:34.044 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:34.087 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:34.088 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:34.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:34.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:34.198 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:34.234 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:34.401 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:34.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:34.505 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:34.531 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:34.577 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:34.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:34.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:34.604 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[223014]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[223014]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:37:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[223014]: osdmap e493: 8 total, 7 up, 8 in 2023-09-23T22:37:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[223014]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.605 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[223014]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:34.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[215775]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.605 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[215775]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:37:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[215775]: osdmap e493: 8 total, 7 up, 8 in 2023-09-23T22:37:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[215775]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:34 smithi026 ceph-mon[215775]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:34 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:34.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:34 smithi070 ceph-mon[192652]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:34 smithi070 ceph-mon[192652]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:34 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:37:34.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:34 smithi070 ceph-mon[192652]: osdmap e493: 8 total, 7 up, 8 in 2023-09-23T22:37:34.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:34 smithi070 ceph-mon[192652]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:34 smithi070 ceph-mon[192652]: failed to encode map e493 with expected crc 2023-09-23T22:37:34.683 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:34.742 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:34.979 INFO:teuthology.orchestra.run.smithi026.stderr:no valid command found; 10 closest matches: 2023-09-23T22:37:34.979 INFO:teuthology.orchestra.run.smithi026.stderr:0 2023-09-23T22:37:34.979 INFO:teuthology.orchestra.run.smithi026.stderr:1 2023-09-23T22:37:34.979 INFO:teuthology.orchestra.run.smithi026.stderr:2 2023-09-23T22:37:34.979 INFO:teuthology.orchestra.run.smithi026.stderr:abort 2023-09-23T22:37:34.979 INFO:teuthology.orchestra.run.smithi026.stderr:assert 2023-09-23T22:37:34.979 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs debug_inject_read_zeros 2023-09-23T22:37:34.979 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs files list 2023-09-23T22:37:34.979 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs stats 2023-09-23T22:37:34.980 INFO:teuthology.orchestra.run.smithi026.stderr:bluestore bluefs device info [] 2023-09-23T22:37:34.980 INFO:teuthology.orchestra.run.smithi026.stderr:config diff 2023-09-23T22:37:34.980 INFO:teuthology.orchestra.run.smithi026.stderr:admin_socket: invalid command 2023-09-23T22:37:35.037 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:37:35.038 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:37:35.063 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:37:35.097 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:37:35.120 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:37:35.125 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:35.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:35.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:35.146 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:37:35.176 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:37:35.176 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:37:35.176 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T22:37:35.176 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:37:35.177 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:37:35.177 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:37:35.177 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:37:35.177 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:37:35.177 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:37:35.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:35.286 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:35.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:35 smithi070 ceph-mon[192652]: pgmap v1275: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 75845/252597 objects degraded (30.026%); 30603/252597 objects misplaced (12.115%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:37:35.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3708138767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:35.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:35 smithi026 ceph-mon[223014]: pgmap v1275: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 75845/252597 objects degraded (30.026%); 30603/252597 objects misplaced (12.115%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:37:35.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3708138767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:35 smithi026 ceph-mon[215775]: pgmap v1275: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 77 op/s; 75845/252597 objects degraded (30.026%); 30603/252597 objects misplaced (12.115%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:37:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3708138767' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:36.349 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:37:36.668 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:37:36.668 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:37:36.671 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T22:37:36.671 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2023-09-23T22:37:36.804 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:36.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:36.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:36.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:36.948 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:37.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:37.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:37.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:37.137 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:37.144 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:37.162 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:37.204 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:37.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:37.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:37.309 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:37.336 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:37.378 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:37.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:37.382 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:37.482 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:37.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:37 smithi026 ceph-mon[215775]: pgmap v1276: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 86 op/s; 75684/251913 objects degraded (30.044%); 30517/251913 objects misplaced (12.114%); 5.5 MiB/s, 89 objects/s recovering 2023-09-23T22:37:37.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:37 smithi026 ceph-mon[223014]: pgmap v1276: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 86 op/s; 75684/251913 objects degraded (30.044%); 30517/251913 objects misplaced (12.114%); 5.5 MiB/s, 89 objects/s recovering 2023-09-23T22:37:37.505 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:37 smithi070 ceph-mon[192652]: pgmap v1276: 109 pgs: 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 86 op/s; 75684/251913 objects degraded (30.044%); 30517/251913 objects misplaced (12.114%); 5.5 MiB/s, 89 objects/s recovering 2023-09-23T22:37:37.507 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:37.549 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:37.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:37.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:37.655 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:37.682 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:37.726 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:37.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:37.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:37.831 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:37.873 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:37.910 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:37:37.910 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:37:37.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:37:37.910 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:37:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114854 7.a 7:514305bc:::benchmark_data_smithi026_410749_object24517:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e305)", 2023-09-23T22:37:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:52.042477+0000", 2023-09-23T22:37:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 585.86338326700002, 2023-09-23T22:37:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.374932499, 2023-09-23T22:37:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:37:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:37.911 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114854 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042477+0000", 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042477+0000", 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042475+0000", 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042482+0000", 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6210000000000004e-06 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042496+0000", 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3213e-05 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042508+0000", 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2310000000000001e-05 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043832+0000", 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013236859999999999 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043845+0000", 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3722e-05 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.392953+0000", 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3491075079999999 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:37.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393122+0000", 2023-09-23T22:37:37.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016948199999999999 2023-09-23T22:37:37.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.417296+0000", 2023-09-23T22:37:37.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024174047000000001 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.417409+0000", 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112722 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.919 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:114867 7.a 7:5bbaaced:::benchmark_data_smithi026_410749_object24530:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e305)", 2023-09-23T22:37:37.919 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:27:52.042515+0000", 2023-09-23T22:37:37.919 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 585.86334475700005, 2023-09-23T22:37:37.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3830621110000001, 2023-09-23T22:37:37.919 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:37.919 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:37.919 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:37.919 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:37:37.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:37:37.920 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 114867 2023-09-23T22:37:37.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.920 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:37.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:37.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042515+0000", 2023-09-23T22:37:37.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:37.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042515+0000", 2023-09-23T22:37:37.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:37.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042514+0000", 2023-09-23T22:37:37.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:37:37.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:37.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042518+0000", 2023-09-23T22:37:37.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0249999999999998e-06 2023-09-23T22:37:37.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:37.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042525+0000", 2023-09-23T22:37:37.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1919999999999999e-06 2023-09-23T22:37:37.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:37.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.042531+0000", 2023-09-23T22:37:37.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8989999999999998e-06 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043871+0000", 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00134023 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:52.043879+0000", 2023-09-23T22:37:37.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7200000000000006e-06 2023-09-23T22:37:37.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393153+0000", 2023-09-23T22:37:37.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3492744459999999 2023-09-23T22:37:37.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:37.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393299+0000", 2023-09-23T22:37:37.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145838 2023-09-23T22:37:37.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,5", 2023-09-23T22:37:37.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.393456+0000", 2023-09-23T22:37:37.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000156928 2023-09-23T22:37:37.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.400332+0000", 2023-09-23T22:37:37.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068758229999999997 2023-09-23T22:37:37.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:37.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.400343+0000", 2023-09-23T22:37:37.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0963e-05 2023-09-23T22:37:37.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.417452+0000", 2023-09-23T22:37:37.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017109017000000001 2023-09-23T22:37:37.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421935+0000", 2023-09-23T22:37:37.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0044831059999999997 2023-09-23T22:37:37.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.421995+0000", 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9889000000000002e-05 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:27:53.425577+0000", 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035825710000000001 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.930 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104714 9.7 9:e7839733:::benchmark_data_smithi026_555954_object9985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.341845+0000", 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.564015069, 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3063670860000001, 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:37.931 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104714 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.341845+0000", 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.341845+0000", 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.341843+0000", 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:37.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.341853+0000", 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8290000000000003e-06 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.341867+0000", 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3929e-05 2023-09-23T22:37:37.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.341874+0000", 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2869999999999999e-06 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.341930+0000", 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6527000000000003e-05 2023-09-23T22:37:37.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.341942+0000", 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1394e-05 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.400783+0000", 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0588411419999999 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.408410+0000", 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076270449999999998 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.471288+0000", 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.062877998000000004 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.488497+0000", 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017208719000000001 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.549533+0000", 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.061036864000000003 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.555444+0000", 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059103209999999996 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.567353+0000", 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011909515000000001 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.607916+0000", 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.040563080000000001 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.609839+0000", 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001922644 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.613128+0000", 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032887960000000001 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.620770+0000", 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076419679999999999 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.648212+0000", 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027441977999999999 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104718 9.7 9:edbf8c0c:::benchmark_data_smithi026_555954_object9989:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.352762+0000", 2023-09-23T22:37:37.942 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.553097478000002, 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.793293687, 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104718 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:37.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.352762+0000", 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.352762+0000", 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.352760+0000", 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.352770+0000", 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6309999999999998e-06 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.352790+0000", 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0035999999999999e-05 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.352799+0000", 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6629999999999999e-06 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.352851+0000", 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2073000000000001e-05 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.352859+0000", 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0700000000000007e-06 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.501735+0000", 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1488759390000001 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.520975+0000", 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019240208000000002 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.649432+0000", 2023-09-23T22:37:37.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.128456824 2023-09-23T22:37:37.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:37.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.663478+0000", 2023-09-23T22:37:37.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014046018 2023-09-23T22:37:37.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.941641+0000", 2023-09-23T22:37:37.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.27816311999999999 2023-09-23T22:37:37.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:37.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.957010+0000", 2023-09-23T22:37:37.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015369534000000001 2023-09-23T22:37:37.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:37.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.991699+0000", 2023-09-23T22:37:37.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034688973999999997 2023-09-23T22:37:37.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:37.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.106863+0000", 2023-09-23T22:37:37.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.11516351599999999 2023-09-23T22:37:37.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.109945+0000", 2023-09-23T22:37:37.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030818669999999999 2023-09-23T22:37:37.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.112442+0000", 2023-09-23T22:37:37.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024971120000000001 2023-09-23T22:37:37.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:37.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.117778+0000", 2023-09-23T22:37:37.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053363660000000004 2023-09-23T22:37:37.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.146056+0000", 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028277767999999998 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.956 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104750 9.7 9:e97fb605:::benchmark_data_smithi026_555954_object10021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:16.536635+0000", 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 21.369224839000001, 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.65138935499999995, 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104750 2023-09-23T22:37:37.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.536635+0000", 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.536635+0000", 2023-09-23T22:37:37.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:37.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.536633+0000", 2023-09-23T22:37:37.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:37:37.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:37.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.536646+0000", 2023-09-23T22:37:37.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2826000000000001e-05 2023-09-23T22:37:37.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:37.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.536661+0000", 2023-09-23T22:37:37.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5610000000000001e-05 2023-09-23T22:37:37.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:37.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.536671+0000", 2023-09-23T22:37:37.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8649999999999996e-06 2023-09-23T22:37:37.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.664927+0000", 2023-09-23T22:37:37.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.128255542 2023-09-23T22:37:37.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:37.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.676599+0000", 2023-09-23T22:37:37.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011672442999999999 2023-09-23T22:37:37.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.738870+0000", 2023-09-23T22:37:37.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.062270699999999998 2023-09-23T22:37:37.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:37.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.752752+0000", 2023-09-23T22:37:37.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013882093 2023-09-23T22:37:37.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.050821+0000", 2023-09-23T22:37:37.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.29806934499999999 2023-09-23T22:37:37.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:37.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.066233+0000", 2023-09-23T22:37:37.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015411681999999999 2023-09-23T22:37:37.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:37.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.091771+0000", 2023-09-23T22:37:37.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025538291000000001 2023-09-23T22:37:37.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:37.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.147677+0000", 2023-09-23T22:37:37.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.055905539999999997 2023-09-23T22:37:37.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.152229+0000", 2023-09-23T22:37:37.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045520760000000004 2023-09-23T22:37:37.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.157211+0000", 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049819230000000001 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.163527+0000", 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063163070000000002 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.188024+0000", 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024497260999999999 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104753 9.7 9:ec61e548:::benchmark_data_smithi026_555954_object10024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:16.543191+0000", 2023-09-23T22:37:37.969 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 21.362668944999999, 2023-09-23T22:37:37.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80967601300000003, 2023-09-23T22:37:37.970 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:37.970 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:37.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:37.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:37.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:37.970 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104753 2023-09-23T22:37:37.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.970 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543191+0000", 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543191+0000", 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543189+0000", 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543197+0000", 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3409999999999997e-06 2023-09-23T22:37:37.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543207+0000", 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0584000000000001e-05 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543214+0000", 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8480000000000003e-06 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.680429+0000", 2023-09-23T22:37:37.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13721514200000001 2023-09-23T22:37:37.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:37.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.689187+0000", 2023-09-23T22:37:37.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087579400000000005 2023-09-23T22:37:37.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.810188+0000", 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.12100058399999999 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.823054+0000", 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012866591 2023-09-23T22:37:37.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.229389+0000", 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.406334578 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.239986+0000", 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010596698999999999 2023-09-23T22:37:37.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.261060+0000", 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021074219000000002 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.325845+0000", 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.064785512000000003 2023-09-23T22:37:37.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.329351+0000", 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003505953 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.331897+0000", 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025458289999999999 2023-09-23T22:37:37.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.337270+0000", 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053730260000000004 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.352867+0000", 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015596601999999999 2023-09-23T22:37:37.979 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104754 9.7 9:e0c383ff:::benchmark_data_smithi026_555954_object10025:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:16.543986+0000", 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 21.361873890999998, 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94067334400000002, 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:37.980 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104754 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543986+0000", 2023-09-23T22:37:37.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543986+0000", 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543985+0000", 2023-09-23T22:37:37.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.543990+0000", 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6800000000000001e-06 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.544001+0000", 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1201e-05 2023-09-23T22:37:37.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.544009+0000", 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4800000000000004e-06 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.823986+0000", 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.27997792300000002 2023-09-23T22:37:37.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.833319+0000", 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093326120000000005 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.879104+0000", 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045785025 2023-09-23T22:37:37.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.899257+0000", 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020152657000000001 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.391108+0000", 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.49185119100000002 2023-09-23T22:37:37.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.407198+0000", 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016090046 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.441019+0000", 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033821478000000002 2023-09-23T22:37:37.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.462931+0000", 2023-09-23T22:37:37.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021911166999999999 2023-09-23T22:37:37.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:37.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.465168+0000", 2023-09-23T22:37:37.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002236947 2023-09-23T22:37:37.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:37.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.466033+0000", 2023-09-23T22:37:37.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00086573100000000001 2023-09-23T22:37:37.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:37.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.471760+0000", 2023-09-23T22:37:37.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005726999 2023-09-23T22:37:37.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:37.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.484659+0000", 2023-09-23T22:37:37.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012898895 2023-09-23T22:37:37.990 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.991 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:37.991 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:37.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.991 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104761 9.7 9:e3393b39:::benchmark_data_smithi026_555954_object10032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:37.991 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.119462+0000", 2023-09-23T22:37:37.991 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 20.786397544, 2023-09-23T22:37:37.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.46030439299999998, 2023-09-23T22:37:37.992 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:37.992 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:37.992 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:37.992 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:37.992 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:37.992 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104761 2023-09-23T22:37:37.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.992 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:37.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:37.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.119462+0000", 2023-09-23T22:37:37.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:37.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.119462+0000", 2023-09-23T22:37:37.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:37.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:37.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.119460+0000", 2023-09-23T22:37:37.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:37:37.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:37.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.119474+0000", 2023-09-23T22:37:37.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3533000000000001e-05 2023-09-23T22:37:37.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:37.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.119491+0000", 2023-09-23T22:37:37.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7615e-05 2023-09-23T22:37:37.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:37.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.119505+0000", 2023-09-23T22:37:37.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3574000000000001e-05 2023-09-23T22:37:37.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.189265+0000", 2023-09-23T22:37:37.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.069760131000000003 2023-09-23T22:37:37.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:37.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.199593+0000", 2023-09-23T22:37:37.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010328407 2023-09-23T22:37:37.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:37.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.313318+0000", 2023-09-23T22:37:37.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.113724672 2023-09-23T22:37:37.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:37.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.325023+0000", 2023-09-23T22:37:37.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011704777 2023-09-23T22:37:37.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:37.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:37.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.512158+0000", 2023-09-23T22:37:38.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.18713526999999999 2023-09-23T22:37:38.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.519419+0000", 2023-09-23T22:37:38.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072607519999999997 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.532254+0000", 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012835087 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.546959+0000", 2023-09-23T22:37:38.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014705621 2023-09-23T22:37:38.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.548976+0000", 2023-09-23T22:37:38.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020168500000000002 2023-09-23T22:37:38.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.549594+0000", 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00061773700000000004 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.553221+0000", 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036267489999999999 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.579767+0000", 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026545816999999999 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104810 9.7 9:e5b67d4d:::benchmark_data_smithi026_555954_object10081:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:17.553397+0000", 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 20.352462418999998, 2023-09-23T22:37:38.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.29403384599999999, 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104810 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.553397+0000", 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.553397+0000", 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.553396+0000", 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.553403+0000", 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5180000000000001e-06 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.553416+0000", 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3044000000000001e-05 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.553424+0000", 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9349999999999994e-06 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.581739+0000", 2023-09-23T22:37:38.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028315257 2023-09-23T22:37:38.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.601173+0000", 2023-09-23T22:37:38.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019434607999999999 2023-09-23T22:37:38.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.654373+0000", 2023-09-23T22:37:38.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.053199634000000003 2023-09-23T22:37:38.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.667916+0000", 2023-09-23T22:37:38.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013543289999999999 2023-09-23T22:37:38.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.746875+0000", 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.078958739 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.754928+0000", 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080526480000000008 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.770445+0000", 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015517311000000001 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.822196+0000", 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.051750649000000003 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.825774+0000", 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035782909999999999 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.829507+0000", 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037328169999999998 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.834412+0000", 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049047520000000001 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.847431+0000", 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01301968 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.014 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104890 9.7 9:e7ecee4a:::benchmark_data_smithi026_555954_object10161:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.000197+0000", 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 19.905662418999999, 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.40980203900000001, 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.015 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104890 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.000197+0000", 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.000197+0000", 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.000196+0000", 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.000202+0000", 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5509999999999996e-06 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.000215+0000", 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2581e-05 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.000222+0000", 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3719999999999999e-06 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.095543+0000", 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.095320364000000005 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.105552+0000", 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010009153999999999 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.176315+0000", 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.070762810999999995 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.183276+0000", 2023-09-23T22:37:38.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069616690000000002 2023-09-23T22:37:38.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.327074+0000", 2023-09-23T22:37:38.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.14379757700000001 2023-09-23T22:37:38.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.332390+0000", 2023-09-23T22:37:38.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053155959999999997 2023-09-23T22:37:38.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.343642+0000", 2023-09-23T22:37:38.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011252228 2023-09-23T22:37:38.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.361153+0000", 2023-09-23T22:37:38.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017511183 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.363207+0000", 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020536769999999998 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.393053+0000", 2023-09-23T22:37:38.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.029846050999999998 2023-09-23T22:37:38.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.396077+0000", 2023-09-23T22:37:38.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00302433 2023-09-23T22:37:38.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.409999+0000", 2023-09-23T22:37:38.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01392241 2023-09-23T22:37:38.026 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.026 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.026 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.026 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104904 9.7 9:e65afb3a:::benchmark_data_smithi026_555954_object10175:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:38.026 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.119133+0000", 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 19.786726757, 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.44483745000000002, 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104904 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.027 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.119133+0000", 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.119133+0000", 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.119131+0000", 2023-09-23T22:37:38.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:38.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.119140+0000", 2023-09-23T22:37:38.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3000000000000007e-06 2023-09-23T22:37:38.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.119157+0000", 2023-09-23T22:37:38.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6540000000000001e-05 2023-09-23T22:37:38.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.119167+0000", 2023-09-23T22:37:38.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0355e-05 2023-09-23T22:37:38.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.215937+0000", 2023-09-23T22:37:38.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.096769187000000007 2023-09-23T22:37:38.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.225908+0000", 2023-09-23T22:37:38.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099718340000000006 2023-09-23T22:37:38.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.253110+0000", 2023-09-23T22:37:38.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027201346000000001 2023-09-23T22:37:38.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.261781+0000", 2023-09-23T22:37:38.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086716799999999993 2023-09-23T22:37:38.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.458078+0000", 2023-09-23T22:37:38.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.19629637699999999 2023-09-23T22:37:38.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.466090+0000", 2023-09-23T22:37:38.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080124879999999999 2023-09-23T22:37:38.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.481749+0000", 2023-09-23T22:37:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015658598999999999 2023-09-23T22:37:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.528726+0000", 2023-09-23T22:37:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.046977181999999999 2023-09-23T22:37:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.534049+0000", 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053225770000000002 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.544748+0000", 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010699166 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.548157+0000", 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0034092469999999998 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.563970+0000", 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015813443999999999 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.038 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104917 9.7 9:ed792beb:::benchmark_data_smithi026_555954_object10188:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.262211+0000", 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 19.643648448, 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.43655643900000002, 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.039 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104917 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.262211+0000", 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.262211+0000", 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.262210+0000", 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.262217+0000", 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4619999999999999e-06 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.262231+0000", 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4284000000000001e-05 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.262240+0000", 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4840000000000004e-06 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.352508+0000", 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.090268227000000006 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.359217+0000", 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006709235 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.385456+0000", 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026238686000000001 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.392438+0000", 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006982416 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.637032+0000", 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.24459383300000001 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.647663+0000", 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010630282 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.658854+0000", 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011191942999999999 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.674224+0000", 2023-09-23T22:37:38.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015369638 2023-09-23T22:37:38.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.677727+0000", 2023-09-23T22:37:38.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035030880000000001 2023-09-23T22:37:38.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.678671+0000", 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00094390800000000003 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.683010+0000", 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0043388089999999999 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.698768+0000", 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015757845999999999 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104927 9.7 9:eed16836:::benchmark_data_smithi026_555954_object10198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e491)", 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.321084+0000", 2023-09-23T22:37:38.049 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 19.584775438000001, 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.51025583200000002, 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104927 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.321084+0000", 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.321084+0000", 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.321083+0000", 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.321089+0000", 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7909999999999999e-06 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.321101+0000", 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1328000000000001e-05 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.321107+0000", 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2929999999999997e-06 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.410649+0000", 2023-09-23T22:37:38.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.08954173 2023-09-23T22:37:38.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.418093+0000", 2023-09-23T22:37:38.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074438760000000003 2023-09-23T22:37:38.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.514548+0000", 2023-09-23T22:37:38.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.096455336000000003 2023-09-23T22:37:38.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.525485+0000", 2023-09-23T22:37:38.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010937359000000001 2023-09-23T22:37:38.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.734027+0000", 2023-09-23T22:37:38.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20854150699999999 2023-09-23T22:37:38.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.743487+0000", 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094602750000000006 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.758598+0000", 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015110646 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.810567+0000", 2023-09-23T22:37:38.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.051969499000000002 2023-09-23T22:37:38.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.813595+0000", 2023-09-23T22:37:38.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030275860000000001 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.814446+0000", 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00085088199999999996 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.818495+0000", 2023-09-23T22:37:38.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004049034 2023-09-23T22:37:38.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.831340+0000", 2023-09-23T22:37:38.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012845389 2023-09-23T22:37:38.060 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.060 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.060 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104934 9.7 9:e2391469:::benchmark_data_smithi026_555954_object10205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e491)", 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:18.430193+0000", 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 19.475667262999998, 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.42387689499999998, 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.061 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.062 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.062 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104934 2023-09-23T22:37:38.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.062 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.430193+0000", 2023-09-23T22:37:38.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.430193+0000", 2023-09-23T22:37:38.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.430190+0000", 2023-09-23T22:37:38.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:37:38.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.430201+0000", 2023-09-23T22:37:38.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0749e-05 2023-09-23T22:37:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.430221+0000", 2023-09-23T22:37:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0434000000000001e-05 2023-09-23T22:37:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.430233+0000", 2023-09-23T22:37:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1564e-05 2023-09-23T22:37:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.535340+0000", 2023-09-23T22:37:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.105106543 2023-09-23T22:37:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.544045+0000", 2023-09-23T22:37:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087054039999999999 2023-09-23T22:37:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.624401+0000", 2023-09-23T22:37:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080355646000000003 2023-09-23T22:37:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.635497+0000", 2023-09-23T22:37:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011096028000000001 2023-09-23T22:37:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.780675+0000", 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.145177997 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.786424+0000", 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057493529999999996 2023-09-23T22:37:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.799303+0000", 2023-09-23T22:37:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012878588 2023-09-23T22:37:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.833781+0000", 2023-09-23T22:37:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034478172000000001 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.836109+0000", 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023284989999999999 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.836796+0000", 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00068623999999999996 2023-09-23T22:37:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.840370+0000", 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035740210000000001 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:18.854069+0000", 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013699855 2023-09-23T22:37:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:105653 9.7 9:e849b9bb:::benchmark_data_smithi026_555954_object10924:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T22:37:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:35.727087+0000", 2023-09-23T22:37:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.178772978, 2023-09-23T22:37:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.29800436299999999, 2023-09-23T22:37:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105653 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.727087+0000", 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.727087+0000", 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.727085+0000", 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.727094+0000", 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3899999999999999e-06 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.727109+0000", 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4561e-05 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.727129+0000", 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9891999999999999e-05 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.761078+0000", 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033949737000000001 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.771136+0000", 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010057626 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.852007+0000", 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.080871252000000005 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.859318+0000", 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073107909999999996 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.916087+0000", 2023-09-23T22:37:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056768950999999998 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.923023+0000", 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069355959999999996 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.936432+0000", 2023-09-23T22:37:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0134092 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.994987+0000", 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.058555268000000001 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.997513+0000", 2023-09-23T22:37:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025261379999999998 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:35.998329+0000", 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081554399999999997 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.001764+0000", 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003434837 2023-09-23T22:37:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.025091+0000", 2023-09-23T22:37:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023327549999999999 2023-09-23T22:37:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:105742 9.7 9:ecaae8fd:::benchmark_data_smithi026_555954_object11013:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:36.778150+0000", 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.127709879, 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.33216320199999999, 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105742 2023-09-23T22:37:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.778150+0000", 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.778150+0000", 2023-09-23T22:37:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.778148+0000", 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.778155+0000", 2023-09-23T22:37:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7490000000000001e-06 2023-09-23T22:37:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.778169+0000", 2023-09-23T22:37:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3952e-05 2023-09-23T22:37:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.778176+0000", 2023-09-23T22:37:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.255e-06 2023-09-23T22:37:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.779741+0000", 2023-09-23T22:37:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00156445 2023-09-23T22:37:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.800156+0000", 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020414730999999998 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.852526+0000", 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.052370275000000001 2023-09-23T22:37:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.860330+0000", 2023-09-23T22:37:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0078041990000000004 2023-09-23T22:37:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.951154+0000", 2023-09-23T22:37:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.090824447000000003 2023-09-23T22:37:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.969666+0000", 2023-09-23T22:37:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018511403999999999 2023-09-23T22:37:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.999697+0000", 2023-09-23T22:37:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030031248999999999 2023-09-23T22:37:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.084794+0000", 2023-09-23T22:37:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.085097031000000004 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.085804+0000", 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0010096899999999999 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.088866+0000", 2023-09-23T22:37:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030616760000000002 2023-09-23T22:37:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.094416+0000", 2023-09-23T22:37:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055501869999999998 2023-09-23T22:37:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.110313+0000", 2023-09-23T22:37:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015897403000000001 2023-09-23T22:37:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:105744 9.7 9:e76d6ff5:::benchmark_data_smithi026_555954_object11015:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T22:37:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:36.800313+0000", 2023-09-23T22:37:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.105546428, 2023-09-23T22:37:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.34851991100000002, 2023-09-23T22:37:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105744 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.800313+0000", 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.800313+0000", 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.800311+0000", 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.800320+0000", 2023-09-23T22:37:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2430000000000005e-06 2023-09-23T22:37:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.800333+0000", 2023-09-23T22:37:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3529e-05 2023-09-23T22:37:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.800342+0000", 2023-09-23T22:37:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1770000000000004e-06 2023-09-23T22:37:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.802352+0000", 2023-09-23T22:37:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002009683 2023-09-23T22:37:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.820612+0000", 2023-09-23T22:37:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018259758000000001 2023-09-23T22:37:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.894039+0000", 2023-09-23T22:37:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.073427100999999995 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.907687+0000", 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013648508 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.050930+0000", 2023-09-23T22:37:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.14324308399999999 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.058900+0000", 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079693760000000002 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.071993+0000", 2023-09-23T22:37:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013093254 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.112975+0000", 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.040982034000000001 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.115512+0000", 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025367319999999999 2023-09-23T22:37:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.116702+0000", 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001190394 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.123851+0000", 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071491339999999997 2023-09-23T22:37:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.148833+0000", 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024981873000000002 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:105745 9.7 9:e4a08971:::benchmark_data_smithi026_555954_object11016:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:36.979391+0000", 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.92646857299999996, 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.57914730999999997, 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105745 2023-09-23T22:37:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.979391+0000", 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.979391+0000", 2023-09-23T22:37:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.979389+0000", 2023-09-23T22:37:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:37:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.979399+0000", 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.041e-05 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.979414+0000", 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4983e-05 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:36.979427+0000", 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3225e-05 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.150871+0000", 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.17144404599999999 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.167734+0000", 2023-09-23T22:37:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016862826000000001 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.237243+0000", 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.069508818 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.245000+0000", 2023-09-23T22:37:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077568009999999998 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.427842+0000", 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.182841896 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.433934+0000", 2023-09-23T22:37:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060917560000000003 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.445298+0000", 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011364703 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.527138+0000", 2023-09-23T22:37:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.081839978999999993 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.531211+0000", 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040726089999999996 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.532504+0000", 2023-09-23T22:37:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012933549999999999 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.538966+0000", 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006461301 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.558539+0000", 2023-09-23T22:37:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019572995999999999 2023-09-23T22:37:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:105747 9.7 9:edfdccb4:::benchmark_data_smithi026_555954_object11018:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T22:37:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:37.098984+0000", 2023-09-23T22:37:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.80687582800000002, 2023-09-23T22:37:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.50629956499999995, 2023-09-23T22:37:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105747 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.098984+0000", 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.098984+0000", 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.098982+0000", 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.098991+0000", 2023-09-23T22:37:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8880000000000003e-06 2023-09-23T22:37:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.099007+0000", 2023-09-23T22:37:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5661000000000001e-05 2023-09-23T22:37:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.099017+0000", 2023-09-23T22:37:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0115000000000001e-05 2023-09-23T22:37:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.168891+0000", 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.069874169 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.180280+0000", 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011388561 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.300055+0000", 2023-09-23T22:37:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.119775622 2023-09-23T22:37:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.311276+0000", 2023-09-23T22:37:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011221088000000001 2023-09-23T22:37:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.488650+0000", 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.177373155 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.497518+0000", 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088685740000000006 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.515430+0000", 2023-09-23T22:37:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017911667999999999 2023-09-23T22:37:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.564108+0000", 2023-09-23T22:37:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.048678626000000003 2023-09-23T22:37:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.569214+0000", 2023-09-23T22:37:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051055400000000004 2023-09-23T22:37:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.570768+0000", 2023-09-23T22:37:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015542080000000001 2023-09-23T22:37:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.576869+0000", 2023-09-23T22:37:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0061011069999999997 2023-09-23T22:37:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.605284+0000", 2023-09-23T22:37:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.028414267999999999 2023-09-23T22:37:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:105750 9.7 9:e504bf03:::benchmark_data_smithi026_555954_object11021:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e493)", 2023-09-23T22:37:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:37.125278+0000", 2023-09-23T22:37:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.78058142699999999, 2023-09-23T22:37:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.580417925, 2023-09-23T22:37:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:37:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:37:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:37:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:37:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:37:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 105750 2023-09-23T22:37:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:37:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:37:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.125278+0000", 2023-09-23T22:37:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:37:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.125278+0000", 2023-09-23T22:37:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:37:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:37:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.125276+0000", 2023-09-23T22:37:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:37:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:37:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.125286+0000", 2023-09-23T22:37:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7019999999999996e-06 2023-09-23T22:37:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:37:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.125307+0000", 2023-09-23T22:37:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1143000000000001e-05 2023-09-23T22:37:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.125318+0000", 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1168999999999999e-05 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.181819+0000", 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.056500492999999999 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.196521+0000", 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014702722 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.355991+0000", 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.159469746 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for rw locks", 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.364243+0000", 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082516250000000003 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.629573+0000", 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26533003500000002 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:37:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.635936+0000", 2023-09-23T22:37:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063627859999999996 2023-09-23T22:37:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:37:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.652763+0000", 2023-09-23T22:37:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016827891000000001 2023-09-23T22:37:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.680190+0000", 2023-09-23T22:37:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.027426769 2023-09-23T22:37:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:37:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.683220+0000", 2023-09-23T22:37:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030295819999999998 2023-09-23T22:37:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.683904+0000", 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00068399800000000001 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.686923+0000", 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030196810000000002 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:37:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:37:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:37:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:37.705696+0000", 2023-09-23T22:37:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018772825 2023-09-23T22:37:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:37:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:37:38.142 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:37:38.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:38.143 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:38.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:38.244 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:38.274 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:38.324 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:38.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:38.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:38.381 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 75684/251913 objects degraded (30.044%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:38.430 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:38.468 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:38.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 75684/251913 objects degraded (30.044%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:38.669 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:37:38.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 75684/251913 objects degraded (30.044%), 22 pgs degraded (PG_DEGRADED) 2023-09-23T22:37:39.149 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:39.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:39.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:39.249 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:37:39.255 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:39.302 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:39.383 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:39.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:39.490 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:39.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:39 smithi026 ceph-mon[215775]: pgmap v1277: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 69 op/s; 75532/251508 objects degraded (30.032%); 30446/251508 objects misplaced (12.105%); 5.8 MiB/s, 95 objects/s recovering 2023-09-23T22:37:39.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:39.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:39.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:39 smithi026 ceph-mon[223014]: pgmap v1277: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 69 op/s; 75532/251508 objects degraded (30.032%); 30446/251508 objects misplaced (12.105%); 5.8 MiB/s, 95 objects/s recovering 2023-09-23T22:37:39.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:39.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:39.530 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:39.561 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:39.601 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:39.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:39.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:39.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:39 smithi070 ceph-mon[192652]: pgmap v1277: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 69 op/s; 75532/251508 objects degraded (30.032%); 30446/251508 objects misplaced (12.105%); 5.8 MiB/s, 95 objects/s recovering 2023-09-23T22:37:39.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:39.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:39.705 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:37:39.706 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:39.741 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:39.825 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:39 smithi026 conmon[584057]: 2023-09-23T22:37:39.631+0000 7f971cf30640 -1 osd.1 487 log_to_monitors true 2023-09-23T22:37:39.829 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:37:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:37:39] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.43.0" 2023-09-23T22:37:39.852 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:39.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:39.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:39.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:39.992 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:40.047 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:40.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:40.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:40.150 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:40.184 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:40.237 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:40.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:40.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:40.343 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:40.369 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:40.410 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:37:40.424 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:40 smithi026 ceph-mon[223014]: from='osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:37:40.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1484896485' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:40.425 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:40 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:40 smithi026 ceph-mon[215775]: from='osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:37:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1484896485' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:40.425 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:40 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:40 smithi070 ceph-mon[192652]: from='osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T22:37:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1484896485' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:40.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:40 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:37:40.735 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:37:40 smithi026 conmon[584057]: 2023-09-23T22:37:40.423+0000 7f970d39b700 -1 osd.1 487 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:37:40.803 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:40.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:40.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:40.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:40.935 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:40.981 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:40.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:41.085 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:41.114 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:41.672 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:37:41.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:41 smithi070 ceph-mon[192652]: pgmap v1278: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 69 op/s; 75532/251508 objects degraded (30.032%); 30446/251508 objects misplaced (12.105%); 5.8 MiB/s, 95 objects/s recovering 2023-09-23T22:37:41.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:41 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:41.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:41 smithi070 ceph-mon[192652]: failed to encode map e494 with expected crc 2023-09-23T22:37:41.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:41 smithi070 ceph-mon[192652]: failed to encode map e494 with expected crc 2023-09-23T22:37:41.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:41 smithi070 ceph-mon[192652]: from='osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:37:41.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:41 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:37:41.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:41 smithi070 ceph-mon[192652]: osdmap e494: 8 total, 7 up, 8 in 2023-09-23T22:37:41.693 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:41 smithi070 ceph-mon[192652]: from='osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:37:41.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[223014]: pgmap v1278: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 69 op/s; 75532/251508 objects degraded (30.032%); 30446/251508 objects misplaced (12.105%); 5.8 MiB/s, 95 objects/s recovering 2023-09-23T22:37:41.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:41.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[223014]: failed to encode map e494 with expected crc 2023-09-23T22:37:41.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[223014]: failed to encode map e494 with expected crc 2023-09-23T22:37:41.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[223014]: from='osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:37:41.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:37:41.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[223014]: osdmap e494: 8 total, 7 up, 8 in 2023-09-23T22:37:41.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[223014]: from='osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:37:41.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[215775]: pgmap v1278: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 69 op/s; 75532/251508 objects degraded (30.032%); 30446/251508 objects misplaced (12.105%); 5.8 MiB/s, 95 objects/s recovering 2023-09-23T22:37:41.704 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:41.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[215775]: failed to encode map e494 with expected crc 2023-09-23T22:37:41.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[215775]: failed to encode map e494 with expected crc 2023-09-23T22:37:41.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[215775]: from='osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T22:37:41.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:37:41.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[215775]: osdmap e494: 8 total, 7 up, 8 in 2023-09-23T22:37:41.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:41 smithi026 ceph-mon[215775]: from='osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:37:41.712 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:41.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:41.725 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:37:41.744 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:37:41.765 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:37:41.817 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:41.846 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:41.902 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:41.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:41.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:42.006 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:42.037 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:42.074 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:42.090 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:42.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:42.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:42.194 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:42.222 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:42.277 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:42.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:42.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:42.382 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:42.423 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:42.476 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:42.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:42.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:42.580 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:42.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[223014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:37:42.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[223014]: failed to encode map e495 with expected crc 2023-09-23T22:37:42.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[223014]: osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431] boot 2023-09-23T22:37:42.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[223014]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T22:37:42.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:37:42.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[223014]: pgmap v1281: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 75392/250947 objects degraded (30.043%); 30389/250947 objects misplaced (12.110%); 3.5 MiB/s, 81 objects/s recovering 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4070435134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[223014]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[215775]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[215775]: failed to encode map e495 with expected crc 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[215775]: osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431] boot 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[215775]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[215775]: pgmap v1281: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 75392/250947 objects degraded (30.043%); 30389/250947 objects misplaced (12.110%); 3.5 MiB/s, 81 objects/s recovering 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4070435134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:42.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:42 smithi026 ceph-mon[215775]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T22:37:42.607 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:42.651 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:42.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:42.755 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:42.779 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:42 smithi070 ceph-mon[192652]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:37:42.779 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:42 smithi070 ceph-mon[192652]: failed to encode map e495 with expected crc 2023-09-23T22:37:42.779 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:42 smithi070 ceph-mon[192652]: osd.1 [v2:172.21.15.26:6810/1220464431,v1:172.21.15.26:6811/1220464431] boot 2023-09-23T22:37:42.779 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:42 smithi070 ceph-mon[192652]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T22:37:42.779 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:42 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T22:37:42.779 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:42 smithi070 ceph-mon[192652]: pgmap v1281: 109 pgs: 1 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 34 active+undersized, 11 active+undersized+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 52 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 68 op/s; 75392/250947 objects degraded (30.043%); 30389/250947 objects misplaced (12.110%); 3.5 MiB/s, 81 objects/s recovering 2023-09-23T22:37:42.780 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4070435134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:42.780 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:42 smithi070 ceph-mon[192652]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T22:37:42.781 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:42.823 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:42.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:42.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:42.926 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:42.961 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:43.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:43.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:43.401 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:37:43.446 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:43.479 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:43.529 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:43.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:43.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:43.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:43.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: 1.5 scrub starts 2023-09-23T22:37:43.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.672 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.672 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.672 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 75392/250947 objects degraded (30.043%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:43.672 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[215775]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T22:37:43.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: 1.5 scrub starts 2023-09-23T22:37:43.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 75392/250947 objects degraded (30.043%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:43.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:43 smithi026 ceph-mon[223014]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T22:37:43.690 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: 1.5 scrub starts 2023-09-23T22:37:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: failed to encode map e496 with expected crc 2023-09-23T22:37:43.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 75392/250947 objects degraded (30.043%), 22 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:43.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:43 smithi070 ceph-mon[192652]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T22:37:43.877 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:43.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:43.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:43.960 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:37:43.960 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:37:43.984 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:44.015 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:44.064 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:44.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:44.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:44.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:44.209 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:44.289 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:44.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:44.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:44.395 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:44.426 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:44.474 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:44.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:44.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:44.578 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:44.607 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:44.652 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:44.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:44.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:44.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: 9.4 continuing backfill to osd.3 from (461'1226,494'6674] MIN to 494'6674 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: 9.a continuing backfill to osd.3 from (460'800,494'6778] MIN to 494'6778 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: 9.2 continuing backfill to osd.3 from (459'600,494'6621] MIN to 494'6621 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: pgmap v1284: 109 pgs: 3 remapped+peering, 4 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 1 active+undersized+remapped, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 57 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 64 op/s; 46946/250737 objects degraded (18.723%); 19932/250737 objects misplaced (7.949%); 1008 KiB/s, 65 objects/s recovering 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[223014]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T22:37:44.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: 9.4 continuing backfill to osd.3 from (461'1226,494'6674] MIN to 494'6674 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: 9.a continuing backfill to osd.3 from (460'800,494'6778] MIN to 494'6778 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: 9.2 continuing backfill to osd.3 from (459'600,494'6621] MIN to 494'6621 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: pgmap v1284: 109 pgs: 3 remapped+peering, 4 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 1 active+undersized+remapped, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 57 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 64 op/s; 46946/250737 objects degraded (18.723%); 19932/250737 objects misplaced (7.949%); 1008 KiB/s, 65 objects/s recovering 2023-09-23T22:37:44.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:44 smithi026 ceph-mon[215775]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T22:37:44.756 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:44.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: 9.4 continuing backfill to osd.3 from (461'1226,494'6674] MIN to 494'6674 2023-09-23T22:37:44.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: 9.a continuing backfill to osd.3 from (460'800,494'6778] MIN to 494'6778 2023-09-23T22:37:44.781 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: 9.2 continuing backfill to osd.3 from (459'600,494'6621] MIN to 494'6621 2023-09-23T22:37:44.782 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: failed to encode map e497 with expected crc 2023-09-23T22:37:44.782 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: pgmap v1284: 109 pgs: 3 remapped+peering, 4 peering, 1 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 1 active+undersized+remapped, 9 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 2 active+recovering+degraded+remapped, 57 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 64 op/s; 46946/250737 objects degraded (18.723%); 19932/250737 objects misplaced (7.949%); 1008 KiB/s, 65 objects/s recovering 2023-09-23T22:37:44.782 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:44 smithi070 ceph-mon[192652]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T22:37:44.783 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:44.826 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:44.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:44.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:44.929 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:44.957 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:44.999 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:45.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:45.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:45.044 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:45.103 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:45.129 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:45.170 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:45.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:45.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:45.274 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:45.299 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:45.340 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:45.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:45.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:45.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:45.485 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:45.658 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:45.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:45.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:45.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:45 smithi026 ceph-mon[215775]: failed to encode map e498 with expected crc 2023-09-23T22:37:45.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:45 smithi026 ceph-mon[215775]: failed to encode map e498 with expected crc 2023-09-23T22:37:45.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:45 smithi026 ceph-mon[223014]: failed to encode map e498 with expected crc 2023-09-23T22:37:45.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:45 smithi026 ceph-mon[223014]: failed to encode map e498 with expected crc 2023-09-23T22:37:45.764 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:45.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:45 smithi070 ceph-mon[192652]: failed to encode map e498 with expected crc 2023-09-23T22:37:45.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:45 smithi070 ceph-mon[192652]: failed to encode map e498 with expected crc 2023-09-23T22:37:45.791 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:37:45 smithi070 conmon[172396]: logger=cleanup t=2023-09-23T22:37:45.677410835Z level=info msg="Completed cleanup jobs" duration=735.439362ms 2023-09-23T22:37:45.793 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:45.839 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:45.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:45.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:45.942 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:45.982 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:46.044 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:46.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:46.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:46.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:46.180 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:46.243 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:46.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:46.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:46.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:46.383 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:46.460 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:46.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:46.564 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:46.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:46 smithi026 ceph-mon[223014]: pgmap v1286: 109 pgs: 4 remapped+peering, 4 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 19 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 67 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 23 op/s; 21541/250653 objects degraded (8.594%); 19861/250653 objects misplaced (7.924%); 2.1 MiB/s, 67 objects/s recovering 2023-09-23T22:37:46.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:46 smithi026 ceph-mon[215775]: pgmap v1286: 109 pgs: 4 remapped+peering, 4 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 19 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 67 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 23 op/s; 21541/250653 objects degraded (8.594%); 19861/250653 objects misplaced (7.924%); 2.1 MiB/s, 67 objects/s recovering 2023-09-23T22:37:46.602 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:46.694 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:46.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:46.798 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:37:46.832 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:46.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:46 smithi070 ceph-mon[192652]: pgmap v1286: 109 pgs: 4 remapped+peering, 4 peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 19 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 67 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 23 op/s; 21541/250653 objects degraded (8.594%); 19861/250653 objects misplaced (7.924%); 2.1 MiB/s, 67 objects/s recovering 2023-09-23T22:37:46.894 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:37:46.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:46.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:37:46.998 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:47.028 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:47.075 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:47.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:47.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:47.178 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:47.209 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:47.265 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:47.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:47.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:47.369 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:47.399 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:47.457 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:47.458 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:47 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1478742219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:47.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:47 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1478742219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:47.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:47.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:47.461 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:37:47.560 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:47.587 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:47.629 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:47.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:47.733 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:47.757 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:47 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1478742219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:37:47.759 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:47.801 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:47.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:47.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:47.905 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:47.936 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:47.985 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:47.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:47.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:48.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:48.116 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:48.347 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:48.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:48.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:48.451 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:48.478 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:48.639 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:48.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:48.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:48.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[215775]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[215775]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1478742219' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[215775]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[215775]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[215775]: pgmap v1288: 109 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 13549/250593 objects degraded (5.407%); 30290/250593 objects misplaced (12.087%); 1.6 MiB/s, 101 objects/s recovering 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[215775]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 21541/250653 objects degraded (8.594%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:37:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[223014]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[223014]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1478742219' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:37:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[223014]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T22:37:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[223014]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[223014]: pgmap v1288: 109 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 13549/250593 objects degraded (5.407%); 30290/250593 objects misplaced (12.087%); 1.6 MiB/s, 101 objects/s recovering 2023-09-23T22:37:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[223014]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 21541/250653 objects degraded (8.594%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:48.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:48.773 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:48.829 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:48.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:48.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:48 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:37:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:48 smithi070 ceph-mon[192652]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:48 smithi070 ceph-mon[192652]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1478742219' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:37:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:48 smithi070 ceph-mon[192652]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T22:37:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:48 smithi070 ceph-mon[192652]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:48 smithi070 ceph-mon[192652]: pgmap v1288: 109 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 13549/250593 objects degraded (5.407%); 30290/250593 objects misplaced (12.087%); 1.6 MiB/s, 101 objects/s recovering 2023-09-23T22:37:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:48 smithi070 ceph-mon[192652]: failed to encode map e499 with expected crc 2023-09-23T22:37:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 21541/250653 objects degraded (8.594%), 13 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:48.933 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:48.960 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:49.013 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:49.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:49.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:49.117 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:49.146 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:49.189 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:49.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:49.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:49.233 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:37:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:37:49.233 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:37:49.233 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:37:49.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:49.324 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:49.376 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:49.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:49.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:49.479 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:49.509 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:49.558 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:49.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:49.561 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:49.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:49 smithi026 ceph-mon[223014]: failed to encode map e500 with expected crc 2023-09-23T22:37:49.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:49 smithi026 ceph-mon[223014]: failed to encode map e500 with expected crc 2023-09-23T22:37:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:49 smithi026 ceph-mon[223014]: failed to encode map e500 with expected crc 2023-09-23T22:37:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:49 smithi026 ceph-mon[223014]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T22:37:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:49 smithi026 ceph-mon[215775]: failed to encode map e500 with expected crc 2023-09-23T22:37:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:49 smithi026 ceph-mon[215775]: failed to encode map e500 with expected crc 2023-09-23T22:37:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:49 smithi026 ceph-mon[215775]: failed to encode map e500 with expected crc 2023-09-23T22:37:49.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:49 smithi026 ceph-mon[215775]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T22:37:49.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:49 smithi070 ceph-mon[192652]: failed to encode map e500 with expected crc 2023-09-23T22:37:49.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:49 smithi070 ceph-mon[192652]: failed to encode map e500 with expected crc 2023-09-23T22:37:49.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:49 smithi070 ceph-mon[192652]: failed to encode map e500 with expected crc 2023-09-23T22:37:49.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:49 smithi070 ceph-mon[192652]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T22:37:49.663 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:49.691 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:49.954 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:49.956 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:37:49.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:49.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:37:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:37:49] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.43.0" 2023-09-23T22:37:50.059 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:50.084 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 3, 5, 6, 7, 2, 4, 1] 2023-09-23T22:37:50.084 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:37:50.084 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T22:37:50.085 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:37:50.085 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:37:50.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T22:37:50.089 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:50.146 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:50.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:50.250 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:50.282 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:50.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:50.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:50.352 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:37:50.399 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:37:50.445 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:50.473 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:50.525 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:50.568 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:50.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:50.572 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:50.673 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:50.701 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:50.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:50 smithi026 ceph-mon[223014]: pgmap v1290: 109 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 7 op/s; 13549/250593 objects degraded (5.407%); 30290/250593 objects misplaced (12.087%); 1.2 MiB/s, 83 objects/s recovering 2023-09-23T22:37:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:50 smithi026 ceph-mon[215775]: pgmap v1290: 109 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 7 op/s; 13549/250593 objects degraded (5.407%); 30290/250593 objects misplaced (12.087%); 1.2 MiB/s, 83 objects/s recovering 2023-09-23T22:37:50.765 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:50 smithi070 ceph-mon[192652]: pgmap v1290: 109 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 1 activating, 2 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 86 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 7 op/s; 13549/250593 objects degraded (5.407%); 30290/250593 objects misplaced (12.087%); 1.2 MiB/s, 83 objects/s recovering 2023-09-23T22:37:50.796 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:50.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:50.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:50.900 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:50.927 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:50.971 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:50.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:50.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:51.074 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:51.109 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:51.165 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:51.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:51.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:51.270 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:51.302 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:51.354 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:51.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:51.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:51.461 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:51.491 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:51.541 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:51.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:51.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:51.645 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:51.675 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:51.729 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:51.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:51.732 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:51.783 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:37:51.811 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:37:51.833 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:51.839 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:37:51.864 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:37:51.873 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:51.892 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:37:51.899 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:37:51.899 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:37:51.899 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:37:51.899 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:37:51.899 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:37:51.899 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:37:51.900 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:37:51.900 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:37:51.945 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:37:51.945 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:37:51.950 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:51.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:51.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:52.060 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:52.093 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:52.145 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:52.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:52.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:52.249 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:52.275 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:52.319 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:52.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:52.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:52.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:52.453 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:52.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:52 smithi026 ceph-mon[215775]: pgmap v1291: 109 pgs: 1 peering, 1 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 87 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 22259/250560 objects degraded (8.884%); 22226/250560 objects misplaced (8.871%); 6.9 MiB/s, 174 objects/s recovering 2023-09-23T22:37:52.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:52 smithi026 ceph-mon[223014]: pgmap v1291: 109 pgs: 1 peering, 1 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 87 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 22259/250560 objects degraded (8.884%); 22226/250560 objects misplaced (8.871%); 6.9 MiB/s, 174 objects/s recovering 2023-09-23T22:37:52.764 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:37:53.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:52 smithi070 ceph-mon[192652]: pgmap v1291: 109 pgs: 1 peering, 1 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 8 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 87 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 22259/250560 objects degraded (8.884%); 22226/250560 objects misplaced (8.871%); 6.9 MiB/s, 174 objects/s recovering 2023-09-23T22:37:53.128 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:37:53.678 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:53.842 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:53.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:53.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:53.926 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 22259/250560 objects degraded (8.884%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:53.926 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 22259/250560 objects degraded (8.884%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:53.947 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:53.971 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 22259/250560 objects degraded (8.884%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:53.975 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:54.248 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:54.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:54.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:54.357 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:54.385 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:54.531 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:54.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:54.547 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:37:54.547 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "" 2023-09-23T22:37:54.547 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:37:54.634 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:54.674 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:54.729 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:37:54.729 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:37:55.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:55 smithi070 ceph-mon[192652]: pgmap v1292: 109 pgs: 1 activating+undersized+remapped, 1 activating, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 96 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 27638/250440 objects degraded (11.036%); 17540/250440 objects misplaced (7.004%); 5.7 MiB/s, 133 objects/s recovering 2023-09-23T22:37:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:55.407 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:37:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:55 smithi026 ceph-mon[223014]: pgmap v1292: 109 pgs: 1 activating+undersized+remapped, 1 activating, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 96 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 27638/250440 objects degraded (11.036%); 17540/250440 objects misplaced (7.004%); 5.7 MiB/s, 133 objects/s recovering 2023-09-23T22:37:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:55.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:55 smithi026 ceph-mon[215775]: pgmap v1292: 109 pgs: 1 activating+undersized+remapped, 1 activating, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 96 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 8 op/s; 27638/250440 objects degraded (11.036%); 17540/250440 objects misplaced (7.004%); 5.7 MiB/s, 133 objects/s recovering 2023-09-23T22:37:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:37:55.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:37:55.764 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:37:55.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:37:56.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:56.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:56.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:56.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:37:56.219 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:56.365 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:37:56.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:56.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:37:56.469 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:56.496 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:56.539 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:56.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:56.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:56.582 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:37:56.642 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:37:56.680 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:56.760 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:37:56.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:56.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:37:56.864 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:37:56.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:57.055 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:37:57.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:57.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:37:57.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:57 smithi070 ceph-mon[192652]: pgmap v1293: 109 pgs: 1 activating+undersized+remapped, 1 activating, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 96 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 27634/250311 objects degraded (11.040%); 17531/250311 objects misplaced (7.004%); 5.6 MiB/s, 100 objects/s recovering 2023-09-23T22:37:57.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1484392641' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:57.105 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:57.108 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:37:57.161 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:57.196 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:57.343 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:57.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:57.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:57.416 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:37:57.416 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:37:57.447 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:37:57.474 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:57.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:57 smithi026 ceph-mon[215775]: pgmap v1293: 109 pgs: 1 activating+undersized+remapped, 1 activating, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 96 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 27634/250311 objects degraded (11.040%); 17531/250311 objects misplaced (7.004%); 5.6 MiB/s, 100 objects/s recovering 2023-09-23T22:37:57.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1484392641' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:57.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:57 smithi026 ceph-mon[223014]: pgmap v1293: 109 pgs: 1 activating+undersized+remapped, 1 activating, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 96 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 12 op/s; 27634/250311 objects degraded (11.040%); 17531/250311 objects misplaced (7.004%); 5.6 MiB/s, 100 objects/s recovering 2023-09-23T22:37:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1484392641' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:37:57.682 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:37:57.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:57.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:37:57.786 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:57.820 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:57.867 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:57.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:57.972 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:58.000 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:58.263 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:37:58.278 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:58.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:58 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:58 smithi070 ceph-mon[192652]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:58 smithi070 ceph-mon[192652]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:37:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:58 smithi070 ceph-mon[192652]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T22:37:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:58 smithi070 ceph-mon[192652]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:58 smithi070 ceph-mon[192652]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.383 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:58.410 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:58.461 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:37:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:58.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:37:58.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:58.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[215775]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[215775]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[215775]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[215775]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[215775]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[223014]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[223014]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[223014]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[223014]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:58 smithi026 ceph-mon[223014]: failed to encode map e501 with expected crc 2023-09-23T22:37:58.567 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:58.601 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:58.645 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:37:58.683 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:58.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:58.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:58.788 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:37:58.824 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:58.928 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:37:58.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:58.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:37:58.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:37:59.013 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:37:59.032 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:37:59.037 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:37:59.059 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:37:59.062 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:59.078 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:37:59.098 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:37:59.124 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:37:59.151 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:37:59.154 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:37:59.154 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:37:59.154 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:37:59.154 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:37:59.154 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:37:59.154 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:37:59.155 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:37:59.155 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:37:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[215775]: pgmap v1295: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 26471/250227 objects degraded (10.579%); 16370/250227 objects misplaced (6.542%); 13 MiB/s, 217 objects/s recovering 2023-09-23T22:37:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[215775]: failed to encode map e502 with expected crc 2023-09-23T22:37:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[215775]: failed to encode map e502 with expected crc 2023-09-23T22:37:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[215775]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T22:37:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[215775]: failed to encode map e502 with expected crc 2023-09-23T22:37:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26471/250227 objects degraded (10.579%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:59.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/360479911' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:59.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[223014]: pgmap v1295: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 26471/250227 objects degraded (10.579%); 16370/250227 objects misplaced (6.542%); 13 MiB/s, 217 objects/s recovering 2023-09-23T22:37:59.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[223014]: failed to encode map e502 with expected crc 2023-09-23T22:37:59.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[223014]: failed to encode map e502 with expected crc 2023-09-23T22:37:59.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[223014]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T22:37:59.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[223014]: failed to encode map e502 with expected crc 2023-09-23T22:37:59.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26471/250227 objects degraded (10.579%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:59.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:37:59 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/360479911' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:59.312 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:37:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:37:59.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:37:59.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:59 smithi070 ceph-mon[192652]: pgmap v1295: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 13 op/s; 26471/250227 objects degraded (10.579%); 16370/250227 objects misplaced (6.542%); 13 MiB/s, 217 objects/s recovering 2023-09-23T22:37:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:59 smithi070 ceph-mon[192652]: failed to encode map e502 with expected crc 2023-09-23T22:37:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:59 smithi070 ceph-mon[192652]: failed to encode map e502 with expected crc 2023-09-23T22:37:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:59 smithi070 ceph-mon[192652]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T22:37:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:59 smithi070 ceph-mon[192652]: failed to encode map e502 with expected crc 2023-09-23T22:37:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:59 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26471/250227 objects degraded (10.579%), 12 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:37:59.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:37:59 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/360479911' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:37:59.416 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:37:59.447 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:37:59.863 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:37:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:37:59] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-09-23T22:38:00.042 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:00.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:00.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:00.146 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:00.173 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:00.217 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:00.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:00.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:00.321 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:00.356 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:00.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:00 smithi070 ceph-mon[192652]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:00 smithi070 ceph-mon[192652]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:00 smithi070 ceph-mon[192652]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T22:38:00.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:00 smithi070 ceph-mon[192652]: 9.4 continuing backfill to osd.3 from (461'1226,501'6693] MIN to 501'6693 2023-09-23T22:38:00.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:00 smithi070 ceph-mon[192652]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:00 smithi070 ceph-mon[192652]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.410 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:00.415 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[223014]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[223014]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[223014]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T22:38:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[223014]: 9.4 continuing backfill to osd.3 from (461'1226,501'6693] MIN to 501'6693 2023-09-23T22:38:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[223014]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[223014]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[215775]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[215775]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[215775]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T22:38:00.416 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[215775]: 9.4 continuing backfill to osd.3 from (461'1226,501'6693] MIN to 501'6693 2023-09-23T22:38:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[215775]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.417 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:00 smithi026 ceph-mon[215775]: failed to encode map e503 with expected crc 2023-09-23T22:38:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:00.417 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:00.518 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:00.545 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:00.591 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:00.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:00.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:00.702 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:38:00.725 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:00.801 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:00.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:00.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:00.904 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:00.939 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:01.013 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:01.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:01.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:01.122 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:01.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:01.161 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:01.204 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:01.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:01.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:01.215 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:38:01.215 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:38:01.215 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:38:01.215 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:38:01.215 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:38:01.221 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 3, 5, 6, 7, 2, 4, 1] 2023-09-23T22:38:01.221 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:38:01.221 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T22:38:01.221 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:38:01.221 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:38:01.221 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T22:38:01.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:01.335 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:01 smithi026 ceph-mon[215775]: pgmap v1298: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 26471/250227 objects degraded (10.579%); 16370/250227 objects misplaced (6.542%); 12 MiB/s, 194 objects/s recovering 2023-09-23T22:38:01.335 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:01 smithi026 ceph-mon[223014]: pgmap v1298: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 26471/250227 objects degraded (10.579%); 16370/250227 objects misplaced (6.542%); 12 MiB/s, 194 objects/s recovering 2023-09-23T22:38:01.339 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:01.402 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:01.404 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:01 smithi070 ceph-mon[192652]: pgmap v1298: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 11 op/s; 26471/250227 objects degraded (10.579%); 16370/250227 objects misplaced (6.542%); 12 MiB/s, 194 objects/s recovering 2023-09-23T22:38:01.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:01.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:01.508 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:01.554 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:01.720 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:01.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:01.822 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:01.854 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:01.865 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:38:02.352 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:02.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:02.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:02.456 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:02.487 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:02.491 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:02.507 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:38:02.541 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:02.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:02.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:02.559 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:02.650 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:02.691 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:02.857 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:02.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:02.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:02.961 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:02.990 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:03.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:03 smithi026 ceph-mon[215775]: pgmap v1299: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 20977/249390 objects degraded (8.411%); 25425/249390 objects misplaced (10.195%); 19 MiB/s, 314 objects/s recovering 2023-09-23T22:38:03.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:03 smithi026 ceph-mon[223014]: pgmap v1299: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 20977/249390 objects degraded (8.411%); 25425/249390 objects misplaced (10.195%); 19 MiB/s, 314 objects/s recovering 2023-09-23T22:38:03.240 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:03.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:03.242 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:03.343 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:03.375 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:03.428 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:03.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:03.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:03.532 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:03.556 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:03 smithi070 ceph-mon[192652]: pgmap v1299: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 51 op/s; 20977/249390 objects degraded (8.411%); 25425/249390 objects misplaced (10.195%); 19 MiB/s, 314 objects/s recovering 2023-09-23T22:38:03.559 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:03.606 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:03.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:03.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:03.709 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:03.744 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:03.859 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:03.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:03.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:03.965 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:03.999 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:04.061 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:04.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:04.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:04.144 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:38:04.165 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:04.205 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:04.234 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:04.301 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2692380854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:04.302 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:04 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:04.302 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:04 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 20977/249390 objects degraded (8.411%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:04.302 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:04 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2692380854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:04 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:04 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 20977/249390 objects degraded (8.411%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:04.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:04 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:04.317 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:04.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:04.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:04.423 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:04.453 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2692380854' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:04.453 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:04 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:04.453 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:04 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 20977/249390 objects degraded (8.411%), 11 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:04.453 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:04 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:38:04.457 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:04.501 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:04.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:04.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:04.604 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:04.638 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:04.865 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:38:04.865 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:38:04.866 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:38:04.866 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:38:04.882 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:38:04.894 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:04.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:04.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:04.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:38:04.935 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:38:04.966 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:38:04.972 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:38:04.972 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:38:04.972 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:38:04.972 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:38:04.972 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:38:04.972 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:38:04.973 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:38:04.973 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:38:04.998 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:05.033 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:05.151 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:38:05.151 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:38:05.152 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:38:05.178 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:05 smithi070 ceph-mon[192652]: pgmap v1300: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 15579/248811 objects degraded (6.261%); 30007/248811 objects misplaced (12.060%); 6.9 MiB/s, 122 objects/s recovering 2023-09-23T22:38:05.178 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:05 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:05.178 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:05 smithi070 ceph-mon[192652]: failed to encode map e504 with expected crc 2023-09-23T22:38:05.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:05 smithi070 ceph-mon[192652]: failed to encode map e504 with expected crc 2023-09-23T22:38:05.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:05 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:05.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:05 smithi070 ceph-mon[192652]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T22:38:05.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:05 smithi070 ceph-mon[192652]: failed to encode map e504 with expected crc 2023-09-23T22:38:05.179 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:05 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3699185609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:05.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[223014]: pgmap v1300: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 15579/248811 objects degraded (6.261%); 30007/248811 objects misplaced (12.060%); 6.9 MiB/s, 122 objects/s recovering 2023-09-23T22:38:05.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:05.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[223014]: failed to encode map e504 with expected crc 2023-09-23T22:38:05.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[223014]: failed to encode map e504 with expected crc 2023-09-23T22:38:05.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:05.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[223014]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T22:38:05.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[223014]: failed to encode map e504 with expected crc 2023-09-23T22:38:05.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3699185609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:05.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[215775]: pgmap v1300: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 75 op/s; 15579/248811 objects degraded (6.261%); 30007/248811 objects misplaced (12.060%); 6.9 MiB/s, 122 objects/s recovering 2023-09-23T22:38:05.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:05.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[215775]: failed to encode map e504 with expected crc 2023-09-23T22:38:05.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[215775]: failed to encode map e504 with expected crc 2023-09-23T22:38:05.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:05.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[215775]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T22:38:05.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[215775]: failed to encode map e504 with expected crc 2023-09-23T22:38:05.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:05 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3699185609' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:06.956 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:38:07.364 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:07.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:07.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:07.442 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:07 smithi070 ceph-mon[192652]: pgmap v1302: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 15544/247983 objects degraded (6.268%); 29603/247983 objects misplaced (11.938%); 9.1 MiB/s, 155 objects/s recovering 2023-09-23T22:38:07.467 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:07 smithi026 ceph-mon[223014]: pgmap v1302: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 15544/247983 objects degraded (6.268%); 29603/247983 objects misplaced (11.938%); 9.1 MiB/s, 155 objects/s recovering 2023-09-23T22:38:07.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:07 smithi026 ceph-mon[215775]: pgmap v1302: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 100 op/s; 15544/247983 objects degraded (6.268%); 29603/247983 objects misplaced (11.938%); 9.1 MiB/s, 155 objects/s recovering 2023-09-23T22:38:07.506 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:07.522 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:07.667 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:38:07.714 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:07.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:07.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:07.820 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:07.855 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:38:07.858 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:38:07.958 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:07.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:07.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:08.066 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:08.098 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:08.250 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:08.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:08.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:08.353 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:08.389 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:08.591 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:08.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:08.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:08.723 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:09.132 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:09.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:09.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:09.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:09.269 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:09.270 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:38:09.289 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:09.333 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:09.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:09.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:09.441 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:09.469 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:09 smithi070 ceph-mon[192652]: pgmap v1303: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 15519/247281 objects degraded (6.276%); 29510/247281 objects misplaced (11.934%); 8.4 MiB/s, 143 objects/s recovering 2023-09-23T22:38:09.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:09 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15519/247281 objects degraded (6.276%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:09.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:09.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:09.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:09.470 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:09.473 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:09.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[215775]: pgmap v1303: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 15519/247281 objects degraded (6.276%); 29510/247281 objects misplaced (11.934%); 8.4 MiB/s, 143 objects/s recovering 2023-09-23T22:38:09.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15519/247281 objects degraded (6.276%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:09.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:09.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:09.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:09.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:09.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[223014]: pgmap v1303: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 15519/247281 objects degraded (6.276%); 29510/247281 objects misplaced (11.934%); 8.4 MiB/s, 143 objects/s recovering 2023-09-23T22:38:09.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15519/247281 objects degraded (6.276%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:09.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:09.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:09.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:09.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:09.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:09.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:09.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:09.635 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:09.673 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:09.886 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:09.896 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:38:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:38:09] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-09-23T22:38:09.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:09.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:09.957 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:09.997 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:10.045 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:10.096 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:10.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:10.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:10.200 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:10.233 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:10.482 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:10.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:10.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:10.586 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:10.614 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:10.713 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:10.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:10.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:10.817 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:10.852 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:10.953 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:10.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:10.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:11.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:11.085 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:38:11.093 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:11.109 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:38:11.137 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:38:11.154 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:38:11.158 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:11.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:11.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:11.168 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:38:11.199 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:38:11.230 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:38:11.251 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:38:11.264 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:11.274 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:38:11.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:38:11.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:38:11.276 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:38:11.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:38:11.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:38:11.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:38:11.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:38:11.277 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:38:11.291 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:11.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:11 smithi026 ceph-mon[215775]: pgmap v1304: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 15519/247281 objects degraded (6.276%); 29510/247281 objects misplaced (11.934%); 7.1 MiB/s, 121 objects/s recovering 2023-09-23T22:38:11.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3436955258' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:38:11.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:38:11.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2803920096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:11.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:11 smithi026 ceph-mon[223014]: pgmap v1304: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 15519/247281 objects degraded (6.276%); 29510/247281 objects misplaced (11.934%); 7.1 MiB/s, 121 objects/s recovering 2023-09-23T22:38:11.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3436955258' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:38:11.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:38:11.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2803920096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:11.543 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:11.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:11 smithi070 ceph-mon[192652]: pgmap v1304: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 15519/247281 objects degraded (6.276%); 29510/247281 objects misplaced (11.934%); 7.1 MiB/s, 121 objects/s recovering 2023-09-23T22:38:11.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3436955258' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:38:11.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:38:11.545 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2803920096' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:11.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:11.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:11.647 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:11.673 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:11.719 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:11.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:11.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:11.823 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:11.849 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:11.898 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:11.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:11.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:12.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:12.037 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:12.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:12.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[223014]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[223014]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:38:12.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[223014]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T22:38:12.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[223014]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[223014]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:12.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[215775]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[215775]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:38:12.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[215775]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T22:38:12.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[215775]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:12 smithi026 ceph-mon[215775]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.444 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 3, 5, 6, 7, 2, 4, 1] 2023-09-23T22:38:12.444 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:38:12.444 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T22:38:12.444 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:38:12.444 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:38:12.445 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T22:38:12.464 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:12.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:12.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:12.574 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:12.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:12 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:12.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:12 smithi070 ceph-mon[192652]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:12 smithi070 ceph-mon[192652]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:12 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:38:12.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:12 smithi070 ceph-mon[192652]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T22:38:12.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:12 smithi070 ceph-mon[192652]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.606 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:12 smithi070 ceph-mon[192652]: failed to encode map e505 with expected crc 2023-09-23T22:38:12.612 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:12.678 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:12.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:12.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:12.782 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:12.815 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:12.868 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:12.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:12.917 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:12.976 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:13.006 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:13.059 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:13.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:13.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:13.163 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:13.195 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:13.251 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:13.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:13.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:13.340 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:38:13.340 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:38:13.341 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:38:13.341 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:38:13.354 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:13.368 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:13 smithi070 ceph-mon[192652]: pgmap v1306: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 15491/246411 objects degraded (6.287%); 29036/246411 objects misplaced (11.784%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T22:38:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:13 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:13.369 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:13 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:13.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:13 smithi026 ceph-mon[223014]: pgmap v1306: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 15491/246411 objects degraded (6.287%); 29036/246411 objects misplaced (11.784%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T22:38:13.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:13 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:13.389 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:13 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:13.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:13 smithi026 ceph-mon[215775]: pgmap v1306: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 99 op/s; 15491/246411 objects degraded (6.287%); 29036/246411 objects misplaced (11.784%); 6.6 MiB/s, 106 objects/s recovering 2023-09-23T22:38:13.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:13 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:13.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:13 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:13.395 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:13.563 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:13.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:13.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:13.669 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:13.701 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:13.746 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:13.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:13.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:13.849 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:13.879 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:13.929 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:13.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:13.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:14.033 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:14.062 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:14.214 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:14.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:14.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:14.321 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:14.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:14 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15491/246411 objects degraded (6.287%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:14.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:14 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:14.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:14 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:14.349 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:14 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:14.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:14 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15491/246411 objects degraded (6.287%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:14.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:14 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:14.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:14 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:14.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:14 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:14.352 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:14.459 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:38:14.463 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:14.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:14.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:14.487 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:38:14.567 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:14.609 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:14.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:14 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15491/246411 objects degraded (6.287%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:14.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:14 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:38:14.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:14 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:38:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:14 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:14.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:14.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:14.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:14.727 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:38:14.727 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "" 2023-09-23T22:38:14.727 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:38:14.789 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:14.828 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:14.842 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:38:14.939 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:14.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:14.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:15.044 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:15.072 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:15.123 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:15.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:15.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:15.147 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:15 smithi026 ceph-mon[215775]: pgmap v1307: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 15455/245901 objects degraded (6.285%); 28966/245901 objects misplaced (11.780%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:38:15.232 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:15.233 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:15.261 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:15.369 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:15.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:15.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:15.439 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:15 smithi026 ceph-mon[215775]: 1.5 scrub starts 2023-09-23T22:38:15.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:15 smithi026 ceph-mon[223014]: pgmap v1307: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 15455/245901 objects degraded (6.285%); 28966/245901 objects misplaced (11.780%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:38:15.441 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:15 smithi026 ceph-mon[223014]: 1.5 scrub starts 2023-09-23T22:38:15.459 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:15 smithi070 ceph-mon[192652]: pgmap v1307: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 102 op/s; 15455/245901 objects degraded (6.285%); 28966/245901 objects misplaced (11.780%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:38:15.459 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:15 smithi070 ceph-mon[192652]: 1.5 scrub starts 2023-09-23T22:38:15.473 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:15.516 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:15.536 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:15.577 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:15.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:15.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:15.582 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:15.682 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:15.711 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:15.760 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:15.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:15.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:15.864 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:15.898 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:16.271 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:16.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:16.376 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:16.414 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:16.930 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:16.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:17.034 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:17.063 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:17.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:17 smithi070 ceph-mon[192652]: pgmap v1308: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 15422/245067 objects degraded (6.293%); 28511/245067 objects misplaced (11.634%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T22:38:17.297 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:17 smithi070 ceph-mon[192652]: 1.d scrub starts 2023-09-23T22:38:17.343 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:17.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:17 smithi026 ceph-mon[223014]: pgmap v1308: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 15422/245067 objects degraded (6.293%); 28511/245067 objects misplaced (11.634%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T22:38:17.346 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:17 smithi026 ceph-mon[223014]: 1.d scrub starts 2023-09-23T22:38:17.346 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:17 smithi026 ceph-mon[215775]: pgmap v1308: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 97 op/s; 15422/245067 objects degraded (6.293%); 28511/245067 objects misplaced (11.634%); 6.1 MiB/s, 96 objects/s recovering 2023-09-23T22:38:17.347 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:17 smithi026 ceph-mon[215775]: 1.d scrub starts 2023-09-23T22:38:17.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:17.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:17.448 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:17.479 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:17.529 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:17.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:17.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:17.593 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:38:17.593 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:38:17.593 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:38:17.593 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:38:17.593 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:38:17.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:17.673 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:17.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:17.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:17.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:17.825 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:38:17.851 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:38:17.851 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:17.880 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:17.882 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:38:17.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:38:17.929 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:17.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:17.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:17.940 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:38:17.961 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:38:17.986 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:38:18.013 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:38:18.017 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:38:18.017 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:38:18.018 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:38:18.018 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:38:18.018 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:38:18.018 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:38:18.018 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:38:18.018 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:38:18.034 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:18.077 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:18.241 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:18.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:18.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:18.346 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:18.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:18.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/490014726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:18.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/490014726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:18.557 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:18.558 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/490014726' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:18.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:18.660 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:18.687 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:18.877 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:18.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:18.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:18.980 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:19.005 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:19.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:19 smithi026 ceph-mon[215775]: pgmap v1309: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 15395/244314 objects degraded (6.301%); 28415/244314 objects misplaced (11.631%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:38:19.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:19 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15395/244314 objects degraded (6.301%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:19.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2294874450' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:19.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:19 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:19.179 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:38:19.200 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:19.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:19.273 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:38:19.305 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:19.337 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:19 smithi026 ceph-mon[223014]: pgmap v1309: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 15395/244314 objects degraded (6.301%); 28415/244314 objects misplaced (11.631%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:38:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:19 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15395/244314 objects degraded (6.301%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2294874450' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:19.416 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:19 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:19.453 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:19.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:19.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:19.560 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:19.575 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:19.587 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:19 smithi070 ceph-mon[192652]: pgmap v1309: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 98 op/s; 15395/244314 objects degraded (6.301%); 28415/244314 objects misplaced (11.631%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T22:38:19.587 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:19 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15395/244314 objects degraded (6.301%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:19.587 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2294874450' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:19.587 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:19 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:19.591 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:19.644 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:19.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:19.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:19.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:38:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:38:19] "GET /metrics HTTP/1.1" 200 36493 "" "Prometheus/2.43.0" 2023-09-23T22:38:19.748 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:19.775 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:19.826 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:19.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:19.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:19.958 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:20.007 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:20.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:20.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:20.111 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:20.141 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:20.190 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:20.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:20.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:20.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:38:20.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[215775]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[215775]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[215775]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[215775]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[215775]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[223014]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[223014]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[223014]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[223014]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T22:38:20.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:20 smithi026 ceph-mon[223014]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.296 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:20.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:20 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:38:20.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:20 smithi070 ceph-mon[192652]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:20 smithi070 ceph-mon[192652]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:20 smithi070 ceph-mon[192652]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.320 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:20 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:20.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:20 smithi070 ceph-mon[192652]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T22:38:20.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:20 smithi070 ceph-mon[192652]: failed to encode map e506 with expected crc 2023-09-23T22:38:20.323 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:20.373 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:20.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:20.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:20.471 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 3, 5, 6, 7, 2, 4, 1] 2023-09-23T22:38:20.471 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:38:20.471 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T22:38:20.471 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:38:20.471 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:38:20.472 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T22:38:20.477 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:20.635 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:20.762 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:20.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:20.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:20.867 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:20.894 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:20.922 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:38:21.034 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:21.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:21.138 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:21.170 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:21.173 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:21.268 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:21.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:21.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:21.350 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:21.373 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:21.409 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:21.504 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:21.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:21.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:21.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:21 smithi026 ceph-mon[223014]: pgmap v1311: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 82 op/s; 15395/244314 objects degraded (6.301%); 28415/244314 objects misplaced (11.631%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:38:21.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:21 smithi026 ceph-mon[223014]: 1.5 scrub starts 2023-09-23T22:38:21.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:21 smithi026 ceph-mon[215775]: pgmap v1311: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 82 op/s; 15395/244314 objects degraded (6.301%); 28415/244314 objects misplaced (11.631%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:38:21.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:21 smithi026 ceph-mon[215775]: 1.5 scrub starts 2023-09-23T22:38:21.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:21.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:21 smithi070 ceph-mon[192652]: pgmap v1311: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 82 op/s; 15395/244314 objects degraded (6.301%); 28415/244314 objects misplaced (11.631%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T22:38:21.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:21 smithi070 ceph-mon[192652]: 1.5 scrub starts 2023-09-23T22:38:21.654 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:21.696 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:38:21.697 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:38:21.706 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:21.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:21.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:21.810 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:21.848 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:21.952 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:38:21.955 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:21.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:21.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:22.062 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:22.093 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:22.778 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:23.317 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:23.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:23.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:23.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:23 smithi070 ceph-mon[192652]: pgmap v1312: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 15356/243159 objects degraded (6.315%); 27855/243159 objects misplaced (11.455%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:38:23.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15356/243159 objects degraded (6.315%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:23.382 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:38:23.412 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:38:23.422 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:23.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:38:23.470 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:23 smithi026 ceph-mon[215775]: pgmap v1312: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 15356/243159 objects degraded (6.315%); 27855/243159 objects misplaced (11.455%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:38:23.470 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15356/243159 objects degraded (6.315%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:23.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:23 smithi026 ceph-mon[223014]: pgmap v1312: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 108 op/s; 15356/243159 objects degraded (6.315%); 27855/243159 objects misplaced (11.455%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:38:23.470 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15356/243159 objects degraded (6.315%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:23.470 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:38:23.471 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:23.503 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:38:23.533 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:38:23.533 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:23.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:23.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:23.554 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:38:23.555 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:38:23.581 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:38:23.589 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:38:23.589 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:38:23.589 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:38:23.589 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:38:23.589 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:38:23.590 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:38:23.590 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:38:23.590 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:38:23.639 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:23.665 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:23.757 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:23.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:23.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:23.860 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:23.891 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:23.975 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:23.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:23.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:24.081 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:24.110 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:24.260 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:24.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:24.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:24.361 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:38:24.361 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "" 2023-09-23T22:38:24.362 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:38:24.364 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:24.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3645889386' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:24.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:24.395 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:24.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3645889386' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:24.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:24.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:24.397 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:24.574 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3645889386' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:24.575 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:24.575 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:24.740 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:24.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:24.846 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:24.874 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:24.920 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:24.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:24.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:24.939 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:25.027 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:25.068 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:25.137 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:25.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:25.245 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:25.274 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:25.353 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:25.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:25.374 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:25 smithi026 ceph-mon[215775]: pgmap v1313: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 15325/242661 objects degraded (6.315%); 27789/242661 objects misplaced (11.452%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:38:25.374 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:25 smithi026 ceph-mon[223014]: pgmap v1313: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 15325/242661 objects degraded (6.315%); 27789/242661 objects misplaced (11.452%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:38:25.458 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:25.502 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:25.567 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:25.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:25.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:25 smithi070 ceph-mon[192652]: pgmap v1313: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 15325/242661 objects degraded (6.315%); 27789/242661 objects misplaced (11.452%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T22:38:25.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:25.704 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:25.754 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:25.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:25.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:25.860 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:25.888 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:25.941 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:25.948 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:26.048 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:26.084 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:26.153 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:26.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:26.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:26.259 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:26.291 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:38:26.298 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:26.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:26 smithi026 ceph-mon[223014]: 1.d scrub starts 2023-09-23T22:38:26.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3072096244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:26.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:26 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:26.414 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:26 smithi026 ceph-mon[215775]: 1.d scrub starts 2023-09-23T22:38:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3072096244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:26.415 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:26 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:26.554 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:26.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:26.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:26.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:26 smithi070 ceph-mon[192652]: 1.d scrub starts 2023-09-23T22:38:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3072096244' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:26 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:26.657 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:26.683 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:26.692 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:38:26.729 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:26.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:26.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:26.837 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:26.862 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:26.907 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:26.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:26.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:27.013 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:27.042 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:27.089 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:27.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:27.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:27.137 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:38:27.137 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:38:27.137 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:38:27.137 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:38:27.194 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:27.222 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:27.271 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:27.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:27.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:27.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[215775]: pgmap v1314: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 15288/241833 objects degraded (6.322%); 27252/241833 objects misplaced (11.269%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:38:27.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:27.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[215775]: failed to encode map e507 with expected crc 2023-09-23T22:38:27.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:38:27.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[215775]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T22:38:27.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[215775]: failed to encode map e507 with expected crc 2023-09-23T22:38:27.286 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[215775]: failed to encode map e507 with expected crc 2023-09-23T22:38:27.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:27 smithi070 ceph-mon[192652]: pgmap v1314: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 15288/241833 objects degraded (6.322%); 27252/241833 objects misplaced (11.269%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:38:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:27 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:27 smithi070 ceph-mon[192652]: failed to encode map e507 with expected crc 2023-09-23T22:38:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:27 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:38:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:27 smithi070 ceph-mon[192652]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T22:38:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:27 smithi070 ceph-mon[192652]: failed to encode map e507 with expected crc 2023-09-23T22:38:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:27 smithi070 ceph-mon[192652]: failed to encode map e507 with expected crc 2023-09-23T22:38:27.376 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:27.406 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:27.455 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:27.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:27.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:27.567 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:27.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[223014]: pgmap v1314: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 107 op/s; 15288/241833 objects degraded (6.322%); 27252/241833 objects misplaced (11.269%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:38:27.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:27.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[223014]: failed to encode map e507 with expected crc 2023-09-23T22:38:27.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:38:27.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[223014]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T22:38:27.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[223014]: failed to encode map e507 with expected crc 2023-09-23T22:38:27.598 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:27 smithi026 ceph-mon[223014]: failed to encode map e507 with expected crc 2023-09-23T22:38:27.602 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:27.874 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:27.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:38:27.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:27.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:27.924 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:27.985 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:28.010 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:28.054 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:28.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:28.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:28.162 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:28.196 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:28.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:28 smithi026 ceph-mon[223014]: failed to encode map e507 with expected crc 2023-09-23T22:38:28.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15288/241833 objects degraded (6.322%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:28.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:28 smithi026 ceph-mon[215775]: failed to encode map e507 with expected crc 2023-09-23T22:38:28.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15288/241833 objects degraded (6.322%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:28.506 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:28.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:28.610 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:28.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:28 smithi070 ceph-mon[192652]: failed to encode map e507 with expected crc 2023-09-23T22:38:28.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15288/241833 objects degraded (6.322%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:28.637 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:28.684 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:28.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:28.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:28.789 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:28.819 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:29.009 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:29.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:29.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:29.113 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:29.121 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:29.162 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:29.219 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:38:29.281 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:29.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:29.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:29.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:29.414 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:29 smithi026 ceph-mon[215775]: pgmap v1316: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 15253/240900 objects degraded (6.332%); 27136/240900 objects misplaced (11.264%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T22:38:29.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:29 smithi026 ceph-mon[223014]: pgmap v1316: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 15253/240900 objects degraded (6.332%); 27136/240900 objects misplaced (11.264%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T22:38:29.451 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:29 smithi070 ceph-mon[192652]: pgmap v1316: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 136 op/s; 15253/240900 objects degraded (6.332%); 27136/240900 objects misplaced (11.264%); 8.4 MiB/s, 134 objects/s recovering 2023-09-23T22:38:29.628 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:29.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:29.654 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:38:29.678 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:38:29.701 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:38:29.723 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:38:29.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:29.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:38:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:38:29] "GET /metrics HTTP/1.1" 200 36489 "" "Prometheus/2.43.0" 2023-09-23T22:38:29.744 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:38:29.761 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:29.766 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:38:29.793 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:38:29.810 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:29.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:29.822 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:38:29.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:38:29.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:38:29.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:38:29.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:38:29.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:38:29.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:38:29.824 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:38:29.825 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:38:29.915 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:29.942 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:29.986 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:29.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:29.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:30.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:30.131 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:30.420 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 3, 5, 6, 7, 2, 4, 1] 2023-09-23T22:38:30.420 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:38:30.420 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T22:38:30.420 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:38:30.420 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:38:30.420 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T22:38:30.449 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:30 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3329540026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:30.449 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:30 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3329540026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:30.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:30 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3329540026' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:30.903 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:30.909 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:30.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:30.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:31.030 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:31.064 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:31.130 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:31.186 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:31.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:31.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:31.291 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:31.329 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:31.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:31 smithi026 ceph-mon[215775]: pgmap v1317: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 15253/240900 objects degraded (6.332%); 27136/240900 objects misplaced (11.264%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:38:31.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:31 smithi026 ceph-mon[223014]: pgmap v1317: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 15253/240900 objects degraded (6.332%); 27136/240900 objects misplaced (11.264%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:38:31.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:31 smithi070 ceph-mon[192652]: pgmap v1317: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 15253/240900 objects degraded (6.332%); 27136/240900 objects misplaced (11.264%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:38:31.711 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:38:31.711 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:38:31.737 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:31.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:31.847 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:31.874 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:31.928 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:31.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:31.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:32.042 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:32.070 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:32.114 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:32.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:32.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:32.220 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:32.261 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:32.944 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:38:33.271 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:38:33.436 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:33.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:33.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:33.541 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:33.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:33 smithi026 ceph-mon[223014]: pgmap v1318: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 15215/239637 objects degraded (6.349%); 26465/239637 objects misplaced (11.044%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:38:33.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15215/239637 objects degraded (6.349%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:33.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:33 smithi026 ceph-mon[215775]: pgmap v1318: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 15215/239637 objects degraded (6.349%); 26465/239637 objects misplaced (11.044%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:38:33.563 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15215/239637 objects degraded (6.349%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:33.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:33 smithi070 ceph-mon[192652]: pgmap v1318: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 117 op/s; 15215/239637 objects degraded (6.349%); 26465/239637 objects misplaced (11.044%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T22:38:33.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15215/239637 objects degraded (6.349%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:33.568 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:33.615 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:33.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:33.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:33.721 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:33.737 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:33.748 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:33.792 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:33.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:33.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:33.897 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:33.934 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:34.242 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:34.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:34.348 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:34.376 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:34.404 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:38:34.404 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:38:34.404 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:38:34.419 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:34.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:34.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:34.425 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:34.524 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:34.563 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:35.213 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:35.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:35.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:35.317 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:35.335 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:38:35.346 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:35.394 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:35.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:35.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:35 smithi026 ceph-mon[223014]: pgmap v1319: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 15187/239175 objects degraded (6.350%); 26392/239175 objects misplaced (11.035%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T22:38:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1844011551' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:35 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:35 smithi026 ceph-mon[215775]: pgmap v1319: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 15187/239175 objects degraded (6.350%); 26392/239175 objects misplaced (11.035%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T22:38:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1844011551' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:35 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:35.500 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:35.536 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:35 smithi070 ceph-mon[192652]: pgmap v1319: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 116 op/s; 15187/239175 objects degraded (6.350%); 26392/239175 objects misplaced (11.035%); 7.5 MiB/s, 120 objects/s recovering 2023-09-23T22:38:35.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1844011551' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:35.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:35 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:36.520 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:36.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:36.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:36.588 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:36.588 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[223014]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.588 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:36.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[223014]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T22:38:36.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[223014]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[223014]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.589 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[223014]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:36.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[215775]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:36.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[215775]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T22:38:36.589 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[215775]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[215775]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:36 smithi026 ceph-mon[215775]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:36 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:36.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:36 smithi070 ceph-mon[192652]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:36 smithi070 ceph-mon[192652]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T22:38:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:36 smithi070 ceph-mon[192652]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:36 smithi070 ceph-mon[192652]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:36 smithi070 ceph-mon[192652]: failed to encode map e508 with expected crc 2023-09-23T22:38:36.625 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:36.667 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:36.772 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:36.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:36.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:36.877 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:36.904 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:36.951 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:36.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:36.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:37.056 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:37.084 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:37.129 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:37.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:37.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:37.235 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:37.274 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:37.454 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:37.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:37.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:37.557 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:37.582 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:37 smithi070 ceph-mon[192652]: pgmap v1321: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 15149/238287 objects degraded (6.357%); 25671/238287 objects misplaced (10.773%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T22:38:37.584 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:37.633 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:37.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:37.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:37.693 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:38:37.723 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:38:37.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:37 smithi026 ceph-mon[223014]: pgmap v1321: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 15149/238287 objects degraded (6.357%); 25671/238287 objects misplaced (10.773%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T22:38:37.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:37 smithi026 ceph-mon[215775]: pgmap v1321: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 114 op/s; 15149/238287 objects degraded (6.357%); 25671/238287 objects misplaced (10.773%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T22:38:37.739 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:37.752 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:38:37.791 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:37.853 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:38:37.874 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:37.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:37.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:37.887 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:38:37.915 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:38:37.938 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:38:37.958 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:38:37.960 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:38:37.960 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:38:37.960 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:38:37.960 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:38:37.961 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:38:37.961 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:38:37.961 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:38:37.961 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:38:37.981 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:38.007 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:38.053 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:38.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:38.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:38.064 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151868 8.f 8.3587fcdf (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.193100+0000", 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 292.86935607300001, 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0010652879999999, 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.065 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151868 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193100+0000", 2023-09-23T22:38:38.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193100+0000", 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193098+0000", 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193104+0000", 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9959999999999999e-06 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193112+0000", 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8630000000000008e-06 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193120+0000", 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8820000000000001e-06 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.205704+0000", 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012583321999999999 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.205719+0000", 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5328999999999998e-05 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.194140+0000", 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98842133099999996 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.194165+0000", 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4830000000000001e-05 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188384 8.e 8:7bd73a78:::benchmark_data_smithi026_497658_object89492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.107077+0000", 2023-09-23T22:38:38.071 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.955378572, 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0056152540000001, 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188384 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107077+0000", 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107077+0000", 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107076+0000", 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107082+0000", 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9000000000000003e-06 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107094+0000", 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2679e-05 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107102+0000", 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3640000000000001e-06 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107151+0000", 2023-09-23T22:38:38.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9509999999999999e-05 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107162+0000", 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0845999999999999e-05 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.104169+0000", 2023-09-23T22:38:38.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99700736599999995 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.104250+0000", 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0839000000000003e-05 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.104359+0000", 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109159 2023-09-23T22:38:38.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112412+0000", 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080528330000000006 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112558+0000", 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014534199999999999 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112597+0000", 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9882000000000002e-05 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112651+0000", 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.363e-05 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112693+0000", 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1485e-05 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188399 8.e 8:77b1158e:::benchmark_data_smithi026_497658_object89507:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.080 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.133354+0000", 2023-09-23T22:38:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.92910204899999, 2023-09-23T22:38:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97957391500000002, 2023-09-23T22:38:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188399 2023-09-23T22:38:38.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133354+0000", 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133354+0000", 2023-09-23T22:38:38.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133352+0000", 2023-09-23T22:38:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:38:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133359+0000", 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.576e-06 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133371+0000", 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1505000000000001e-05 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133380+0000", 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0850000000000003e-06 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133426+0000", 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6149000000000001e-05 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133436+0000", 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6930000000000002e-06 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.106563+0000", 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.973127305 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.106691+0000", 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127429 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.106810+0000", 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011979799999999999 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112773+0000", 2023-09-23T22:38:38.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059625190000000003 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112784+0000", 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1277999999999999e-05 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112859+0000", 2023-09-23T22:38:38.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5185999999999995e-05 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112891+0000", 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1646000000000002e-05 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112928+0000", 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6625999999999998e-05 2023-09-23T22:38:38.090 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188411 8.e 8:767a8bc9:::benchmark_data_smithi026_497658_object89519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.143178+0000", 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.91927813500001, 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97762692500000004, 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.091 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188411 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143178+0000", 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143178+0000", 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143176+0000", 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143184+0000", 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7600000000000002e-06 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143197+0000", 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3035e-05 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143206+0000", 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7339999999999997e-06 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143334+0000", 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128713 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143345+0000", 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0165e-05 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.108581+0000", 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96523684499999995 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.108722+0000", 2023-09-23T22:38:38.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014076299999999999 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.108854+0000", 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013138000000000001 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112790+0000", 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039360590000000004 2023-09-23T22:38:38.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112821+0000", 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.15e-05 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120715+0000", 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007893466 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120762+0000", 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7178000000000001e-05 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120805+0000", 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2877000000000003e-05 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.099 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188418 8.e 8:72a1b165:::benchmark_data_smithi026_497658_object89526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.147012+0000", 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.915443471, 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98431729099999998, 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.100 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188418 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147012+0000", 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147012+0000", 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147011+0000", 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147016+0000", 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2360000000000004e-06 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147028+0000", 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1131999999999999e-05 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147035+0000", 2023-09-23T22:38:38.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0099999999999998e-06 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147095+0000", 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0294000000000002e-05 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147104+0000", 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4949999999999997e-06 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.110529+0000", 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96342510100000001 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.110634+0000", 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104362 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.110768+0000", 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133886 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112828+0000", 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002059988 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112973+0000", 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145816 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131241+0000", 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018267965000000001 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131290+0000", 2023-09-23T22:38:38.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8979e-05 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131330+0000", 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9229e-05 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188423 8.e 8:79446d78:::benchmark_data_smithi026_497658_object89531:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.155111+0000", 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.90734437200001, 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98102083900000003, 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.109 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188423 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155111+0000", 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155111+0000", 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155111+0000", 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155114+0000", 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.382e-06 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155122+0000", 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7270000000000007e-06 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155128+0000", 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0360000000000003e-06 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155201+0000", 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2506e-05 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155209+0000", 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8100000000000004e-06 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.113014+0000", 2023-09-23T22:38:38.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95780411099999996 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.113100+0000", 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6584999999999999e-05 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.113176+0000", 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6187999999999996e-05 2023-09-23T22:38:38.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120640+0000", 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074639049999999998 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.123359+0000", 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027189860000000001 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136058+0000", 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012698521000000001 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136105+0000", 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7261e-05 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136132+0000", 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7339e-05 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188425 8.e 8:7aac6c1a:::benchmark_data_smithi026_497658_object89533:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.155148+0000", 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.90730797399999, 2023-09-23T22:38:38.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98108856700000002, 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188425 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155148+0000", 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155148+0000", 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155147+0000", 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155150+0000", 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0929999999999999e-06 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155158+0000", 2023-09-23T22:38:38.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7659999999999998e-06 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155164+0000", 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8130000000000001e-06 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155232+0000", 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8145999999999994e-05 2023-09-23T22:38:38.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155238+0000", 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9680000000000003e-06 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.115278+0000", 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96004005299999995 2023-09-23T22:38:38.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.115389+0000", 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000110332 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.115491+0000", 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010268999999999999 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120676+0000", 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051844309999999998 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.125970+0000", 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052943809999999999 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136166+0000", 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010195533 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136200+0000", 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4057999999999998e-05 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136236+0000", 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6789999999999998e-05 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.126 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188426 8.e 8:7e42bbf5:::benchmark_data_smithi026_497658_object89534:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.907286795, 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98118840799999996, 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188426 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:38:38.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155171+0000", 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6089999999999999e-06 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155179+0000", 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9230000000000002e-06 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155185+0000", 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.468e-06 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155258+0000", 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3825999999999994e-05 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155277+0000", 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8732999999999998e-05 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.117481+0000", 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96220415800000003 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.117631+0000", 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149794 2023-09-23T22:38:38.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.117792+0000", 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016125699999999999 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120845+0000", 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030528819999999998 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126058+0000", 2023-09-23T22:38:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052129309999999996 2023-09-23T22:38:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136274+0000", 2023-09-23T22:38:38.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010215504 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136323+0000", 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9487000000000002e-05 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136357+0000", 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4270999999999998e-05 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.134 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188468 8.e 8:707983ca:::benchmark_data_smithi026_497658_object89576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.203282+0000", 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.85917340399999, 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93560747200000005, 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.135 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188468 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203282+0000", 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203282+0000", 2023-09-23T22:38:38.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203281+0000", 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203287+0000", 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.558e-06 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203301+0000", 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4270000000000001e-05 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203309+0000", 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7570000000000004e-06 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203404+0000", 2023-09-23T22:38:38.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5044999999999995e-05 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203414+0000", 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.009e-05 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120892+0000", 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91747810299999999 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.121025+0000", 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132965 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.121149+0000", 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012415200000000001 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126021+0000", 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048717930000000001 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126064+0000", 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2577999999999998e-05 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138822+0000", 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012758472999999999 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138853+0000", 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0626000000000001e-05 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138890+0000", 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7178999999999998e-05 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188494 8.e 8:77384446:::benchmark_data_smithi026_497658_object89602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.231123+0000", 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.831332734, 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.907887483, 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.143 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188494 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231123+0000", 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231123+0000", 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231122+0000", 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231129+0000", 2023-09-23T22:38:38.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.849e-06 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231147+0000", 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8558000000000001e-05 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231154+0000", 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5889999999999999e-06 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231190+0000", 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5886999999999997e-05 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231199+0000", 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6150000000000003e-06 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.123384+0000", 2023-09-23T22:38:38.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89218471700000002 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.123494+0000", 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011040900000000001 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.123601+0000", 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106308 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126048+0000", 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024475690000000001 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138928+0000", 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012879639 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138939+0000", 2023-09-23T22:38:38.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.064e-05 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138977+0000", 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8488000000000001e-05 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139011+0000", 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3550999999999998e-05 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.150 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188560 8.e 8:7d1dd26f:::benchmark_data_smithi026_497658_object89668:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.299766+0000", 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.76269019899999, 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84735557699999997, 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.151 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188560 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299766+0000", 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299766+0000", 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299764+0000", 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299770+0000", 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.356e-06 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299783+0000", 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3609e-05 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299789+0000", 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7710000000000003e-06 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299829+0000", 2023-09-23T22:38:38.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0308000000000001e-05 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299837+0000", 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2529999999999998e-06 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126098+0000", 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82626175000000002 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126228+0000", 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012984800000000001 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126361+0000", 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013277800000000001 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131369+0000", 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050074830000000001 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139019+0000", 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076508490000000004 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147050+0000", 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080308989999999993 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147088+0000", 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7812999999999999e-05 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147121+0000", 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3114999999999997e-05 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188585 8.e 8:741c7c02:::benchmark_data_smithi026_497658_object89693:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.328240+0000", 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.73421582399999, 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81961890599999998, 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.159 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188585 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.328240+0000", 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.328240+0000", 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.328239+0000", 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.328245+0000", 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7760000000000003e-06 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.328259+0000", 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4929000000000001e-05 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.328267+0000", 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7740000000000004e-06 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.328321+0000", 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3384999999999998e-05 2023-09-23T22:38:38.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.328333+0000", 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1915e-05 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.128532+0000", 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.80019907999999995 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.128651+0000", 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011895 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.128770+0000", 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118952 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136017+0000", 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072476349999999997 2023-09-23T22:38:38.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139024+0000", 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003007025 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147787+0000", 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087632130000000006 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147832+0000", 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5058000000000001e-05 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147859+0000", 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6429000000000001e-05 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.166 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188594 8.e 8:72b9e1b0:::benchmark_data_smithi026_497658_object89702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.341081+0000", 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.72137453600001, 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96898526399999996, 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188594 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341081+0000", 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341081+0000", 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341080+0000", 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341086+0000", 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0730000000000001e-06 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341100+0000", 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3634e-05 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341109+0000", 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1039999999999996e-06 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341162+0000", 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2799000000000001e-05 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341172+0000", 2023-09-23T22:38:38.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0117e-05 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131392+0000", 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.79022011400000003 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131504+0000", 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011189800000000001 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131612+0000", 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107944 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136037+0000", 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004425544 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139029+0000", 2023-09-23T22:38:38.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029920530000000002 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.309970+0000", 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.170940913 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310027+0000", 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6858999999999999e-05 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310067+0000", 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9443999999999997e-05 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188614 8.e 8:7bd220a9:::benchmark_data_smithi026_497658_object89722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.369895+0000", 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.69256126100001, 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94027976199999996, 2023-09-23T22:38:38.174 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188614 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369895+0000", 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369895+0000", 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369893+0000", 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369898+0000", 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7949999999999998e-06 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369907+0000", 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5560000000000008e-06 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369912+0000", 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1200000000000001e-06 2023-09-23T22:38:38.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369951+0000", 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9601000000000003e-05 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369962+0000", 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0267e-05 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.133818+0000", 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.76385622200000003 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.133886+0000", 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7848000000000005e-05 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.133963+0000", 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7553999999999996e-05 2023-09-23T22:38:38.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138783+0000", 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004819416 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139033+0000", 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025017000000000003 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310106+0000", 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.17107334900000001 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310144+0000", 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7375e-05 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310174+0000", 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0691999999999997e-05 2023-09-23T22:38:38.181 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188632 8.e 8:7f064a7d:::benchmark_data_smithi026_497658_object89740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.492803+0000", 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.56965241500001, 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.81747772100000005, 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.182 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188632 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.492803+0000", 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.492803+0000", 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.492802+0000", 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.492807+0000", 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7579999999999998e-06 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.492821+0000", 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3327e-05 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.492829+0000", 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8930000000000005e-06 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.493331+0000", 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050263900000000004 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.493344+0000", 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2486999999999999e-05 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136386+0000", 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.643042161 2023-09-23T22:38:38.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136500+0000", 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113681 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136600+0000", 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010005000000000001 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147134+0000", 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010534262000000001 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147173+0000", 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8940999999999999e-05 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310214+0000", 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.163040824 2023-09-23T22:38:38.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310251+0000", 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6971000000000002e-05 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310281+0000", 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0397000000000001e-05 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.189 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188668 8.e 8:76500427:::benchmark_data_smithi026_497658_object89776:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.570702+0000", 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.49175403300001, 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.73968478400000004, 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.190 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188668 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.570702+0000", 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.570702+0000", 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.570700+0000", 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.570707+0000", 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5610000000000004e-06 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.570735+0000", 2023-09-23T22:38:38.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8526000000000001e-05 2023-09-23T22:38:38.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.570752+0000", 2023-09-23T22:38:38.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6328999999999999e-05 2023-09-23T22:38:38.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.570804+0000", 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2213000000000003e-05 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.570811+0000", 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7909999999999999e-06 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139068+0000", 2023-09-23T22:38:38.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.56825771800000002 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139182+0000", 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113948 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139291+0000", 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108436 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147205+0000", 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0079141860000000001 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147212+0000", 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2660000000000004e-06 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310322+0000", 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16310936100000001 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310352+0000", 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0141000000000002e-05 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310387+0000", 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4913000000000003e-05 2023-09-23T22:38:38.197 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188684 8.e 8:7ec78580:::benchmark_data_smithi026_497658_object89792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.625224+0000", 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 234.43723162200001, 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68527995500000005, 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.198 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188684 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.625224+0000", 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.625224+0000", 2023-09-23T22:38:38.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.625223+0000", 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.625229+0000", 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.19e-06 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.625245+0000", 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6285000000000001e-05 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.625253+0000", 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0970000000000006e-06 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.625303+0000", 2023-09-23T22:38:38.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9685000000000003e-05 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.625313+0000", 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8239999999999995e-06 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.141794+0000", 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.51648112300000004 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.141903+0000", 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109443 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.142018+0000", 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114641 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147217+0000", 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051988390000000002 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.309901+0000", 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.16268445200000001 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310426+0000", 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052441399999999996 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310469+0000", 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.333e-05 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310504+0000", 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5029e-05 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104685 9.2 9:46fec5fe:::benchmark_data_smithi026_555954_object9956:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.292511+0000", 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.769944586999998, 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0057272559999999, 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104685 2023-09-23T22:38:38.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292511+0000", 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292511+0000", 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292509+0000", 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292518+0000", 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4230000000000005e-06 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292537+0000", 2023-09-23T22:38:38.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9038999999999998e-05 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292550+0000", 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3446e-05 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292613+0000", 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2655000000000002e-05 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292625+0000", 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1747e-05 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298147+0000", 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0055220330000001 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298238+0000", 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1748000000000003e-05 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104686 9.2 9:4bc296ca:::benchmark_data_smithi026_555954_object9957:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.769885958000003, 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.032939702, 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.211 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104686 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292569+0000", 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292573+0000", 2023-09-23T22:38:38.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6500000000000002e-06 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292581+0000", 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8080000000000005e-06 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292589+0000", 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4209999999999995e-06 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292656+0000", 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7243999999999995e-05 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292666+0000", 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3850000000000008e-06 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298265+0000", 2023-09-23T22:38:38.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.005598754 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298394+0000", 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012897599999999999 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298532+0000", 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001381 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.308571+0000", 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010039792000000001 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.308872+0000", 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030035000000000001 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325328+0000", 2023-09-23T22:38:38.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016456529000000001 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325391+0000", 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2385999999999995e-05 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325510+0000", 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118841 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.218 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104695 9.6 9:6983d852:::benchmark_data_smithi026_555954_object9966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.320260+0000", 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.742196320000005, 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.000856502, 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:38.219 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104695 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320260+0000", 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320260+0000", 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:38:38.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320258+0000", 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320267+0000", 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6309999999999992e-06 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:38:38.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320286+0000", 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.893e-05 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320294+0000", 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1999999999999994e-06 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320349+0000", 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5631999999999999e-05 2023-09-23T22:38:38.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320362+0000", 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3037000000000001e-05 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.309531+0000", 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98916838299999998 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.309666+0000", 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135649 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6", 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.309801+0000", 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134764 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.317002+0000", 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072011929999999998 2023-09-23T22:38:38.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.320998+0000", 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039957480000000004 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.321034+0000", 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5407999999999997e-05 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:38:38.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.321116+0000", 2023-09-23T22:38:38.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2433000000000002e-05 2023-09-23T22:38:38.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.226 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:38:38.226 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:38:38.226 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:38:38.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:38.265 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:38.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:38 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2887192339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:38.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15149/238287 objects degraded (6.357%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:38 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2887192339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:38.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15149/238287 objects degraded (6.357%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:38.613 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:38:38.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:38 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2887192339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:38.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15149/238287 objects degraded (6.357%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:38.724 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:38.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:38.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:38.833 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:38.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:39.223 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:38:39.290 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:39.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:39.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:39.325 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:39.397 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:39.425 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:39.480 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:39.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:39 smithi026 ceph-mon[223014]: pgmap v1322: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 15112/237534 objects degraded (6.362%); 25609/237534 objects misplaced (10.781%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T22:38:39.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:39.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:39.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:39 smithi026 ceph-mon[215775]: pgmap v1322: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 15112/237534 objects degraded (6.362%); 25609/237534 objects misplaced (10.781%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T22:38:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:39.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:39.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:39.589 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:39.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:39 smithi070 ceph-mon[192652]: pgmap v1322: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 112 op/s; 15112/237534 objects degraded (6.362%); 25609/237534 objects misplaced (10.781%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T22:38:39.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:39.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:39.631 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:39.637 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:39.740 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:39.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:39.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:39.844 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:39.870 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:38:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:38:39] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-09-23T22:38:39.873 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:39.921 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:39.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:39.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:40.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:40.053 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:40.098 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:40.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:40.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:40.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:40.235 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:40.495 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:40.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:40.607 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:40.641 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:40.693 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:40.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:40.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:40.799 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:40.827 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:40.876 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:40.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:40.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:40.980 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:41.032 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:41.106 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:41.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:41.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:41.195 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:38:41.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:41.236 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:41.279 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:41.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:41.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:41.382 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:41.389 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 3, 5, 6, 7, 2, 4, 1] 2023-09-23T22:38:41.389 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:38:41.389 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T22:38:41.389 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:38:41.389 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:38:41.389 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T22:38:41.410 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:41.465 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:41.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:41.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:41 smithi026 ceph-mon[223014]: pgmap v1323: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 15112/237441 objects degraded (6.365%); 25576/237441 objects misplaced (10.772%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:38:41.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:41 smithi026 ceph-mon[215775]: pgmap v1323: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 15112/237441 objects degraded (6.365%); 25576/237441 objects misplaced (10.772%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:38:41.569 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:41.611 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:41.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:41 smithi070 ceph-mon[192652]: pgmap v1323: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 115 op/s; 15112/237441 objects degraded (6.365%); 25576/237441 objects misplaced (10.772%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T22:38:41.645 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:41.673 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:41.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:41.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:41.777 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:41.803 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:42.331 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:38:42.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2990964988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:38:42.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2990964988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:38:42.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2990964988' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:38:43.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[223014]: pgmap v1324: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 15072/236235 objects degraded (6.380%); 24672/236235 objects misplaced (10.444%); 10 MiB/s, 163 objects/s recovering 2023-09-23T22:38:43.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:43.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[223014]: failed to encode map e509 with expected crc 2023-09-23T22:38:43.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2990964988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:38:43.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[223014]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T22:38:43.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[223014]: 1.5 scrub starts 2023-09-23T22:38:43.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15072/236235 objects degraded (6.380%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:43.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[215775]: pgmap v1324: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 15072/236235 objects degraded (6.380%); 24672/236235 objects misplaced (10.444%); 10 MiB/s, 163 objects/s recovering 2023-09-23T22:38:43.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:43.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[215775]: failed to encode map e509 with expected crc 2023-09-23T22:38:43.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2990964988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:38:43.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[215775]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T22:38:43.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[215775]: 1.5 scrub starts 2023-09-23T22:38:43.601 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15072/236235 objects degraded (6.380%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:43.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:43 smithi070 ceph-mon[192652]: pgmap v1324: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 15072/236235 objects degraded (6.380%); 24672/236235 objects misplaced (10.444%); 10 MiB/s, 163 objects/s recovering 2023-09-23T22:38:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:43 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:43 smithi070 ceph-mon[192652]: failed to encode map e509 with expected crc 2023-09-23T22:38:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2990964988' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:38:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:43 smithi070 ceph-mon[192652]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T22:38:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:43 smithi070 ceph-mon[192652]: 1.5 scrub starts 2023-09-23T22:38:43.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15072/236235 objects degraded (6.380%), 10 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:43.855 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:38:43.874 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:38:43.896 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:38:43.917 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:38:43.936 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:38:43.956 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:38:43.976 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:38:44.001 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:38:44.002 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:38:44.002 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:38:44.002 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:38:44.003 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:38:44.003 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:38:44.003 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:38:44.003 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:38:44.003 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:38:44.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:44 smithi026 ceph-mon[215775]: failed to encode map e509 with expected crc 2023-09-23T22:38:44.329 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:44 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2677716921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:44.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:44 smithi026 ceph-mon[223014]: failed to encode map e509 with expected crc 2023-09-23T22:38:44.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:44 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2677716921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:44.615 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:44.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:44 smithi070 ceph-mon[192652]: failed to encode map e509 with expected crc 2023-09-23T22:38:44.616 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:44 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2677716921' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:44.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:44.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:44.718 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:44.756 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:44.781 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:44.809 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:44.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:44.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:44.952 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:45.002 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:45.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:45.109 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:45.138 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:45.285 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:45.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:45.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:45.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[215775]: pgmap v1326: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 15030/235662 objects degraded (6.378%); 24615/235662 objects misplaced (10.445%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:38:45.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[215775]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.333 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[215775]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[215775]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[215775]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T22:38:45.334 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[215775]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.388 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:45.426 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:45.451 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:38:45.499 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:45.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:45.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:45.613 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:45 smithi070 ceph-mon[192652]: pgmap v1326: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 15030/235662 objects degraded (6.378%); 24615/235662 objects misplaced (10.445%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:38:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:45 smithi070 ceph-mon[192652]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:45 smithi070 ceph-mon[192652]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:45 smithi070 ceph-mon[192652]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:45 smithi070 ceph-mon[192652]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T22:38:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:45 smithi070 ceph-mon[192652]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.639 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:38:45.639 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:38:45.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[223014]: pgmap v1326: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 123 op/s; 15030/235662 objects degraded (6.378%); 24615/235662 objects misplaced (10.445%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T22:38:45.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[223014]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[223014]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[223014]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[223014]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T22:38:45.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:45 smithi026 ceph-mon[223014]: failed to encode map e510 with expected crc 2023-09-23T22:38:45.650 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:45.724 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:45.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:45.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:45.831 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:45.875 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:45.933 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:45.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:45.982 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:46.039 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:46.069 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:46.129 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:46.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:46.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:46.234 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:46.263 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:46.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[223014]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[223014]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[223014]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[223014]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T22:38:46.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[223014]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[215775]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[215775]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[215775]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[215775]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T22:38:46.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:46 smithi026 ceph-mon[215775]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.602 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:46 smithi070 ceph-mon[192652]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.602 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:46 smithi070 ceph-mon[192652]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.602 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:46 smithi070 ceph-mon[192652]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.602 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:46 smithi070 ceph-mon[192652]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T22:38:46.602 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:46 smithi070 ceph-mon[192652]: failed to encode map e511 with expected crc 2023-09-23T22:38:46.662 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:46.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:46.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:46.678 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:38:46.767 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:46.797 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:46.806 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:38:46.855 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:46.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:46.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:46.961 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:46.992 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:47.093 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:47.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:47.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:47.111 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:38:47.111 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T22:38:47.111 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:38:47.189 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:47.197 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:47.231 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:47.293 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:47.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:47.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:47.366 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:47.393 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:47 smithi070 ceph-mon[192652]: pgmap v1329: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 14963/234618 objects degraded (6.378%); 24084/234618 objects misplaced (10.265%); 14 MiB/s, 288 objects/s recovering 2023-09-23T22:38:47.401 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:47.434 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:47.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:47 smithi026 ceph-mon[215775]: pgmap v1329: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 14963/234618 objects degraded (6.378%); 24084/234618 objects misplaced (10.265%); 14 MiB/s, 288 objects/s recovering 2023-09-23T22:38:47.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:47 smithi026 ceph-mon[223014]: pgmap v1329: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 156 op/s; 14963/234618 objects degraded (6.378%); 24084/234618 objects misplaced (10.265%); 14 MiB/s, 288 objects/s recovering 2023-09-23T22:38:47.551 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:47.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:47.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:47.658 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:47.689 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:48.076 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:48.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:48.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:48.179 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:48.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:48.257 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:48.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:48.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:48.355 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:38:48.362 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:48.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3008545609' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:48.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:48 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:48.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 14963/234618 objects degraded (6.378%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:48.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3008545609' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:48.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:48 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:48.380 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 14963/234618 objects degraded (6.378%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:48.389 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:48.439 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:48.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:48.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:48.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:48.570 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:48.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3008545609' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:48 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:38:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 14963/234618 objects degraded (6.378%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:48.667 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:48.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:48.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:48.770 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:48.797 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:48.839 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:48.841 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:48.942 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:48.984 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:49.286 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:49.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:49.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:49.389 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:49.418 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:49.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:49 smithi070 ceph-mon[192652]: pgmap v1330: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 14915/233721 objects degraded (6.382%); 23987/233721 objects misplaced (10.263%); 6.2 MiB/s, 160 objects/s recovering 2023-09-23T22:38:49.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:49 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:38:49.494 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:49 smithi070 ceph-mon[192652]: failed to encode map e512 with expected crc 2023-09-23T22:38:49.494 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:49 smithi070 ceph-mon[192652]: failed to encode map e512 with expected crc 2023-09-23T22:38:49.494 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:49 smithi070 ceph-mon[192652]: failed to encode map e512 with expected crc 2023-09-23T22:38:49.494 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:49 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:49.494 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:49 smithi070 ceph-mon[192652]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T22:38:49.523 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:49.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:49.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:49.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[223014]: pgmap v1330: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 14915/233721 objects degraded (6.382%); 23987/233721 objects misplaced (10.263%); 6.2 MiB/s, 160 objects/s recovering 2023-09-23T22:38:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:38:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[223014]: failed to encode map e512 with expected crc 2023-09-23T22:38:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[223014]: failed to encode map e512 with expected crc 2023-09-23T22:38:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[223014]: failed to encode map e512 with expected crc 2023-09-23T22:38:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[223014]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T22:38:49.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:38:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:38:49] "GET /metrics HTTP/1.1" 200 36491 "" "Prometheus/2.43.0" 2023-09-23T22:38:49.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[215775]: pgmap v1330: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 139 op/s; 14915/233721 objects degraded (6.382%); 23987/233721 objects misplaced (10.263%); 6.2 MiB/s, 160 objects/s recovering 2023-09-23T22:38:49.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:38:49.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[215775]: failed to encode map e512 with expected crc 2023-09-23T22:38:49.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[215775]: failed to encode map e512 with expected crc 2023-09-23T22:38:49.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[215775]: failed to encode map e512 with expected crc 2023-09-23T22:38:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:38:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:49 smithi026 ceph-mon[215775]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T22:38:49.628 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:49.660 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:49.679 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:38:49.704 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:38:49.716 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:49.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:49.719 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:49.731 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:38:49.736 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:38:49.737 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:38:49.765 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:38:49.789 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:38:49.808 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:38:49.819 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:49.828 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:38:49.852 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:38:49.855 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:49.855 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:38:49.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:38:49.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:38:49.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:38:49.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:38:49.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:38:49.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:38:49.856 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:38:50.009 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:50.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:50.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:50.252 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:50.281 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:50.486 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:38:50.501 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:50.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:50.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:50.528 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3551280253' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:50.607 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:50.628 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3551280253' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:50.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3551280253' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:50.647 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:50.711 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:50.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:50.719 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:50.820 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:50.860 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:50.917 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:50.918 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:38:50.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:50.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:51.024 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:51.069 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:51.132 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:51.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:51.238 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:51.276 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:51.297 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:38:51.348 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:51.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:51.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:51.453 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:51.488 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:51.560 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:51.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:51 smithi070 ceph-mon[192652]: pgmap v1332: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14915/233616 objects degraded (6.384%); 23951/233616 objects misplaced (10.252%); 5.8 MiB/s, 154 objects/s recovering 2023-09-23T22:38:51.666 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:38:51.701 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:51.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:51 smithi026 ceph-mon[223014]: pgmap v1332: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14915/233616 objects degraded (6.384%); 23951/233616 objects misplaced (10.252%); 5.8 MiB/s, 154 objects/s recovering 2023-09-23T22:38:51.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:51 smithi026 ceph-mon[215775]: pgmap v1332: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 97 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 113 op/s; 14915/233616 objects degraded (6.384%); 23951/233616 objects misplaced (10.252%); 5.8 MiB/s, 154 objects/s recovering 2023-09-23T22:38:51.775 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:51.870 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:38:51.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:51.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:38:51.974 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:52.011 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:52.068 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:52.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:52.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:52.173 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:52.228 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:52.342 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:52.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:52.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:52.448 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:52.478 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:52.631 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:52.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:52.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:52.656 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:38:52.736 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:52.769 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:52.848 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:38:52.848 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:38:52.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:38:52.848 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:38:52.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48942 9.c 9.a863b59c (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.682106+0000", 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 181.16223082400001, 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99119241000000002, 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48942 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.849 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682106+0000", 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682106+0000", 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682105+0000", 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682183+0000", 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8666999999999997e-05 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682195+0000", 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2023e-05 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682203+0000", 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9549999999999992e-06 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691884+0000", 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096805209999999992 2023-09-23T22:38:52.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691901+0000", 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7312e-05 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673292+0000", 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98139073399999999 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673299+0000", 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7959999999999998e-06 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48944 9.c 9.f6505a7c (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.682215+0000", 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 181.162122243, 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99109994300000004, 2023-09-23T22:38:52.854 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48944 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682215+0000", 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682215+0000", 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682214+0000", 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682280+0000", 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5757000000000001e-05 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682288+0000", 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2889999999999998e-06 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682296+0000", 2023-09-23T22:38:52.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2710000000000003e-06 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691924+0000", 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096284600000000001 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691930+0000", 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.122e-06 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673310+0000", 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.981380107 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673315+0000", 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5009999999999998e-06 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48956 9.e 9.222599de (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.682302+0000", 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 181.162035594, 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98747526900000004, 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48956 2023-09-23T22:38:52.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682302+0000", 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682302+0000", 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682301+0000", 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682355+0000", 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3656999999999998e-05 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682362+0000", 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2960000000000001e-06 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682367+0000", 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7790000000000003e-06 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.693185+0000", 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010818533 2023-09-23T22:38:52.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.693190+0000", 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9110000000000001e-06 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.669765+0000", 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97657507700000001 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.669777+0000", 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1501999999999999e-05 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48967 9.e 9.ebcd835e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.704140+0000", 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 181.14019684799999, 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96566792199999996, 2023-09-23T22:38:52.865 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48967 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.704140+0000", 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.704140+0000", 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.704139+0000", 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.704199+0000", 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0081000000000003e-05 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.704209+0000", 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0137e-05 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.704219+0000", 2023-09-23T22:38:52.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0018e-05 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.704290+0000", 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0710999999999994e-05 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.704308+0000", 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8717999999999999e-05 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.669800+0000", 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96549157699999999 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.669808+0000", 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3580000000000003e-06 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48974 9.c 9.c8bb748c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.712465+0000", 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 181.13187184899999, 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96086489500000005, 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48974 2023-09-23T22:38:52.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.712465+0000", 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.712465+0000", 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.712463+0000", 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.712519+0000", 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5701000000000001e-05 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.712537+0000", 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.774e-05 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.712545+0000", 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4149999999999999e-06 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.712622+0000", 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6460999999999999e-05 2023-09-23T22:38:52.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.712634+0000", 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2257e-05 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673326+0000", 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96069219699999997 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673330+0000", 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2189999999999998e-06 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48987 9.e 9.6a3361de (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.728127+0000", 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 181.116210712, 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94522904699999999, 2023-09-23T22:38:52.876 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48987 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.728127+0000", 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.728127+0000", 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.728125+0000", 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.728179+0000", 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4656e-05 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.728202+0000", 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2187e-05 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.728210+0000", 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3950000000000001e-06 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.728267+0000", 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7500000000000002e-05 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.728278+0000", 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0129000000000001e-05 2023-09-23T22:38:52.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673350+0000", 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94507253199999997 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673356+0000", 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4280000000000004e-06 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.881 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75839 9.e 9:747fb0cf:::benchmark_data_smithi026_555954_object75838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.254321+0000", 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 149.59001606300001, 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.009841719, 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.882 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75839 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254321+0000", 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254321+0000", 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254319+0000", 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254433+0000", 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113632 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254450+0000", 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6667000000000002e-05 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254456+0000", 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7610000000000002e-06 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254520+0000", 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4104999999999997e-05 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254529+0000", 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8799999999999997e-06 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.239678+0000", 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98514882599999998 2023-09-23T22:38:52.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.239842+0000", 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016353099999999999 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.240035+0000", 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019364800000000001 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.249930+0000", 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.009894524 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264057+0000", 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014127293000000001 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264105+0000", 2023-09-23T22:38:52.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7617999999999997e-05 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264138+0000", 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3302000000000001e-05 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264163+0000", 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4776000000000001e-05 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.889 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75851 9.e 9:72e98536:::benchmark_data_smithi026_555954_object75850:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.266486+0000", 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 149.57785100000001, 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.997817069, 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.890 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75851 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266486+0000", 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266486+0000", 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266485+0000", 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266560+0000", 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4641999999999994e-05 2023-09-23T22:38:52.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266591+0000", 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1872999999999998e-05 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266600+0000", 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0960000000000007e-06 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266665+0000", 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4324000000000003e-05 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266675+0000", 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0036e-05 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.241436+0000", 2023-09-23T22:38:52.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97476125800000002 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.241566+0000", 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012998400000000001 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.241731+0000", 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000164981 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.249937+0000", 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082056939999999995 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264197+0000", 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014260324 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264251+0000", 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3449000000000003e-05 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264281+0000", 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0531000000000003e-05 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264303+0000", 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2221999999999998e-05 2023-09-23T22:38:52.897 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75870 9.e 9:7ffb9527:::benchmark_data_smithi026_555954_object75869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.291109+0000", 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 149.55322782299999, 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97326064000000001, 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.898 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75870 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291109+0000", 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.899 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291109+0000", 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291108+0000", 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291155+0000", 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6782e-05 2023-09-23T22:38:52.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291172+0000", 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7436e-05 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291180+0000", 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.994e-06 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291269+0000", 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8403e-05 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291288+0000", 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9711000000000002e-05 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.243282+0000", 2023-09-23T22:38:52.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.951993172 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.243406+0000", 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000124828 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.243551+0000", 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014428499999999999 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.253042+0000", 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094910329999999994 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264225+0000", 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011183383 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.904 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264322+0000", 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6733999999999999e-05 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264349+0000", 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6846999999999999e-05 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264370+0000", 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1325999999999999e-05 2023-09-23T22:38:52.905 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75902 9.e 9:7aab8295:::benchmark_data_smithi026_555954_object75901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.319436+0000", 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 149.52490079500001, 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.945147132, 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.906 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75902 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.319436+0000", 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.907 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.319436+0000", 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.319435+0000", 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.319545+0000", 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011063799999999999 2023-09-23T22:38:52.908 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.319563+0000", 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7136999999999999e-05 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.319570+0000", 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4580000000000004e-06 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.909 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.319630+0000", 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0180000000000003e-05 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.319640+0000", 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0108999999999999e-05 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.910 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.245222+0000", 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92558137299999999 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.245299+0000", 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7537e-05 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.245383+0000", 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4011000000000006e-05 2023-09-23T22:38:52.911 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.253052+0000", 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.007668795 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264418+0000", 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011366008 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.912 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264506+0000", 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8479000000000004e-05 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264558+0000", 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1409999999999997e-05 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264584+0000", 2023-09-23T22:38:52.913 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5675999999999999e-05 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75977 9.e 9:733440cf:::benchmark_data_smithi026_555954_object75976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.389369+0000", 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 149.45496816599999, 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.87535491099999996, 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.914 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75977 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.389369+0000", 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.915 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.389369+0000", 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.389362+0000", 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999928 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.389392+0000", 2023-09-23T22:38:52.916 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9989e-05 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.389403+0000", 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1428e-05 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.389411+0000", 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4919999999999996e-06 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.917 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.389475+0000", 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3633999999999994e-05 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.389485+0000", 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9890000000000004e-06 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.918 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.246465+0000", 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.85698057299999997 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.246583+0000", 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000117565 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.246677+0000", 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4235999999999997e-05 2023-09-23T22:38:52.919 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.253057+0000", 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0063799640000000001 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264445+0000", 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011387711999999999 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.920 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264642+0000", 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000197288 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264685+0000", 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3118000000000003e-05 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264724+0000", 2023-09-23T22:38:52.921 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9014000000000003e-05 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104674 9.e 9:7c43a090:::benchmark_data_smithi026_555954_object9945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 98.591418942999994, 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0412800680000001, 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:38:52.922 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104674 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.923 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252921+0000", 2023-09-23T22:38:52.924 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6509999999999999e-06 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252938+0000", 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6416999999999999e-05 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252943+0000", 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8679999999999998e-06 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.925 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.254941+0000", 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001998395 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.255082+0000", 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014090000000000001 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:38:52.926 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.255225+0000", 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014290599999999999 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.260880+0000", 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056549929999999997 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.261192+0000", 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031205499999999999 2023-09-23T22:38:52.927 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.280742+0000", 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019549997 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.280749+0000", 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2590000000000003e-06 2023-09-23T22:38:52.928 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.294129+0000", 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0133797609999999 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.294198+0000", 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9536999999999995e-05 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.929 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104670 9.4 9:28c1ceab:::benchmark_data_smithi026_555954_object9941:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.293142+0000", 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 98.551195000999996, 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0530680600000002, 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.930 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104670 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293142+0000", 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293142+0000", 2023-09-23T22:38:52.931 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293141+0000", 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293146+0000", 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6229999999999996e-06 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.932 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293155+0000", 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4839999999999993e-06 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293163+0000", 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8669999999999994e-06 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.933 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293213+0000", 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9536999999999997e-05 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293222+0000", 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1740000000000006e-06 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.261387+0000", 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9681656429999999 2023-09-23T22:38:52.934 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.265588+0000", 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042009059999999999 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.342211+0000", 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.076622814999999997 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.935 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.346210+0000", 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039992270000000002 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104676 9.4 9:2b72389e:::benchmark_data_smithi026_555954_object9947:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 98.551160181, 2023-09-23T22:38:52.936 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0644994749999999, 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104676 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:38:52.937 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:38:52.938 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293180+0000", 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.898e-06 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293186+0000", 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5080000000000002e-06 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.939 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293191+0000", 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7829999999999998e-06 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293244+0000", 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2927999999999999e-05 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293249+0000", 2023-09-23T22:38:52.940 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1440000000000002e-06 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.265969+0000", 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9727195919999998 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.270138+0000", 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041699170000000004 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.941 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.277809+0000", 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0076701460000000001 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.310996+0000", 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033187649999999999 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.942 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.347606+0000", 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036609292000000002 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.349036+0000", 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014300789999999999 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.351072+0000", 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020369070000000001 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.943 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.357677+0000", 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066040279999999996 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104680 9.4 9:208db86f:::benchmark_data_smithi026_555954_object9951:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.293206+0000", 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 98.551131685000001, 2023-09-23T22:38:52.944 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0756980610000002, 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104680 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293206+0000", 2023-09-23T22:38:52.945 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293206+0000", 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293205+0000", 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.946 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293208+0000", 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5409999999999999e-06 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293214+0000", 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2210000000000002e-06 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.947 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293218+0000", 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0640000000000004e-06 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293268+0000", 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9852999999999999e-05 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293273+0000", 2023-09-23T22:38:52.948 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8940000000000001e-06 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.287851+0000", 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9945780060000002 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.293072+0000", 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052214150000000001 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.949 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.300397+0000", 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073247130000000001 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.317253+0000", 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016856401 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.950 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.358723+0000", 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.041469682000000001 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.359839+0000", 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011156460000000001 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.361542+0000", 2023-09-23T22:38:52.951 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001703148 2023-09-23T22:38:52.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.953 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.953 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.953 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.368904+0000", 2023-09-23T22:38:52.953 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073618570000000003 2023-09-23T22:38:52.953 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.953 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.953 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.953 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(9.c 489/478 [PushOp(9:31531dd3:::benchmark_data_smithi026_555954_object9982:head, version: 460'627, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:31531dd3:::benchmark_data_smithi026_555954_object9982:head@460'627, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.334517+0000", 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 97.509820336999994, 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6523379540000001, 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334517+0000", 2023-09-23T22:38:52.954 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334517+0000", 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334516+0000", 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.955 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334620+0000", 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010427799999999999 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334637+0000", 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6889999999999999e-05 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.956 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334646+0000", 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9209999999999998e-06 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.984277+0000", 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6496310489999999 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.984284+0000", 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8950000000000001e-06 2023-09-23T22:38:52.957 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.986855+0000", 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002571041 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104715 9.4 9:22773e84:::benchmark_data_smithi026_555954_object9986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.341862+0000", 2023-09-23T22:38:52.958 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 97.502475458000006, 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2123820830000001, 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104715 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.959 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341862+0000", 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341862+0000", 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341859+0000", 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:38:52.960 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341870+0000", 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0808999999999999e-05 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341889+0000", 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8604e-05 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.961 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341900+0000", 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1754e-05 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.306973+0000", 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.965072514 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:38:52.962 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.310651+0000", 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036783609999999998 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.383378+0000", 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.072726329000000006 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.387721+0000", 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043429649999999998 2023-09-23T22:38:52.963 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.502428+0000", 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114707641 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.506100+0000", 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036718440000000001 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.964 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.512882+0000", 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067814260000000001 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.543660+0000", 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030778920000000001 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.544694+0000", 2023-09-23T22:38:52.965 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001033574 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.545296+0000", 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00060233699999999999 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.547534+0000", 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022377349999999998 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.966 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.554244+0000", 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067097160000000001 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(9.c 489/478 [PushOp(9:3a91d401:::benchmark_data_smithi026_555954_object9993:head, version: 460'628, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:3a91d401:::benchmark_data_smithi026_555954_object9993:head@460'628, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.362751+0000", 2023-09-23T22:38:52.967 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 97.481586668999995, 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6258491149999998, 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362751+0000", 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.968 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362751+0000", 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362750+0000", 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362865+0000", 2023-09-23T22:38:52.969 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115596 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362882+0000", 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7062e-05 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362897+0000", 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4198e-05 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.970 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.986936+0000", 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6240388779999999 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.986943+0000", 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7989999999999994e-06 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.971 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.988600+0000", 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001656364 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104723 9.4 9:24a0630c:::benchmark_data_smithi026_555954_object9994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.365986+0000", 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 97.478350739000007, 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2005458440000001, 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.972 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104723 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365986+0000", 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.973 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365986+0000", 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365985+0000", 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.974 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365992+0000", 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6150000000000002e-06 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.366002+0000", 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0249e-05 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.366007+0000", 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1039999999999998e-06 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.975 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.393428+0000", 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.027421414 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.398310+0000", 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048813049999999998 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.976 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.425313+0000", 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027003367 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.429163+0000", 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038503410000000002 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.527354+0000", 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098190762000000001 2023-09-23T22:38:52.977 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.530473+0000", 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031186880000000001 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.537672+0000", 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0071995219999999999 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.978 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.555295+0000", 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017622819000000001 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.556659+0000", 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013637510000000001 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.557146+0000", 2023-09-23T22:38:52.979 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048692499999999998 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.559282+0000", 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021364320000000002 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.566532+0000", 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00725007 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.980 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104756 9.4 9:258d171c:::benchmark_data_smithi026_555954_object10027:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:16.626866+0000", 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 96.217471316000001, 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.026819157, 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:38:52.981 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104756 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626866+0000", 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626866+0000", 2023-09-23T22:38:52.982 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626865+0000", 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626872+0000", 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2779999999999996e-06 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.983 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626886+0000", 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4470999999999999e-05 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626893+0000", 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5139999999999998e-06 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.984 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.398840+0000", 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.77194742100000002 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.402776+0000", 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039360929999999999 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.442574+0000", 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039797499 2023-09-23T22:38:52.985 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.446766+0000", 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041918859999999997 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.575609+0000", 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12884293299999999 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.986 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.578674+0000", 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00306552 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.586425+0000", 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077511760000000002 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.616679+0000", 2023-09-23T22:38:52.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030253802 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.617743+0000", 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010636319999999999 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.644498+0000", 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026755226999999999 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.646717+0000", 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022190299999999999 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.653685+0000", 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069680569999999997 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.989 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:38:52.990 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:38:52.990 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:38:53.052 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:53.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:53.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:53.078 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [] live_osds: [0, 3, 5, 6, 7, 2, 4, 1] 2023-09-23T22:38:53.078 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:38:53.078 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 3, 5, 6, 7, 2, 4, 1] 2023-09-23T22:38:53.078 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T22:38:53.078 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T22:38:53.142 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:53.159 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:53.192 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:53.311 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:53.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:53.421 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:53.447 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:53 smithi070 ceph-mon[192652]: pgmap v1333: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 14873/232269 objects degraded (6.403%); 23832/232269 objects misplaced (10.261%); 5.2 MiB/s, 133 objects/s recovering 2023-09-23T22:38:53.448 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 14873/232269 objects degraded (6.403%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:53.448 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:38:53 smithi070 systemd[1]: Stopping Ceph osd.6 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:38:53.451 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:53.539 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:53.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:53.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:53.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:53 smithi026 ceph-mon[223014]: pgmap v1333: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 14873/232269 objects degraded (6.403%); 23832/232269 objects misplaced (10.261%); 5.2 MiB/s, 133 objects/s recovering 2023-09-23T22:38:53.566 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 14873/232269 objects degraded (6.403%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:53.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:53 smithi026 ceph-mon[215775]: pgmap v1333: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 155 op/s; 14873/232269 objects degraded (6.403%); 23832/232269 objects misplaced (10.261%); 5.2 MiB/s, 133 objects/s recovering 2023-09-23T22:38:53.566 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 14873/232269 objects degraded (6.403%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:53.607 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:38:53.643 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:53.678 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:53.739 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:53.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:53.847 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:53.875 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:53.927 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:53.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:53.930 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:54.031 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:54.055 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:38:53 smithi070 conmon[131227]: 2023-09-23T22:38:53.826+0000 7fb776f6e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:38:54.056 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:38:53 smithi070 conmon[131227]: 2023-09-23T22:38:53.826+0000 7fb776f6e700 -1 osd.6 512 *** Got signal Terminated *** 2023-09-23T22:38:54.056 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:38:53 smithi070 conmon[131227]: 2023-09-23T22:38:53.826+0000 7fb776f6e700 -1 osd.6 512 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:38:54.058 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:54.110 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:54.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:54.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:54.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:54.258 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:54.396 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:38:54.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[215775]: 1.d scrub starts 2023-09-23T22:38:54.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[215775]: osd.6 marked itself down and dead 2023-09-23T22:38:54.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3900601997' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:54.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:54.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:54.561 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:54.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[223014]: 1.d scrub starts 2023-09-23T22:38:54.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[223014]: osd.6 marked itself down and dead 2023-09-23T22:38:54.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3900601997' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:54.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:54.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:54.562 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:54.570 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:54.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:54.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:54.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:54 smithi070 ceph-mon[192652]: 1.d scrub starts 2023-09-23T22:38:54.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:54 smithi070 ceph-mon[192652]: osd.6 marked itself down and dead 2023-09-23T22:38:54.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3900601997' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:54.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:54 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:38:54.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:38:54.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:38:54.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:54.711 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:55.088 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:55.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:55.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:55.193 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:55.229 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:55.370 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:55 smithi070 ceph-mon[192652]: pgmap v1334: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 14823/231588 objects degraded (6.401%); 23745/231588 objects misplaced (10.253%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:38:55.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:55 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:55.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:55 smithi070 ceph-mon[192652]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:38:55.371 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:55 smithi070 ceph-mon[192652]: failed to encode map e513 with expected crc 2023-09-23T22:38:55.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:55 smithi070 ceph-mon[192652]: failed to encode map e513 with expected crc 2023-09-23T22:38:55.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:55 smithi070 ceph-mon[192652]: failed to encode map e513 with expected crc 2023-09-23T22:38:55.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:55 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:38:55.372 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:55 smithi070 ceph-mon[192652]: osdmap e513: 8 total, 7 up, 8 in 2023-09-23T22:38:55.461 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:55.495 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:55.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:55.498 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:55.596 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:38:55.599 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:38:55.622 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:38:55.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[223014]: pgmap v1334: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 14823/231588 objects degraded (6.401%); 23745/231588 objects misplaced (10.253%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:38:55.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:55.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[223014]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:38:55.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[223014]: failed to encode map e513 with expected crc 2023-09-23T22:38:55.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[223014]: failed to encode map e513 with expected crc 2023-09-23T22:38:55.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[223014]: failed to encode map e513 with expected crc 2023-09-23T22:38:55.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:38:55.642 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[223014]: osdmap e513: 8 total, 7 up, 8 in 2023-09-23T22:38:55.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[215775]: pgmap v1334: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 122 op/s; 14823/231588 objects degraded (6.401%); 23745/231588 objects misplaced (10.253%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T22:38:55.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:38:55.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[215775]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T22:38:55.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[215775]: failed to encode map e513 with expected crc 2023-09-23T22:38:55.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[215775]: failed to encode map e513 with expected crc 2023-09-23T22:38:55.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[215775]: failed to encode map e513 with expected crc 2023-09-23T22:38:55.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:38:55.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:55 smithi026 ceph-mon[215775]: osdmap e513: 8 total, 7 up, 8 in 2023-09-23T22:38:55.645 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:55.649 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:38:55.679 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:38:55.706 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:38:55.738 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:38:55.746 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:38:55.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:55.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:38:55.767 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:38:55.768 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:38:55.768 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:38:55.768 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:38:55.768 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:38:55.768 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:38:55.768 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:38:55.768 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:38:55.768 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:38:55.853 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:55.887 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:55.939 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:55.945 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:38:55 smithi070 bash[372258]: 5b472aeb7f6995e3ce85a3f582277d08ce93bf455d38239112a31015d4a031da 2023-09-23T22:38:55.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:55.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:56.047 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:56.077 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:56.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:56 smithi070 ceph-mon[192652]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:56 smithi070 ceph-mon[192652]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:56 smithi070 ceph-mon[192652]: osdmap e514: 8 total, 7 up, 8 in 2023-09-23T22:38:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:56 smithi070 ceph-mon[192652]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:56 smithi070 ceph-mon[192652]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:56 smithi070 ceph-mon[192652]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:56 smithi070 ceph-mon[192652]: 9.a continuing backfill to osd.3 from (461'1300,512'7236] MIN to 512'7236 2023-09-23T22:38:56.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3029962143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:56.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[215775]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[215775]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[215775]: osdmap e514: 8 total, 7 up, 8 in 2023-09-23T22:38:56.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[215775]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[215775]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[215775]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[215775]: 9.a continuing backfill to osd.3 from (461'1300,512'7236] MIN to 512'7236 2023-09-23T22:38:56.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3029962143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[223014]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[223014]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[223014]: osdmap e514: 8 total, 7 up, 8 in 2023-09-23T22:38:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[223014]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[223014]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[223014]: failed to encode map e514 with expected crc 2023-09-23T22:38:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[223014]: 9.a continuing backfill to osd.3 from (461'1300,512'7236] MIN to 512'7236 2023-09-23T22:38:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3029962143' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:38:57.056 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:38:57.057 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:38:57.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:57.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:57.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:57.145 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:38:57.189 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:38:57.225 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:57.339 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:38:57.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:57.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:38:57.444 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:57.483 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:57.541 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:57.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[215775]: pgmap v1337: 109 pgs: 2 remapped+peering, 8 peering, 17 stale+active+clean, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 12161/230520 objects degraded (5.275%); 18770/230520 objects misplaced (8.142%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:38:57.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T22:38:57.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[215775]: failed to encode map e515 with expected crc 2023-09-23T22:38:57.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[215775]: failed to encode map e515 with expected crc 2023-09-23T22:38:57.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[215775]: failed to encode map e515 with expected crc 2023-09-23T22:38:57.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[215775]: osdmap e515: 8 total, 7 up, 8 in 2023-09-23T22:38:57.544 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[223014]: pgmap v1337: 109 pgs: 2 remapped+peering, 8 peering, 17 stale+active+clean, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 12161/230520 objects degraded (5.275%); 18770/230520 objects misplaced (8.142%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:38:57.544 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T22:38:57.545 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[223014]: failed to encode map e515 with expected crc 2023-09-23T22:38:57.545 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[223014]: failed to encode map e515 with expected crc 2023-09-23T22:38:57.545 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[223014]: failed to encode map e515 with expected crc 2023-09-23T22:38:57.545 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:57 smithi026 ceph-mon[223014]: osdmap e515: 8 total, 7 up, 8 in 2023-09-23T22:38:57.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:57.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:57.552 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:38:57.552 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:38:57.593 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:57.646 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:57.678 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:57.708 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:57 smithi070 ceph-mon[192652]: pgmap v1337: 109 pgs: 2 remapped+peering, 8 peering, 17 stale+active+clean, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 73 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 142 op/s; 12161/230520 objects degraded (5.275%); 18770/230520 objects misplaced (8.142%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T22:38:57.709 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:57 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T22:38:57.709 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:57 smithi070 ceph-mon[192652]: failed to encode map e515 with expected crc 2023-09-23T22:38:57.709 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:57 smithi070 ceph-mon[192652]: failed to encode map e515 with expected crc 2023-09-23T22:38:57.709 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:57 smithi070 ceph-mon[192652]: failed to encode map e515 with expected crc 2023-09-23T22:38:57.709 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:57 smithi070 ceph-mon[192652]: osdmap e515: 8 total, 7 up, 8 in 2023-09-23T22:38:57.719 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T22:38:57.763 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:38:57.763 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T22:38:58.222 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:58.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:58.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:58.326 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:58.341 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:38:58.357 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:58.412 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:58.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:58.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12161/230520 objects degraded (5.275%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:58.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12161/230520 objects degraded (5.275%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:58.518 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:38:58.544 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:58.586 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:38:58.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:58.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:38:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12161/230520 objects degraded (5.275%), 7 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T22:38:58.693 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:38:58.720 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:58.767 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:38:58.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:58.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:38:58.873 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:38:58.901 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:58.942 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:38:58.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:58.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:38:59.050 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:38:59.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:38:59.083 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:59.245 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:38:59.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:59.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:38:59.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:59.368 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:38:59.391 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:59.531 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:38:59.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:38:59.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:38:59.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:38:59 smithi070 ceph-mon[192652]: pgmap v1339: 109 pgs: 2 remapped+peering, 8 peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 18 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 143 op/s; 21652/229884 objects degraded (9.419%); 18707/229884 objects misplaced (8.138%); 1.6 MiB/s, 23 objects/s recovering 2023-09-23T22:38:59.636 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:38:59.672 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:38:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:38:59] "GET /metrics HTTP/1.1" 200 36490 "" "Prometheus/2.43.0" 2023-09-23T22:38:59.673 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:38:59 smithi026 ceph-mon[215775]: pgmap v1339: 109 pgs: 2 remapped+peering, 8 peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 18 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 143 op/s; 21652/229884 objects degraded (9.419%); 18707/229884 objects misplaced (8.138%); 1.6 MiB/s, 23 objects/s recovering 2023-09-23T22:38:59.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:38:59 smithi026 ceph-mon[223014]: pgmap v1339: 109 pgs: 2 remapped+peering, 8 peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 18 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 143 op/s; 21652/229884 objects degraded (9.419%); 18707/229884 objects misplaced (8.138%); 1.6 MiB/s, 23 objects/s recovering 2023-09-23T22:38:59.675 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:38:59.902 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:00.234 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:00.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:00.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:00.340 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:00.373 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:00.412 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:39:00.431 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:00.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:00.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:00.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2566424926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:00.483 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2566424926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:00.539 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:00.568 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:00.611 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:00.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:00.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:00.715 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:00.739 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2566424926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:00.742 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:00.785 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:00.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:00.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:00.892 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:00.927 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:01.389 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:01.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:01.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:01.493 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:01.527 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:01.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[215775]: pgmap v1340: 109 pgs: 2 remapped+peering, 8 peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 18 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 105 op/s; 21652/229884 objects degraded (9.419%); 18707/229884 objects misplaced (8.138%); 687 KiB/s, 9 objects/s recovering 2023-09-23T22:39:01.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:01.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[215775]: failed to encode map e516 with expected crc 2023-09-23T22:39:01.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[215775]: failed to encode map e516 with expected crc 2023-09-23T22:39:01.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2566424926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:01.648 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[215775]: osdmap e516: 8 total, 7 up, 8 in 2023-09-23T22:39:01.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[223014]: pgmap v1340: 109 pgs: 2 remapped+peering, 8 peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 18 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 105 op/s; 21652/229884 objects degraded (9.419%); 18707/229884 objects misplaced (8.138%); 687 KiB/s, 9 objects/s recovering 2023-09-23T22:39:01.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:01.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[223014]: failed to encode map e516 with expected crc 2023-09-23T22:39:01.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[223014]: failed to encode map e516 with expected crc 2023-09-23T22:39:01.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2566424926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:01.649 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:01 smithi026 ceph-mon[223014]: osdmap e516: 8 total, 7 up, 8 in 2023-09-23T22:39:01.673 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:01 smithi070 ceph-mon[192652]: pgmap v1340: 109 pgs: 2 remapped+peering, 8 peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 18 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 105 op/s; 21652/229884 objects degraded (9.419%); 18707/229884 objects misplaced (8.138%); 687 KiB/s, 9 objects/s recovering 2023-09-23T22:39:01.674 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:01 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:01.674 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:01 smithi070 ceph-mon[192652]: failed to encode map e516 with expected crc 2023-09-23T22:39:01.674 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:01 smithi070 ceph-mon[192652]: failed to encode map e516 with expected crc 2023-09-23T22:39:01.674 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2566424926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:01.674 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:01 smithi070 ceph-mon[192652]: osdmap e516: 8 total, 7 up, 8 in 2023-09-23T22:39:01.732 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:01.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:01.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:01.836 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:01.862 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:01.904 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:01.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:01.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:01.981 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:01.982 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:02.009 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:02.044 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:02.361 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:02.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:02.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:02.469 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:02.498 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:02.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:02 smithi026 ceph-mon[223014]: pgmap v1342: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 23 active+undersized, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 13 KiB/s rd, 35 KiB/s wr, 53 op/s; 31474/229719 objects degraded (13.701%); 18698/229719 objects misplaced (8.140%); 282 KiB/s, 2 objects/s recovering 2023-09-23T22:39:02.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:02 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T22:39:02.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:02 smithi026 ceph-mon[215775]: pgmap v1342: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 23 active+undersized, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 13 KiB/s rd, 35 KiB/s wr, 53 op/s; 31474/229719 objects degraded (13.701%); 18698/229719 objects misplaced (8.140%); 282 KiB/s, 2 objects/s recovering 2023-09-23T22:39:02.625 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:02 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T22:39:02.764 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [6] live_osds: [0, 3, 5, 7, 2, 4, 1] 2023-09-23T22:39:02.764 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:39:02.765 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 3, 5, 7, 2, 4, 1] 2023-09-23T22:39:02.765 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T22:39:02.765 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T22:39:02.859 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:02.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:02.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:02.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:02 smithi070 ceph-mon[192652]: pgmap v1342: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 23 active+undersized, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 13 KiB/s rd, 35 KiB/s wr, 53 op/s; 31474/229719 objects degraded (13.701%); 18698/229719 objects misplaced (8.140%); 282 KiB/s, 2 objects/s recovering 2023-09-23T22:39:02.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:02 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2023-09-23T22:39:02.966 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:02.985 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:39:02 smithi026 systemd[1]: Stopping Ceph osd.3 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:39:02.993 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:03.042 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:03.043 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:39:03.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:03.046 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:03.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:03.172 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:03.221 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:03.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:03.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:03.325 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:03.364 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:03.377 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:39:03.408 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:39:03.429 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:39:03.435 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:39:03.469 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:03.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:03.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:03.575 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:03.598 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:03.601 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:03.643 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:03.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:03.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:03.735 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:39:03 smithi026 conmon[261933]: 2023-09-23T22:39:03.421+0000 7fd68326f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:39:03.736 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:39:03 smithi026 conmon[261933]: 2023-09-23T22:39:03.421+0000 7fd68326f700 -1 osd.3 516 *** Got signal Terminated *** 2023-09-23T22:39:03.736 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:39:03 smithi026 conmon[261933]: 2023-09-23T22:39:03.421+0000 7fd68326f700 -1 osd.3 516 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:39:03.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 31474/229719 objects degraded (13.701%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:39:03.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3187275614' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:03.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:03 smithi026 ceph-mon[215775]: osd.3 marked itself down and dead 2023-09-23T22:39:03.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 31474/229719 objects degraded (13.701%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:39:03.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3187275614' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:03.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:03 smithi026 ceph-mon[223014]: osd.3 marked itself down and dead 2023-09-23T22:39:03.748 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:03.779 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:03.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 31474/229719 objects degraded (13.701%), 14 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T22:39:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3187275614' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:03 smithi070 ceph-mon[192652]: osd.3 marked itself down and dead 2023-09-23T22:39:03.890 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:03.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:03.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:03.995 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:04.020 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:04.062 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:04.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:04.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:04.168 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:04.194 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:04.240 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:04.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:04.347 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:04.373 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:04.414 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:04.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:04.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:04.520 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:04.552 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:04.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:04 smithi070 ceph-mon[192652]: pgmap v1343: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 23 active+undersized, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 41 op/s; 31474/229719 objects degraded (13.701%); 18698/229719 objects misplaced (8.140%); 220 KiB/s, 2 objects/s recovering 2023-09-23T22:39:04.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:04 smithi070 ceph-mon[192652]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T22:39:04.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:04 smithi070 ceph-mon[192652]: failed to encode map e517 with expected crc 2023-09-23T22:39:04.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:04 smithi070 ceph-mon[192652]: osdmap e517: 8 total, 6 up, 8 in 2023-09-23T22:39:04.726 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:04.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:04 smithi026 ceph-mon[223014]: pgmap v1343: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 23 active+undersized, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 41 op/s; 31474/229719 objects degraded (13.701%); 18698/229719 objects misplaced (8.140%); 220 KiB/s, 2 objects/s recovering 2023-09-23T22:39:04.728 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:04 smithi026 ceph-mon[223014]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T22:39:04.728 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:04 smithi026 ceph-mon[223014]: failed to encode map e517 with expected crc 2023-09-23T22:39:04.728 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:04 smithi026 ceph-mon[223014]: osdmap e517: 8 total, 6 up, 8 in 2023-09-23T22:39:04.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:04 smithi026 ceph-mon[215775]: pgmap v1343: 109 pgs: 1 active+recovery_wait+degraded, 1 remapped+peering, 6 stale+active+clean, 1 active+recovery_wait+undersized+degraded, 23 active+undersized, 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+undersized+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 63 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 41 op/s; 31474/229719 objects degraded (13.701%); 18698/229719 objects misplaced (8.140%); 220 KiB/s, 2 objects/s recovering 2023-09-23T22:39:04.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:04 smithi026 ceph-mon[215775]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T22:39:04.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:04 smithi026 ceph-mon[215775]: failed to encode map e517 with expected crc 2023-09-23T22:39:04.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:04 smithi026 ceph-mon[215775]: osdmap e517: 8 total, 6 up, 8 in 2023-09-23T22:39:04.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:04.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:04.745 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:04.830 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:04.856 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:04.898 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:04.899 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:39:04.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:04.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:04.917 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:39:04.936 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:39:04.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:39:04.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:39:04.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:39:04.937 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: osd down 2023-09-23T22:39:04.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:39:04.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:39:04.937 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:39:04.937 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:39:05.004 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:05.039 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:05.454 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:39:05 smithi026 bash[608288]: 6e18c70e050b1d77ca511a46eed9a69087e83b68146296fb0808d1f570a606cf 2023-09-23T22:39:05.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[223014]: failed to encode map e517 with expected crc 2023-09-23T22:39:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[223014]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[223014]: osdmap e518: 8 total, 6 up, 8 in 2023-09-23T22:39:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[223014]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[223014]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[223014]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[215775]: failed to encode map e517 with expected crc 2023-09-23T22:39:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[215775]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[215775]: osdmap e518: 8 total, 6 up, 8 in 2023-09-23T22:39:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[215775]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[215775]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:05 smithi026 ceph-mon[215775]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:05 smithi070 ceph-mon[192652]: failed to encode map e517 with expected crc 2023-09-23T22:39:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:05 smithi070 ceph-mon[192652]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:05 smithi070 ceph-mon[192652]: osdmap e518: 8 total, 6 up, 8 in 2023-09-23T22:39:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:05 smithi070 ceph-mon[192652]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:05 smithi070 ceph-mon[192652]: failed to encode map e518 with expected crc 2023-09-23T22:39:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:05 smithi070 ceph-mon[192652]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.370 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:06.370 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:06.711 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:06 smithi070 ceph-mon[192652]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.711 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:06 smithi070 ceph-mon[192652]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:06 smithi070 ceph-mon[192652]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:06 smithi070 ceph-mon[192652]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:06 smithi070 ceph-mon[192652]: pgmap v1346: 109 pgs: 3 remapped+peering, 22 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded, 29 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 43 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 15 op/s; 19359/229602 objects degraded (8.432%); 13679/229602 objects misplaced (5.958%); 1.7 MiB/s, 37 objects/s recovering 2023-09-23T22:39:06.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:06 smithi070 ceph-mon[192652]: failed to encode map e519 with expected crc 2023-09-23T22:39:06.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:06 smithi070 ceph-mon[192652]: osdmap e519: 8 total, 6 up, 8 in 2023-09-23T22:39:06.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[223014]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[223014]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[223014]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[223014]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[223014]: pgmap v1346: 109 pgs: 3 remapped+peering, 22 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded, 29 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 43 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 15 op/s; 19359/229602 objects degraded (8.432%); 13679/229602 objects misplaced (5.958%); 1.7 MiB/s, 37 objects/s recovering 2023-09-23T22:39:06.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[223014]: failed to encode map e519 with expected crc 2023-09-23T22:39:06.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[223014]: osdmap e519: 8 total, 6 up, 8 in 2023-09-23T22:39:06.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[215775]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[215775]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[215775]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[215775]: failed to encode map e518 with expected crc 2023-09-23T22:39:06.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[215775]: pgmap v1346: 109 pgs: 3 remapped+peering, 22 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded, 29 active+undersized, 1 active+remapped+backfill_wait, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 43 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 15 op/s; 19359/229602 objects degraded (8.432%); 13679/229602 objects misplaced (5.958%); 1.7 MiB/s, 37 objects/s recovering 2023-09-23T22:39:06.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[215775]: failed to encode map e519 with expected crc 2023-09-23T22:39:06.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:06 smithi026 ceph-mon[215775]: osdmap e519: 8 total, 6 up, 8 in 2023-09-23T22:39:06.843 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119469 7.c 7:37fe5ea5:::benchmark_data_smithi026_410749_object29132:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.322397+0000", 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 593.39402910599995, 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9204321049999997, 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.844 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119469 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322397+0000", 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.845 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322397+0000", 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322396+0000", 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.846 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322399+0000", 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.548e-06 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322408+0000", 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2410000000000001e-06 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322413+0000", 2023-09-23T22:39:06.847 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.135e-06 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322457+0000", 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3455999999999999e-05 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.322472+0000", 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5319999999999999e-05 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.848 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053967+0000", 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.73149523100000002 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053971+0000", 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.737e-06 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.849 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.223011+0000", 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1690396930000002 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.224476+0000", 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001465273 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.226493+0000", 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020173280000000001 2023-09-23T22:39:06.850 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.238434+0000", 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011941267 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.238582+0000", 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147921 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.851 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.238990+0000", 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040729999999999998 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.239462+0000", 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00047265700000000003 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.240135+0000", 2023-09-23T22:39:06.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00067237199999999997 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.242829+0000", 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026940850000000001 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119490 7.c 7:3eaefa36:::benchmark_data_smithi026_410749_object29153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:39:06.853 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.356832+0000", 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 593.35959383700003, 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8905167999999999, 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119490 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356832+0000", 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356832+0000", 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356831+0000", 2023-09-23T22:39:06.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356836+0000", 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1410000000000004e-06 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356845+0000", 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8470000000000008e-06 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356852+0000", 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8990000000000004e-06 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356902+0000", 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0081999999999999e-05 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.356914+0000", 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1416e-05 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053982+0000", 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.69706834200000001 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053986+0000", 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3809999999999999e-06 2023-09-23T22:39:06.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.230378+0000", 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1763928799999999 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.231993+0000", 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001614376 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.234257+0000", 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022642140000000001 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.242966+0000", 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0087088479999999999 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.243295+0000", 2023-09-23T22:39:06.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032858300000000002 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.243696+0000", 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040189499999999999 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.244112+0000", 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00041517399999999998 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.244703+0000", 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059169699999999997 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.247349+0000", 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002645484 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119497 7.c 7:362bd5ae:::benchmark_data_smithi026_410749_object29160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.362138+0000", 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 593.35428786600005, 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9043871029999999, 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119497 2023-09-23T22:39:06.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362138+0000", 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362138+0000", 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362136+0000", 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362143+0000", 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1210000000000001e-06 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362156+0000", 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2310000000000001e-05 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362165+0000", 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7140000000000005e-06 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362228+0000", 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2415999999999993e-05 2023-09-23T22:39:06.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.362236+0000", 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8429999999999999e-06 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053996+0000", 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.69175968499999996 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.053999+0000", 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3210000000000001e-06 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.247536+0000", 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1935366379999999 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.248998+0000", 2023-09-23T22:39:06.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001461681 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.251332+0000", 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023344030000000001 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.262650+0000", 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011318004 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.262974+0000", 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032428099999999999 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.263074+0000", 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9837e-05 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.263467+0000", 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039298099999999998 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.264022+0000", 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055437699999999999 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.266525+0000", 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025033540000000002 2023-09-23T22:39:06.871 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119502 7.c 7:39b9d062:::benchmark_data_smithi026_410749_object29165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.368183+0000", 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 593.34824320600001, 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9164363140000003, 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.872 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119502 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368183+0000", 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368183+0000", 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368182+0000", 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368187+0000", 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6529999999999999e-06 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368199+0000", 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1875000000000001e-05 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368207+0000", 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0499999999999992e-06 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368265+0000", 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8074e-05 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.368277+0000", 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1640999999999999e-05 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054010+0000", 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68573267699999996 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054013+0000", 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3349999999999998e-06 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.254782+0000", 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2007688540000001 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.256184+0000", 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014024669999999999 2023-09-23T22:39:06.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.258340+0000", 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021561169999999999 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.266945+0000", 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086043339999999999 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.879 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.267354+0000", 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040936299999999999 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.267803+0000", 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044911400000000002 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.282079+0000", 2023-09-23T22:39:06.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014275512000000001 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.282581+0000", 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050250700000000002 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.284619+0000", 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002037681 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.881 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119523 7.c 7:3440ea60:::benchmark_data_smithi026_410749_object29186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.498288+0000", 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 593.218138232, 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7907858110000001, 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:39:06.882 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119523 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498288+0000", 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498288+0000", 2023-09-23T22:39:06.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498287+0000", 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498292+0000", 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4890000000000003e-06 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498301+0000", 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9789999999999999e-06 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498315+0000", 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3635000000000001e-05 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498356+0000", 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1279999999999998e-05 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.498362+0000", 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1199999999999999e-06 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054024+0000", 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.555662137 2023-09-23T22:39:06.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054028+0000", 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.343e-06 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.267979+0000", 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2139512649999999 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.887 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.269550+0000", 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001571645 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.271850+0000", 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022990599999999999 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285119+0000", 2023-09-23T22:39:06.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013269311000000001 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285481+0000", 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036202899999999999 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285593+0000", 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111935 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.286135+0000", 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00054262699999999995 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.286669+0000", 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00053313299999999998 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.289073+0000", 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002404822 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119530 7.c 7:36830f67:::benchmark_data_smithi026_410749_object29193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.567375+0000", 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 593.14905037300002, 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7261192759999999, 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.891 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119530 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567375+0000", 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567375+0000", 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567374+0000", 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567380+0000", 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1850000000000003e-06 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567389+0000", 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5410000000000006e-06 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567394+0000", 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1220000000000003e-06 2023-09-23T22:39:06.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567454+0000", 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9857e-05 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.567465+0000", 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1219e-05 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054039+0000", 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.48657352799999998 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054042+0000", 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5599999999999998e-06 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.274794+0000", 2023-09-23T22:39:06.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2207516300000001 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.276186+0000", 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001392162 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.278086+0000", 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001899502 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.285725+0000", 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076398430000000003 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.289503+0000", 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037773630000000002 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.289936+0000", 2023-09-23T22:39:06.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043365299999999999 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.290271+0000", 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033435799999999999 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.290826+0000", 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000554742 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.293495+0000", 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002669121 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.35406.0:119565 7.c 7:366e6103:::benchmark_data_smithi026_410749_object29228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:29:13.701580+0000", 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 593.01484621700001, 2023-09-23T22:39:06.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6102326480000002, 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.35406", 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/3431811944", 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119565 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701580+0000", 2023-09-23T22:39:06.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701580+0000", 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701579+0000", 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:39:06.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701583+0000", 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4100000000000001e-06 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701596+0000", 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3318e-05 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701602+0000", 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2179999999999998e-06 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701649+0000", 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7822999999999999e-05 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:13.701661+0000", 2023-09-23T22:39:06.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1739000000000001e-05 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054055+0000", 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.35239350400000002 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:14.054059+0000", 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.605e-06 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.293665+0000", 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2396055229999998 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.295090+0000", 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001425078 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.297158+0000", 2023-09-23T22:39:06.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002068318 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.307433+0000", 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010274817 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.307802+0000", 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00036851599999999999 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.308198+0000", 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039608500000000001 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.308594+0000", 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00039657100000000002 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.309219+0000", 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00062512600000000005 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:29:16.311812+0000", 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025928660000000001 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151873 8.6 8:675a56d2:::benchmark_data_smithi026_497658_object52981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 321.66800870700001, 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6652286869999999, 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151873 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048419+0000", 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7810000000000001e-06 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048428+0000", 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2649999999999996e-06 2023-09-23T22:39:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048434+0000", 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2419999999999997e-06 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048492+0000", 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7899999999999998e-05 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048608+0000", 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116111 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048734+0000", 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012566199999999999 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.057680+0000", 2023-09-23T22:39:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008946173 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202205+0000", 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.144524878 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202215+0000", 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0217000000000001e-05 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198539+0000", 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99632416099999999 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198553+0000", 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4236999999999999e-05 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713565+0000", 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5150114879999999 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713646+0000", 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1019000000000005e-05 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151874 8.e 8:70ae8a39:::benchmark_data_smithi026_497658_object52982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 321.66165331100001, 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6486478249999998, 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151874 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054771+0000", 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054778+0000", 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7599999999999997e-06 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054794+0000", 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6138999999999999e-05 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054803+0000", 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8610000000000004e-06 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054857+0000", 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3900000000000002e-05 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.055039+0000", 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018265100000000001 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.055175+0000", 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013610799999999999 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.057826+0000", 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00265037 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.060056+0000", 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022297520000000002 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202153+0000", 2023-09-23T22:39:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.142097219 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202169+0000", 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5707000000000001e-05 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198586+0000", 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99641781500000004 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198596+0000", 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.662e-06 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703316+0000", 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.504720442 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703420+0000", 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010388800000000001 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151883 8.e 8:73a3339c:::benchmark_data_smithi026_497658_object52991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 321.417858599, 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4185350429999999, 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151883 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298566+0000", 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298574+0000", 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6419999999999996e-06 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298592+0000", 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.749e-05 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298603+0000", 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1551e-05 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298672+0000", 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8804000000000001e-05 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298688+0000", 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6004000000000001e-05 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198619+0000", 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89993140699999996 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198625+0000", 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0419999999999999e-06 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703671+0000", 2023-09-23T22:39:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.505045454 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703821+0000", 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014994300000000001 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703937+0000", 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116397 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713345+0000", 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094075259999999994 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713414+0000", 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9202999999999995e-05 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713603+0000", 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000189373 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716935+0000", 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033319270000000002 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716988+0000", 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3121000000000002e-05 2023-09-23T22:39:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.717102+0000", 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011384599999999999 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151885 8.e 8:7d6f345f:::benchmark_data_smithi026_497658_object52993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:39:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 321.41686783699998, 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4256615969999999, 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151885 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299557+0000", 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299563+0000", 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1279999999999996e-06 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299579+0000", 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5736e-05 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299589+0000", 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9720000000000004e-06 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299662+0000", 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3312000000000006e-05 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299675+0000", 2023-09-23T22:39:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3414999999999999e-05 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198647+0000", 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89897187599999995 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198655+0000", 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1219999999999995e-06 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706694+0000", 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5080385459999999 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706833+0000", 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013966099999999999 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:39:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706985+0000", 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151915 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713514+0000", 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065285760000000003 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713609+0000", 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5025000000000006e-05 2023-09-23T22:39:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716876+0000", 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032670960000000001 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725068+0000", 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0081923059999999999 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725134+0000", 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5544999999999997e-05 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725220+0000", 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5604000000000002e-05 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151886 8.e 8:74052c51:::benchmark_data_smithi026_497658_object52994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 321.41458627700001, 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4236035519999999, 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151886 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301838+0000", 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301844+0000", 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2099999999999998e-06 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301857+0000", 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2943e-05 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301867+0000", 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3810000000000005e-06 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301924+0000", 2023-09-23T22:39:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7370000000000001e-05 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301936+0000", 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2551999999999999e-05 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198678+0000", 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89674120599999996 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198686+0000", 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0330000000000008e-06 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.709941+0000", 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5112555969999999 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.710081+0000", 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013977400000000001 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.710232+0000", 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015061799999999999 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713594+0000", 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033623500000000001 2023-09-23T22:39:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716839+0000", 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032453930000000001 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716906+0000", 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7041999999999999e-05 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725274+0000", 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083672850000000003 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725329+0000", 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5031000000000002e-05 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725443+0000", 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114305 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104711 9.c 9:31531dd3:::benchmark_data_smithi026_555954_object9982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.333837+0000", 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 111.382588756, 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9901661960000001, 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104711 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333837+0000", 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333837+0000", 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333835+0000", 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333844+0000", 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9309999999999997e-06 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333865+0000", 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0749000000000001e-05 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333875+0000", 2023-09-23T22:39:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9089999999999995e-06 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333951+0000", 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5932000000000004e-05 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.334444+0000", 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049377199999999996 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.997213+0000", 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6627689000000001 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.997330+0000", 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011688699999999999 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:39:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.997482+0000", 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015183899999999999 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.014559+0000", 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017076560000000001 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045008+0000", 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030448994 2023-09-23T22:39:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045090+0000", 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2562e-05 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.320023+0000", 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.27493322999999997 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.320097+0000", 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3244000000000005e-05 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.324003+0000", 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039067069999999997 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104722 9.c 9:3a91d401:::benchmark_data_smithi026_555954_object9993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.362114+0000", 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 111.354312257, 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9830620269999999, 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104722 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362114+0000", 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362114+0000", 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362112+0000", 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362119+0000", 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5490000000000003e-06 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362132+0000", 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3563e-05 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362140+0000", 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5209999999999997e-06 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362187+0000", 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7746e-05 2023-09-23T22:39:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362696+0000", 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050887800000000004 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.009860+0000", 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6471635419999999 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.009977+0000", 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011697300000000001 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.010139+0000", 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000161762 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.022087+0000", 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011948652000000001 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045050+0000", 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022963063999999998 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045126+0000", 2023-09-23T22:39:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5426000000000006e-05 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.342067+0000", 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.29694130099999999 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.342108+0000", 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1276999999999998e-05 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.345176+0000", 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030672630000000002 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:110944 9.c 9:3acbf7b4:::benchmark_data_smithi026_555954_object16215:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e508)", 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:38:40.685898+0000", 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 26.030528031999999, 2023-09-23T22:39:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.047804976999999998, 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 110944 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.685898+0000", 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.685898+0000", 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.685896+0000", 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.685903+0000", 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0230000000000004e-06 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.685918+0000", 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4450000000000001e-05 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.685925+0000", 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9909999999999996e-06 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.700612+0000", 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014687683 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.701125+0000", 2023-09-23T22:39:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000512518 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.715815+0000", 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01469005 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.715944+0000", 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129116 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.716128+0000", 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000184036 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.720896+0000", 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047675879999999997 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.720963+0000", 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7127999999999998e-05 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.721536+0000", 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057358400000000003 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.730390+0000", 2023-09-23T22:39:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0088530280000000006 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.730459+0000", 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9803999999999998e-05 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:40.733703+0000", 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032434360000000001 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:111247 9.c 9:3d0ce5c5:::benchmark_data_smithi026_555954_object16518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e509)", 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:38:43.350037+0000", 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 23.366388841999999, 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.046443621999999997, 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 111247 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.350037+0000", 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.350037+0000", 2023-09-23T22:39:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.350035+0000", 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.350044+0000", 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8389999999999996e-06 2023-09-23T22:39:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.350061+0000", 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7065999999999999e-05 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.350071+0000", 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8630000000000003e-06 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.353289+0000", 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032179769999999999 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.353806+0000", 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00051734000000000003 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.376191+0000", 2023-09-23T22:39:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022385019999999999 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.376335+0000", 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014418800000000001 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.376501+0000", 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016539500000000001 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.392676+0000", 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016175295999999999 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.392743+0000", 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6771000000000002e-05 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.392757+0000", 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3827e-05 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.392812+0000", 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5755999999999999e-05 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.392886+0000", 2023-09-23T22:39:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4048999999999995e-05 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:43.396481+0000", 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035941319999999999 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112272 9.c 9:31ba4fe5:::benchmark_data_smithi026_555954_object17543:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T22:39:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:38:50.675502+0000", 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 16.040924123, 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.035552750000000001, 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112272 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.675502+0000", 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.675502+0000", 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.675500+0000", 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.675509+0000", 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4149999999999999e-06 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.675528+0000", 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9408000000000001e-05 2023-09-23T22:39:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.675538+0000", 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0083e-05 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.675601+0000", 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2674000000000003e-05 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.676155+0000", 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055470599999999997 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.698829+0000", 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022674071000000001 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.698912+0000", 2023-09-23T22:39:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2835000000000002e-05 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.699012+0000", 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9364999999999995e-05 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.706465+0000", 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074537479999999996 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.706501+0000", 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5172000000000001e-05 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.707922+0000", 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001421664 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.707935+0000", 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2996999999999999e-05 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.707979+0000", 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3346000000000001e-05 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:50.711054+0000", 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003075833 2023-09-23T22:39:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112714 9.c 9:3aaf472e:::benchmark_data_smithi026_555954_object17985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:38:53.777846+0000", 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 12.938579907999999, 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.036073998000000003, 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112714 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.777846+0000", 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.777846+0000", 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.777844+0000", 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.777855+0000", 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0375e-05 2023-09-23T22:39:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.777878+0000", 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3175000000000001e-05 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.777889+0000", 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1525000000000001e-05 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.777962+0000", 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2731999999999994e-05 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.778867+0000", 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00090545999999999997 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.799504+0000", 2023-09-23T22:39:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020636134 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.799655+0000", 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015180700000000001 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.799821+0000", 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016532999999999999 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.806750+0000", 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069292260000000001 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.807563+0000", 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081306100000000001 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.808044+0000", 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000480901 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.810650+0000", 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026056859999999999 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.810722+0000", 2023-09-23T22:39:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2816000000000002e-05 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:53.813920+0000", 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031974500000000001 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112807 9.c 9:37a3517b:::benchmark_data_smithi026_555954_object18078:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e512)", 2023-09-23T22:39:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:38:54.354357+0000", 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 12.362068812, 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.058412578999999999, 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112807 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.354357+0000", 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.354357+0000", 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.354356+0000", 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.354361+0000", 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4689999999999996e-06 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.354372+0000", 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0134e-05 2023-09-23T22:39:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.354377+0000", 2023-09-23T22:39:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2839999999999997e-06 2023-09-23T22:39:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.356966+0000", 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025896069999999998 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.357620+0000", 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00065372300000000004 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.371114+0000", 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013493689 2023-09-23T22:39:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.371231+0000", 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011698500000000001 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.371389+0000", 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015799299999999999 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.377486+0000", 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0060972830000000002 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.378735+0000", 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001249109 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.399097+0000", 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020361396 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.409055+0000", 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0099581919999999994 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.409113+0000", 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7803999999999997e-05 2023-09-23T22:39:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.412770+0000", 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036569850000000002 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112851 9.9 9:9529eda2:::benchmark_data_smithi026_555954_object18122:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e514)", 2023-09-23T22:39:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:38:55.433548+0000", 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 11.282878102, 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041328792000000003, 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112851 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.433548+0000", 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.433548+0000", 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.433547+0000", 2023-09-23T22:39:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.433550+0000", 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1999999999999999e-06 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.433557+0000", 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6959999999999999e-06 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.433563+0000", 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0229999999999998e-06 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.442012+0000", 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0084485770000000005 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.442175+0000", 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016360699999999999 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.442321+0000", 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014544800000000001 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.447096+0000", 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047748699999999996 2023-09-23T22:39:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.448623+0000", 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015269599999999999 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.471465+0000", 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022842224000000001 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.471525+0000", 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0225e-05 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.474876+0000", 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033513929999999998 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:07.022 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:39:07.177 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:07.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:07.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:07.291 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:07.327 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:07.462 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[223014]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.463 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[223014]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.463 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[223014]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[215775]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[215775]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[215775]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[215775]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 12 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2023-09-23T22:39:07.672 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:07.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:39:07.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:07.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:07.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[223014]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:07 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 12 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2023-09-23T22:39:07.778 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:07.826 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:07 smithi070 ceph-mon[192652]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:07 smithi070 ceph-mon[192652]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:07 smithi070 ceph-mon[192652]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:07 smithi070 ceph-mon[192652]: failed to encode map e519 with expected crc 2023-09-23T22:39:07.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:07 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 12 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2023-09-23T22:39:07.915 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:07.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:07.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:08.021 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:08.064 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:08.136 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:08.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:08.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:08.188 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:08.244 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:08.290 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:08.371 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:39:08.422 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:08.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:08.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:08.529 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:08.569 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:08.657 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:08.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:08.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:08.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:08 smithi026 ceph-mon[223014]: pgmap v1348: 109 pgs: 1 active+clean+remapped, 3 remapped+peering, 23 peering, 6 undersized+peered, 39 active+undersized, 1 active+undersized+remapped, 8 active+undersized+degraded, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 26 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 19622/229410 objects degraded (8.553%); 10982/229410 objects misplaced (4.787%); 373 KiB/s, 28 objects/s recovering 2023-09-23T22:39:08.663 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 19359/229602 objects degraded (8.432%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:39:08.663 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:08 smithi026 ceph-mon[215775]: pgmap v1348: 109 pgs: 1 active+clean+remapped, 3 remapped+peering, 23 peering, 6 undersized+peered, 39 active+undersized, 1 active+undersized+remapped, 8 active+undersized+degraded, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 26 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 19622/229410 objects degraded (8.553%); 10982/229410 objects misplaced (4.787%); 373 KiB/s, 28 objects/s recovering 2023-09-23T22:39:08.664 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 19359/229602 objects degraded (8.432%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:39:08.764 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:08.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:08 smithi070 ceph-mon[192652]: pgmap v1348: 109 pgs: 1 active+clean+remapped, 3 remapped+peering, 23 peering, 6 undersized+peered, 39 active+undersized, 1 active+undersized+remapped, 8 active+undersized+degraded, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 26 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 17 op/s; 19622/229410 objects degraded (8.553%); 10982/229410 objects misplaced (4.787%); 373 KiB/s, 28 objects/s recovering 2023-09-23T22:39:08.790 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 19359/229602 objects degraded (8.432%), 10 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:39:08.803 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:08.863 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:08.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:08.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:08.877 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:08.970 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:09.003 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:09.063 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:09.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:09.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:09.168 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:09.200 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:09.264 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:09.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:09.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:09.369 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:09.400 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:09.463 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:09.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:09.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:09.497 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:39:09.567 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:09.605 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:09.675 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:09.678 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:09 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3946888560' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:09.678 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:09.678 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:09.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:09.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:09.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:39:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:39:09] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.43.0" 2023-09-23T22:39:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:09 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3946888560' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:09.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:09 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3946888560' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:09.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:09.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:09.781 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:09.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:09.885 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:09.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:09.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:09.990 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:10.025 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:10.079 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:10.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:10.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:10.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:10.218 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:10.408 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T22:39:10.484 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:39:10 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T22:39:10.485 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:39:10 smithi026 systemd[1]: Stopped Ceph osd.3 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:39:10.506 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:39:10.507 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T22:39:10.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:10.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[215775]: failed to encode map e520 with expected crc 2023-09-23T22:39:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[215775]: failed to encode map e520 with expected crc 2023-09-23T22:39:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[215775]: failed to encode map e520 with expected crc 2023-09-23T22:39:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3946888560' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:39:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[215775]: osdmap e520: 8 total, 6 up, 8 in 2023-09-23T22:39:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[215775]: pgmap v1350: 109 pgs: 1 active+clean+remapped, 3 remapped+peering, 23 peering, 6 undersized+peered, 39 active+undersized, 1 active+undersized+remapped, 8 active+undersized+degraded, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 26 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 19 op/s; 19622/229410 objects degraded (8.553%); 10982/229410 objects misplaced (4.787%); 421 KiB/s, 31 objects/s recovering 2023-09-23T22:39:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[223014]: failed to encode map e520 with expected crc 2023-09-23T22:39:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[223014]: failed to encode map e520 with expected crc 2023-09-23T22:39:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[223014]: failed to encode map e520 with expected crc 2023-09-23T22:39:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3946888560' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:39:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[223014]: osdmap e520: 8 total, 6 up, 8 in 2023-09-23T22:39:10.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:10 smithi026 ceph-mon[223014]: pgmap v1350: 109 pgs: 1 active+clean+remapped, 3 remapped+peering, 23 peering, 6 undersized+peered, 39 active+undersized, 1 active+undersized+remapped, 8 active+undersized+degraded, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 26 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 19 op/s; 19622/229410 objects degraded (8.553%); 10982/229410 objects misplaced (4.787%); 421 KiB/s, 31 objects/s recovering 2023-09-23T22:39:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:10 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:10 smithi070 ceph-mon[192652]: failed to encode map e520 with expected crc 2023-09-23T22:39:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:10 smithi070 ceph-mon[192652]: failed to encode map e520 with expected crc 2023-09-23T22:39:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:10 smithi070 ceph-mon[192652]: failed to encode map e520 with expected crc 2023-09-23T22:39:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3946888560' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:39:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:10 smithi070 ceph-mon[192652]: osdmap e520: 8 total, 6 up, 8 in 2023-09-23T22:39:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:10 smithi070 ceph-mon[192652]: pgmap v1350: 109 pgs: 1 active+clean+remapped, 3 remapped+peering, 23 peering, 6 undersized+peered, 39 active+undersized, 1 active+undersized+remapped, 8 active+undersized+degraded, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 26 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 19 op/s; 19622/229410 objects degraded (8.553%); 10982/229410 objects misplaced (4.787%); 421 KiB/s, 31 objects/s recovering 2023-09-23T22:39:11.093 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:11.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:11.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:11.198 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:11.234 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:11.298 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:11.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:11.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:11.404 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:11.445 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:11.570 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:11.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:11.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:11.674 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:11.705 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:11.927 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:11.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:11.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:11.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:11 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 7 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:39:11.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:11 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 7 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:39:12.041 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:12.076 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:12.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:11 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 7 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:39:12.307 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:12.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:12.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:12.415 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:12.465 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:12.526 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:12.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:12.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:12.588 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:39:12.588 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:39:12.589 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:39:12.589 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:39:12.634 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:12.665 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:12.719 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:12.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:12.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:12.733 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:39:12.754 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:39:12.780 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:39:12.804 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:39:12.824 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:12.834 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:39:12.855 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:12 smithi026 ceph-mon[215775]: pgmap v1351: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 43024/227310 objects degraded (18.927%); 20215/227310 objects misplaced (8.893%); 8.7 MiB/s, 305 objects/s recovering 2023-09-23T22:39:12.855 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:12 smithi026 ceph-mon[223014]: pgmap v1351: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 43024/227310 objects degraded (18.927%); 20215/227310 objects misplaced (8.893%); 8.7 MiB/s, 305 objects/s recovering 2023-09-23T22:39:12.858 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:12.862 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:39:12.864 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:39:12.864 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:39:12.865 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:39:12.865 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:39:12.865 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:39:12.865 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:39:12.865 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:39:12.865 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:39:12.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:12 smithi070 ceph-mon[192652]: pgmap v1351: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 1 active+undersized+remapped, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 120 op/s; 43024/227310 objects degraded (18.927%); 20215/227310 objects misplaced (8.893%); 8.7 MiB/s, 305 objects/s recovering 2023-09-23T22:39:12.963 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:12.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:13.068 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:13.099 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.150 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:13.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:13.153 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:13.253 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:13.283 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.330 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:13.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:13.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:13.434 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:13.464 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.471 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:39:13.525 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:13.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:13.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:13.632 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:13.662 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.771 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:13.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:13.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:13.816 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2186845024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:13.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 43024/227310 objects degraded (18.927%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:13.817 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:13 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:39:13.818 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2186845024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:13.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 43024/227310 objects degraded (18.927%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:13.819 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:13 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:39:13.843 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:13.844 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:13.876 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:13.920 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:13.951 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:13.993 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:13.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:14.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:14.101 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:14.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2186845024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:14.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 43024/227310 objects degraded (18.927%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:14.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:13 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:39:14.140 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:14.203 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:39:14.209 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:14.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:14.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:14.322 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:14.378 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:14.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:14.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:14.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:14.620 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:14.654 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:14.863 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:14.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:14.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:14.920 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:14.969 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:15.007 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:15.119 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:15.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:15.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:15.227 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:15.270 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:15.346 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:15.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:15.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:15.444 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:15 smithi070 ceph-mon[192652]: pgmap v1352: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 39970/226296 objects degraded (17.663%); 22918/226296 objects misplaced (10.127%); 7.6 MiB/s, 252 objects/s recovering 2023-09-23T22:39:15.457 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:15.507 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:39:15.508 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:39:15.508 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-09-23T22:39:15.508 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:39:15.511 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:15.543 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:15 smithi026 ceph-mon[215775]: pgmap v1352: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 39970/226296 objects degraded (17.663%); 22918/226296 objects misplaced (10.127%); 7.6 MiB/s, 252 objects/s recovering 2023-09-23T22:39:15.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:15 smithi026 ceph-mon[223014]: pgmap v1352: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 137 op/s; 39970/226296 objects degraded (17.663%); 22918/226296 objects misplaced (10.127%); 7.6 MiB/s, 252 objects/s recovering 2023-09-23T22:39:15.756 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:15.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:15.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:15.845 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:39:15.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:15.900 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:16.118 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:16.890 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:16.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:16.895 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:16.996 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:17.028 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:17.188 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:17.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:17.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:17.295 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:17.302 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:39:17.337 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:17.409 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:17.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:17.526 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:17.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:17 smithi070 ceph-mon[192652]: pgmap v1353: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 39619/224679 objects degraded (17.634%); 22832/224679 objects misplaced (10.162%); 12 MiB/s, 307 objects/s recovering 2023-09-23T22:39:17.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:17 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1772038160' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:17.572 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:17.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:17 smithi026 ceph-mon[215775]: pgmap v1353: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 39619/224679 objects degraded (17.634%); 22832/224679 objects misplaced (10.162%); 12 MiB/s, 307 objects/s recovering 2023-09-23T22:39:17.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:17 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1772038160' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:17.623 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:17 smithi026 ceph-mon[223014]: pgmap v1353: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 175 op/s; 39619/224679 objects degraded (17.634%); 22832/224679 objects misplaced (10.162%); 12 MiB/s, 307 objects/s recovering 2023-09-23T22:39:17.623 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:17 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1772038160' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:17.644 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:17.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:17.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:17.748 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:17.784 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:17.855 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:17.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:17.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:17.897 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:17.897 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:17.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:17.996 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:18.104 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:18.128 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:18.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:18.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:18.233 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:18.271 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:18.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:39:18.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[223014]: failed to encode map e521 with expected crc 2023-09-23T22:39:18.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[223014]: failed to encode map e521 with expected crc 2023-09-23T22:39:18.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[223014]: failed to encode map e521 with expected crc 2023-09-23T22:39:18.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1772038160' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:18.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[223014]: osdmap e521: 8 total, 6 up, 8 in 2023-09-23T22:39:18.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 39619/224679 objects degraded (17.634%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:18.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:39:18.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[215775]: failed to encode map e521 with expected crc 2023-09-23T22:39:18.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[215775]: failed to encode map e521 with expected crc 2023-09-23T22:39:18.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[215775]: failed to encode map e521 with expected crc 2023-09-23T22:39:18.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1772038160' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:18.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[215775]: osdmap e521: 8 total, 6 up, 8 in 2023-09-23T22:39:18.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 39619/224679 objects degraded (17.634%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:18.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:18 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:39:18.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:18 smithi070 ceph-mon[192652]: failed to encode map e521 with expected crc 2023-09-23T22:39:18.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:18 smithi070 ceph-mon[192652]: failed to encode map e521 with expected crc 2023-09-23T22:39:18.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:18 smithi070 ceph-mon[192652]: failed to encode map e521 with expected crc 2023-09-23T22:39:18.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:18 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1772038160' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:18.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:18 smithi070 ceph-mon[192652]: osdmap e521: 8 total, 6 up, 8 in 2023-09-23T22:39:18.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 39619/224679 objects degraded (17.634%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:19.099 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:39:19.101 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:39:19.108 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:19.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:19.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:19.212 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:39:19.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:19.252 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:19.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:19 smithi070 ceph-mon[192652]: pgmap v1355: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 39170/221934 objects degraded (17.649%); 22455/221934 objects misplaced (10.118%); 16 MiB/s, 443 objects/s recovering 2023-09-23T22:39:19.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:19 smithi026 ceph-mon[223014]: pgmap v1355: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 39170/221934 objects degraded (17.649%); 22455/221934 objects misplaced (10.118%); 16 MiB/s, 443 objects/s recovering 2023-09-23T22:39:19.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:19 smithi026 ceph-mon[215775]: pgmap v1355: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 307 op/s; 39170/221934 objects degraded (17.649%); 22455/221934 objects misplaced (10.118%); 16 MiB/s, 443 objects/s recovering 2023-09-23T22:39:19.793 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:19.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:39:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:39:19] "GET /metrics HTTP/1.1" 200 36495 "" "Prometheus/2.43.0" 2023-09-23T22:39:20.014 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:20.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:20.018 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:20.118 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:20.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:39:20.147 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:20.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[223014]: from='client.36552 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:20.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[223014]: failed to encode map e522 with expected crc 2023-09-23T22:39:20.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[223014]: failed to encode map e522 with expected crc 2023-09-23T22:39:20.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[223014]: osdmap e522: 8 total, 6 up, 8 in 2023-09-23T22:39:20.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[223014]: failed to encode map e522 with expected crc 2023-09-23T22:39:20.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[215775]: from='client.36552 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:20.313 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[215775]: failed to encode map e522 with expected crc 2023-09-23T22:39:20.314 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[215775]: failed to encode map e522 with expected crc 2023-09-23T22:39:20.314 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[215775]: osdmap e522: 8 total, 6 up, 8 in 2023-09-23T22:39:20.314 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:20 smithi026 ceph-mon[215775]: failed to encode map e522 with expected crc 2023-09-23T22:39:20.315 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:20 smithi070 ceph-mon[192652]: from='client.36552 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:20.315 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:20 smithi070 ceph-mon[192652]: failed to encode map e522 with expected crc 2023-09-23T22:39:20.315 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:20 smithi070 ceph-mon[192652]: failed to encode map e522 with expected crc 2023-09-23T22:39:20.315 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:20 smithi070 ceph-mon[192652]: osdmap e522: 8 total, 6 up, 8 in 2023-09-23T22:39:20.315 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:20 smithi070 ceph-mon[192652]: failed to encode map e522 with expected crc 2023-09-23T22:39:20.352 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:20.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:20.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:20.456 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:20.463 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1355,"stamp":"2023-09-23T22:39:17.605950+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4851165553,"num_objects":73978,"num_object_clones":0,"num_object_copies":221934,"num_objects_missing_on_primary":1793,"num_objects_missing":1793,"num_objects_degraded":39170,"num_objects_misplaced":22455,"num_objects_unfound":0,"num_objects_dirty":73978,"num_whiteouts":0,"num_read":6027,"num_read_kb":12268,"num_write":116433,"num_write_kb":6075410,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23589,"num_bytes_recovered":1357314513,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104290,"ondisk_log_size":104290,"up":224,"acting":231,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":340,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18038500,"kb_used_data":14996196,"kb_used_omap":52,"kb_used_meta":3042187,"kb_avail":731922716,"statfs":{"total":767960285184,"available":749488861184,"internally_reserved":0,"allocated":15356104704,"data_stored":15338430772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53844,"internal_metadata":3115199916},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-163315712,"num_objects":-2492,"num_object_clones":0,"num_object_copies":-7476,"num_objects_missing_on_primary":-4998,"num_objects_missing":-7757,"num_objects_degraded":19548,"num_objects_misplaced":11473,"num_objects_unfound":0,"num_objects_dirty":-2492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2492,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3602,"num_bytes_recovered":135266304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.116633"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":580,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.288874+0000","last_change":"2023-09-23T22:37:42.441352+0000","last_active":"2023-09-23T22:39:06.288874+0000","last_peered":"2023-09-23T22:39:06.288874+0000","last_clean":"2023-09-23T22:39:06.288874+0000","last_became_active":"2023-09-23T22:37:42.440853+0000","last_became_peered":"2023-09-23T22:37:42.440853+0000","last_unstale":"2023-09-23T22:39:06.288874+0000","last_undegraded":"2023-09-23T22:39:06.288874+0000","last_fullsized":"2023-09-23T22:39:06.288874+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:11.866578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":219,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.290442+0000","last_change":"2023-09-23T22:39:05.290466+0000","last_active":"2023-09-23T22:39:06.290442+0000","last_peered":"2023-09-23T22:39:06.290442+0000","last_clean":"2023-09-23T22:37:07.670854+0000","last_became_active":"2023-09-23T22:39:05.290466+0000","last_became_peered":"2023-09-23T22:39:05.290466+0000","last_unstale":"2023-09-23T22:39:06.290442+0000","last_undegraded":"2023-09-23T22:39:06.290442+0000","last_fullsized":"2023-09-23T22:39:05.285697+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:14.593594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":263,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.285920+0000","last_change":"2023-09-23T22:39:05.289963+0000","last_active":"2023-09-23T22:39:06.285920+0000","last_peered":"2023-09-23T22:39:06.285920+0000","last_clean":"2023-09-23T22:39:00.403392+0000","last_became_active":"2023-09-23T22:39:05.289963+0000","last_became_peered":"2023-09-23T22:39:05.289963+0000","last_unstale":"2023-09-23T22:39:06.285920+0000","last_undegraded":"2023-09-23T22:39:06.285920+0000","last_fullsized":"2023-09-23T22:39:05.284569+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.806799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":322,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.306573+0000","last_change":"2023-09-23T22:39:05.307660+0000","last_active":"2023-09-23T22:39:06.306573+0000","last_peered":"2023-09-23T22:39:06.306573+0000","last_clean":"2023-09-23T22:35:17.625896+0000","last_became_active":"2023-09-23T22:39:05.307660+0000","last_became_peered":"2023-09-23T22:39:05.307660+0000","last_unstale":"2023-09-23T22:39:06.306573+0000","last_undegraded":"2023-09-23T22:39:06.306573+0000","last_fullsized":"2023-09-23T22:39:05.302351+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:13.440047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":544,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286069+0000","last_change":"2023-09-23T22:39:05.304373+0000","last_active":"2023-09-23T22:39:06.286069+0000","last_peered":"2023-09-23T22:39:06.286069+0000","last_clean":"2023-09-23T22:37:07.670321+0000","last_became_active":"2023-09-23T22:39:05.304373+0000","last_became_peered":"2023-09-23T22:39:05.304373+0000","last_unstale":"2023-09-23T22:39:06.286069+0000","last_undegraded":"2023-09-23T22:39:06.286069+0000","last_fullsized":"2023-09-23T22:39:05.284343+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:51.889095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":317,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.289846+0000","last_change":"2023-09-23T22:38:55.388832+0000","last_active":"2023-09-23T22:39:06.289846+0000","last_peered":"2023-09-23T22:39:06.289846+0000","last_clean":"2023-09-23T22:37:07.661100+0000","last_became_active":"2023-09-23T22:38:55.388832+0000","last_became_peered":"2023-09-23T22:38:55.388832+0000","last_unstale":"2023-09-23T22:39:06.289846+0000","last_undegraded":"2023-09-23T22:39:06.289846+0000","last_fullsized":"2023-09-23T22:38:55.382522+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:09.227034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":349,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.306815+0000","last_change":"2023-09-23T22:39:05.302151+0000","last_active":"2023-09-23T22:38:54.405779+0000","last_peered":"2023-09-23T22:39:06.306815+0000","last_clean":"2023-09-23T22:38:48.372435+0000","last_became_active":"2023-09-23T22:36:36.406686+0000","last_became_peered":"2023-09-23T22:39:05.302151+0000","last_unstale":"2023-09-23T22:39:06.306815+0000","last_undegraded":"2023-09-23T22:39:06.306815+0000","last_fullsized":"2023-09-23T22:39:05.301734+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:12.647131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":300,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.286075+0000","last_change":"2023-09-23T22:39:05.285935+0000","last_active":"2023-09-23T22:38:54.400276+0000","last_peered":"2023-09-23T22:39:06.286075+0000","last_clean":"2023-09-23T22:38:48.357505+0000","last_became_active":"2023-09-23T22:37:42.458398+0000","last_became_peered":"2023-09-23T22:39:05.285935+0000","last_unstale":"2023-09-23T22:39:06.286075+0000","last_undegraded":"2023-09-23T22:39:06.286075+0000","last_fullsized":"2023-09-23T22:39:05.284557+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:22.365837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":637,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.288051+0000","last_change":"2023-09-23T22:39:05.294537+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:39:06.288051+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.294537+0000","last_unstale":"2023-09-23T22:39:06.288051+0000","last_undegraded":"2023-09-23T22:39:06.288051+0000","last_fullsized":"2023-09-23T22:39:05.292289+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":537,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.287854+0000","last_change":"2023-09-23T22:39:05.294737+0000","last_active":"2023-09-23T22:39:06.287854+0000","last_peered":"2023-09-23T22:39:06.287854+0000","last_clean":"2023-09-23T22:39:00.404530+0000","last_became_active":"2023-09-23T22:39:05.294737+0000","last_became_peered":"2023-09-23T22:39:05.294737+0000","last_unstale":"2023-09-23T22:39:06.287854+0000","last_undegraded":"2023-09-23T22:39:06.287854+0000","last_fullsized":"2023-09-23T22:39:05.286342+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:12.866122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":158,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.288297+0000","last_change":"2023-09-23T22:39:05.298634+0000","last_active":"2023-09-23T22:38:54.377414+0000","last_peered":"2023-09-23T22:39:06.288297+0000","last_clean":"2023-09-23T22:38:48.357759+0000","last_became_active":"2023-09-23T22:36:36.521405+0000","last_became_peered":"2023-09-23T22:39:05.298634+0000","last_unstale":"2023-09-23T22:39:06.288297+0000","last_undegraded":"2023-09-23T22:39:06.288297+0000","last_fullsized":"2023-09-23T22:39:05.297707+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:35.327335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":65,"reported_epoch":519,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:06.287873+0000","last_change":"2023-09-23T22:39:05.295204+0000","last_active":"2023-09-23T22:39:06.287873+0000","last_peered":"2023-09-23T22:39:06.287873+0000","last_clean":"2023-09-23T22:36:31.355768+0000","last_became_active":"2023-09-23T22:39:05.295204+0000","last_became_peered":"2023-09-23T22:39:05.295204+0000","last_unstale":"2023-09-23T22:39:06.287873+0000","last_undegraded":"2023-09-23T22:39:05.291232+0000","last_fullsized":"2023-09-23T22:39:05.291128+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:25.561328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":160,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.287148+0000","last_change":"2023-09-23T22:38:55.383671+0000","last_active":"2023-09-23T22:39:06.287148+0000","last_peered":"2023-09-23T22:39:06.287148+0000","last_clean":"2023-09-23T22:38:48.410000+0000","last_became_active":"2023-09-23T22:38:55.383671+0000","last_became_peered":"2023-09-23T22:38:55.383671+0000","last_unstale":"2023-09-23T22:39:06.287148+0000","last_undegraded":"2023-09-23T22:39:06.287148+0000","last_fullsized":"2023-09-23T22:38:55.381036+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:27.813898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":509,"reported_epoch":519,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:06.304322+0000","last_change":"2023-09-23T22:38:55.387313+0000","last_active":"2023-09-23T22:39:06.304322+0000","last_peered":"2023-09-23T22:39:06.304322+0000","last_clean":"2023-09-23T22:33:24.367826+0000","last_became_active":"2023-09-23T22:38:55.387313+0000","last_became_peered":"2023-09-23T22:38:55.387313+0000","last_unstale":"2023-09-23T22:39:06.304322+0000","last_undegraded":"2023-09-23T22:38:55.380607+0000","last_fullsized":"2023-09-23T22:38:55.380500+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:05.769113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":543,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.289660+0000","last_change":"2023-09-23T22:38:55.397043+0000","last_active":"2023-09-23T22:39:06.289660+0000","last_peered":"2023-09-23T22:39:06.289660+0000","last_clean":"2023-09-23T22:38:48.354081+0000","last_became_active":"2023-09-23T22:38:55.397043+0000","last_became_peered":"2023-09-23T22:38:55.397043+0000","last_unstale":"2023-09-23T22:39:06.289660+0000","last_undegraded":"2023-09-23T22:39:06.289660+0000","last_fullsized":"2023-09-23T22:38:55.379386+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:38.446121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":900,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T22:39:09.500251+0000","last_change":"2023-09-23T22:36:36.374902+0000","last_active":"2023-09-23T22:39:09.500251+0000","last_peered":"2023-09-23T22:39:09.500251+0000","last_clean":"2023-09-23T22:39:09.500251+0000","last_became_active":"2023-09-23T22:36:22.309281+0000","last_became_peered":"2023-09-23T22:36:22.309281+0000","last_unstale":"2023-09-23T22:39:09.500251+0000","last_undegraded":"2023-09-23T22:39:09.500251+0000","last_fullsized":"2023-09-23T22:39:09.500251+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":154,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T22:39:09.500438+0000","last_change":"2023-09-23T22:35:52.674513+0000","last_active":"2023-09-23T22:39:09.500438+0000","last_peered":"2023-09-23T22:39:09.500438+0000","last_clean":"2023-09-23T22:39:09.500438+0000","last_became_active":"2023-09-23T22:35:52.674194+0000","last_became_peered":"2023-09-23T22:35:52.674194+0000","last_unstale":"2023-09-23T22:39:09.500438+0000","last_undegraded":"2023-09-23T22:39:09.500438+0000","last_fullsized":"2023-09-23T22:39:09.500438+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:54.383524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":345,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286160+0000","last_change":"2023-09-23T22:39:05.294773+0000","last_active":"2023-09-23T22:39:06.286160+0000","last_peered":"2023-09-23T22:39:06.286160+0000","last_clean":"2023-09-23T22:37:07.671236+0000","last_became_active":"2023-09-23T22:39:05.294773+0000","last_became_peered":"2023-09-23T22:39:05.294773+0000","last_unstale":"2023-09-23T22:39:06.286160+0000","last_undegraded":"2023-09-23T22:39:06.286160+0000","last_fullsized":"2023-09-23T22:39:05.285024+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:47.212090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":742,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286327+0000","last_change":"2023-09-23T22:38:55.391529+0000","last_active":"2023-09-23T22:39:06.286327+0000","last_peered":"2023-09-23T22:39:06.286327+0000","last_clean":"2023-09-23T22:35:16.261819+0000","last_became_active":"2023-09-23T22:38:55.391529+0000","last_became_peered":"2023-09-23T22:38:55.391529+0000","last_unstale":"2023-09-23T22:39:06.286327+0000","last_undegraded":"2023-09-23T22:39:06.286327+0000","last_fullsized":"2023-09-23T22:38:55.384595+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":380,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.309022+0000","last_change":"2023-09-23T22:39:05.307619+0000","last_active":"2023-09-23T22:39:06.309022+0000","last_peered":"2023-09-23T22:39:06.309022+0000","last_clean":"2023-09-23T22:36:31.476578+0000","last_became_active":"2023-09-23T22:39:05.307619+0000","last_became_peered":"2023-09-23T22:39:05.307619+0000","last_unstale":"2023-09-23T22:39:06.309022+0000","last_undegraded":"2023-09-23T22:39:06.309022+0000","last_fullsized":"2023-09-23T22:39:05.284898+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:50.408737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":430,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.287536+0000","last_change":"2023-09-23T22:39:05.294555+0000","last_active":"2023-09-23T22:38:54.405186+0000","last_peered":"2023-09-23T22:39:06.287536+0000","last_clean":"2023-09-23T22:38:48.357907+0000","last_became_active":"2023-09-23T22:36:36.406789+0000","last_became_peered":"2023-09-23T22:39:05.294555+0000","last_unstale":"2023-09-23T22:39:06.287536+0000","last_undegraded":"2023-09-23T22:39:06.287536+0000","last_fullsized":"2023-09-23T22:39:05.293080+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:45.131265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":566,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.288826+0000","last_change":"2023-09-23T22:38:55.396680+0000","last_active":"2023-09-23T22:39:06.288826+0000","last_peered":"2023-09-23T22:39:06.288826+0000","last_clean":"2023-09-23T22:28:58.571704+0000","last_became_active":"2023-09-23T22:38:55.396680+0000","last_became_peered":"2023-09-23T22:38:55.396680+0000","last_unstale":"2023-09-23T22:39:06.288826+0000","last_undegraded":"2023-09-23T22:39:06.288826+0000","last_fullsized":"2023-09-23T22:38:55.378990+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":483,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.289199+0000","last_change":"2023-09-23T22:38:55.387267+0000","last_active":"2023-09-23T22:39:06.289199+0000","last_peered":"2023-09-23T22:39:06.289199+0000","last_clean":"2023-09-23T22:38:48.353526+0000","last_became_active":"2023-09-23T22:38:55.387267+0000","last_became_peered":"2023-09-23T22:38:55.387267+0000","last_unstale":"2023-09-23T22:39:06.289199+0000","last_undegraded":"2023-09-23T22:39:06.289199+0000","last_fullsized":"2023-09-23T22:38:55.379616+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:39.130605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":366,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.306772+0000","last_change":"2023-09-23T22:39:05.307686+0000","last_active":"2023-09-23T22:39:06.306772+0000","last_peered":"2023-09-23T22:39:06.306772+0000","last_clean":"2023-09-23T22:35:17.626249+0000","last_became_active":"2023-09-23T22:39:05.307686+0000","last_became_peered":"2023-09-23T22:39:05.307686+0000","last_unstale":"2023-09-23T22:39:06.306772+0000","last_undegraded":"2023-09-23T22:39:06.306772+0000","last_fullsized":"2023-09-23T22:39:05.302436+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:24.866314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":894,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.286569+0000","last_change":"2023-09-23T22:36:22.311443+0000","last_active":"2023-09-23T22:39:06.286569+0000","last_peered":"2023-09-23T22:39:06.286569+0000","last_clean":"2023-09-23T22:39:06.286569+0000","last_became_active":"2023-09-23T22:36:22.311137+0000","last_became_peered":"2023-09-23T22:36:22.311137+0000","last_unstale":"2023-09-23T22:39:06.286569+0000","last_undegraded":"2023-09-23T22:39:06.286569+0000","last_fullsized":"2023-09-23T22:39:06.286569+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":394,"reported_epoch":519,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:39:06.306353+0000","last_change":"2023-09-23T22:39:05.302168+0000","last_active":"2023-09-23T22:39:04.291704+0000","last_peered":"2023-09-23T22:39:06.306353+0000","last_clean":"2023-09-23T22:28:58.571564+0000","last_became_active":"2023-09-23T22:28:05.154059+0000","last_became_peered":"2023-09-23T22:39:05.302168+0000","last_unstale":"2023-09-23T22:39:06.306353+0000","last_undegraded":"2023-09-23T22:39:05.286285+0000","last_fullsized":"2023-09-23T22:39:05.286213+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:03.826266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":62,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286661+0000","last_change":"2023-09-23T22:39:05.295415+0000","last_active":"2023-09-23T22:39:06.286661+0000","last_peered":"2023-09-23T22:39:06.286661+0000","last_clean":"2023-09-23T22:37:07.674004+0000","last_became_active":"2023-09-23T22:39:05.295415+0000","last_became_peered":"2023-09-23T22:39:05.295415+0000","last_unstale":"2023-09-23T22:39:06.286661+0000","last_undegraded":"2023-09-23T22:39:06.286661+0000","last_fullsized":"2023-09-23T22:39:05.293230+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:45.148758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":886,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.291077+0000","last_change":"2023-09-23T22:38:55.382695+0000","last_active":"2023-09-23T22:39:06.291077+0000","last_peered":"2023-09-23T22:39:06.291077+0000","last_clean":"2023-09-23T22:38:53.587260+0000","last_became_active":"2023-09-23T22:38:55.382695+0000","last_became_peered":"2023-09-23T22:38:55.382695+0000","last_unstale":"2023-09-23T22:39:06.291077+0000","last_undegraded":"2023-09-23T22:39:06.291077+0000","last_fullsized":"2023-09-23T22:38:55.379918+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.5","version":"520'7145","reported_seq":7389,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T22:39:15.073138+0000","last_change":"2023-09-23T22:39:04.622783+0000","last_active":"2023-09-23T22:39:15.073138+0000","last_peered":"2023-09-23T22:39:15.073138+0000","last_clean":"2023-09-23T22:39:15.073138+0000","last_became_active":"2023-09-23T22:37:42.446611+0000","last_became_peered":"2023-09-23T22:37:42.446611+0000","last_unstale":"2023-09-23T22:39:15.073138+0000","last_undegraded":"2023-09-23T22:39:15.073138+0000","last_fullsized":"2023-09-23T22:39:15.073138+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7145,"log_dups_size":0,"ondisk_log_size":7145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:58.345441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301268992,"num_objects":4597,"num_object_clones":0,"num_object_copies":13791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7145,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":546,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.290656+0000","last_change":"2023-09-23T22:35:52.675241+0000","last_active":"2023-09-23T22:39:06.290656+0000","last_peered":"2023-09-23T22:39:06.290656+0000","last_clean":"2023-09-23T22:39:06.290656+0000","last_became_active":"2023-09-23T22:35:52.674769+0000","last_became_peered":"2023-09-23T22:35:52.674769+0000","last_unstale":"2023-09-23T22:39:06.290656+0000","last_undegraded":"2023-09-23T22:39:06.290656+0000","last_fullsized":"2023-09-23T22:39:06.290656+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:43.611334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":456,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286522+0000","last_change":"2023-09-23T22:38:55.387756+0000","last_active":"2023-09-23T22:39:06.286522+0000","last_peered":"2023-09-23T22:39:06.286522+0000","last_clean":"2023-09-23T22:35:23.290223+0000","last_became_active":"2023-09-23T22:38:55.387756+0000","last_became_peered":"2023-09-23T22:38:55.387756+0000","last_unstale":"2023-09-23T22:39:06.286522+0000","last_undegraded":"2023-09-23T22:39:06.286522+0000","last_fullsized":"2023-09-23T22:38:55.384253+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.075495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":637,"reported_epoch":519,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:39:06.288205+0000","last_change":"2023-09-23T22:39:05.294786+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:39:06.288205+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.294786+0000","last_unstale":"2023-09-23T22:39:06.288205+0000","last_undegraded":"2023-09-23T22:39:05.292513+0000","last_fullsized":"2023-09-23T22:39:05.292435+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"520'7259","reported_seq":12142,"reported_epoch":520,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:15.536176+0000","last_change":"2023-09-23T22:39:05.315684+0000","last_active":"2023-09-23T22:39:15.536176+0000","last_peered":"2023-09-23T22:39:15.536176+0000","last_clean":"2023-09-23T22:39:15.536176+0000","last_became_active":"2023-09-23T22:39:05.304526+0000","last_became_peered":"2023-09-23T22:39:05.304526+0000","last_unstale":"2023-09-23T22:39:15.536176+0000","last_undegraded":"2023-09-23T22:39:15.536176+0000","last_fullsized":"2023-09-23T22:39:15.536176+0000","mapping_epoch":517,"log_start":"461'1300","ondisk_log_start":"461'1300","created":451,"last_epoch_clean":518,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5959,"log_dups_size":0,"ondisk_log_size":5959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:07.663671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304021504,"num_objects":4639,"num_object_clones":0,"num_object_copies":13917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4639,"num_objects_unfound":0,"num_objects_dirty":4639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7259,"num_write_kb":380736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2514,"num_bytes_recovered":158728192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":4639}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":361,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.287962+0000","last_change":"2023-09-23T22:37:42.438301+0000","last_active":"2023-09-23T22:39:06.287962+0000","last_peered":"2023-09-23T22:39:06.287962+0000","last_clean":"2023-09-23T22:39:06.287962+0000","last_became_active":"2023-09-23T22:37:42.437753+0000","last_became_peered":"2023-09-23T22:37:42.437753+0000","last_unstale":"2023-09-23T22:39:06.287962+0000","last_undegraded":"2023-09-23T22:39:06.287962+0000","last_fullsized":"2023-09-23T22:39:06.287962+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:21.146523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":493,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.285912+0000","last_change":"2023-09-23T22:39:05.289891+0000","last_active":"2023-09-23T22:39:06.285912+0000","last_peered":"2023-09-23T22:39:06.285912+0000","last_clean":"2023-09-23T22:39:00.403311+0000","last_became_active":"2023-09-23T22:39:05.289891+0000","last_became_peered":"2023-09-23T22:39:05.289891+0000","last_unstale":"2023-09-23T22:39:06.285912+0000","last_undegraded":"2023-09-23T22:39:06.285912+0000","last_fullsized":"2023-09-23T22:39:05.284229+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:50.581117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":785,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.287280+0000","last_change":"2023-09-23T22:39:05.294943+0000","last_active":"2023-09-23T22:35:19.273163+0000","last_peered":"2023-09-23T22:39:06.287280+0000","last_clean":"2023-09-23T22:35:16.262487+0000","last_became_active":"2023-09-23T22:34:36.010161+0000","last_became_peered":"2023-09-23T22:39:05.294943+0000","last_unstale":"2023-09-23T22:39:06.287280+0000","last_undegraded":"2023-09-23T22:39:06.287280+0000","last_fullsized":"2023-09-23T22:39:05.292735+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.7","version":"521'7258","reported_seq":8386,"reported_epoch":521,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:39:17.336257+0000","last_change":"2023-09-23T22:39:06.345952+0000","last_active":"2023-09-23T22:39:17.336257+0000","last_peered":"2023-09-23T22:39:17.336257+0000","last_clean":"2023-09-23T22:36:35.361199+0000","last_became_active":"2023-09-23T22:39:06.319908+0000","last_became_peered":"2023-09-23T22:39:06.319908+0000","last_unstale":"2023-09-23T22:39:17.336257+0000","last_undegraded":"2023-09-23T22:39:06.295290+0000","last_fullsized":"2023-09-23T22:39:17.336257+0000","mapping_epoch":518,"log_start":"461'893","ondisk_log_start":"461'893","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6365,"log_dups_size":0,"ondisk_log_size":6365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:55.819710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302776320,"num_objects":4620,"num_object_clones":0,"num_object_copies":13860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":507,"num_objects_misplaced":4113,"num_objects_unfound":0,"num_objects_dirty":4620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7258,"num_write_kb":380096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":507,"num_bytes_recovered":33226752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,1],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4620}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":322,"reported_epoch":519,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:39:06.287658+0000","last_change":"2023-09-23T22:39:05.294595+0000","last_active":"2023-09-23T22:35:19.265438+0000","last_peered":"2023-09-23T22:39:06.287658+0000","last_clean":"2023-09-23T22:35:16.262186+0000","last_became_active":"2023-09-23T22:34:32.963996+0000","last_became_peered":"2023-09-23T22:39:05.294595+0000","last_unstale":"2023-09-23T22:39:06.287658+0000","last_undegraded":"2023-09-23T22:39:05.293265+0000","last_fullsized":"2023-09-23T22:39:05.293222+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:57.686355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":220,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286955+0000","last_change":"2023-09-23T22:38:55.397046+0000","last_active":"2023-09-23T22:39:06.286955+0000","last_peered":"2023-09-23T22:39:06.286955+0000","last_clean":"2023-09-23T22:37:07.662421+0000","last_became_active":"2023-09-23T22:38:55.397046+0000","last_became_peered":"2023-09-23T22:38:55.397046+0000","last_unstale":"2023-09-23T22:39:06.286955+0000","last_undegraded":"2023-09-23T22:39:06.286955+0000","last_fullsized":"2023-09-23T22:38:55.385491+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:27.922878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":684,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.306235+0000","last_change":"2023-09-23T22:39:05.302770+0000","last_active":"2023-09-23T22:39:04.293181+0000","last_peered":"2023-09-23T22:39:06.306235+0000","last_clean":"2023-09-23T22:28:58.573726+0000","last_became_active":"2023-09-23T22:29:02.583400+0000","last_became_peered":"2023-09-23T22:39:05.302770+0000","last_unstale":"2023-09-23T22:39:06.306235+0000","last_undegraded":"2023-09-23T22:39:06.306235+0000","last_fullsized":"2023-09-23T22:39:05.302009+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.6","version":"521'7145","reported_seq":7396,"reported_epoch":521,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:17.330175+0000","last_change":"2023-09-23T22:39:04.475370+0000","last_active":"2023-09-23T22:39:17.330175+0000","last_peered":"2023-09-23T22:39:17.330175+0000","last_clean":"2023-09-23T22:37:14.247564+0000","last_became_active":"2023-09-23T22:38:55.393928+0000","last_became_peered":"2023-09-23T22:38:55.393928+0000","last_unstale":"2023-09-23T22:39:17.330175+0000","last_undegraded":"2023-09-23T22:38:55.379302+0000","last_fullsized":"2023-09-23T22:38:55.379093+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7145,"log_dups_size":0,"ondisk_log_size":7145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:41:43.946933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301662208,"num_objects":4603,"num_object_clones":0,"num_object_copies":13809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4603,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7145,"num_write_kb":375936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":74,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":4603}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":364,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.286757+0000","last_change":"2023-09-23T22:39:05.286647+0000","last_active":"2023-09-23T22:38:54.408907+0000","last_peered":"2023-09-23T22:39:06.286757+0000","last_clean":"2023-09-23T22:38:53.017852+0000","last_became_active":"2023-09-23T22:37:42.458233+0000","last_became_peered":"2023-09-23T22:39:05.286647+0000","last_unstale":"2023-09-23T22:39:06.286757+0000","last_undegraded":"2023-09-23T22:39:06.286757+0000","last_fullsized":"2023-09-23T22:39:05.285527+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:29.003765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":284,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.288219+0000","last_change":"2023-09-23T22:37:42.432660+0000","last_active":"2023-09-23T22:39:06.288219+0000","last_peered":"2023-09-23T22:39:06.288219+0000","last_clean":"2023-09-23T22:39:06.288219+0000","last_became_active":"2023-09-23T22:37:42.431969+0000","last_became_peered":"2023-09-23T22:37:42.431969+0000","last_unstale":"2023-09-23T22:39:06.288219+0000","last_undegraded":"2023-09-23T22:39:06.288219+0000","last_fullsized":"2023-09-23T22:39:06.288219+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:54.642557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"520'6950","reported_seq":17586,"reported_epoch":520,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:39:14.916878+0000","last_change":"2023-09-23T22:39:05.324504+0000","last_active":"2023-09-23T22:39:14.916878+0000","last_peered":"2023-09-23T22:39:14.916878+0000","last_clean":"2023-09-23T22:35:27.126327+0000","last_became_active":"2023-09-23T22:39:05.299442+0000","last_became_peered":"2023-09-23T22:39:05.299442+0000","last_unstale":"2023-09-23T22:39:14.916878+0000","last_undegraded":"2023-09-23T22:39:05.295724+0000","last_fullsized":"2023-09-23T22:39:05.291664+0000","mapping_epoch":517,"log_start":"461'1100","ondisk_log_start":"461'1100","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5850,"log_dups_size":0,"ondisk_log_size":5850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:42:20.524365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295305216,"num_objects":4506,"num_object_clones":0,"num_object_copies":13518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9012,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6950,"num_write_kb":366592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5255,"num_bytes_recovered":344326144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7,4],"avail_no_missing":["7"],"object_location_counts":[{"shards":"4,7","objects":4149},{"shards":"7","objects":357}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":821,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T22:39:09.500946+0000","last_change":"2023-09-23T22:39:05.299735+0000","last_active":"2023-09-23T22:39:09.500946+0000","last_peered":"2023-09-23T22:39:09.500946+0000","last_clean":"2023-09-23T22:36:31.469549+0000","last_became_active":"2023-09-23T22:39:05.299735+0000","last_became_peered":"2023-09-23T22:39:05.299735+0000","last_unstale":"2023-09-23T22:39:09.500946+0000","last_undegraded":"2023-09-23T22:39:09.500946+0000","last_fullsized":"2023-09-23T22:39:05.286634+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:10.963206+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":335,"reported_epoch":519,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:06.307385+0000","last_change":"2023-09-23T22:39:05.304092+0000","last_active":"2023-09-23T22:39:06.307385+0000","last_peered":"2023-09-23T22:39:06.307385+0000","last_clean":"2023-09-23T22:36:31.477294+0000","last_became_active":"2023-09-23T22:39:05.304092+0000","last_became_peered":"2023-09-23T22:39:05.304092+0000","last_unstale":"2023-09-23T22:39:06.307385+0000","last_undegraded":"2023-09-23T22:39:05.285659+0000","last_fullsized":"2023-09-23T22:39:05.285557+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:15.273429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":428,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.303852+0000","last_change":"2023-09-23T22:39:05.290625+0000","last_active":"2023-09-23T22:38:54.402963+0000","last_peered":"2023-09-23T22:39:06.303852+0000","last_clean":"2023-09-23T22:38:53.017791+0000","last_became_active":"2023-09-23T22:36:36.405891+0000","last_became_peered":"2023-09-23T22:39:05.290625+0000","last_unstale":"2023-09-23T22:39:06.303852+0000","last_undegraded":"2023-09-23T22:39:06.303852+0000","last_fullsized":"2023-09-23T22:39:05.289355+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:02.717357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"520'7149","reported_seq":7318,"reported_epoch":520,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:15.695321+0000","last_change":"2023-09-23T22:38:55.383275+0000","last_active":"2023-09-23T22:39:15.695321+0000","last_peered":"2023-09-23T22:39:15.695321+0000","last_clean":"2023-09-23T22:38:54.291969+0000","last_became_active":"2023-09-23T22:38:55.383275+0000","last_became_peered":"2023-09-23T22:38:55.383275+0000","last_unstale":"2023-09-23T22:39:15.695321+0000","last_undegraded":"2023-09-23T22:38:55.381280+0000","last_fullsized":"2023-09-23T22:38:55.381176+0000","mapping_epoch":513,"log_start":"461'1200","ondisk_log_start":"461'1200","created":451,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5949,"log_dups_size":0,"ondisk_log_size":5949,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:08:03.051898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300744704,"num_objects":4589,"num_object_clones":0,"num_object_copies":13767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4589,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7149,"num_write_kb":375616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":4589}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":955,"reported_epoch":520,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:39:09.500161+0000","last_change":"2023-09-23T22:39:05.288550+0000","last_active":"2023-09-23T22:39:09.500161+0000","last_peered":"2023-09-23T22:39:09.500161+0000","last_clean":"2023-09-23T22:36:31.469347+0000","last_became_active":"2023-09-23T22:39:05.288550+0000","last_became_peered":"2023-09-23T22:39:05.288550+0000","last_unstale":"2023-09-23T22:39:09.500161+0000","last_undegraded":"2023-09-23T22:39:09.500161+0000","last_fullsized":"2023-09-23T22:39:05.284160+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:48.075424+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":288,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.289117+0000","last_change":"2023-09-23T22:38:55.396620+0000","last_active":"2023-09-23T22:39:06.289117+0000","last_peered":"2023-09-23T22:39:06.289117+0000","last_clean":"2023-09-23T22:37:07.662537+0000","last_became_active":"2023-09-23T22:38:55.396620+0000","last_became_peered":"2023-09-23T22:38:55.396620+0000","last_unstale":"2023-09-23T22:39:06.289117+0000","last_undegraded":"2023-09-23T22:39:06.289117+0000","last_fullsized":"2023-09-23T22:38:55.379490+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:47.749424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":148,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T22:39:09.500872+0000","last_change":"2023-09-23T22:37:42.423759+0000","last_active":"2023-09-23T22:39:09.500872+0000","last_peered":"2023-09-23T22:39:09.500872+0000","last_clean":"2023-09-23T22:39:09.500872+0000","last_became_active":"2023-09-23T22:37:42.423441+0000","last_became_peered":"2023-09-23T22:37:42.423441+0000","last_unstale":"2023-09-23T22:39:09.500872+0000","last_undegraded":"2023-09-23T22:39:09.500872+0000","last_fullsized":"2023-09-23T22:39:09.500872+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:04.848811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":566,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.289527+0000","last_change":"2023-09-23T22:38:55.396840+0000","last_active":"2023-09-23T22:39:06.289527+0000","last_peered":"2023-09-23T22:39:06.289527+0000","last_clean":"2023-09-23T22:28:58.571704+0000","last_became_active":"2023-09-23T22:38:55.396840+0000","last_became_peered":"2023-09-23T22:38:55.396840+0000","last_unstale":"2023-09-23T22:39:06.289527+0000","last_undegraded":"2023-09-23T22:39:06.289527+0000","last_fullsized":"2023-09-23T22:38:55.379140+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"520'7218","reported_seq":10684,"reported_epoch":520,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T22:39:15.769089+0000","last_change":"2023-09-23T22:39:07.006283+0000","last_active":"2023-09-23T22:39:15.769089+0000","last_peered":"2023-09-23T22:39:15.769089+0000","last_clean":"2023-09-23T22:36:35.349924+0000","last_became_active":"2023-09-23T22:39:06.853186+0000","last_became_peered":"2023-09-23T22:39:06.853186+0000","last_unstale":"2023-09-23T22:39:15.769089+0000","last_undegraded":"2023-09-23T22:39:06.844058+0000","last_fullsized":"2023-09-23T22:39:06.288558+0000","mapping_epoch":518,"log_start":"461'1287","ondisk_log_start":"461'1287","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5931,"log_dups_size":0,"ondisk_log_size":5931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:30.355747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303169536,"num_objects":4626,"num_object_clones":0,"num_object_copies":13878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4626,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7218,"num_write_kb":379008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1671,"num_bytes_recovered":67305472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[1,5],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,5","objects":4626}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":534,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.288865+0000","last_change":"2023-09-23T22:35:52.681932+0000","last_active":"2023-09-23T22:39:06.288865+0000","last_peered":"2023-09-23T22:39:06.288865+0000","last_clean":"2023-09-23T22:39:06.288865+0000","last_became_active":"2023-09-23T22:35:52.681575+0000","last_became_peered":"2023-09-23T22:35:52.681575+0000","last_unstale":"2023-09-23T22:39:06.288865+0000","last_undegraded":"2023-09-23T22:39:06.288865+0000","last_fullsized":"2023-09-23T22:39:06.288865+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:25.183658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":507,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.287517+0000","last_change":"2023-09-23T22:38:55.388007+0000","last_active":"2023-09-23T22:39:06.287517+0000","last_peered":"2023-09-23T22:39:06.287517+0000","last_clean":"2023-09-23T22:37:07.660692+0000","last_became_active":"2023-09-23T22:38:55.388007+0000","last_became_peered":"2023-09-23T22:38:55.388007+0000","last_unstale":"2023-09-23T22:39:06.287517+0000","last_undegraded":"2023-09-23T22:39:06.287517+0000","last_fullsized":"2023-09-23T22:38:55.384033+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:14.119720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"520'7215","reported_seq":7331,"reported_epoch":520,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:39:15.554895+0000","last_change":"2023-09-23T22:39:06.312054+0000","last_active":"2023-09-23T22:39:15.554895+0000","last_peered":"2023-09-23T22:39:15.554895+0000","last_clean":"2023-09-23T22:36:35.536592+0000","last_became_active":"2023-09-23T22:39:06.292018+0000","last_became_peered":"2023-09-23T22:39:06.292018+0000","last_unstale":"2023-09-23T22:39:15.554895+0000","last_undegraded":"2023-09-23T22:39:06.287438+0000","last_fullsized":"2023-09-23T22:39:15.554895+0000","mapping_epoch":518,"log_start":"461'1281","ondisk_log_start":"461'1281","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5934,"log_dups_size":0,"ondisk_log_size":5934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:30:10.623604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302317568,"num_objects":4613,"num_object_clones":0,"num_object_copies":13839,"num_objects_missing_on_primary":81,"num_objects_missing":81,"num_objects_degraded":81,"num_objects_misplaced":4613,"num_objects_unfound":0,"num_objects_dirty":4613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7215,"num_write_kb":378496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,2,7","objects":4613}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":901,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.287522+0000","last_change":"2023-09-23T22:38:55.383101+0000","last_active":"2023-09-23T22:39:06.287522+0000","last_peered":"2023-09-23T22:39:06.287522+0000","last_clean":"2023-09-23T22:38:48.353691+0000","last_became_active":"2023-09-23T22:38:55.383101+0000","last_became_peered":"2023-09-23T22:38:55.383101+0000","last_unstale":"2023-09-23T22:39:06.287522+0000","last_undegraded":"2023-09-23T22:39:06.287522+0000","last_fullsized":"2023-09-23T22:38:55.381406+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:55.111338+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":238,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286250+0000","last_change":"2023-09-23T22:39:05.297947+0000","last_active":"2023-09-23T22:39:06.286250+0000","last_peered":"2023-09-23T22:39:06.286250+0000","last_clean":"2023-09-23T22:37:07.670248+0000","last_became_active":"2023-09-23T22:39:05.297947+0000","last_became_peered":"2023-09-23T22:39:05.297947+0000","last_unstale":"2023-09-23T22:39:06.286250+0000","last_undegraded":"2023-09-23T22:39:06.286250+0000","last_fullsized":"2023-09-23T22:39:05.284753+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:18.514918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":281,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286635+0000","last_change":"2023-09-23T22:39:05.294341+0000","last_active":"2023-09-23T22:39:06.286635+0000","last_peered":"2023-09-23T22:39:06.286635+0000","last_clean":"2023-09-23T22:39:00.403604+0000","last_became_active":"2023-09-23T22:39:05.294341+0000","last_became_peered":"2023-09-23T22:39:05.294341+0000","last_unstale":"2023-09-23T22:39:06.286635+0000","last_undegraded":"2023-09-23T22:39:06.286635+0000","last_fullsized":"2023-09-23T22:39:05.290023+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:21.644681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"520'7031","reported_seq":20693,"reported_epoch":520,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:15.107442+0000","last_change":"2023-09-23T22:39:09.972071+0000","last_active":"2023-09-23T22:39:15.107442+0000","last_peered":"2023-09-23T22:39:15.107442+0000","last_clean":"2023-09-23T22:39:15.107442+0000","last_became_active":"2023-09-23T22:39:06.318194+0000","last_became_peered":"2023-09-23T22:39:06.318194+0000","last_unstale":"2023-09-23T22:39:15.107442+0000","last_undegraded":"2023-09-23T22:39:15.107442+0000","last_fullsized":"2023-09-23T22:39:15.107442+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:31.028708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294322176,"num_objects":4491,"num_object_clones":0,"num_object_copies":13473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4491,"num_objects_unfound":0,"num_objects_dirty":4491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7031,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6634,"num_bytes_recovered":336199680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4491}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":785,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.286896+0000","last_change":"2023-09-23T22:39:05.295156+0000","last_active":"2023-09-23T22:35:19.273163+0000","last_peered":"2023-09-23T22:39:06.286896+0000","last_clean":"2023-09-23T22:35:16.262487+0000","last_became_active":"2023-09-23T22:34:36.010161+0000","last_became_peered":"2023-09-23T22:39:05.295156+0000","last_unstale":"2023-09-23T22:39:06.286896+0000","last_undegraded":"2023-09-23T22:39:06.286896+0000","last_fullsized":"2023-09-23T22:39:05.292995+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":958,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.286467+0000","last_change":"2023-09-23T22:37:42.433834+0000","last_active":"2023-09-23T22:39:06.286467+0000","last_peered":"2023-09-23T22:39:06.286467+0000","last_clean":"2023-09-23T22:39:06.286467+0000","last_became_active":"2023-09-23T22:37:42.432400+0000","last_became_peered":"2023-09-23T22:37:42.432400+0000","last_unstale":"2023-09-23T22:39:06.286467+0000","last_undegraded":"2023-09-23T22:39:06.286467+0000","last_fullsized":"2023-09-23T22:39:06.286467+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:17.145403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":560,"reported_epoch":520,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:09.500682+0000","last_change":"2023-09-23T22:39:05.287940+0000","last_active":"2023-09-23T22:38:54.405816+0000","last_peered":"2023-09-23T22:39:09.500682+0000","last_clean":"2023-09-23T22:38:48.357982+0000","last_became_active":"2023-09-23T22:36:36.407307+0000","last_became_peered":"2023-09-23T22:39:05.287940+0000","last_unstale":"2023-09-23T22:39:09.500682+0000","last_undegraded":"2023-09-23T22:39:09.500682+0000","last_fullsized":"2023-09-23T22:39:05.287229+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:59.639645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":236,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.286774+0000","last_change":"2023-09-23T22:37:42.444779+0000","last_active":"2023-09-23T22:39:06.286774+0000","last_peered":"2023-09-23T22:39:06.286774+0000","last_clean":"2023-09-23T22:39:06.286774+0000","last_became_active":"2023-09-23T22:37:42.437935+0000","last_became_peered":"2023-09-23T22:37:42.437935+0000","last_unstale":"2023-09-23T22:39:06.286774+0000","last_undegraded":"2023-09-23T22:39:06.286774+0000","last_fullsized":"2023-09-23T22:39:06.286774+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:18.444049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"520'7099","reported_seq":7484,"reported_epoch":520,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:15.079868+0000","last_change":"2023-09-23T22:39:05.317964+0000","last_active":"2023-09-23T22:39:15.079868+0000","last_peered":"2023-09-23T22:39:15.079868+0000","last_clean":"2023-09-23T22:36:35.270317+0000","last_became_active":"2023-09-23T22:39:05.317964+0000","last_became_peered":"2023-09-23T22:39:05.317964+0000","last_unstale":"2023-09-23T22:39:15.079868+0000","last_undegraded":"2023-09-23T22:39:05.287172+0000","last_fullsized":"2023-09-23T22:39:05.287061+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7099,"log_dups_size":0,"ondisk_log_size":7099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:53.385651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301400064,"num_objects":4599,"num_object_clones":0,"num_object_copies":13797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4599,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7099,"num_write_kb":374336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":9502720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":4599}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":901,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T22:39:09.500351+0000","last_change":"2023-09-23T22:36:35.383077+0000","last_active":"2023-09-23T22:39:09.500351+0000","last_peered":"2023-09-23T22:39:09.500351+0000","last_clean":"2023-09-23T22:39:09.500351+0000","last_became_active":"2023-09-23T22:36:22.300226+0000","last_became_peered":"2023-09-23T22:36:22.300226+0000","last_unstale":"2023-09-23T22:39:09.500351+0000","last_undegraded":"2023-09-23T22:39:09.500351+0000","last_fullsized":"2023-09-23T22:39:09.500351+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1422,"reported_epoch":519,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:06.287673+0000","last_change":"2023-09-23T22:38:55.389270+0000","last_active":"2023-09-23T22:39:06.287673+0000","last_peered":"2023-09-23T22:39:06.287673+0000","last_clean":"2023-09-23T22:38:48.353781+0000","last_became_active":"2023-09-23T22:38:55.389270+0000","last_became_peered":"2023-09-23T22:38:55.389270+0000","last_unstale":"2023-09-23T22:39:06.287673+0000","last_undegraded":"2023-09-23T22:38:55.382069+0000","last_fullsized":"2023-09-23T22:38:55.381954+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:28.932588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":464,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.290397+0000","last_change":"2023-09-23T22:38:55.397986+0000","last_active":"2023-09-23T22:39:06.290397+0000","last_peered":"2023-09-23T22:39:06.290397+0000","last_clean":"2023-09-23T22:35:23.289972+0000","last_became_active":"2023-09-23T22:38:55.397986+0000","last_became_peered":"2023-09-23T22:38:55.397986+0000","last_unstale":"2023-09-23T22:39:06.290397+0000","last_undegraded":"2023-09-23T22:39:06.290397+0000","last_fullsized":"2023-09-23T22:38:55.379810+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:58.839783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":148,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T22:39:09.500225+0000","last_change":"2023-09-23T22:39:05.291228+0000","last_active":"2023-09-23T22:39:09.500225+0000","last_peered":"2023-09-23T22:39:09.500225+0000","last_clean":"2023-09-23T22:39:01.247643+0000","last_became_active":"2023-09-23T22:39:05.291228+0000","last_became_peered":"2023-09-23T22:39:05.291228+0000","last_unstale":"2023-09-23T22:39:09.500225+0000","last_undegraded":"2023-09-23T22:39:09.500225+0000","last_fullsized":"2023-09-23T22:39:05.285277+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:47.583378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"520'7417","reported_seq":7733,"reported_epoch":520,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:15.712603+0000","last_change":"2023-09-23T22:39:06.557390+0000","last_active":"2023-09-23T22:39:15.712603+0000","last_peered":"2023-09-23T22:39:15.712603+0000","last_clean":"2023-09-23T22:36:35.350237+0000","last_became_active":"2023-09-23T22:39:06.291946+0000","last_became_peered":"2023-09-23T22:39:06.291946+0000","last_unstale":"2023-09-23T22:39:15.712603+0000","last_undegraded":"2023-09-23T22:39:06.286510+0000","last_fullsized":"2023-09-23T22:39:06.285916+0000","mapping_epoch":518,"log_start":"461'1500","ondisk_log_start":"461'1500","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5917,"log_dups_size":0,"ondisk_log_size":5917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:34.797366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308740096,"num_objects":4711,"num_object_clones":0,"num_object_copies":14133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4711,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7417,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":4711}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":895,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.287016+0000","last_change":"2023-09-23T22:36:22.310802+0000","last_active":"2023-09-23T22:39:06.287016+0000","last_peered":"2023-09-23T22:39:06.287016+0000","last_clean":"2023-09-23T22:39:06.287016+0000","last_became_active":"2023-09-23T22:36:22.310477+0000","last_became_peered":"2023-09-23T22:36:22.310477+0000","last_unstale":"2023-09-23T22:39:06.287016+0000","last_undegraded":"2023-09-23T22:39:06.287016+0000","last_fullsized":"2023-09-23T22:39:06.287016+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":194,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T22:39:09.500805+0000","last_change":"2023-09-23T22:39:05.309059+0000","last_active":"2023-09-23T22:39:09.500805+0000","last_peered":"2023-09-23T22:39:09.500805+0000","last_clean":"2023-09-23T22:39:01.248920+0000","last_became_active":"2023-09-23T22:39:05.309059+0000","last_became_peered":"2023-09-23T22:39:05.309059+0000","last_unstale":"2023-09-23T22:39:09.500805+0000","last_undegraded":"2023-09-23T22:39:09.500805+0000","last_fullsized":"2023-09-23T22:39:05.286315+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:26.012539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":139,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T22:39:09.500611+0000","last_change":"2023-09-23T22:38:55.382188+0000","last_active":"2023-09-23T22:39:09.500611+0000","last_peered":"2023-09-23T22:39:09.500611+0000","last_clean":"2023-09-23T22:38:48.445651+0000","last_became_active":"2023-09-23T22:38:55.382188+0000","last_became_peered":"2023-09-23T22:38:55.382188+0000","last_unstale":"2023-09-23T22:39:09.500611+0000","last_undegraded":"2023-09-23T22:39:09.500611+0000","last_fullsized":"2023-09-23T22:38:55.378777+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:42.378244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":160,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.286333+0000","last_change":"2023-09-23T22:37:42.437482+0000","last_active":"2023-09-23T22:39:06.286333+0000","last_peered":"2023-09-23T22:39:06.286333+0000","last_clean":"2023-09-23T22:39:06.286333+0000","last_became_active":"2023-09-23T22:37:42.433915+0000","last_became_peered":"2023-09-23T22:37:42.433915+0000","last_unstale":"2023-09-23T22:39:06.286333+0000","last_undegraded":"2023-09-23T22:39:06.286333+0000","last_fullsized":"2023-09-23T22:39:06.286333+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.522090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"520'7408","reported_seq":7582,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T22:39:15.657470+0000","last_change":"2023-09-23T22:37:43.048884+0000","last_active":"2023-09-23T22:39:15.657470+0000","last_peered":"2023-09-23T22:39:15.657470+0000","last_clean":"2023-09-23T22:39:15.657470+0000","last_became_active":"2023-09-23T22:37:42.457023+0000","last_became_peered":"2023-09-23T22:37:42.457023+0000","last_unstale":"2023-09-23T22:39:15.657470+0000","last_undegraded":"2023-09-23T22:39:15.657470+0000","last_fullsized":"2023-09-23T22:39:15.657470+0000","mapping_epoch":495,"log_start":"461'1500","ondisk_log_start":"461'1500","created":451,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:39.190808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305266688,"num_objects":4658,"num_object_clones":0,"num_object_copies":13974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7408,"num_write_kb":386112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1400,"reported_epoch":520,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:09.500293+0000","last_change":"2023-09-23T22:39:05.299849+0000","last_active":"2023-09-23T22:39:09.500293+0000","last_peered":"2023-09-23T22:39:09.500293+0000","last_clean":"2023-09-23T22:39:01.247439+0000","last_became_active":"2023-09-23T22:39:05.299849+0000","last_became_peered":"2023-09-23T22:39:05.299849+0000","last_unstale":"2023-09-23T22:39:09.500293+0000","last_undegraded":"2023-09-23T22:39:05.286114+0000","last_fullsized":"2023-09-23T22:39:05.286006+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:43.518374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":742,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286502+0000","last_change":"2023-09-23T22:38:55.391540+0000","last_active":"2023-09-23T22:39:06.286502+0000","last_peered":"2023-09-23T22:39:06.286502+0000","last_clean":"2023-09-23T22:35:16.261819+0000","last_became_active":"2023-09-23T22:38:55.391540+0000","last_became_peered":"2023-09-23T22:38:55.391540+0000","last_unstale":"2023-09-23T22:39:06.286502+0000","last_undegraded":"2023-09-23T22:39:06.286502+0000","last_fullsized":"2023-09-23T22:38:55.384846+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":268,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.289015+0000","last_change":"2023-09-23T22:39:05.288671+0000","last_active":"2023-09-23T22:38:54.401344+0000","last_peered":"2023-09-23T22:39:06.289015+0000","last_clean":"2023-09-23T22:38:48.357699+0000","last_became_active":"2023-09-23T22:36:36.407197+0000","last_became_peered":"2023-09-23T22:39:05.288671+0000","last_unstale":"2023-09-23T22:39:06.289015+0000","last_undegraded":"2023-09-23T22:39:06.289015+0000","last_fullsized":"2023-09-23T22:39:05.287235+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.518505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":228,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.291202+0000","last_change":"2023-09-23T22:38:55.387206+0000","last_active":"2023-09-23T22:39:06.291202+0000","last_peered":"2023-09-23T22:39:06.291202+0000","last_clean":"2023-09-23T22:35:23.291202+0000","last_became_active":"2023-09-23T22:38:55.387206+0000","last_became_peered":"2023-09-23T22:38:55.387206+0000","last_unstale":"2023-09-23T22:39:06.291202+0000","last_undegraded":"2023-09-23T22:39:06.291202+0000","last_fullsized":"2023-09-23T22:38:55.380731+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:29.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"520'7250","reported_seq":7333,"reported_epoch":520,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:15.063584+0000","last_change":"2023-09-23T22:38:55.398058+0000","last_active":"2023-09-23T22:39:15.063584+0000","last_peered":"2023-09-23T22:39:15.063584+0000","last_clean":"2023-09-23T22:38:54.300575+0000","last_became_active":"2023-09-23T22:38:55.398058+0000","last_became_peered":"2023-09-23T22:38:55.398058+0000","last_unstale":"2023-09-23T22:39:15.063584+0000","last_undegraded":"2023-09-23T22:38:55.382246+0000","last_fullsized":"2023-09-23T22:38:55.382121+0000","mapping_epoch":513,"log_start":"461'1400","ondisk_log_start":"461'1400","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5850,"log_dups_size":0,"ondisk_log_size":5850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:50.649091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308281344,"num_objects":4704,"num_object_clones":0,"num_object_copies":14112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4704,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7250,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":4704}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":566,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.290124+0000","last_change":"2023-09-23T22:38:55.397258+0000","last_active":"2023-09-23T22:39:06.290124+0000","last_peered":"2023-09-23T22:39:06.290124+0000","last_clean":"2023-09-23T22:28:58.571704+0000","last_became_active":"2023-09-23T22:38:55.397258+0000","last_became_peered":"2023-09-23T22:38:55.397258+0000","last_unstale":"2023-09-23T22:39:06.290124+0000","last_undegraded":"2023-09-23T22:39:06.290124+0000","last_fullsized":"2023-09-23T22:38:55.379621+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":925,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.286194+0000","last_change":"2023-09-23T22:37:42.457358+0000","last_active":"2023-09-23T22:39:06.286194+0000","last_peered":"2023-09-23T22:39:06.286194+0000","last_clean":"2023-09-23T22:39:06.286194+0000","last_became_active":"2023-09-23T22:37:42.437821+0000","last_became_peered":"2023-09-23T22:37:42.437821+0000","last_unstale":"2023-09-23T22:39:06.286194+0000","last_undegraded":"2023-09-23T22:39:06.286194+0000","last_fullsized":"2023-09-23T22:39:06.286194+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:17.445534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":279,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.288982+0000","last_change":"2023-09-23T22:39:05.307919+0000","last_active":"2023-09-23T22:39:06.288982+0000","last_peered":"2023-09-23T22:39:06.288982+0000","last_clean":"2023-09-23T22:39:00.405520+0000","last_became_active":"2023-09-23T22:39:05.307919+0000","last_became_peered":"2023-09-23T22:39:05.307919+0000","last_unstale":"2023-09-23T22:39:06.288982+0000","last_undegraded":"2023-09-23T22:39:06.288982+0000","last_fullsized":"2023-09-23T22:39:05.287534+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:26.565338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":394,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.290222+0000","last_change":"2023-09-23T22:35:52.675187+0000","last_active":"2023-09-23T22:39:06.290222+0000","last_peered":"2023-09-23T22:39:06.290222+0000","last_clean":"2023-09-23T22:39:06.290222+0000","last_became_active":"2023-09-23T22:35:52.674444+0000","last_became_peered":"2023-09-23T22:35:52.674444+0000","last_unstale":"2023-09-23T22:39:06.290222+0000","last_undegraded":"2023-09-23T22:39:06.290222+0000","last_fullsized":"2023-09-23T22:39:06.290222+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:17.802244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.e","version":"520'7286","reported_seq":12634,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T22:39:16.537454+0000","last_change":"2023-09-23T22:37:46.112536+0000","last_active":"2023-09-23T22:39:16.537454+0000","last_peered":"2023-09-23T22:39:16.537454+0000","last_clean":"2023-09-23T22:39:16.537454+0000","last_became_active":"2023-09-23T22:37:42.470866+0000","last_became_peered":"2023-09-23T22:37:42.470866+0000","last_unstale":"2023-09-23T22:39:16.537454+0000","last_undegraded":"2023-09-23T22:39:16.537454+0000","last_fullsized":"2023-09-23T22:39:16.537454+0000","mapping_epoch":495,"log_start":"461'1400","ondisk_log_start":"461'1400","created":451,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5886,"log_dups_size":0,"ondisk_log_size":5886,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:22.071126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302645248,"num_objects":4618,"num_object_clones":0,"num_object_copies":13854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7286,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2609,"num_bytes_recovered":166002688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":684,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.306368+0000","last_change":"2023-09-23T22:39:05.302767+0000","last_active":"2023-09-23T22:39:04.293769+0000","last_peered":"2023-09-23T22:39:06.306368+0000","last_clean":"2023-09-23T22:28:58.573726+0000","last_became_active":"2023-09-23T22:29:02.583400+0000","last_became_peered":"2023-09-23T22:39:05.302767+0000","last_unstale":"2023-09-23T22:39:06.306368+0000","last_undegraded":"2023-09-23T22:39:06.306368+0000","last_fullsized":"2023-09-23T22:39:05.302023+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":112,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.287960+0000","last_change":"2023-09-23T22:35:52.688113+0000","last_active":"2023-09-23T22:39:06.287960+0000","last_peered":"2023-09-23T22:39:06.287960+0000","last_clean":"2023-09-23T22:39:06.287960+0000","last_became_active":"2023-09-23T22:35:52.687708+0000","last_became_peered":"2023-09-23T22:35:52.687708+0000","last_unstale":"2023-09-23T22:39:06.287960+0000","last_undegraded":"2023-09-23T22:39:06.287960+0000","last_fullsized":"2023-09-23T22:39:06.287960+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:07.029149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":142,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T22:39:09.500357+0000","last_change":"2023-09-23T22:37:42.424518+0000","last_active":"2023-09-23T22:39:09.500357+0000","last_peered":"2023-09-23T22:39:09.500357+0000","last_clean":"2023-09-23T22:39:09.500357+0000","last_became_active":"2023-09-23T22:37:42.424144+0000","last_became_peered":"2023-09-23T22:37:42.424144+0000","last_unstale":"2023-09-23T22:39:09.500357+0000","last_undegraded":"2023-09-23T22:39:09.500357+0000","last_fullsized":"2023-09-23T22:39:09.500357+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:52:51.255509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":325,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.290534+0000","last_change":"2023-09-23T22:37:42.432892+0000","last_active":"2023-09-23T22:39:06.290534+0000","last_peered":"2023-09-23T22:39:06.290534+0000","last_clean":"2023-09-23T22:39:06.290534+0000","last_became_active":"2023-09-23T22:37:42.432356+0000","last_became_peered":"2023-09-23T22:37:42.432356+0000","last_unstale":"2023-09-23T22:39:06.290534+0000","last_undegraded":"2023-09-23T22:39:06.290534+0000","last_fullsized":"2023-09-23T22:39:06.290534+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:14.288445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.d","version":"520'7335","reported_seq":12778,"reported_epoch":520,"state":"active+clean","last_fresh":"2023-09-23T22:39:14.752342+0000","last_change":"2023-09-23T22:37:42.930274+0000","last_active":"2023-09-23T22:39:14.752342+0000","last_peered":"2023-09-23T22:39:14.752342+0000","last_clean":"2023-09-23T22:39:14.752342+0000","last_became_active":"2023-09-23T22:37:42.448382+0000","last_became_peered":"2023-09-23T22:37:42.448382+0000","last_unstale":"2023-09-23T22:39:14.752342+0000","last_undegraded":"2023-09-23T22:39:14.752342+0000","last_fullsized":"2023-09-23T22:39:14.752342+0000","mapping_epoch":495,"log_start":"461'1400","ondisk_log_start":"461'1400","created":451,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5935,"log_dups_size":0,"ondisk_log_size":5935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:20.785382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313131032,"num_objects":4779,"num_object_clones":0,"num_object_copies":14337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4779,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7335,"num_write_kb":387585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2659,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":890,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.291337+0000","last_change":"2023-09-23T22:38:55.383034+0000","last_active":"2023-09-23T22:39:06.291337+0000","last_peered":"2023-09-23T22:39:06.291337+0000","last_clean":"2023-09-23T22:38:48.354625+0000","last_became_active":"2023-09-23T22:38:55.383034+0000","last_became_peered":"2023-09-23T22:38:55.383034+0000","last_unstale":"2023-09-23T22:39:06.291337+0000","last_undegraded":"2023-09-23T22:39:06.291337+0000","last_fullsized":"2023-09-23T22:38:55.380898+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":157,"reported_epoch":519,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:06.287059+0000","last_change":"2023-09-23T22:39:05.312608+0000","last_active":"2023-09-23T22:39:06.287059+0000","last_peered":"2023-09-23T22:39:06.287059+0000","last_clean":"2023-09-23T22:35:17.625026+0000","last_became_active":"2023-09-23T22:39:05.312608+0000","last_became_peered":"2023-09-23T22:39:05.312608+0000","last_unstale":"2023-09-23T22:39:06.287059+0000","last_undegraded":"2023-09-23T22:39:05.293584+0000","last_fullsized":"2023-09-23T22:39:05.293517+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:47.778707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":1021,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.287717+0000","last_change":"2023-09-23T22:38:55.388290+0000","last_active":"2023-09-23T22:39:06.287717+0000","last_peered":"2023-09-23T22:39:06.287717+0000","last_clean":"2023-09-23T22:38:48.410284+0000","last_became_active":"2023-09-23T22:38:55.388290+0000","last_became_peered":"2023-09-23T22:38:55.388290+0000","last_unstale":"2023-09-23T22:39:06.287717+0000","last_undegraded":"2023-09-23T22:39:06.287717+0000","last_fullsized":"2023-09-23T22:38:55.384357+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:40.082337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":42,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.286654+0000","last_change":"2023-09-23T22:39:05.312578+0000","last_active":"2023-09-23T22:39:06.286654+0000","last_peered":"2023-09-23T22:39:06.286654+0000","last_clean":"2023-09-23T22:31:26.131914+0000","last_became_active":"2023-09-23T22:39:05.312578+0000","last_became_peered":"2023-09-23T22:39:05.312578+0000","last_unstale":"2023-09-23T22:39:06.286654+0000","last_undegraded":"2023-09-23T22:39:06.286654+0000","last_fullsized":"2023-09-23T22:39:05.291104+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:52.614783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"520'7339","reported_seq":9606,"reported_epoch":520,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:39:17.003084+0000","last_change":"2023-09-23T22:39:07.012094+0000","last_active":"2023-09-23T22:39:17.003084+0000","last_peered":"2023-09-23T22:39:17.003084+0000","last_clean":"2023-09-23T22:35:27.348348+0000","last_became_active":"2023-09-23T22:39:06.327496+0000","last_became_peered":"2023-09-23T22:39:06.327496+0000","last_unstale":"2023-09-23T22:39:17.003084+0000","last_undegraded":"2023-09-23T22:39:06.297790+0000","last_fullsized":"2023-09-23T22:39:17.003084+0000","mapping_epoch":518,"log_start":"461'1365","ondisk_log_start":"461'1365","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:42.584301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301268992,"num_objects":4597,"num_object_clones":0,"num_object_copies":13791,"num_objects_missing_on_primary":1712,"num_objects_missing":1712,"num_objects_degraded":1712,"num_objects_misplaced":4597,"num_objects_unfound":0,"num_objects_dirty":4597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7339,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":777,"num_bytes_recovered":50921472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,0],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,4,7","objects":2874},{"shards":"0,7","objects":1712}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":637,"reported_epoch":519,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:06.288350+0000","last_change":"2023-09-23T22:39:05.295060+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:39:06.288350+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.295060+0000","last_unstale":"2023-09-23T22:39:06.288350+0000","last_undegraded":"2023-09-23T22:39:06.288350+0000","last_fullsized":"2023-09-23T22:39:05.292715+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":144,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T22:39:09.501735+0000","last_change":"2023-09-23T22:38:55.388004+0000","last_active":"2023-09-23T22:39:09.501735+0000","last_peered":"2023-09-23T22:39:09.501735+0000","last_clean":"2023-09-23T22:33:41.182644+0000","last_became_active":"2023-09-23T22:38:55.388004+0000","last_became_peered":"2023-09-23T22:38:55.388004+0000","last_unstale":"2023-09-23T22:39:09.501735+0000","last_undegraded":"2023-09-23T22:39:09.501735+0000","last_fullsized":"2023-09-23T22:38:55.379533+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:15.982785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":259,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.288047+0000","last_change":"2023-09-23T22:37:42.460308+0000","last_active":"2023-09-23T22:39:06.288047+0000","last_peered":"2023-09-23T22:39:06.288047+0000","last_clean":"2023-09-23T22:39:06.288047+0000","last_became_active":"2023-09-23T22:37:42.438206+0000","last_became_peered":"2023-09-23T22:37:42.438206+0000","last_unstale":"2023-09-23T22:39:06.288047+0000","last_undegraded":"2023-09-23T22:39:06.288047+0000","last_fullsized":"2023-09-23T22:39:06.288047+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:54.493150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":398,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.289463+0000","last_change":"2023-09-23T22:38:55.382690+0000","last_active":"2023-09-23T22:39:06.289463+0000","last_peered":"2023-09-23T22:39:06.289463+0000","last_clean":"2023-09-23T22:35:23.293559+0000","last_became_active":"2023-09-23T22:38:55.382690+0000","last_became_peered":"2023-09-23T22:38:55.382690+0000","last_unstale":"2023-09-23T22:39:06.289463+0000","last_undegraded":"2023-09-23T22:39:06.289463+0000","last_fullsized":"2023-09-23T22:38:55.380032+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:58.155649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":175,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.307175+0000","last_change":"2023-09-23T22:35:52.675772+0000","last_active":"2023-09-23T22:39:06.307175+0000","last_peered":"2023-09-23T22:39:06.307175+0000","last_clean":"2023-09-23T22:39:06.307175+0000","last_became_active":"2023-09-23T22:35:52.675279+0000","last_became_peered":"2023-09-23T22:35:52.675279+0000","last_unstale":"2023-09-23T22:39:06.307175+0000","last_undegraded":"2023-09-23T22:39:06.307175+0000","last_fullsized":"2023-09-23T22:39:06.307175+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.513213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":148,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T22:39:09.500446+0000","last_change":"2023-09-23T22:39:05.291652+0000","last_active":"2023-09-23T22:39:09.500446+0000","last_peered":"2023-09-23T22:39:09.500446+0000","last_clean":"2023-09-23T22:39:01.247800+0000","last_became_active":"2023-09-23T22:39:05.291652+0000","last_became_peered":"2023-09-23T22:39:05.291652+0000","last_unstale":"2023-09-23T22:39:09.500446+0000","last_undegraded":"2023-09-23T22:39:09.500446+0000","last_fullsized":"2023-09-23T22:39:05.287811+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:03.849150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":544,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T22:39:06.287875+0000","last_change":"2023-09-23T22:37:42.419112+0000","last_active":"2023-09-23T22:39:06.287875+0000","last_peered":"2023-09-23T22:39:06.287875+0000","last_clean":"2023-09-23T22:39:06.287875+0000","last_became_active":"2023-09-23T22:37:42.418593+0000","last_became_peered":"2023-09-23T22:37:42.418593+0000","last_unstale":"2023-09-23T22:39:06.287875+0000","last_undegraded":"2023-09-23T22:39:06.287875+0000","last_fullsized":"2023-09-23T22:39:06.287875+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:46.031051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":508,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.288367+0000","last_change":"2023-09-23T22:39:05.290461+0000","last_active":"2023-09-23T22:39:06.288367+0000","last_peered":"2023-09-23T22:39:06.288367+0000","last_clean":"2023-09-23T22:37:07.670799+0000","last_became_active":"2023-09-23T22:39:05.290461+0000","last_became_peered":"2023-09-23T22:39:05.290461+0000","last_unstale":"2023-09-23T22:39:06.288367+0000","last_undegraded":"2023-09-23T22:39:06.288367+0000","last_fullsized":"2023-09-23T22:39:05.285092+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:31.927928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":559,"reported_epoch":519,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:06.288000+0000","last_change":"2023-09-23T22:38:55.382794+0000","last_active":"2023-09-23T22:39:06.288000+0000","last_peered":"2023-09-23T22:39:06.288000+0000","last_clean":"2023-09-23T22:38:48.353999+0000","last_became_active":"2023-09-23T22:38:55.382794+0000","last_became_peered":"2023-09-23T22:38:55.382794+0000","last_unstale":"2023-09-23T22:39:06.288000+0000","last_undegraded":"2023-09-23T22:38:55.381634+0000","last_fullsized":"2023-09-23T22:38:55.381551+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:52.591257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":285,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.289223+0000","last_change":"2023-09-23T22:39:05.307911+0000","last_active":"2023-09-23T22:39:06.289223+0000","last_peered":"2023-09-23T22:39:06.289223+0000","last_clean":"2023-09-23T22:35:23.286304+0000","last_became_active":"2023-09-23T22:39:05.307911+0000","last_became_peered":"2023-09-23T22:39:05.307911+0000","last_unstale":"2023-09-23T22:39:06.289223+0000","last_undegraded":"2023-09-23T22:39:06.289223+0000","last_fullsized":"2023-09-23T22:39:05.287595+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:03.410439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":119,"reported_epoch":520,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:09.500998+0000","last_change":"2023-09-23T22:39:05.288020+0000","last_active":"2023-09-23T22:33:45.291092+0000","last_peered":"2023-09-23T22:39:09.500998+0000","last_clean":"2023-09-23T22:33:41.181859+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:39:05.288020+0000","last_unstale":"2023-09-23T22:39:09.500998+0000","last_undegraded":"2023-09-23T22:39:09.500998+0000","last_fullsized":"2023-09-23T22:39:05.287481+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.969554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":520,"state":"active+undersized","last_fresh":"2023-09-23T22:39:09.501086+0000","last_change":"2023-09-23T22:39:05.309675+0000","last_active":"2023-09-23T22:39:09.501086+0000","last_peered":"2023-09-23T22:39:09.501086+0000","last_clean":"2023-09-23T22:34:30.962054+0000","last_became_active":"2023-09-23T22:39:05.309675+0000","last_became_peered":"2023-09-23T22:39:05.309675+0000","last_unstale":"2023-09-23T22:39:09.501086+0000","last_undegraded":"2023-09-23T22:39:09.501086+0000","last_fullsized":"2023-09-23T22:39:05.286965+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:26.850825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":471,"reported_epoch":520,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:09.501879+0000","last_change":"2023-09-23T22:38:55.394101+0000","last_active":"2023-09-23T22:39:09.501879+0000","last_peered":"2023-09-23T22:39:09.501879+0000","last_clean":"2023-09-23T22:38:48.446310+0000","last_became_active":"2023-09-23T22:38:55.394101+0000","last_became_peered":"2023-09-23T22:38:55.394101+0000","last_unstale":"2023-09-23T22:39:09.501879+0000","last_undegraded":"2023-09-23T22:38:55.380019+0000","last_fullsized":"2023-09-23T22:38:55.379907+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:57.436984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":603,"reported_epoch":519,"state":"active+undersized","last_fresh":"2023-09-23T22:39:06.288596+0000","last_change":"2023-09-23T22:38:55.384159+0000","last_active":"2023-09-23T22:39:06.288596+0000","last_peered":"2023-09-23T22:39:06.288596+0000","last_clean":"2023-09-23T22:38:48.354354+0000","last_became_active":"2023-09-23T22:38:55.384159+0000","last_became_peered":"2023-09-23T22:38:55.384159+0000","last_unstale":"2023-09-23T22:39:06.288596+0000","last_undegraded":"2023-09-23T22:39:06.288596+0000","last_fullsized":"2023-09-23T22:38:55.378819+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:01.489638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":66,"acting":66,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":31744,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48807,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":66,"acting":66,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1211762,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":38,"acting":39,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4846321688,"num_objects":73950,"num_object_clones":0,"num_object_copies":221850,"num_objects_missing_on_primary":1793,"num_objects_missing":1793,"num_objects_degraded":39144,"num_objects_misplaced":22453,"num_objects_unfound":0,"num_objects_dirty":73950,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":115504,"num_write_kb":6062465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23177,"num_bytes_recovered":1335099392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15288381440,"data_stored":15288369224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99878,"ondisk_log_size":99878,"up":35,"acting":41,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":314,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":19,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234418,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3729344,"kb_used_data":3286584,"kb_used_omap":0,"kb_used_meta":442752,"kb_avail":90015808,"statfs":{"total":95995035648,"available":92176187392,"internally_reserved":0,"allocated":3365462016,"data_stored":3363726193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":453378048},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:31 2023","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":2.8730000000000002,"15min":3.8519999999999999},"min":{"1min":0.28899999999999998,"5min":0.24199999999999999,"15min":0.20300000000000001},"max":{"1min":1.0009999999999999,"5min":88.974000000000004,"15min":220.66},"last":1.9550000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":3.0489999999999999,"15min":4.194},"min":{"1min":0.28399999999999997,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.0309999999999997,"5min":89.236000000000004,"15min":220.59899999999999},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.4700000000000002,"5min":3.4700000000000002,"15min":3.4700000000000002},"last":1.913},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.9500000000000002,"5min":2.9500000000000002,"15min":2.9500000000000002},"last":0.495}]},{"osd":2,"last update":"Sat Sep 23 22:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.6870000000000001,"15min":1.5980000000000001},"min":{"1min":0.28199999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.9889999999999999,"5min":52.622,"15min":70.183999999999997},"last":1.9950000000000001},{"interface":"front","average":{"1min":0.62,"5min":2.0219999999999998,"15min":1.835},"min":{"1min":0.307,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.0110000000000001,"5min":52.744,"15min":70.140000000000001},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.2969999999999999,"15min":2.5390000000000001},"min":{"1min":0.35999999999999999,"5min":0.316,"15min":0.26900000000000002},"max":{"1min":3.5710000000000002,"5min":9.8989999999999991,"15min":219.90700000000001},"last":0.435},{"interface":"front","average":{"1min":0.82699999999999996,"5min":2.101,"15min":2.9540000000000002},"min":{"1min":0.38,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":3.3719999999999999,"5min":48.866999999999997,"15min":219.976},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.65100000000000002,"15min":0.72299999999999998},"min":{"1min":0.27900000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68000000000000005,"5min":2.8090000000000002,"15min":2.8090000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.8680000000000001,"15min":1.8220000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.2509999999999999,"5min":48.960000000000001,"15min":48.960000000000001},"last":0.46800000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:38:22 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":3.6629999999999998,"15min":1.7689999999999999},"min":{"1min":0.29899999999999999,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":0.997,"5min":125.083,"15min":125.083},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.66200000000000003,"5min":3.4550000000000001,"15min":1.8180000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.27300000000000002},"max":{"1min":2.2170000000000001,"5min":122.351,"15min":122.351},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:38:22 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":2.5499999999999998,"15min":1.389},"min":{"1min":0.32000000000000001,"5min":0.311,"15min":0.29499999999999998},"max":{"1min":0.77500000000000002,"5min":88.284999999999997,"15min":88.284999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.623,"5min":2.4009999999999998,"15min":1.4339999999999999},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.33700000000000002},"max":{"1min":2.3050000000000002,"5min":88.355000000000004,"15min":88.355000000000004},"last":0.54900000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496049,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2171296,"kb_used_data":1893144,"kb_used_omap":0,"kb_used_meta":278144,"kb_avail":91573856,"statfs":{"total":95995035648,"available":93771628544,"internally_reserved":0,"allocated":1938579456,"data_stored":1936876687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":284819456},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":4.9400000000000004,"15min":4.5979999999999999},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":0.623,"5min":173.291,"15min":213.23699999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":5.2850000000000001,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.22600000000000001},"max":{"1min":0.59799999999999998,"5min":174.232,"15min":213.304},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.4570000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:38:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":3.8730000000000002,"15min":2.5680000000000001},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.2770000000000001,"5min":103.548,"15min":103.548},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":3.8069999999999999,"15min":2.657},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.583,"5min":105.99299999999999,"15min":105.99299999999999},"last":2.5049999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:37:58 2023","interfaces":[{"interface":"back","average":{"1min":4.3259999999999996,"5min":2.2690000000000001,"15min":2.9209999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":82.584000000000003,"5min":82.584000000000003,"15min":125.125},"last":0.58399999999999996},{"interface":"front","average":{"1min":4.6360000000000001,"5min":2.2040000000000002,"15min":2.5070000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":82.316000000000003,"5min":82.316000000000003,"15min":82.316000000000003},"last":2.3290000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:37:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.99099999999999999,"15min":1.042},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.5099999999999998,"5min":7.5099999999999998,"15min":7.5099999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.4299999999999999,"15min":1.5589999999999999},"min":{"1min":0.309,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0569999999999999,"5min":6.3490000000000002,"15min":6.3490000000000002},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 22:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.7110000000000001,"15min":1.6779999999999999},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.4359999999999999,"5min":15.032999999999999,"15min":88.194999999999993},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.1279999999999999,"15min":1.359},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69499999999999995,"5min":7.0839999999999996,"15min":88.025000000000006},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.117,"15min":1.353},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.314},"max":{"1min":0.71199999999999997,"5min":7.5670000000000002,"15min":82.801000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.2649999999999999,"15min":1.508},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.29699999999999999},"max":{"1min":0.65400000000000003,"5min":13.465999999999999,"15min":83.052999999999997},"last":0.65800000000000003}]}]},{"osd":1,"up_from":495,"seq":2126008811540,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3742220,"kb_used_data":3607684,"kb_used_omap":12,"kb_used_meta":134515,"kb_avail":90002932,"statfs":{"total":95995035648,"available":92163002368,"internally_reserved":0,"allocated":3694268416,"data_stored":3691579137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":137744046},"hb_peers":[0,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":2.2189999999999999,"5min":2.2189999999999999,"15min":2.2189999999999999},"last":0.254},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.5860000000000003,"5min":5.5860000000000003,"15min":5.5860000000000003},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.8370000000000002,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.1840000000000002,"5min":2.1840000000000002,"15min":2.1840000000000002},"last":0.629}]},{"osd":3,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.6390000000000002,"5min":5.6390000000000002,"15min":5.6390000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.032,"15min":1.032},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.5739999999999998,"5min":4.5739999999999998,"15min":4.5739999999999998},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":5.6740000000000004,"5min":5.6740000000000004,"15min":5.6740000000000004},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.23,"5min":2.23,"15min":2.23},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.36099999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.503},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":5.7050000000000001,"5min":5.7050000000000001,"15min":5.7050000000000001},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.444,"5min":2.444,"15min":2.444},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.40000000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958855,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2632324,"kb_used_data":1793020,"kb_used_omap":8,"kb_used_meta":839287,"kb_avail":91112828,"statfs":{"total":95995035648,"available":93299535872,"internally_reserved":0,"allocated":1836052480,"data_stored":1833309349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":859430762},"hb_peers":[1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.8599999999999999,"5min":3.8599999999999999,"15min":3.8599999999999999},"last":0.16600000000000001},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.6539999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.26700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:38:13 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":4.0650000000000004,"15min":3.1819999999999999},"min":{"1min":0.248,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.6379999999999999,"5min":175.89400000000001,"15min":175.89400000000001},"last":0.248},{"interface":"front","average":{"1min":0.495,"5min":3.8010000000000002,"15min":2.3500000000000001},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.659,"5min":175.66300000000001,"15min":175.66300000000001},"last":0.32000000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.024,"15min":2.4399999999999999},"min":{"1min":0.28799999999999998,"5min":0.23100000000000001,"15min":0.22500000000000001},"max":{"1min":1.698,"5min":7.0579999999999998,"15min":146.87299999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.149,"15min":2.516},"min":{"1min":0.311,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.1459999999999999,"5min":10.026,"15min":145.48099999999999},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.2010000000000001,"15min":1.262},"min":{"1min":0.32700000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.1899999999999999,"5min":9.968,"15min":9.968},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.2430000000000001,"15min":1.2989999999999999},"min":{"1min":0.378,"5min":0.373,"15min":0.373},"max":{"1min":3.25,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":0.33300000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":3.6520000000000001,"15min":2.7170000000000001},"min":{"1min":0.36899999999999999,"5min":0.32700000000000001,"15min":0.29299999999999998},"max":{"1min":4.0599999999999996,"5min":126.872,"15min":126.872},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.59399999999999997,"5min":2.1469999999999998,"15min":1.8200000000000001},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":1.9690000000000001,"5min":126.521,"15min":126.521},"last":0.35099999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:38:23 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":2.25,"15min":2.3490000000000002},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.308},"max":{"1min":4.1280000000000001,"5min":74.355000000000004,"15min":74.355000000000004},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.8320000000000001,"15min":1.7529999999999999},"min":{"1min":0.39100000000000001,"5min":0.38400000000000001,"15min":0.35099999999999998},"max":{"1min":1.7330000000000001,"5min":73.150000000000006,"15min":73.150000000000006},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:38:23 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.2889999999999999,"15min":1.5800000000000001},"min":{"1min":0.46899999999999997,"5min":0.38600000000000001,"15min":0.29999999999999999},"max":{"1min":2.6469999999999998,"5min":6.2320000000000002,"15min":22.338999999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":1.6080000000000001,"15min":2.008},"min":{"1min":0.41999999999999998,"5min":0.38800000000000001,"15min":0.307},"max":{"1min":4.0990000000000002,"5min":9.0879999999999992,"15min":16.381},"last":0.36599999999999999}]}]},{"osd":2,"up_from":349,"seq":1498943586419,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1695812,"kb_used_data":1469628,"kb_used_omap":15,"kb_used_meta":226160,"kb_avail":92049340,"statfs":{"total":95995035648,"available":94258524160,"internally_reserved":0,"allocated":1504899072,"data_stored":1502272729,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":231588598},"hb_peers":[0,1,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":2.0800000000000001,"15min":10.228999999999999},"min":{"1min":0.23499999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":1.5649999999999999,"5min":86.620000000000005,"15min":194.06299999999999},"last":7.9400000000000004},{"interface":"front","average":{"1min":0.73199999999999998,"5min":2.464,"15min":11.512},"min":{"1min":0.22,"5min":0.22,"15min":0.215},"max":{"1min":3.9609999999999999,"5min":86.355000000000004,"15min":196.947},"last":1.1479999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.4350000000000001,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":8.0050000000000008},{"interface":"front","average":{"1min":1.097,"5min":1.097,"15min":1.097},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":4.9829999999999997,"5min":4.9829999999999997,"15min":4.9829999999999997},"last":1.2070000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:38:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.6870000000000001,"15min":1.5680000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.0309999999999997,"5min":15.585000000000001,"15min":15.585000000000001},"last":0.745},{"interface":"front","average":{"1min":0.502,"5min":1.3320000000000001,"15min":1.0289999999999999},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0609999999999999,"5min":14.128,"15min":14.128},"last":0.629}]},{"osd":4,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.7649999999999999,"15min":1.917},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.4130000000000003,"5min":11.669,"15min":11.669},"last":2.1120000000000001},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.073,"15min":1.173},"min":{"1min":0.36799999999999999,"5min":0.311,"15min":0.311},"max":{"1min":2.661,"5min":5.4969999999999999,"15min":5.4969999999999999},"last":1.244}]},{"osd":5,"last update":"Sat Sep 23 22:39:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":4.2830000000000004,"15min":3.3860000000000001},"min":{"1min":0.40300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.4560000000000004,"5min":163.56399999999999,"15min":163.56399999999999},"last":2.1640000000000001},{"interface":"front","average":{"1min":0.57199999999999995,"5min":3.7200000000000002,"15min":2.9049999999999998},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.9159999999999999,"5min":163.541,"15min":163.541},"last":1.1870000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":2.851,"15min":1.958},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":2.0979999999999999,"5min":169.798,"15min":169.798},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.169,"5min":3.7669999999999999,"15min":2.411},"min":{"1min":0.42399999999999999,"5min":0.373,"15min":0.373},"max":{"1min":7.1130000000000004,"5min":168.44200000000001,"15min":168.44200000000001},"last":1.49}]},{"osd":7,"last update":"Sat Sep 23 22:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":2.2890000000000001,"15min":1.6719999999999999},"min":{"1min":0.40999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":3.996,"5min":108.113,"15min":108.113},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.218,"5min":3.48,"15min":2.0329999999999999},"min":{"1min":0.42399999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":4.5039999999999996,"5min":108.093,"15min":108.093},"last":2.2000000000000002}]}]},{"osd":3,"up_from":108,"seq":463856468266,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1136452,"kb_used_data":582012,"kb_used_omap":4,"kb_used_meta":554427,"kb_avail":92608700,"statfs":{"total":95995035648,"available":94831308800,"internally_reserved":0,"allocated":595980288,"data_stored":593258702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":567733805},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.2639999999999998,"15min":3.7320000000000002},"min":{"1min":0.218,"5min":0.192,"15min":0.192},"max":{"1min":31.917000000000002,"5min":84.239000000000004,"15min":124.282},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.9399999999999999,"5min":2.9689999999999999,"15min":4.0309999999999997},"min":{"1min":0.28499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":31.984000000000002,"5min":84.188000000000002,"15min":124.30500000000001},"last":0.89400000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.4650000000000001,"5min":1.4650000000000001,"15min":1.4650000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.9570000000000001,"5min":1.9570000000000001,"15min":1.9570000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":31.856999999999999,"5min":31.856999999999999,"15min":31.856999999999999},"last":0.84899999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:39:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":3.048,"15min":2.3900000000000001},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":31.968,"5min":157.571,"15min":157.571},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.1269999999999998,"5min":3.6549999999999998,"15min":2.423},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.223},"max":{"1min":31.899999999999999,"5min":158.33000000000001,"15min":158.33000000000001},"last":0.96199999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.4059999999999999,"15min":1.3580000000000001},"min":{"1min":0.34999999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":32.015000000000001,"5min":32.015000000000001,"15min":32.015000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.04,"5min":1.3680000000000001,"15min":1.2929999999999999},"min":{"1min":0.34999999999999998,"5min":0.31,"15min":0.31},"max":{"1min":32.094999999999999,"5min":32.094999999999999,"15min":32.094999999999999},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":2.097,"15min":2.2320000000000002},"min":{"1min":0.32500000000000001,"5min":0.318,"15min":0.315},"max":{"1min":51.286999999999999,"5min":51.286999999999999,"15min":51.286999999999999},"last":0.92800000000000005},{"interface":"front","average":{"1min":2.9670000000000001,"5min":2.077,"15min":2.2679999999999998},"min":{"1min":0.39000000000000001,"5min":0.36299999999999999,"15min":0.29099999999999998},"max":{"1min":51.270000000000003,"5min":51.270000000000003,"15min":51.270000000000003},"last":0.81699999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:37:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.21,"15min":1.9670000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34200000000000003},"max":{"1min":7.5140000000000002,"5min":24.957000000000001,"15min":24.957000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.0379999999999998,"15min":2.206},"min":{"1min":0.376,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.4509999999999996,"5min":18.387,"15min":75.564999999999998},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.694,"15min":2.1110000000000002},"min":{"1min":0.436,"5min":0.38600000000000001,"15min":0.32400000000000001},"max":{"1min":32.040999999999997,"5min":32.040999999999997,"15min":75.525000000000006},"last":0.77700000000000002},{"interface":"front","average":{"1min":2.012,"5min":1.5600000000000001,"15min":1.804},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.379},"max":{"1min":32.058,"5min":32.058,"15min":32.058},"last":0.49199999999999999}]}]},{"osd":4,"up_from":465,"seq":1997159792682,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1518532,"kb_used_data":1430972,"kb_used_omap":12,"kb_used_meta":87539,"kb_avail":92226620,"statfs":{"total":95995035648,"available":94440058880,"internally_reserved":0,"allocated":1465315328,"data_stored":1463576206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":89640561},"hb_peers":[0,1,2,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.98699999999999999,"15min":1.2549999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.56100000000000005,"5min":4.9909999999999997,"15min":4.9909999999999997},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.006,"15min":1.0780000000000001},"min":{"1min":0.27000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.3940000000000001,"5min":6.7839999999999998,"15min":6.7839999999999998},"last":0.38100000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.7549999999999999,"5min":2.7549999999999999,"15min":2.7549999999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":3.6059999999999999,"5min":3.6059999999999999,"15min":3.6059999999999999},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.2050000000000001,"15min":1.3939999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.0019999999999998,"5min":4.8540000000000001,"15min":4.8540000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.70399999999999996,"5min":1.5760000000000001,"15min":1.798},"min":{"1min":0.33000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.968,"5min":15.632999999999999,"15min":15.632999999999999},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.1339999999999999,"15min":1.2869999999999999},"min":{"1min":0.34699999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.9489999999999998,"5min":4.766,"15min":4.766},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.8839999999999999,"15min":2.407},"min":{"1min":0.36099999999999999,"5min":0.247,"15min":0.247},"max":{"1min":2.327,"5min":8.141,"15min":8.141},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.135,"15min":1.4359999999999999},"min":{"1min":0.28399999999999997,"5min":0.248,"15min":0.248},"max":{"1min":0.97499999999999998,"5min":5.5140000000000002,"15min":5.5140000000000002},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.1319999999999999,"15min":1.3720000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.9740000000000002,"5min":7.5999999999999996,"15min":7.5999999999999996},"last":0.36599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:37:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":4.944,"15min":5.6680000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.343,"5min":104.521,"15min":104.521},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.58199999999999996,"5min":4.9610000000000003,"15min":5.6909999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.7969999999999999,"5min":101.419,"15min":101.419},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.333,"15min":1.649},"min":{"1min":0.32100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.4690000000000003,"5min":8.1950000000000003,"15min":8.1950000000000003},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.6990000000000001,"15min":2.181},"min":{"1min":0.35699999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.4820000000000002,"5min":8.6140000000000008,"15min":8.6140000000000008},"last":0.51000000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757691,"num_pgs":8,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1412520,"kb_used_data":933152,"kb_used_omap":0,"kb_used_meta":479360,"kb_avail":92332632,"statfs":{"total":95995035648,"available":94548615168,"internally_reserved":0,"allocated":955547648,"data_stored":953831769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":490864640},"hb_peers":[0,1,2,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":5.2110000000000003,"15min":5.8090000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.22},"max":{"1min":3.2250000000000001,"5min":145.46600000000001,"15min":163.54499999999999},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.46200000000000002,"5min":5.5350000000000001,"15min":5.6269999999999998},"min":{"1min":0.248,"5min":0.24099999999999999,"15min":0.23499999999999999},"max":{"1min":1.3240000000000001,"5min":144.429,"15min":162.81800000000001},"last":0.28999999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0099999999999998,"5min":3.0099999999999998,"15min":3.0099999999999998},"last":2.0289999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.696,"5min":1.696,"15min":1.696},"last":0.252}]},{"osd":2,"last update":"Sat Sep 23 22:39:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":2.9470000000000001,"15min":2.419},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.8650000000000002,"5min":70.153000000000006,"15min":70.153000000000006},"last":2.1440000000000001},{"interface":"front","average":{"1min":0.498,"5min":2.7250000000000001,"15min":2.4340000000000002},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.0680000000000001,"5min":70.078999999999994,"15min":70.078999999999994},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:38:47 2023","interfaces":[{"interface":"back","average":{"1min":13.23,"5min":4.5650000000000004,"15min":4.0369999999999999},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":98.882000000000005,"5min":98.882000000000005,"15min":132.59},"last":0.39400000000000002},{"interface":"front","average":{"1min":13.026999999999999,"5min":4.0209999999999999,"15min":3.8980000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29299999999999998},"max":{"1min":98.856999999999999,"5min":98.856999999999999,"15min":129.381},"last":0.30599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:57 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":2.6619999999999999,"15min":3.6619999999999999},"min":{"1min":0.26700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.032,"5min":61.173999999999999,"15min":61.173999999999999},"last":2.1070000000000002},{"interface":"front","average":{"1min":0.48999999999999999,"5min":2.7290000000000001,"15min":3.7229999999999999},"min":{"1min":0.376,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.0589999999999999,"5min":61.203000000000003,"15min":61.203000000000003},"last":2.173}]},{"osd":6,"last update":"Sat Sep 23 22:38:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.9319999999999999,"15min":1.4990000000000001},"min":{"1min":0.27000000000000002,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.375,"5min":56.667000000000002,"15min":70.581999999999994},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.71,"15min":1.478},"min":{"1min":0.35399999999999998,"5min":0.29099999999999998,"15min":0.248},"max":{"1min":1.1240000000000001,"5min":57.185000000000002,"15min":71.093000000000004},"last":0.53400000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:38:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":1.655,"15min":1.2929999999999999},"min":{"1min":0.38400000000000001,"5min":0.313,"15min":0.28000000000000003},"max":{"1min":1.2829999999999999,"5min":59.101999999999997,"15min":59.101999999999997},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":1.792,"15min":1.413},"min":{"1min":0.32800000000000001,"5min":0.32300000000000001,"15min":0.308},"max":{"1min":0.78200000000000003,"5min":59.151000000000003,"15min":59.151000000000003},"last":2.0739999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1825636352,"data_stored":1825636352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3680768000,"data_stored":3680763928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1495793664,"data_stored":1495793664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":595787776,"data_stored":595787776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1466634240,"data_stored":1466630168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":942800896,"data_stored":942800896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1928134656,"data_stored":1928134656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3352825856,"data_stored":3352821784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:39:20.470 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T22:39:20.471 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-09-23T22:39:20.505 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:20.889 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:21.262 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:21.286 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:39:21.312 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[215775]: pgmap v1357: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 39170/221934 objects degraded (17.649%); 22455/221934 objects misplaced (10.118%); 8.3 MiB/s, 203 objects/s recovering 2023-09-23T22:39:21.313 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[223014]: pgmap v1357: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 39170/221934 objects degraded (17.649%); 22455/221934 objects misplaced (10.118%); 8.3 MiB/s, 203 objects/s recovering 2023-09-23T22:39:21.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[223014]: failed to encode map e523 with expected crc 2023-09-23T22:39:21.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[223014]: failed to encode map e523 with expected crc 2023-09-23T22:39:21.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[223014]: failed to encode map e523 with expected crc 2023-09-23T22:39:21.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[223014]: osdmap e523: 8 total, 6 up, 8 in 2023-09-23T22:39:21.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:21.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:21.314 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1689964093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:21.315 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:39:21.335 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:39:21.355 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:39:21.389 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:39:21.418 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:39:21.418 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:39:21.418 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:39:21.418 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:39:21.418 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:39:21.418 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:39:21.418 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:39:21.419 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:39:21.419 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:39:21.441 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:21.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:21.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:21.548 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:21.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[215775]: failed to encode map e523 with expected crc 2023-09-23T22:39:21.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[215775]: failed to encode map e523 with expected crc 2023-09-23T22:39:21.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[215775]: failed to encode map e523 with expected crc 2023-09-23T22:39:21.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[215775]: osdmap e523: 8 total, 6 up, 8 in 2023-09-23T22:39:21.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:21.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:21.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:21 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1689964093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:21.591 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:21.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:21 smithi070 ceph-mon[192652]: pgmap v1357: 109 pgs: 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 2 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 223 op/s; 39170/221934 objects degraded (17.649%); 22455/221934 objects misplaced (10.118%); 8.3 MiB/s, 203 objects/s recovering 2023-09-23T22:39:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:21 smithi070 ceph-mon[192652]: failed to encode map e523 with expected crc 2023-09-23T22:39:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:21 smithi070 ceph-mon[192652]: failed to encode map e523 with expected crc 2023-09-23T22:39:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:21 smithi070 ceph-mon[192652]: failed to encode map e523 with expected crc 2023-09-23T22:39:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:21 smithi070 ceph-mon[192652]: osdmap e523: 8 total, 6 up, 8 in 2023-09-23T22:39:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:21 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:21.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:21 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1689964093' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:22.016 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:22.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:22.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:22.121 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:22.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:22.204 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:22.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:22.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:22.304 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:39:22.304 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:39:22.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:39:22.304 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:39:22.304 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.304 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179625 8.d 8:bd963f3c:::benchmark_data_smithi026_497658_object80733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T22:39:22.304 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.331387+0000", 2023-09-23T22:39:22.304 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.96766072000003, 2023-09-23T22:39:22.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.046390226, 2023-09-23T22:39:22.305 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.305 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:39:22.305 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.305 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.305 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.305 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179625 2023-09-23T22:39:22.305 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331387+0000", 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331387+0000", 2023-09-23T22:39:22.306 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331386+0000", 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331389+0000", 2023-09-23T22:39:22.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6369999999999999e-06 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331396+0000", 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.516e-06 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331400+0000", 2023-09-23T22:39:22.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1180000000000002e-06 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331465+0000", 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5592000000000005e-05 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331589+0000", 2023-09-23T22:39:22.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000123306 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2,6", 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331711+0000", 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012258500000000001 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.340951+0000", 2023-09-23T22:39:22.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0092401549999999999 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.340962+0000", 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0322999999999999e-05 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377694+0000", 2023-09-23T22:39:22.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.036732049 2023-09-23T22:39:22.312 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.312 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.312 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.312 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377777+0000", 2023-09-23T22:39:22.312 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3311000000000002e-05 2023-09-23T22:39:22.312 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.312 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.312 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.313 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.313 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.313 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179634 8.d 8:b7c2e7fe:::benchmark_data_smithi026_497658_object80742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T22:39:22.313 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.331413+0000", 2023-09-23T22:39:22.313 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.96763500399999, 2023-09-23T22:39:22.313 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.046453466, 2023-09-23T22:39:22.313 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.313 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:39:22.313 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.314 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.314 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.314 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179634 2023-09-23T22:39:22.314 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.314 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.314 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.314 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.314 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331413+0000", 2023-09-23T22:39:22.314 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331413+0000", 2023-09-23T22:39:22.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331412+0000", 2023-09-23T22:39:22.316 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:22.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.316 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331415+0000", 2023-09-23T22:39:22.316 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5009999999999999e-06 2023-09-23T22:39:22.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331422+0000", 2023-09-23T22:39:22.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2089999999999999e-06 2023-09-23T22:39:22.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.317 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331426+0000", 2023-09-23T22:39:22.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.318e-06 2023-09-23T22:39:22.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.335320+0000", 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038939280000000001 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.335421+0000", 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010108299999999999 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2,6", 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.335493+0000", 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1797000000000004e-05 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.340994+0000", 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055008339999999996 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.319 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.341019+0000", 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5037000000000001e-05 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377815+0000", 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.036795911 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377866+0000", 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1248000000000001e-05 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179638 8.d 8:b6313638:::benchmark_data_smithi026_497658_object80746:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.321 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.353396+0000", 2023-09-23T22:39:22.322 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.94565159699999, 2023-09-23T22:39:22.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.061055393, 2023-09-23T22:39:22.322 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.322 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.322 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.322 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.322 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.322 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179638 2023-09-23T22:39:22.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353396+0000", 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353396+0000", 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.324 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.324 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.324 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353396+0000", 2023-09-23T22:39:22.324 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:22.324 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.324 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.324 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.324 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353398+0000", 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.323e-06 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353404+0000", 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0750000000000003e-06 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.325 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353408+0000", 2023-09-23T22:39:22.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8419999999999998e-06 2023-09-23T22:39:22.326 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.326 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.326 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.326 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353461+0000", 2023-09-23T22:39:22.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2778000000000002e-05 2023-09-23T22:39:22.326 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.326 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.326 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353471+0000", 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0149000000000001e-05 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377887+0000", 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0244162080000001 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.327 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.378011+0000", 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012360800000000001 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.378162+0000", 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015116000000000001 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.328 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395006+0000", 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016844624999999998 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414290+0000", 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019283582000000001 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.329 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414387+0000", 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6844999999999999e-05 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414426+0000", 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9545999999999997e-05 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.330 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.331 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.331 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414451+0000", 2023-09-23T22:39:22.331 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5034000000000001e-05 2023-09-23T22:39:22.331 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.331 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.331 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.331 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.331 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.331 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179642 8.d 8:b7bd1a79:::benchmark_data_smithi026_497658_object80750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.358239+0000", 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.94080816899998, 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0563148179999999, 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179642 2023-09-23T22:39:22.332 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358239+0000", 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358239+0000", 2023-09-23T22:39:22.333 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.334 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.334 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358239+0000", 2023-09-23T22:39:22.334 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:22.334 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358242+0000", 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4400000000000001e-06 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358251+0000", 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5439999999999999e-06 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.335 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358255+0000", 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4710000000000001e-06 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358302+0000", 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6727999999999997e-05 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.336 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358312+0000", 2023-09-23T22:39:22.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9790000000000005e-06 2023-09-23T22:39:22.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.381332+0000", 2023-09-23T22:39:22.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0230205020000001 2023-09-23T22:39:22.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.381469+0000", 2023-09-23T22:39:22.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013704400000000001 2023-09-23T22:39:22.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.381594+0000", 2023-09-23T22:39:22.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012458900000000001 2023-09-23T22:39:22.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395011+0000", 2023-09-23T22:39:22.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013417247 2023-09-23T22:39:22.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414343+0000", 2023-09-23T22:39:22.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019331883000000001 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414489+0000", 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014642799999999999 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414520+0000", 2023-09-23T22:39:22.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0605999999999998e-05 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414554+0000", 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4215999999999999e-05 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.342 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179657 8.d 8:bab24576:::benchmark_data_smithi026_497658_object80765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.342 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.371809+0000", 2023-09-23T22:39:22.342 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.92723815199997, 2023-09-23T22:39:22.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.263966353, 2023-09-23T22:39:22.342 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.342 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.342 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.342 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.342 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179657 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371809+0000", 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.344 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371809+0000", 2023-09-23T22:39:22.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.344 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.344 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371809+0000", 2023-09-23T22:39:22.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:22.344 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.344 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371811+0000", 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.249e-06 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371818+0000", 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2480000000000001e-06 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371823+0000", 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2809999999999999e-06 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371869+0000", 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6317999999999999e-05 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371877+0000", 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8299999999999996e-06 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.385058+0000", 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0131806569999999 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.385156+0000", 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8345999999999997e-05 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.385288+0000", 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013212900000000001 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400220+0000", 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014931542000000001 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414579+0000", 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014359764000000001 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635719+0000", 2023-09-23T22:39:22.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.221139108 2023-09-23T22:39:22.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635749+0000", 2023-09-23T22:39:22.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0750000000000002e-05 2023-09-23T22:39:22.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635776+0000", 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6500999999999999e-05 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.351 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179686 8.d 8:b2459d19:::benchmark_data_smithi026_497658_object80794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.352 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.423416+0000", 2023-09-23T22:39:22.352 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.87563109400003, 2023-09-23T22:39:22.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2124665779999999, 2023-09-23T22:39:22.352 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.352 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.352 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.352 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.352 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.352 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179686 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423416+0000", 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.353 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423416+0000", 2023-09-23T22:39:22.354 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.354 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423415+0000", 2023-09-23T22:39:22.354 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:22.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423422+0000", 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6440000000000003e-06 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423433+0000", 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1664999999999999e-05 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423440+0000", 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8889999999999996e-06 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.424713+0000", 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001272537 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.424720+0000", 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8380000000000002e-06 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.388740+0000", 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96401941300000005 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.388878+0000", 2023-09-23T22:39:22.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013780300000000001 2023-09-23T22:39:22.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.358 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.389037+0000", 2023-09-23T22:39:22.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015977600000000001 2023-09-23T22:39:22.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400224+0000", 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011186125999999999 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414667+0000", 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014443754 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635827+0000", 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22116000899999999 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635860+0000", 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2710999999999999e-05 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635883+0000", 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3011999999999999e-05 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179691 8.d 8:b7906ebe:::benchmark_data_smithi026_497658_object80799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.437137+0000", 2023-09-23T22:39:22.361 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.86191043299999, 2023-09-23T22:39:22.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.198820869, 2023-09-23T22:39:22.363 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.363 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.363 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.363 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.363 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.363 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179691 2023-09-23T22:39:22.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.363 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437137+0000", 2023-09-23T22:39:22.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437137+0000", 2023-09-23T22:39:22.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437135+0000", 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437145+0000", 2023-09-23T22:39:22.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3409999999999992e-06 2023-09-23T22:39:22.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437163+0000", 2023-09-23T22:39:22.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7986e-05 2023-09-23T22:39:22.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437171+0000", 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5490000000000006e-06 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437226+0000", 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4580999999999998e-05 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437235+0000", 2023-09-23T22:39:22.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3279999999999995e-06 2023-09-23T22:39:22.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.368 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.391971+0000", 2023-09-23T22:39:22.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95473568900000005 2023-09-23T22:39:22.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.392111+0000", 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014051100000000001 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.392262+0000", 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015037500000000001 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402309+0000", 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010047512999999999 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414716+0000", 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012406531 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635907+0000", 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.221191161 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635933+0000", 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5809e-05 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635958+0000", 2023-09-23T22:39:22.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5369999999999999e-05 2023-09-23T22:39:22.373 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.373 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.373 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.374 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179708 8.d 8:baab2a80:::benchmark_data_smithi026_497658_object80816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.374 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.457089+0000", 2023-09-23T22:39:22.374 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.84195837700003, 2023-09-23T22:39:22.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1789815729999999, 2023-09-23T22:39:22.374 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.374 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.374 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.374 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.375 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.375 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179708 2023-09-23T22:39:22.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.375 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457089+0000", 2023-09-23T22:39:22.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457089+0000", 2023-09-23T22:39:22.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457087+0000", 2023-09-23T22:39:22.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:22.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457095+0000", 2023-09-23T22:39:22.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.148e-06 2023-09-23T22:39:22.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457108+0000", 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3293e-05 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457114+0000", 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.596e-06 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457169+0000", 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4818999999999998e-05 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457179+0000", 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8709999999999992e-06 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395053+0000", 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93787340500000005 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395194+0000", 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014127500000000001 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395338+0000", 2023-09-23T22:39:22.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014388000000000001 2023-09-23T22:39:22.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402314+0000", 2023-09-23T22:39:22.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069762490000000003 2023-09-23T22:39:22.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414739+0000", 2023-09-23T22:39:22.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012424663000000001 2023-09-23T22:39:22.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635996+0000", 2023-09-23T22:39:22.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22125688900000001 2023-09-23T22:39:22.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.636041+0000", 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5510000000000003e-05 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.636071+0000", 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9723999999999999e-05 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.383 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.384 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.384 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179715 8.d 8:bf536315:::benchmark_data_smithi026_497658_object80823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.384 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.470395+0000", 2023-09-23T22:39:22.384 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.82865292100001, 2023-09-23T22:39:22.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.170438106, 2023-09-23T22:39:22.384 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.384 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179715 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470395+0000", 2023-09-23T22:39:22.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.386 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.386 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470395+0000", 2023-09-23T22:39:22.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.386 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.386 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470393+0000", 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470401+0000", 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1629999999999996e-06 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.387 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470414+0000", 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2549e-05 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470420+0000", 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3879999999999997e-06 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470473+0000", 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2553000000000003e-05 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470482+0000", 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1449999999999997e-06 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.397798+0000", 2023-09-23T22:39:22.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92731558999999997 2023-09-23T22:39:22.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.390 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.397927+0000", 2023-09-23T22:39:22.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128928 2023-09-23T22:39:22.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.398079+0000", 2023-09-23T22:39:22.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000152218 2023-09-23T22:39:22.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.408311+0000", 2023-09-23T22:39:22.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010232196000000001 2023-09-23T22:39:22.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414784+0000", 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064727860000000003 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640770+0000", 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22598658799999999 2023-09-23T22:39:22.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640804+0000", 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4106999999999997e-05 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640833+0000", 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8300999999999999e-05 2023-09-23T22:39:22.393 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.394 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.394 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.394 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179746 8.d 8:bb9d95f1:::benchmark_data_smithi026_497658_object80854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.394 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.513114+0000", 2023-09-23T22:39:22.394 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.78593350699998, 2023-09-23T22:39:22.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1278254560000001, 2023-09-23T22:39:22.394 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.395 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.395 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.395 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.395 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.395 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179746 2023-09-23T22:39:22.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.395 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513114+0000", 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513114+0000", 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513113+0000", 2023-09-23T22:39:22.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:22.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513119+0000", 2023-09-23T22:39:22.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5379999999999999e-06 2023-09-23T22:39:22.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513133+0000", 2023-09-23T22:39:22.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3495e-05 2023-09-23T22:39:22.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513140+0000", 2023-09-23T22:39:22.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8700000000000003e-06 2023-09-23T22:39:22.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513255+0000", 2023-09-23T22:39:22.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115285 2023-09-23T22:39:22.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.399 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.399 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513261+0000", 2023-09-23T22:39:22.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7089999999999999e-06 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400241+0000", 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.886980027 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400312+0000", 2023-09-23T22:39:22.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1092000000000003e-05 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400404+0000", 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2526000000000001e-05 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.413998+0000", 2023-09-23T22:39:22.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013593684999999999 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414805+0000", 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00080711400000000005 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640877+0000", 2023-09-23T22:39:22.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22607189499999999 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640910+0000", 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3482999999999998e-05 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640940+0000", 2023-09-23T22:39:22.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9099e-05 2023-09-23T22:39:22.404 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.404 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.404 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.404 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179749 8.d 8:b1665d54:::benchmark_data_smithi026_497658_object80857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.404 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.517186+0000", 2023-09-23T22:39:22.404 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.78186128300001, 2023-09-23T22:39:22.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.123864177, 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179749 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517186+0000", 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517186+0000", 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517185+0000", 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517191+0000", 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8329999999999999e-06 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517205+0000", 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3961000000000001e-05 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517210+0000", 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1139999999999997e-06 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517267+0000", 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7213999999999998e-05 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517277+0000", 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9899999999999992e-06 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.409 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402354+0000", 2023-09-23T22:39:22.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88507623999999996 2023-09-23T22:39:22.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402479+0000", 2023-09-23T22:39:22.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000125093 2023-09-23T22:39:22.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402556+0000", 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7556000000000001e-05 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414007+0000", 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011450514 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414840+0000", 2023-09-23T22:39:22.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00083372199999999996 2023-09-23T22:39:22.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640988+0000", 2023-09-23T22:39:22.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.226147288 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.641029+0000", 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1393999999999997e-05 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.641050+0000", 2023-09-23T22:39:22.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1396e-05 2023-09-23T22:39:22.414 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.414 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.414 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.414 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179761 8.d 8:b0ca34a6:::benchmark_data_smithi026_497658_object80869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.414 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.535617+0000", 2023-09-23T22:39:22.414 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.76343024900001, 2023-09-23T22:39:22.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.12096347, 2023-09-23T22:39:22.415 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.415 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.415 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.415 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.415 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.415 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179761 2023-09-23T22:39:22.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.415 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535617+0000", 2023-09-23T22:39:22.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535617+0000", 2023-09-23T22:39:22.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535617+0000", 2023-09-23T22:39:22.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:22.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535620+0000", 2023-09-23T22:39:22.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7050000000000001e-06 2023-09-23T22:39:22.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535633+0000", 2023-09-23T22:39:22.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3096e-05 2023-09-23T22:39:22.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535640+0000", 2023-09-23T22:39:22.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2020000000000001e-06 2023-09-23T22:39:22.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535706+0000", 2023-09-23T22:39:22.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6019999999999995e-05 2023-09-23T22:39:22.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535716+0000", 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0825000000000001e-05 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.404445+0000", 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.86872851699999998 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.404517+0000", 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2002999999999995e-05 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.404606+0000", 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8734999999999997e-05 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414015+0000", 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094092859999999993 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414876+0000", 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00086061600000000003 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656491+0000", 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.24161501499999999 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656537+0000", 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5915999999999999e-05 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656581+0000", 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4224000000000002e-05 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179767 8.d 8:ba13a7eb:::benchmark_data_smithi026_497658_object80875:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:39:22.424 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.545488+0000", 2023-09-23T22:39:22.425 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 296.75355993599999, 2023-09-23T22:39:22.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1111883810000001, 2023-09-23T22:39:22.425 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.425 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.425 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.425 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:39:22.425 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:22.425 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179767 2023-09-23T22:39:22.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545488+0000", 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545488+0000", 2023-09-23T22:39:22.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545486+0000", 2023-09-23T22:39:22.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:22.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545492+0000", 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3219999999999998e-06 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545507+0000", 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4817000000000001e-05 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545512+0000", 2023-09-23T22:39:22.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4859999999999999e-06 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545551+0000", 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8937999999999999e-05 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545556+0000", 2023-09-23T22:39:22.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4580000000000001e-06 2023-09-23T22:39:22.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.406356+0000", 2023-09-23T22:39:22.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.86079989899999998 2023-09-23T22:39:22.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.406432+0000", 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5601999999999999e-05 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.406530+0000", 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8529999999999993e-05 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414042+0000", 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075122269999999998 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414912+0000", 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00086938100000000002 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656633+0000", 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.24172130999999999 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656656+0000", 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2836e-05 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656676+0000", 2023-09-23T22:39:22.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0058999999999999e-05 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104682 9.f 9:f58eaa2e:::benchmark_data_smithi026_555954_object9953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.265869+0000", 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 128.033178687, 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1940709429999998, 2023-09-23T22:39:22.434 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104682 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265869+0000", 2023-09-23T22:39:22.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265869+0000", 2023-09-23T22:39:22.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265868+0000", 2023-09-23T22:39:22.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:22.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265875+0000", 2023-09-23T22:39:22.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.52e-06 2023-09-23T22:39:22.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265888+0000", 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2493e-05 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265892+0000", 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6199999999999998e-06 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265936+0000", 2023-09-23T22:39:22.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3751999999999997e-05 2023-09-23T22:39:22.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266080+0000", 2023-09-23T22:39:22.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014391999999999999 2023-09-23T22:39:22.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266249+0000", 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000168729 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.269497+0000", 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003247757 2023-09-23T22:39:22.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292030+0000", 2023-09-23T22:39:22.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022533972999999999 2023-09-23T22:39:22.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292040+0000", 2023-09-23T22:39:22.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6260000000000007e-06 2023-09-23T22:39:22.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276089+0000", 2023-09-23T22:39:22.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98404894300000001 2023-09-23T22:39:22.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276099+0000", 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8539999999999992e-06 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425242+0000", 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1491429769999999 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:22.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425390+0000", 2023-09-23T22:39:22.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000148601 2023-09-23T22:39:22.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.459827+0000", 2023-09-23T22:39:22.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034436178999999997 2023-09-23T22:39:22.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.459940+0000", 2023-09-23T22:39:22.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011315900000000001 2023-09-23T22:39:22.445 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.445 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.445 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.446 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104687 9.f 9:fb4465c7:::benchmark_data_smithi026_555954_object9958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:39:22.446 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.279126+0000", 2023-09-23T22:39:22.446 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 128.01992126100001, 2023-09-23T22:39:22.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1856558540000002, 2023-09-23T22:39:22.446 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.446 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:39:22.446 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.446 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104687 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279126+0000", 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279126+0000", 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279125+0000", 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279132+0000", 2023-09-23T22:39:22.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.968e-06 2023-09-23T22:39:22.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279147+0000", 2023-09-23T22:39:22.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5252000000000001e-05 2023-09-23T22:39:22.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279155+0000", 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4139999999999997e-06 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279205+0000", 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0002e-05 2023-09-23T22:39:22.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279383+0000", 2023-09-23T22:39:22.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017872 2023-09-23T22:39:22.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:39:22.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279550+0000", 2023-09-23T22:39:22.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000166343 2023-09-23T22:39:22.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292069+0000", 2023-09-23T22:39:22.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012519417 2023-09-23T22:39:22.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292074+0000", 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1499999999999998e-06 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276124+0000", 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98404924299999996 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276129+0000", 2023-09-23T22:39:22.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6949999999999997e-06 2023-09-23T22:39:22.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425447+0000", 2023-09-23T22:39:22.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1493180839999999 2023-09-23T22:39:22.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:22.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425628+0000", 2023-09-23T22:39:22.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018074899999999999 2023-09-23T22:39:22.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.455 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.455 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464649+0000", 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039020688999999997 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464782+0000", 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013333600000000001 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104699 9.f 9:fa6f98c4:::benchmark_data_smithi026_555954_object9970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.355308+0000", 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 127.943739994, 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1458075220000001, 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:22.457 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:22.458 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104699 2023-09-23T22:39:22.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.458 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355308+0000", 2023-09-23T22:39:22.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355308+0000", 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355306+0000", 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355314+0000", 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2239999999999995e-06 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355332+0000", 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7728999999999999e-05 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355355+0000", 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3218999999999999e-05 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355394+0000", 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8661999999999997e-05 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355401+0000", 2023-09-23T22:39:22.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9550000000000003e-06 2023-09-23T22:39:22.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276149+0000", 2023-09-23T22:39:22.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92074822599999995 2023-09-23T22:39:22.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276155+0000", 2023-09-23T22:39:22.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5539999999999997e-06 2023-09-23T22:39:22.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425668+0000", 2023-09-23T22:39:22.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.149513765 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426016+0000", 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000347487 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.445038+0000", 2023-09-23T22:39:22.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0190222 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.445155+0000", 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011651599999999999 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.445326+0000", 2023-09-23T22:39:22.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017155499999999999 2023-09-23T22:39:22.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464769+0000", 2023-09-23T22:39:22.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019442566000000001 2023-09-23T22:39:22.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464804+0000", 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5648999999999997e-05 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.497550+0000", 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.032745511999999997 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.497594+0000", 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4270000000000001e-05 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.501115+0000", 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035209569999999999 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.468 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.469 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104701 9.f 9:f9048e3a:::benchmark_data_smithi026_555954_object9972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:39:22.469 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.295854+0000", 2023-09-23T22:39:22.469 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 127.003193373, 2023-09-23T22:39:22.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.208795673, 2023-09-23T22:39:22.469 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.469 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.470 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.470 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:22.470 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:22.470 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104701 2023-09-23T22:39:22.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.470 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295854+0000", 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295854+0000", 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295853+0000", 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295861+0000", 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3720000000000005e-06 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295877+0000", 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5875e-05 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295885+0000", 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5109999999999997e-06 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295932+0000", 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6916999999999998e-05 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295940+0000", 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0020000000000006e-06 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426058+0000", 2023-09-23T22:39:22.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1301181819999999 2023-09-23T22:39:22.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:22.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426486+0000", 2023-09-23T22:39:22.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00042859000000000001 2023-09-23T22:39:22.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.452638+0000", 2023-09-23T22:39:22.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026151917 2023-09-23T22:39:22.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.452720+0000", 2023-09-23T22:39:22.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1389000000000003e-05 2023-09-23T22:39:22.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:39:22.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.452832+0000", 2023-09-23T22:39:22.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112194 2023-09-23T22:39:22.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464779+0000", 2023-09-23T22:39:22.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011947107 2023-09-23T22:39:22.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464856+0000", 2023-09-23T22:39:22.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6678e-05 2023-09-23T22:39:22.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.501167+0000", 2023-09-23T22:39:22.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036311450000000002 2023-09-23T22:39:22.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.479 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.501211+0000", 2023-09-23T22:39:22.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4246999999999998e-05 2023-09-23T22:39:22.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.479 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.504650+0000", 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034385829999999998 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104709 9.f 9:fe76147f:::benchmark_data_smithi026_555954_object9980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.348800+0000", 2023-09-23T22:39:22.480 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 126.95024804000001, 2023-09-23T22:39:22.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.175744452, 2023-09-23T22:39:22.481 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.481 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.481 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.481 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:22.481 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:22.481 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104709 2023-09-23T22:39:22.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348800+0000", 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348800+0000", 2023-09-23T22:39:22.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348798+0000", 2023-09-23T22:39:22.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:22.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348807+0000", 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7070000000000004e-06 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348822+0000", 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4944e-05 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348831+0000", 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7280000000000001e-06 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.389358+0000", 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.040527173999999999 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.389371+0000", 2023-09-23T22:39:22.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2469e-05 2023-09-23T22:39:22.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426523+0000", 2023-09-23T22:39:22.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0371526820000001 2023-09-23T22:39:22.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426916+0000", 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039232699999999999 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.493074+0000", 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.066158162000000006 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.493164+0000", 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0309000000000006e-05 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.493254+0000", 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0066000000000001e-05 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.504667+0000", 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011412558999999999 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.504705+0000", 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8373e-05 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.521152+0000", 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016447092999999999 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.521194+0000", 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1467999999999997e-05 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.524544+0000", 2023-09-23T22:39:22.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033503230000000001 2023-09-23T22:39:22.491 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.491 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.491 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.491 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:105990 9.f 9:fc198dae:::benchmark_data_smithi026_555954_object11261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T22:39:22.491 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:43.319569+0000", 2023-09-23T22:39:22.492 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 98.979478817, 2023-09-23T22:39:22.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4132751570000002, 2023-09-23T22:39:22.492 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.492 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.492 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.492 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:22.492 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:22.492 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 105990 2023-09-23T22:39:22.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319569+0000", 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319569+0000", 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319566+0000", 2023-09-23T22:39:22.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999971 2023-09-23T22:39:22.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319582+0000", 2023-09-23T22:39:22.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5855999999999999e-05 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319622+0000", 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0615999999999999e-05 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319637+0000", 2023-09-23T22:39:22.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4922999999999999e-05 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319766+0000", 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012936299999999999 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319786+0000", 2023-09-23T22:39:22.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9723000000000001e-05 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.680894+0000", 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3611081509999998 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.681104+0000", 2023-09-23T22:39:22.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020942399999999999 2023-09-23T22:39:22.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:39:22.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.681225+0000", 2023-09-23T22:39:22.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012102199999999999 2023-09-23T22:39:22.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.729953+0000", 2023-09-23T22:39:22.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048728018999999997 2023-09-23T22:39:22.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730035+0000", 2023-09-23T22:39:22.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2010999999999998e-05 2023-09-23T22:39:22.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730114+0000", 2023-09-23T22:39:22.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9276000000000005e-05 2023-09-23T22:39:22.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730166+0000", 2023-09-23T22:39:22.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1437000000000002e-05 2023-09-23T22:39:22.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.501 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.501 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.732844+0000", 2023-09-23T22:39:22.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026783789999999998 2023-09-23T22:39:22.501 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.501 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.501 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:105991 9.f 9:f5926516:::benchmark_data_smithi026_555954_object11262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 98.979381183000001, 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4159624549999998, 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:22.502 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 105991 2023-09-23T22:39:22.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.503 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:22.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:22.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:39:22.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:22.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319671+0000", 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4060000000000004e-06 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319681+0000", 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0563e-05 2023-09-23T22:39:22.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:22.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319688+0000", 2023-09-23T22:39:22.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8000000000000001e-06 2023-09-23T22:39:22.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319843+0000", 2023-09-23T22:39:22.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015483200000000001 2023-09-23T22:39:22.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:39:22.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319865+0000", 2023-09-23T22:39:22.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1933000000000002e-05 2023-09-23T22:39:22.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.507 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:22.507 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.684141+0000", 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.364276378 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.684269+0000", 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012795800000000001 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:39:22.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.684369+0000", 2023-09-23T22:39:22.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9584000000000001e-05 2023-09-23T22:39:22.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730094+0000", 2023-09-23T22:39:22.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045724787000000003 2023-09-23T22:39:22.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:22.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.732880+0000", 2023-09-23T22:39:22.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002786186 2023-09-23T22:39:22.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:39:22.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.732981+0000", 2023-09-23T22:39:22.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100999 2023-09-23T22:39:22.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:39:22.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.733032+0000", 2023-09-23T22:39:22.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0701000000000001e-05 2023-09-23T22:39:22.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:22.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:22.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:39:22.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.735629+0000", 2023-09-23T22:39:22.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025971940000000002 2023-09-23T22:39:22.511 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.512 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.512 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.512 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:22.512 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:22.512 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:39:22.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:22.546 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:22.606 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:22.607 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:23.154 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:39:23.573 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:23 smithi070 ceph-mon[192652]: pgmap v1359: 109 pgs: 1 remapped+peering, 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 34037/218841 objects degraded (15.553%); 22201/218841 objects misplaced (10.145%); 6.3 MiB/s, 387 objects/s recovering 2023-09-23T22:39:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:23.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:39:23.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:39:23.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:23.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 34037/218841 objects degraded (15.553%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:23.625 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:23.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[215775]: pgmap v1359: 109 pgs: 1 remapped+peering, 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 34037/218841 objects degraded (15.553%); 22201/218841 objects misplaced (10.145%); 6.3 MiB/s, 387 objects/s recovering 2023-09-23T22:39:23.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:23.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:23.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:39:23.628 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:39:23.628 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:23.628 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 34037/218841 objects degraded (15.553%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:23.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[223014]: pgmap v1359: 109 pgs: 1 remapped+peering, 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 3 active+recovery_wait+degraded+remapped, 26 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 324 op/s; 34037/218841 objects degraded (15.553%); 22201/218841 objects misplaced (10.145%); 6.3 MiB/s, 387 objects/s recovering 2023-09-23T22:39:23.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:23.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:23.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:39:23.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:39:23.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:23.629 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 34037/218841 objects degraded (15.553%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:23.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:23.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:23.730 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:23.770 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:23.886 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:23.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:23.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:23.991 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:24.022 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:24.121 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:24.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:24.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:24.231 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:24.278 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:24.336 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:24.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:24.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:24.490 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:24.534 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:24.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:39:24.609 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:24.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:24.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:24.713 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:24.745 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:24.840 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:24.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:24.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:24.944 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:24.977 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:25.029 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:25.085 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:25.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:25.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:25.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:25 smithi070 ceph-mon[192652]: pgmap v1360: 109 pgs: 1 remapped+peering, 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 36707/217548 objects degraded (16.873%); 19108/217548 objects misplaced (8.783%); 4.5 MiB/s, 254 objects/s recovering 2023-09-23T22:39:25.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:25.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:25.103 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2674361268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:25.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:25 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:25.104 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/756121261' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T22:39:25.154 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:39:25.154 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:39:25.154 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:39:25.154 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:39:25.193 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:25.222 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:25.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[223014]: pgmap v1360: 109 pgs: 1 remapped+peering, 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 36707/217548 objects degraded (16.873%); 19108/217548 objects misplaced (8.783%); 4.5 MiB/s, 254 objects/s recovering 2023-09-23T22:39:25.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:25.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:25.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2674361268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:25.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:25.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/756121261' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T22:39:25.233 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[215775]: pgmap v1360: 109 pgs: 1 remapped+peering, 2 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 36707/217548 objects degraded (16.873%); 19108/217548 objects misplaced (8.783%); 4.5 MiB/s, 254 objects/s recovering 2023-09-23T22:39:25.233 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:25.233 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:25.233 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2674361268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:25.233 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:25.233 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/756121261' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T22:39:25.274 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:25.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:25.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:25.340 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 9 pg_num to 26 2023-09-23T22:39:25.340 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:39:25.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:25.411 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:25.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:39:26.103 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:26.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[215775]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[215775]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:26.283 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:26.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:26.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:26.356 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:26.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:26 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:26.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:26 smithi070 ceph-mon[192652]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:26 smithi070 ceph-mon[192652]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:26 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:26.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/756121261' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T22:39:26.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:26 smithi070 ceph-mon[192652]: osdmap e524: 8 total, 6 up, 8 in 2023-09-23T22:39:26.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:26 smithi070 ceph-mon[192652]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:26 smithi070 ceph-mon[192652]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.376 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:26 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T22:39:26.386 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:26.412 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/756121261' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T22:39:26.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[215775]: osdmap e524: 8 total, 6 up, 8 in 2023-09-23T22:39:26.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[215775]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[215775]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.413 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T22:39:26.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:26.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[223014]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[223014]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:26.413 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/756121261' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T22:39:26.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[223014]: osdmap e524: 8 total, 6 up, 8 in 2023-09-23T22:39:26.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[223014]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[223014]: failed to encode map e524 with expected crc 2023-09-23T22:39:26.414 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:26 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T22:39:26.416 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:27.252 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:27.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:27.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:27.355 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:27.387 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:27 smithi070 ceph-mon[192652]: pgmap v1362: 109 pgs: 3 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+undersized+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 368 op/s; 27574/214959 objects degraded (12.828%); 32072/214959 objects misplaced (14.920%); 4.5 MiB/s, 316 objects/s recovering 2023-09-23T22:39:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:27 smithi070 ceph-mon[192652]: failed to encode map e525 with expected crc 2023-09-23T22:39:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:27 smithi070 ceph-mon[192652]: failed to encode map e525 with expected crc 2023-09-23T22:39:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:27 smithi070 ceph-mon[192652]: failed to encode map e525 with expected crc 2023-09-23T22:39:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:27 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T22:39:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:27 smithi070 ceph-mon[192652]: osdmap e525: 8 total, 6 up, 8 in 2023-09-23T22:39:27.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[223014]: pgmap v1362: 109 pgs: 3 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+undersized+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 368 op/s; 27574/214959 objects degraded (12.828%); 32072/214959 objects misplaced (14.920%); 4.5 MiB/s, 316 objects/s recovering 2023-09-23T22:39:27.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[223014]: failed to encode map e525 with expected crc 2023-09-23T22:39:27.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[223014]: failed to encode map e525 with expected crc 2023-09-23T22:39:27.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[223014]: failed to encode map e525 with expected crc 2023-09-23T22:39:27.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T22:39:27.648 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[223014]: osdmap e525: 8 total, 6 up, 8 in 2023-09-23T22:39:27.649 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[215775]: pgmap v1362: 109 pgs: 3 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+undersized+remapped, 3 undersized+degraded+peered, 13 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 368 op/s; 27574/214959 objects degraded (12.828%); 32072/214959 objects misplaced (14.920%); 4.5 MiB/s, 316 objects/s recovering 2023-09-23T22:39:27.649 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[215775]: failed to encode map e525 with expected crc 2023-09-23T22:39:27.649 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[215775]: failed to encode map e525 with expected crc 2023-09-23T22:39:27.649 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[215775]: failed to encode map e525 with expected crc 2023-09-23T22:39:27.649 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T22:39:27.649 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:27 smithi026 ceph-mon[215775]: osdmap e525: 8 total, 6 up, 8 in 2023-09-23T22:39:27.661 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:27.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:27.665 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:27.766 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:27.798 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:27.845 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:27.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:27.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:27.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:27.978 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:27.982 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:39:28.006 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:39:28.025 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:28.027 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:39:28.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:28.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:28.047 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:39:28.065 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:39:28.085 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:39:28.086 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:39:28.086 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:39:28.086 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:39:28.086 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:39:28.086 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:39:28.086 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:39:28.086 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:39:28.086 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:39:28.120 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:28.120 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:28.128 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:28.160 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:28.262 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:28.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:28.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:28.365 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:28.395 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:28.440 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:28.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:28.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:28.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[223014]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[223014]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[223014]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[223014]: osdmap e526: 8 total, 6 up, 8 in 2023-09-23T22:39:28.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[223014]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1656380455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:28.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 27574/214959 objects degraded (12.828%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:28.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[223014]: osdmap e527: 8 total, 6 up, 8 in 2023-09-23T22:39:28.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[215775]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[215775]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[215775]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[215775]: osdmap e526: 8 total, 6 up, 8 in 2023-09-23T22:39:28.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[215775]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1656380455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:28.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 27574/214959 objects degraded (12.828%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:28.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:28 smithi026 ceph-mon[215775]: osdmap e527: 8 total, 6 up, 8 in 2023-09-23T22:39:28.544 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:28.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:28 smithi070 ceph-mon[192652]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:28 smithi070 ceph-mon[192652]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:28 smithi070 ceph-mon[192652]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:28 smithi070 ceph-mon[192652]: osdmap e526: 8 total, 6 up, 8 in 2023-09-23T22:39:28.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:28 smithi070 ceph-mon[192652]: failed to encode map e526 with expected crc 2023-09-23T22:39:28.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:28 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1656380455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:28.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 27574/214959 objects degraded (12.828%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:28.569 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:28 smithi070 ceph-mon[192652]: osdmap e527: 8 total, 6 up, 8 in 2023-09-23T22:39:28.571 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:28.807 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:28.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:28.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:28.811 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:28.812 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:28.850 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:39:28.911 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:28.962 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.011 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:29.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:29.116 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:29.145 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.245 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:29.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:29.265 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:29.351 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:29.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:29 smithi070 ceph-mon[192652]: pgmap v1365: 119 pgs: 9 unknown, 3 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 26872/212124 objects degraded (12.668%); 29720/212124 objects misplaced (14.011%); 2.2 MiB/s, 101 objects/s recovering 2023-09-23T22:39:29.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:29 smithi070 ceph-mon[192652]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:29 smithi070 ceph-mon[192652]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:29 smithi070 ceph-mon[192652]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:29 smithi070 ceph-mon[192652]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.396 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:29 smithi070 ceph-mon[192652]: osdmap e528: 8 total, 6 up, 8 in 2023-09-23T22:39:29.399 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.447 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:39:29.568 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:29.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[215775]: pgmap v1365: 119 pgs: 9 unknown, 3 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 26872/212124 objects degraded (12.668%); 29720/212124 objects misplaced (14.011%); 2.2 MiB/s, 101 objects/s recovering 2023-09-23T22:39:29.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[215775]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[215775]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[215775]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[215775]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[215775]: osdmap e528: 8 total, 6 up, 8 in 2023-09-23T22:39:29.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[223014]: pgmap v1365: 119 pgs: 9 unknown, 3 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 26872/212124 objects degraded (12.668%); 29720/212124 objects misplaced (14.011%); 2.2 MiB/s, 101 objects/s recovering 2023-09-23T22:39:29.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[223014]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[223014]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[223014]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[223014]: failed to encode map e527 with expected crc 2023-09-23T22:39:29.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:29 smithi026 ceph-mon[223014]: osdmap e528: 8 total, 6 up, 8 in 2023-09-23T22:39:29.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:29.715 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.785 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:29.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:29.893 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:29.924 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:39:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:39:29] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-23T22:39:29.928 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:29.966 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:29.987 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:29.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:29.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:30.092 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:30.120 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:30.163 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:30.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:30.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:30.267 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:30.301 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:30.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:30 smithi070 ceph-mon[192652]: failed to encode map e528 with expected crc 2023-09-23T22:39:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:30 smithi070 ceph-mon[192652]: failed to encode map e528 with expected crc 2023-09-23T22:39:30.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:30 smithi070 ceph-mon[192652]: failed to encode map e528 with expected crc 2023-09-23T22:39:30.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:30 smithi026 ceph-mon[223014]: failed to encode map e528 with expected crc 2023-09-23T22:39:30.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:30 smithi026 ceph-mon[223014]: failed to encode map e528 with expected crc 2023-09-23T22:39:30.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:30 smithi026 ceph-mon[223014]: failed to encode map e528 with expected crc 2023-09-23T22:39:30.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:30 smithi026 ceph-mon[215775]: failed to encode map e528 with expected crc 2023-09-23T22:39:30.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:30 smithi026 ceph-mon[215775]: failed to encode map e528 with expected crc 2023-09-23T22:39:30.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:30 smithi026 ceph-mon[215775]: failed to encode map e528 with expected crc 2023-09-23T22:39:30.600 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:30.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:30.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:30.705 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:30.738 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:30.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:39:30.821 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:30.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:30.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:30.927 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:30.983 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:31.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:31.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:31.213 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:31.214 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:31.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:31.269 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:31.362 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:31.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:31.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:31.478 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:31.509 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:31.511 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:31 smithi070 ceph-mon[192652]: pgmap v1368: 119 pgs: 9 unknown, 3 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 236 op/s; 26872/212124 objects degraded (12.668%); 29720/212124 objects misplaced (14.011%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T22:39:31.513 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:31.562 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:31.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:31.572 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:31 smithi026 ceph-mon[223014]: pgmap v1368: 119 pgs: 9 unknown, 3 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 236 op/s; 26872/212124 objects degraded (12.668%); 29720/212124 objects misplaced (14.011%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T22:39:31.572 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:31 smithi026 ceph-mon[215775]: pgmap v1368: 119 pgs: 9 unknown, 3 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 2 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 26 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 236 op/s; 26872/212124 objects degraded (12.668%); 29720/212124 objects misplaced (14.011%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T22:39:31.666 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:31.707 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:32.020 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:32.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:32.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:32.054 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:39:32.132 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:32.169 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:32.239 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:32.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:32.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:32.345 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:32.371 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:39:32.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:32 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3546054617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:32.379 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:32 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:32.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:32 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3546054617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:32.379 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:32 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:32.382 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:32.405 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:39:32.406 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:39:32.406 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T22:39:32.406 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:39:32.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:32 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3546054617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:32.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:32 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:32.874 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:32.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:32.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:32.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:33.007 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:33.052 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:33.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:33.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:33.156 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:33.190 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:33.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[223014]: pgmap v1369: 119 pgs: 1 unknown, 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 28 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 248 op/s; 26561/209298 objects degraded (12.691%); 29257/209298 objects misplaced (13.979%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:39:33.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:33.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[223014]: failed to encode map e529 with expected crc 2023-09-23T22:39:33.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[223014]: failed to encode map e529 with expected crc 2023-09-23T22:39:33.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[223014]: failed to encode map e529 with expected crc 2023-09-23T22:39:33.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:33.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[223014]: osdmap e529: 8 total, 6 up, 8 in 2023-09-23T22:39:33.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26561/209298 objects degraded (12.691%), 23 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:33.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[215775]: pgmap v1369: 119 pgs: 1 unknown, 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 28 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 248 op/s; 26561/209298 objects degraded (12.691%); 29257/209298 objects misplaced (13.979%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:39:33.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:33.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[215775]: failed to encode map e529 with expected crc 2023-09-23T22:39:33.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[215775]: failed to encode map e529 with expected crc 2023-09-23T22:39:33.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[215775]: failed to encode map e529 with expected crc 2023-09-23T22:39:33.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:33.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[215775]: osdmap e529: 8 total, 6 up, 8 in 2023-09-23T22:39:33.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26561/209298 objects degraded (12.691%), 23 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:33.515 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:33.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:33.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:33 smithi070 ceph-mon[192652]: pgmap v1369: 119 pgs: 1 unknown, 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 28 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 248 op/s; 26561/209298 objects degraded (12.691%); 29257/209298 objects misplaced (13.979%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:39:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:33 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:33 smithi070 ceph-mon[192652]: failed to encode map e529 with expected crc 2023-09-23T22:39:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:33 smithi070 ceph-mon[192652]: failed to encode map e529 with expected crc 2023-09-23T22:39:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:33 smithi070 ceph-mon[192652]: failed to encode map e529 with expected crc 2023-09-23T22:39:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:33 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:33 smithi070 ceph-mon[192652]: osdmap e529: 8 total, 6 up, 8 in 2023-09-23T22:39:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26561/209298 objects degraded (12.691%), 23 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:33.631 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:33.658 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:33.666 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:33.703 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:33.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:33.809 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:33.854 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:33.866 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:33.945 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:33.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:33.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:34.003 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:39:34.034 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:39:34.050 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:34.058 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:39:34.077 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:34.083 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:39:34.106 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:39:34.128 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:34.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:34.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:34.138 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:39:34.138 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:39:34.138 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:39:34.138 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:39:34.139 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:39:34.139 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:39:34.139 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:39:34.139 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:39:34.139 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:39:34.237 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:34.267 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:34.316 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:34.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:34.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:34.424 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:34.458 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:34.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:34 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2930154817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:34.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:34 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2930154817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:34.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:34.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:34.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:34.616 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:34.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:34 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2930154817' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:34.643 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:34.699 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:34.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:34.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:34.807 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:34.834 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:34.878 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:34.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:34.988 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:35.014 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:35.061 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:35.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:35.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:35.168 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:35.199 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:35.229 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:35.230 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:35.259 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:35.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:35.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:35.364 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:35.395 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:35.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:35 smithi026 ceph-mon[223014]: pgmap v1371: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 26550/207519 objects degraded (12.794%); 30896/207519 objects misplaced (14.888%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:39:35.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:35 smithi026 ceph-mon[215775]: pgmap v1371: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 26550/207519 objects degraded (12.794%); 30896/207519 objects misplaced (14.888%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:39:35.527 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:39:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:35 smithi070 ceph-mon[192652]: pgmap v1371: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 26550/207519 objects degraded (12.794%); 30896/207519 objects misplaced (14.888%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T22:39:36.262 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:36.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:36.368 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:36.394 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:36.443 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:36.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:36.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:36.507 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:36.549 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:36.580 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:36.637 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:36.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:36.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:36.743 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:36.772 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:36.819 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:36.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:36.826 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:36.927 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:36.954 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:36.970 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:39:36.970 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:39:36.970 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:39:36.971 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:39:37.007 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:37.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:37.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:37.113 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:37.161 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:37.231 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:39:37.331 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:37.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:37.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:37.426 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:39:37.428 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:39:37.437 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:37.464 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:37.511 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:37.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:37.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:37.616 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:37.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:37 smithi070 ceph-mon[192652]: pgmap v1372: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 26149/203115 objects degraded (12.874%); 30068/203115 objects misplaced (14.803%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T22:39:37.651 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:37 smithi026 ceph-mon[223014]: pgmap v1372: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 26149/203115 objects degraded (12.874%); 30068/203115 objects misplaced (14.803%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T22:39:37.651 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:37 smithi026 ceph-mon[215775]: pgmap v1372: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 409 op/s; 26149/203115 objects degraded (12.874%); 30068/203115 objects misplaced (14.803%); 1.2 MiB/s, 18 objects/s recovering 2023-09-23T22:39:37.654 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:38.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:38 smithi026 ceph-mon[215775]: from='client.46484 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:38.626 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 26149/203115 objects degraded (12.874%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:38.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:38 smithi026 ceph-mon[223014]: from='client.46484 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:38.627 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 26149/203115 objects degraded (12.874%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:38 smithi070 ceph-mon[192652]: from='client.46484 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:39:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 26149/203115 objects degraded (12.874%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:38.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:39:38.903 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1372,"stamp":"2023-09-23T22:39:35.611636+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4440058225,"num_objects":67705,"num_object_clones":0,"num_object_copies":203115,"num_objects_missing_on_primary":1450,"num_objects_missing":1450,"num_objects_degraded":26149,"num_objects_misplaced":30068,"num_objects_unfound":0,"num_objects_dirty":67705,"num_whiteouts":0,"num_read":6027,"num_read_kb":12268,"num_write":122706,"num_write_kb":6075410,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25843,"num_bytes_recovered":1403779537,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":169664,"ondisk_log_size":169664,"up":245,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":366,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17036600,"kb_used_data":13890296,"kb_used_omap":52,"kb_used_meta":3146187,"kb_avail":732924616,"statfs":{"total":767960285184,"available":750514806784,"internally_reserved":0,"allocated":14223663104,"data_stored":14205743900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53844,"internal_metadata":3221695916},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,5,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-196804608,"num_objects":-3003,"num_object_clones":0,"num_object_copies":-9009,"num_objects_missing_on_primary":-138,"num_objects_missing":-138,"num_objects_degraded":-723,"num_objects_misplaced":348,"num_objects_unfound":0,"num_objects_dirty":-3003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3003,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":136,"num_bytes_recovered":8912896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.326458"},"pg_stats":[{"pgid":"9.10","version":"528'7401","reported_seq":7585,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:30.738642+0000","last_change":"2023-09-23T22:39:27.389970+0000","last_active":"2023-09-23T22:39:30.738642+0000","last_peered":"2023-09-23T22:39:30.738642+0000","last_clean":"2023-09-23T22:38:54.291969+0000","last_became_active":"2023-09-23T22:39:27.389970+0000","last_became_peered":"2023-09-23T22:39:27.389970+0000","last_unstale":"2023-09-23T22:39:30.738642+0000","last_undegraded":"2023-09-23T22:39:27.360689+0000","last_fullsized":"2023-09-23T22:39:27.346831+0000","mapping_epoch":525,"log_start":"461'1400","ondisk_log_start":"461'1400","created":525,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6001,"log_dups_size":0,"ondisk_log_size":6001,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140967936,"num_objects":2151,"num_object_clones":0,"num_object_copies":6453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2151,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3718,"num_write_kb":187808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":2151}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":588,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.288896+0000","last_change":"2023-09-23T22:37:42.441352+0000","last_active":"2023-09-23T22:39:28.288896+0000","last_peered":"2023-09-23T22:39:28.288896+0000","last_clean":"2023-09-23T22:39:28.288896+0000","last_became_active":"2023-09-23T22:37:42.440853+0000","last_became_peered":"2023-09-23T22:37:42.440853+0000","last_unstale":"2023-09-23T22:39:28.288896+0000","last_undegraded":"2023-09-23T22:39:28.288896+0000","last_fullsized":"2023-09-23T22:39:28.288896+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:11.866578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":227,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.290516+0000","last_change":"2023-09-23T22:39:05.290466+0000","last_active":"2023-09-23T22:39:28.290516+0000","last_peered":"2023-09-23T22:39:28.290516+0000","last_clean":"2023-09-23T22:37:07.670854+0000","last_became_active":"2023-09-23T22:39:05.290466+0000","last_became_peered":"2023-09-23T22:39:05.290466+0000","last_unstale":"2023-09-23T22:39:28.290516+0000","last_undegraded":"2023-09-23T22:39:28.290516+0000","last_fullsized":"2023-09-23T22:39:05.285697+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:14.593594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.13","version":"529'7572","reported_seq":12011,"reported_epoch":529,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:34.803922+0000","last_change":"2023-09-23T22:39:29.317804+0000","last_active":"2023-09-23T22:39:34.803922+0000","last_peered":"2023-09-23T22:39:34.803922+0000","last_clean":"2023-09-23T22:39:34.803922+0000","last_became_active":"2023-09-23T22:39:29.299257+0000","last_became_peered":"2023-09-23T22:39:29.299257+0000","last_unstale":"2023-09-23T22:39:34.803922+0000","last_undegraded":"2023-09-23T22:39:34.803922+0000","last_fullsized":"2023-09-23T22:39:34.803922+0000","mapping_epoch":527,"log_start":"461'1487","ondisk_log_start":"461'1487","created":525,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6085,"log_dups_size":0,"ondisk_log_size":6085,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135626752,"num_objects":2070,"num_object_clones":0,"num_object_copies":6210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2070,"num_objects_unfound":0,"num_objects_dirty":2070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3853,"num_write_kb":189504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1071,"num_bytes_recovered":33652736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":2070}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":271,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.288914+0000","last_change":"2023-09-23T22:39:05.289963+0000","last_active":"2023-09-23T22:39:28.288914+0000","last_peered":"2023-09-23T22:39:28.288914+0000","last_clean":"2023-09-23T22:39:00.403392+0000","last_became_active":"2023-09-23T22:39:05.289963+0000","last_became_peered":"2023-09-23T22:39:05.289963+0000","last_unstale":"2023-09-23T22:39:28.288914+0000","last_undegraded":"2023-09-23T22:39:28.288914+0000","last_fullsized":"2023-09-23T22:39:05.284569+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.806799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":331,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.299624+0000","last_change":"2023-09-23T22:39:05.307660+0000","last_active":"2023-09-23T22:39:29.299624+0000","last_peered":"2023-09-23T22:39:29.299624+0000","last_clean":"2023-09-23T22:35:17.625896+0000","last_became_active":"2023-09-23T22:39:05.307660+0000","last_became_peered":"2023-09-23T22:39:05.307660+0000","last_unstale":"2023-09-23T22:39:29.299624+0000","last_undegraded":"2023-09-23T22:39:29.299624+0000","last_fullsized":"2023-09-23T22:39:05.302351+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:13.440047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"528'7458","reported_seq":7591,"reported_epoch":528,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:39:30.681542+0000","last_change":"2023-09-23T22:39:29.309442+0000","last_active":"2023-09-23T22:39:30.681542+0000","last_peered":"2023-09-23T22:39:30.681542+0000","last_clean":"2023-09-23T22:36:35.536592+0000","last_became_active":"2023-09-23T22:39:29.298891+0000","last_became_peered":"2023-09-23T22:39:29.298891+0000","last_unstale":"2023-09-23T22:39:30.681542+0000","last_undegraded":"2023-09-23T22:39:29.296084+0000","last_fullsized":"2023-09-23T22:39:30.681542+0000","mapping_epoch":527,"log_start":"461'1481","ondisk_log_start":"461'1481","created":525,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5977,"log_dups_size":0,"ondisk_log_size":5977,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142082048,"num_objects":2168,"num_object_clones":0,"num_object_copies":6504,"num_objects_missing_on_primary":39,"num_objects_missing":39,"num_objects_degraded":39,"num_objects_misplaced":2168,"num_objects_unfound":0,"num_objects_dirty":2168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3746,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2168}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":552,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289219+0000","last_change":"2023-09-23T22:39:05.304373+0000","last_active":"2023-09-23T22:39:28.289219+0000","last_peered":"2023-09-23T22:39:28.289219+0000","last_clean":"2023-09-23T22:37:07.670321+0000","last_became_active":"2023-09-23T22:39:05.304373+0000","last_became_peered":"2023-09-23T22:39:05.304373+0000","last_unstale":"2023-09-23T22:39:28.289219+0000","last_undegraded":"2023-09-23T22:39:28.289219+0000","last_fullsized":"2023-09-23T22:39:05.284343+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:51.889095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":326,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.296822+0000","last_change":"2023-09-23T22:38:55.388832+0000","last_active":"2023-09-23T22:39:29.296822+0000","last_peered":"2023-09-23T22:39:29.296822+0000","last_clean":"2023-09-23T22:37:07.661100+0000","last_became_active":"2023-09-23T22:38:55.388832+0000","last_became_peered":"2023-09-23T22:38:55.388832+0000","last_unstale":"2023-09-23T22:39:29.296822+0000","last_undegraded":"2023-09-23T22:39:29.296822+0000","last_fullsized":"2023-09-23T22:38:55.382522+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:09.227034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":358,"reported_epoch":528,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:29.300217+0000","last_change":"2023-09-23T22:39:05.302151+0000","last_active":"2023-09-23T22:38:54.405779+0000","last_peered":"2023-09-23T22:39:29.300217+0000","last_clean":"2023-09-23T22:38:48.372435+0000","last_became_active":"2023-09-23T22:36:36.406686+0000","last_became_peered":"2023-09-23T22:39:05.302151+0000","last_unstale":"2023-09-23T22:39:29.300217+0000","last_undegraded":"2023-09-23T22:39:29.300217+0000","last_fullsized":"2023-09-23T22:39:05.301734+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:12.647131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":308,"reported_epoch":527,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:28.289419+0000","last_change":"2023-09-23T22:39:05.285935+0000","last_active":"2023-09-23T22:38:54.400276+0000","last_peered":"2023-09-23T22:39:28.289419+0000","last_clean":"2023-09-23T22:38:48.357505+0000","last_became_active":"2023-09-23T22:37:42.458398+0000","last_became_peered":"2023-09-23T22:39:05.285935+0000","last_unstale":"2023-09-23T22:39:28.289419+0000","last_undegraded":"2023-09-23T22:39:28.289419+0000","last_fullsized":"2023-09-23T22:39:05.284557+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:22.365837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":646,"reported_epoch":528,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:29.294915+0000","last_change":"2023-09-23T22:39:05.294537+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:39:29.294915+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.294537+0000","last_unstale":"2023-09-23T22:39:29.294915+0000","last_undegraded":"2023-09-23T22:39:29.294915+0000","last_fullsized":"2023-09-23T22:39:05.292289+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":545,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.291357+0000","last_change":"2023-09-23T22:39:05.294737+0000","last_active":"2023-09-23T22:39:28.291357+0000","last_peered":"2023-09-23T22:39:28.291357+0000","last_clean":"2023-09-23T22:39:00.404530+0000","last_became_active":"2023-09-23T22:39:05.294737+0000","last_became_peered":"2023-09-23T22:39:05.294737+0000","last_unstale":"2023-09-23T22:39:28.291357+0000","last_undegraded":"2023-09-23T22:39:28.291357+0000","last_fullsized":"2023-09-23T22:39:05.286342+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:12.866122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":167,"reported_epoch":528,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:29.294703+0000","last_change":"2023-09-23T22:39:05.298634+0000","last_active":"2023-09-23T22:38:54.377414+0000","last_peered":"2023-09-23T22:39:29.294703+0000","last_clean":"2023-09-23T22:38:48.357759+0000","last_became_active":"2023-09-23T22:36:36.521405+0000","last_became_peered":"2023-09-23T22:39:05.298634+0000","last_unstale":"2023-09-23T22:39:29.294703+0000","last_undegraded":"2023-09-23T22:39:29.294703+0000","last_fullsized":"2023-09-23T22:39:05.297707+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:35.327335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":74,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:29.295620+0000","last_change":"2023-09-23T22:39:05.295204+0000","last_active":"2023-09-23T22:39:29.295620+0000","last_peered":"2023-09-23T22:39:29.295620+0000","last_clean":"2023-09-23T22:36:31.355768+0000","last_became_active":"2023-09-23T22:39:05.295204+0000","last_became_peered":"2023-09-23T22:39:05.295204+0000","last_unstale":"2023-09-23T22:39:29.295620+0000","last_undegraded":"2023-09-23T22:39:05.291232+0000","last_fullsized":"2023-09-23T22:39:05.291128+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:25.561328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":168,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.290665+0000","last_change":"2023-09-23T22:38:55.383671+0000","last_active":"2023-09-23T22:39:28.290665+0000","last_peered":"2023-09-23T22:39:28.290665+0000","last_clean":"2023-09-23T22:38:48.410000+0000","last_became_active":"2023-09-23T22:38:55.383671+0000","last_became_peered":"2023-09-23T22:38:55.383671+0000","last_unstale":"2023-09-23T22:39:28.290665+0000","last_undegraded":"2023-09-23T22:39:28.290665+0000","last_fullsized":"2023-09-23T22:38:55.381036+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:27.813898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":517,"reported_epoch":527,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:28.305654+0000","last_change":"2023-09-23T22:38:55.387313+0000","last_active":"2023-09-23T22:39:28.305654+0000","last_peered":"2023-09-23T22:39:28.305654+0000","last_clean":"2023-09-23T22:33:24.367826+0000","last_became_active":"2023-09-23T22:38:55.387313+0000","last_became_peered":"2023-09-23T22:38:55.387313+0000","last_unstale":"2023-09-23T22:39:28.305654+0000","last_undegraded":"2023-09-23T22:38:55.380607+0000","last_fullsized":"2023-09-23T22:38:55.380500+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:05.769113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":551,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289128+0000","last_change":"2023-09-23T22:38:55.397043+0000","last_active":"2023-09-23T22:39:28.289128+0000","last_peered":"2023-09-23T22:39:28.289128+0000","last_clean":"2023-09-23T22:38:48.354081+0000","last_became_active":"2023-09-23T22:38:55.397043+0000","last_became_peered":"2023-09-23T22:38:55.397043+0000","last_unstale":"2023-09-23T22:39:28.289128+0000","last_undegraded":"2023-09-23T22:39:28.289128+0000","last_fullsized":"2023-09-23T22:38:55.379386+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:38.446121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.19","version":"529'7460","reported_seq":7861,"reported_epoch":529,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:35.202938+0000","last_change":"2023-09-23T22:39:27.374950+0000","last_active":"2023-09-23T22:39:35.202938+0000","last_peered":"2023-09-23T22:39:35.202938+0000","last_clean":"2023-09-23T22:36:35.270317+0000","last_became_active":"2023-09-23T22:39:27.374950+0000","last_became_peered":"2023-09-23T22:39:27.374950+0000","last_unstale":"2023-09-23T22:39:35.202938+0000","last_undegraded":"2023-09-23T22:39:27.361410+0000","last_fullsized":"2023-09-23T22:39:27.349419+0000","mapping_epoch":525,"log_start":"458'200","ondisk_log_start":"458'200","created":525,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7260,"log_dups_size":0,"ondisk_log_size":7260,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135069696,"num_objects":2061,"num_object_clones":0,"num_object_copies":6183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2061,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3788,"num_write_kb":187168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4751360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2061}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":907,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.286289+0000","last_change":"2023-09-23T22:36:36.374902+0000","last_active":"2023-09-23T22:39:28.286289+0000","last_peered":"2023-09-23T22:39:28.286289+0000","last_clean":"2023-09-23T22:39:28.286289+0000","last_became_active":"2023-09-23T22:36:22.309281+0000","last_became_peered":"2023-09-23T22:36:22.309281+0000","last_unstale":"2023-09-23T22:39:28.286289+0000","last_undegraded":"2023-09-23T22:39:28.286289+0000","last_fullsized":"2023-09-23T22:39:28.286289+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":161,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.286533+0000","last_change":"2023-09-23T22:35:52.674513+0000","last_active":"2023-09-23T22:39:28.286533+0000","last_peered":"2023-09-23T22:39:28.286533+0000","last_clean":"2023-09-23T22:39:28.286533+0000","last_became_active":"2023-09-23T22:35:52.674194+0000","last_became_peered":"2023-09-23T22:35:52.674194+0000","last_unstale":"2023-09-23T22:39:28.286533+0000","last_undegraded":"2023-09-23T22:39:28.286533+0000","last_fullsized":"2023-09-23T22:39:28.286533+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:54.383524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":353,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.291108+0000","last_change":"2023-09-23T22:39:05.294773+0000","last_active":"2023-09-23T22:39:28.291108+0000","last_peered":"2023-09-23T22:39:28.291108+0000","last_clean":"2023-09-23T22:37:07.671236+0000","last_became_active":"2023-09-23T22:39:05.294773+0000","last_became_peered":"2023-09-23T22:39:05.294773+0000","last_unstale":"2023-09-23T22:39:28.291108+0000","last_undegraded":"2023-09-23T22:39:28.291108+0000","last_fullsized":"2023-09-23T22:39:05.285024+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:47.212090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.18","version":"528'7677","reported_seq":7869,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T22:39:30.663906+0000","last_change":"2023-09-23T22:39:27.401907+0000","last_active":"2023-09-23T22:39:30.663906+0000","last_peered":"2023-09-23T22:39:30.663906+0000","last_clean":"2023-09-23T22:39:30.663906+0000","last_became_active":"2023-09-23T22:39:27.390503+0000","last_became_peered":"2023-09-23T22:39:27.390503+0000","last_unstale":"2023-09-23T22:39:30.663906+0000","last_undegraded":"2023-09-23T22:39:30.663906+0000","last_fullsized":"2023-09-23T22:39:30.663906+0000","mapping_epoch":525,"log_start":"461'1700","ondisk_log_start":"461'1700","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5977,"log_dups_size":0,"ondisk_log_size":5977,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142344192,"num_objects":2172,"num_object_clones":0,"num_object_copies":6516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3861,"num_write_kb":193056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":751,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.294272+0000","last_change":"2023-09-23T22:38:55.391529+0000","last_active":"2023-09-23T22:39:29.294272+0000","last_peered":"2023-09-23T22:39:29.294272+0000","last_clean":"2023-09-23T22:35:16.261819+0000","last_became_active":"2023-09-23T22:38:55.391529+0000","last_became_peered":"2023-09-23T22:38:55.391529+0000","last_unstale":"2023-09-23T22:39:29.294272+0000","last_undegraded":"2023-09-23T22:39:29.294272+0000","last_fullsized":"2023-09-23T22:38:55.384595+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":389,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.299272+0000","last_change":"2023-09-23T22:39:05.307619+0000","last_active":"2023-09-23T22:39:29.299272+0000","last_peered":"2023-09-23T22:39:29.299272+0000","last_clean":"2023-09-23T22:36:31.476578+0000","last_became_active":"2023-09-23T22:39:05.307619+0000","last_became_peered":"2023-09-23T22:39:05.307619+0000","last_unstale":"2023-09-23T22:39:29.299272+0000","last_undegraded":"2023-09-23T22:39:29.299272+0000","last_fullsized":"2023-09-23T22:39:05.284898+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:50.408737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":439,"reported_epoch":528,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:29.294890+0000","last_change":"2023-09-23T22:39:05.294555+0000","last_active":"2023-09-23T22:38:54.405186+0000","last_peered":"2023-09-23T22:39:29.294890+0000","last_clean":"2023-09-23T22:38:48.357907+0000","last_became_active":"2023-09-23T22:36:36.406789+0000","last_became_peered":"2023-09-23T22:39:05.294555+0000","last_unstale":"2023-09-23T22:39:29.294890+0000","last_undegraded":"2023-09-23T22:39:29.294890+0000","last_fullsized":"2023-09-23T22:39:05.293080+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:45.131265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":574,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.288904+0000","last_change":"2023-09-23T22:38:55.396680+0000","last_active":"2023-09-23T22:39:28.288904+0000","last_peered":"2023-09-23T22:39:28.288904+0000","last_clean":"2023-09-23T22:28:58.571704+0000","last_became_active":"2023-09-23T22:38:55.396680+0000","last_became_peered":"2023-09-23T22:38:55.396680+0000","last_unstale":"2023-09-23T22:39:28.288904+0000","last_undegraded":"2023-09-23T22:39:28.288904+0000","last_fullsized":"2023-09-23T22:38:55.378990+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":491,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289226+0000","last_change":"2023-09-23T22:38:55.387267+0000","last_active":"2023-09-23T22:39:28.289226+0000","last_peered":"2023-09-23T22:39:28.289226+0000","last_clean":"2023-09-23T22:38:48.353526+0000","last_became_active":"2023-09-23T22:38:55.387267+0000","last_became_peered":"2023-09-23T22:38:55.387267+0000","last_unstale":"2023-09-23T22:39:28.289226+0000","last_undegraded":"2023-09-23T22:39:28.289226+0000","last_fullsized":"2023-09-23T22:38:55.379616+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:39.130605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":375,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.299831+0000","last_change":"2023-09-23T22:39:05.307686+0000","last_active":"2023-09-23T22:39:29.299831+0000","last_peered":"2023-09-23T22:39:29.299831+0000","last_clean":"2023-09-23T22:35:17.626249+0000","last_became_active":"2023-09-23T22:39:05.307686+0000","last_became_peered":"2023-09-23T22:39:05.307686+0000","last_unstale":"2023-09-23T22:39:29.299831+0000","last_undegraded":"2023-09-23T22:39:29.299831+0000","last_fullsized":"2023-09-23T22:39:05.302436+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:24.866314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":903,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T22:39:29.295138+0000","last_change":"2023-09-23T22:36:22.311443+0000","last_active":"2023-09-23T22:39:29.295138+0000","last_peered":"2023-09-23T22:39:29.295138+0000","last_clean":"2023-09-23T22:39:29.295138+0000","last_became_active":"2023-09-23T22:36:22.311137+0000","last_became_peered":"2023-09-23T22:36:22.311137+0000","last_unstale":"2023-09-23T22:39:29.295138+0000","last_undegraded":"2023-09-23T22:39:29.295138+0000","last_fullsized":"2023-09-23T22:39:29.295138+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":403,"reported_epoch":528,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:39:29.298817+0000","last_change":"2023-09-23T22:39:05.302168+0000","last_active":"2023-09-23T22:39:04.291704+0000","last_peered":"2023-09-23T22:39:29.298817+0000","last_clean":"2023-09-23T22:28:58.571564+0000","last_became_active":"2023-09-23T22:28:05.154059+0000","last_became_peered":"2023-09-23T22:39:05.302168+0000","last_unstale":"2023-09-23T22:39:29.298817+0000","last_undegraded":"2023-09-23T22:39:05.286285+0000","last_fullsized":"2023-09-23T22:39:05.286213+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:03.826266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.296346+0000","last_change":"2023-09-23T22:39:05.295415+0000","last_active":"2023-09-23T22:39:29.296346+0000","last_peered":"2023-09-23T22:39:29.296346+0000","last_clean":"2023-09-23T22:37:07.674004+0000","last_became_active":"2023-09-23T22:39:05.295415+0000","last_became_peered":"2023-09-23T22:39:05.295415+0000","last_unstale":"2023-09-23T22:39:29.296346+0000","last_undegraded":"2023-09-23T22:39:29.296346+0000","last_fullsized":"2023-09-23T22:39:05.293230+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:45.148758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":894,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289146+0000","last_change":"2023-09-23T22:38:55.382695+0000","last_active":"2023-09-23T22:39:28.289146+0000","last_peered":"2023-09-23T22:39:28.289146+0000","last_clean":"2023-09-23T22:38:53.587260+0000","last_became_active":"2023-09-23T22:38:55.382695+0000","last_became_peered":"2023-09-23T22:38:55.382695+0000","last_unstale":"2023-09-23T22:39:28.289146+0000","last_undegraded":"2023-09-23T22:39:28.289146+0000","last_fullsized":"2023-09-23T22:38:55.379918+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.5","version":"529'7524","reported_seq":7788,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T22:39:35.096944+0000","last_change":"2023-09-23T22:39:27.379732+0000","last_active":"2023-09-23T22:39:35.096944+0000","last_peered":"2023-09-23T22:39:35.096944+0000","last_clean":"2023-09-23T22:39:35.096944+0000","last_became_active":"2023-09-23T22:39:27.359019+0000","last_became_peered":"2023-09-23T22:39:27.359019+0000","last_unstale":"2023-09-23T22:39:35.096944+0000","last_undegraded":"2023-09-23T22:39:35.096944+0000","last_fullsized":"2023-09-23T22:39:35.096944+0000","mapping_epoch":495,"log_start":"458'200","ondisk_log_start":"458'200","created":451,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7324,"log_dups_size":0,"ondisk_log_size":7324,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133988352,"num_objects":2044,"num_object_clones":0,"num_object_copies":6132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3826,"num_write_kb":187872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":554,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.290177+0000","last_change":"2023-09-23T22:35:52.675241+0000","last_active":"2023-09-23T22:39:28.290177+0000","last_peered":"2023-09-23T22:39:28.290177+0000","last_clean":"2023-09-23T22:39:28.290177+0000","last_became_active":"2023-09-23T22:35:52.674769+0000","last_became_peered":"2023-09-23T22:35:52.674769+0000","last_unstale":"2023-09-23T22:39:28.290177+0000","last_undegraded":"2023-09-23T22:39:28.290177+0000","last_fullsized":"2023-09-23T22:39:28.290177+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:43.611334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":465,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.294583+0000","last_change":"2023-09-23T22:38:55.387756+0000","last_active":"2023-09-23T22:39:29.294583+0000","last_peered":"2023-09-23T22:39:29.294583+0000","last_clean":"2023-09-23T22:35:23.290223+0000","last_became_active":"2023-09-23T22:38:55.387756+0000","last_became_peered":"2023-09-23T22:38:55.387756+0000","last_unstale":"2023-09-23T22:39:29.294583+0000","last_undegraded":"2023-09-23T22:39:29.294583+0000","last_fullsized":"2023-09-23T22:38:55.384253+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.075495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":646,"reported_epoch":528,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:39:29.295177+0000","last_change":"2023-09-23T22:39:05.294786+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:39:29.295177+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.294786+0000","last_unstale":"2023-09-23T22:39:29.295177+0000","last_undegraded":"2023-09-23T22:39:05.292513+0000","last_fullsized":"2023-09-23T22:39:05.292435+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"528'7519","reported_seq":12419,"reported_epoch":528,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:30.585665+0000","last_change":"2023-09-23T22:39:27.388674+0000","last_active":"2023-09-23T22:39:30.585665+0000","last_peered":"2023-09-23T22:39:30.585665+0000","last_clean":"2023-09-23T22:39:30.585665+0000","last_became_active":"2023-09-23T22:39:27.358268+0000","last_became_peered":"2023-09-23T22:39:27.358268+0000","last_unstale":"2023-09-23T22:39:30.585665+0000","last_undegraded":"2023-09-23T22:39:30.585665+0000","last_fullsized":"2023-09-23T22:39:30.585665+0000","mapping_epoch":517,"log_start":"461'1500","ondisk_log_start":"461'1500","created":451,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6019,"log_dups_size":0,"ondisk_log_size":6019,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142278656,"num_objects":2171,"num_object_clones":0,"num_object_copies":6513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2171,"num_objects_unfound":0,"num_objects_dirty":2171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3778,"num_write_kb":190368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1257,"num_bytes_recovered":79364096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":2171}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":369,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.290943+0000","last_change":"2023-09-23T22:37:42.438301+0000","last_active":"2023-09-23T22:39:28.290943+0000","last_peered":"2023-09-23T22:39:28.290943+0000","last_clean":"2023-09-23T22:39:28.290943+0000","last_became_active":"2023-09-23T22:37:42.437753+0000","last_became_peered":"2023-09-23T22:37:42.437753+0000","last_unstale":"2023-09-23T22:39:28.290943+0000","last_undegraded":"2023-09-23T22:39:28.290943+0000","last_fullsized":"2023-09-23T22:39:28.290943+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:21.146523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":501,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.288751+0000","last_change":"2023-09-23T22:39:05.289891+0000","last_active":"2023-09-23T22:39:28.288751+0000","last_peered":"2023-09-23T22:39:28.288751+0000","last_clean":"2023-09-23T22:39:00.403311+0000","last_became_active":"2023-09-23T22:39:05.289891+0000","last_became_peered":"2023-09-23T22:39:05.289891+0000","last_unstale":"2023-09-23T22:39:28.288751+0000","last_undegraded":"2023-09-23T22:39:28.288751+0000","last_fullsized":"2023-09-23T22:39:05.284229+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:50.581117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":794,"reported_epoch":528,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:29.295229+0000","last_change":"2023-09-23T22:39:05.294943+0000","last_active":"2023-09-23T22:35:19.273163+0000","last_peered":"2023-09-23T22:39:29.295229+0000","last_clean":"2023-09-23T22:35:16.262487+0000","last_became_active":"2023-09-23T22:34:36.010161+0000","last_became_peered":"2023-09-23T22:39:05.294943+0000","last_unstale":"2023-09-23T22:39:29.295229+0000","last_undegraded":"2023-09-23T22:39:29.295229+0000","last_fullsized":"2023-09-23T22:39:05.292735+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.7","version":"528'7528","reported_seq":8672,"reported_epoch":528,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:39:32.304011+0000","last_change":"2023-09-23T22:39:27.368856+0000","last_active":"2023-09-23T22:39:32.304011+0000","last_peered":"2023-09-23T22:39:32.304011+0000","last_clean":"2023-09-23T22:36:35.361199+0000","last_became_active":"2023-09-23T22:39:27.360907+0000","last_became_peered":"2023-09-23T22:39:27.360907+0000","last_unstale":"2023-09-23T22:39:32.304011+0000","last_undegraded":"2023-09-23T22:39:27.346656+0000","last_fullsized":"2023-09-23T22:39:32.304011+0000","mapping_epoch":518,"log_start":"461'893","ondisk_log_start":"461'893","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6635,"log_dups_size":0,"ondisk_log_size":6635,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140083200,"num_objects":2137,"num_object_clones":0,"num_object_copies":6411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":235,"num_objects_misplaced":1902,"num_objects_unfound":0,"num_objects_dirty":2137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3801,"num_write_kb":190048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":253,"num_bytes_recovered":16613376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,1],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2137}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":331,"reported_epoch":528,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:39:29.294987+0000","last_change":"2023-09-23T22:39:05.294595+0000","last_active":"2023-09-23T22:35:19.265438+0000","last_peered":"2023-09-23T22:39:29.294987+0000","last_clean":"2023-09-23T22:35:16.262186+0000","last_became_active":"2023-09-23T22:34:32.963996+0000","last_became_peered":"2023-09-23T22:39:05.294595+0000","last_unstale":"2023-09-23T22:39:29.294987+0000","last_undegraded":"2023-09-23T22:39:05.293265+0000","last_fullsized":"2023-09-23T22:39:05.293222+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:57.686355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":228,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.290443+0000","last_change":"2023-09-23T22:38:55.397046+0000","last_active":"2023-09-23T22:39:28.290443+0000","last_peered":"2023-09-23T22:39:28.290443+0000","last_clean":"2023-09-23T22:37:07.662421+0000","last_became_active":"2023-09-23T22:38:55.397046+0000","last_became_peered":"2023-09-23T22:38:55.397046+0000","last_unstale":"2023-09-23T22:39:28.290443+0000","last_undegraded":"2023-09-23T22:39:28.290443+0000","last_fullsized":"2023-09-23T22:38:55.385491+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:27.922878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":693,"reported_epoch":528,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:29.299364+0000","last_change":"2023-09-23T22:39:05.302770+0000","last_active":"2023-09-23T22:39:04.293181+0000","last_peered":"2023-09-23T22:39:29.299364+0000","last_clean":"2023-09-23T22:28:58.573726+0000","last_became_active":"2023-09-23T22:29:02.583400+0000","last_became_peered":"2023-09-23T22:39:05.302770+0000","last_unstale":"2023-09-23T22:39:29.299364+0000","last_undegraded":"2023-09-23T22:39:29.299364+0000","last_fullsized":"2023-09-23T22:39:05.302009+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.6","version":"528'7432","reported_seq":7697,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:32.281118+0000","last_change":"2023-09-23T22:39:27.360835+0000","last_active":"2023-09-23T22:39:32.281118+0000","last_peered":"2023-09-23T22:39:32.281118+0000","last_clean":"2023-09-23T22:37:14.247564+0000","last_became_active":"2023-09-23T22:39:27.360835+0000","last_became_peered":"2023-09-23T22:39:27.360835+0000","last_unstale":"2023-09-23T22:39:32.281118+0000","last_undegraded":"2023-09-23T22:39:27.345966+0000","last_fullsized":"2023-09-23T22:39:27.345858+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7432,"log_dups_size":0,"ondisk_log_size":7432,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138706944,"num_objects":2116,"num_object_clones":0,"num_object_copies":6348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2116,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3757,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":2116}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":372,"reported_epoch":527,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:28.291298+0000","last_change":"2023-09-23T22:39:05.286647+0000","last_active":"2023-09-23T22:38:54.408907+0000","last_peered":"2023-09-23T22:39:28.291298+0000","last_clean":"2023-09-23T22:38:53.017852+0000","last_became_active":"2023-09-23T22:37:42.458233+0000","last_became_peered":"2023-09-23T22:39:05.286647+0000","last_unstale":"2023-09-23T22:39:28.291298+0000","last_undegraded":"2023-09-23T22:39:28.291298+0000","last_fullsized":"2023-09-23T22:39:05.285527+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:29.003765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":292,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.289138+0000","last_change":"2023-09-23T22:37:42.432660+0000","last_active":"2023-09-23T22:39:28.289138+0000","last_peered":"2023-09-23T22:39:28.289138+0000","last_clean":"2023-09-23T22:39:28.289138+0000","last_became_active":"2023-09-23T22:37:42.431969+0000","last_became_peered":"2023-09-23T22:37:42.431969+0000","last_unstale":"2023-09-23T22:39:28.289138+0000","last_undegraded":"2023-09-23T22:39:28.289138+0000","last_fullsized":"2023-09-23T22:39:28.289138+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:54.642557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"529'7303","reported_seq":20821,"reported_epoch":529,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:39:34.703813+0000","last_change":"2023-09-23T22:39:27.348267+0000","last_active":"2023-09-23T22:39:34.703813+0000","last_peered":"2023-09-23T22:39:34.703813+0000","last_clean":"2023-09-23T22:35:27.126327+0000","last_became_active":"2023-09-23T22:39:27.348267+0000","last_became_peered":"2023-09-23T22:39:27.348267+0000","last_unstale":"2023-09-23T22:39:34.703813+0000","last_undegraded":"2023-09-23T22:39:34.703813+0000","last_fullsized":"2023-09-23T22:39:27.346428+0000","mapping_epoch":517,"log_start":"461'1200","ondisk_log_start":"461'1200","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6103,"log_dups_size":0,"ondisk_log_size":6103,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131432448,"num_objects":2005,"num_object_clones":0,"num_object_copies":6015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4010,"num_objects_unfound":0,"num_objects_dirty":2005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3722,"num_write_kb":183296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3343,"num_bytes_recovered":183861248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2005}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":828,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.286479+0000","last_change":"2023-09-23T22:39:05.299735+0000","last_active":"2023-09-23T22:39:28.286479+0000","last_peered":"2023-09-23T22:39:28.286479+0000","last_clean":"2023-09-23T22:36:31.469549+0000","last_became_active":"2023-09-23T22:39:05.299735+0000","last_became_peered":"2023-09-23T22:39:05.299735+0000","last_unstale":"2023-09-23T22:39:28.286479+0000","last_undegraded":"2023-09-23T22:39:28.286479+0000","last_fullsized":"2023-09-23T22:39:05.286634+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:10.963206+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":344,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:29.300478+0000","last_change":"2023-09-23T22:39:05.304092+0000","last_active":"2023-09-23T22:39:29.300478+0000","last_peered":"2023-09-23T22:39:29.300478+0000","last_clean":"2023-09-23T22:36:31.477294+0000","last_became_active":"2023-09-23T22:39:05.304092+0000","last_became_peered":"2023-09-23T22:39:05.304092+0000","last_unstale":"2023-09-23T22:39:29.300478+0000","last_undegraded":"2023-09-23T22:39:05.285659+0000","last_fullsized":"2023-09-23T22:39:05.285557+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:15.273429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":436,"reported_epoch":527,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:28.298992+0000","last_change":"2023-09-23T22:39:05.290625+0000","last_active":"2023-09-23T22:38:54.402963+0000","last_peered":"2023-09-23T22:39:28.298992+0000","last_clean":"2023-09-23T22:38:53.017791+0000","last_became_active":"2023-09-23T22:36:36.405891+0000","last_became_peered":"2023-09-23T22:39:05.290625+0000","last_unstale":"2023-09-23T22:39:28.298992+0000","last_undegraded":"2023-09-23T22:39:28.298992+0000","last_fullsized":"2023-09-23T22:39:05.289355+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:02.717357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"528'7405","reported_seq":7589,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:30.695255+0000","last_change":"2023-09-23T22:39:27.347160+0000","last_active":"2023-09-23T22:39:30.695255+0000","last_peered":"2023-09-23T22:39:30.695255+0000","last_clean":"2023-09-23T22:38:54.291969+0000","last_became_active":"2023-09-23T22:39:27.347160+0000","last_became_peered":"2023-09-23T22:39:27.347160+0000","last_unstale":"2023-09-23T22:39:30.695255+0000","last_undegraded":"2023-09-23T22:39:27.342633+0000","last_fullsized":"2023-09-23T22:39:27.342514+0000","mapping_epoch":513,"log_start":"461'1400","ondisk_log_start":"461'1400","created":451,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6005,"log_dups_size":0,"ondisk_log_size":6005,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140705792,"num_objects":2147,"num_object_clones":0,"num_object_copies":6441,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2147,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2147,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3722,"num_write_kb":187808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":2147}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":962,"reported_epoch":527,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:39:28.286335+0000","last_change":"2023-09-23T22:39:05.288550+0000","last_active":"2023-09-23T22:39:28.286335+0000","last_peered":"2023-09-23T22:39:28.286335+0000","last_clean":"2023-09-23T22:36:31.469347+0000","last_became_active":"2023-09-23T22:39:05.288550+0000","last_became_peered":"2023-09-23T22:39:05.288550+0000","last_unstale":"2023-09-23T22:39:28.286335+0000","last_undegraded":"2023-09-23T22:39:28.286335+0000","last_fullsized":"2023-09-23T22:39:05.284160+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:48.075424+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":296,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289174+0000","last_change":"2023-09-23T22:38:55.396620+0000","last_active":"2023-09-23T22:39:28.289174+0000","last_peered":"2023-09-23T22:39:28.289174+0000","last_clean":"2023-09-23T22:37:07.662537+0000","last_became_active":"2023-09-23T22:38:55.396620+0000","last_became_peered":"2023-09-23T22:38:55.396620+0000","last_unstale":"2023-09-23T22:39:28.289174+0000","last_undegraded":"2023-09-23T22:39:28.289174+0000","last_fullsized":"2023-09-23T22:38:55.379490+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:47.749424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":155,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.286830+0000","last_change":"2023-09-23T22:37:42.423759+0000","last_active":"2023-09-23T22:39:28.286830+0000","last_peered":"2023-09-23T22:39:28.286830+0000","last_clean":"2023-09-23T22:39:28.286830+0000","last_became_active":"2023-09-23T22:37:42.423441+0000","last_became_peered":"2023-09-23T22:37:42.423441+0000","last_unstale":"2023-09-23T22:39:28.286830+0000","last_undegraded":"2023-09-23T22:39:28.286830+0000","last_fullsized":"2023-09-23T22:39:28.286830+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:04.848811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":574,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289072+0000","last_change":"2023-09-23T22:38:55.396840+0000","last_active":"2023-09-23T22:39:28.289072+0000","last_peered":"2023-09-23T22:39:28.289072+0000","last_clean":"2023-09-23T22:28:58.571704+0000","last_became_active":"2023-09-23T22:38:55.396840+0000","last_became_peered":"2023-09-23T22:38:55.396840+0000","last_unstale":"2023-09-23T22:39:28.289072+0000","last_undegraded":"2023-09-23T22:39:28.289072+0000","last_fullsized":"2023-09-23T22:38:55.379140+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"529'7561","reported_seq":11999,"reported_epoch":529,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:34.806282+0000","last_change":"2023-09-23T22:39:27.367715+0000","last_active":"2023-09-23T22:39:34.806282+0000","last_peered":"2023-09-23T22:39:34.806282+0000","last_clean":"2023-09-23T22:39:34.806282+0000","last_became_active":"2023-09-23T22:39:27.357626+0000","last_became_peered":"2023-09-23T22:39:27.357626+0000","last_unstale":"2023-09-23T22:39:34.806282+0000","last_undegraded":"2023-09-23T22:39:34.806282+0000","last_fullsized":"2023-09-23T22:39:34.806282+0000","mapping_epoch":522,"log_start":"461'1487","ondisk_log_start":"461'1487","created":451,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6074,"log_dups_size":0,"ondisk_log_size":6074,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136347648,"num_objects":2080,"num_object_clones":0,"num_object_copies":6240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2080,"num_objects_unfound":0,"num_objects_dirty":2080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3841,"num_write_kb":189504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1070,"num_bytes_recovered":33652736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":2080}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":542,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.288890+0000","last_change":"2023-09-23T22:35:52.681932+0000","last_active":"2023-09-23T22:39:28.288890+0000","last_peered":"2023-09-23T22:39:28.288890+0000","last_clean":"2023-09-23T22:39:28.288890+0000","last_became_active":"2023-09-23T22:35:52.681575+0000","last_became_peered":"2023-09-23T22:35:52.681575+0000","last_unstale":"2023-09-23T22:39:28.288890+0000","last_undegraded":"2023-09-23T22:39:28.288890+0000","last_fullsized":"2023-09-23T22:39:28.288890+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:25.183658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":515,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.290628+0000","last_change":"2023-09-23T22:38:55.388007+0000","last_active":"2023-09-23T22:39:28.290628+0000","last_peered":"2023-09-23T22:39:28.290628+0000","last_clean":"2023-09-23T22:37:07.660692+0000","last_became_active":"2023-09-23T22:38:55.388007+0000","last_became_peered":"2023-09-23T22:38:55.388007+0000","last_unstale":"2023-09-23T22:39:28.290628+0000","last_undegraded":"2023-09-23T22:39:28.290628+0000","last_fullsized":"2023-09-23T22:38:55.384033+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:14.119720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"528'7457","reported_seq":7589,"reported_epoch":528,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:39:30.699886+0000","last_change":"2023-09-23T22:39:27.369980+0000","last_active":"2023-09-23T22:39:30.699886+0000","last_peered":"2023-09-23T22:39:30.699886+0000","last_clean":"2023-09-23T22:36:35.536592+0000","last_became_active":"2023-09-23T22:39:27.352556+0000","last_became_peered":"2023-09-23T22:39:27.352556+0000","last_unstale":"2023-09-23T22:39:30.699886+0000","last_undegraded":"2023-09-23T22:39:27.343413+0000","last_fullsized":"2023-09-23T22:39:30.699886+0000","mapping_epoch":518,"log_start":"461'1481","ondisk_log_start":"461'1481","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5976,"log_dups_size":0,"ondisk_log_size":5976,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142147584,"num_objects":2169,"num_object_clones":0,"num_object_copies":6507,"num_objects_missing_on_primary":42,"num_objects_missing":42,"num_objects_degraded":42,"num_objects_misplaced":2169,"num_objects_unfound":0,"num_objects_dirty":2169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3745,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2169}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":910,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.296244+0000","last_change":"2023-09-23T22:38:55.383101+0000","last_active":"2023-09-23T22:39:29.296244+0000","last_peered":"2023-09-23T22:39:29.296244+0000","last_clean":"2023-09-23T22:38:48.353691+0000","last_became_active":"2023-09-23T22:38:55.383101+0000","last_became_peered":"2023-09-23T22:38:55.383101+0000","last_unstale":"2023-09-23T22:39:29.296244+0000","last_undegraded":"2023-09-23T22:39:29.296244+0000","last_fullsized":"2023-09-23T22:38:55.381406+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:55.111338+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":246,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289631+0000","last_change":"2023-09-23T22:39:05.297947+0000","last_active":"2023-09-23T22:39:28.289631+0000","last_peered":"2023-09-23T22:39:28.289631+0000","last_clean":"2023-09-23T22:37:07.670248+0000","last_became_active":"2023-09-23T22:39:05.297947+0000","last_became_peered":"2023-09-23T22:39:05.297947+0000","last_unstale":"2023-09-23T22:39:28.289631+0000","last_undegraded":"2023-09-23T22:39:28.289631+0000","last_fullsized":"2023-09-23T22:39:05.284753+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:18.514918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":289,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.290347+0000","last_change":"2023-09-23T22:39:05.294341+0000","last_active":"2023-09-23T22:39:28.290347+0000","last_peered":"2023-09-23T22:39:28.290347+0000","last_clean":"2023-09-23T22:39:00.403604+0000","last_became_active":"2023-09-23T22:39:05.294341+0000","last_became_peered":"2023-09-23T22:39:05.294341+0000","last_unstale":"2023-09-23T22:39:28.290347+0000","last_undegraded":"2023-09-23T22:39:28.290347+0000","last_fullsized":"2023-09-23T22:39:05.290023+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:21.644681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"529'7479","reported_seq":21150,"reported_epoch":529,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:35.245553+0000","last_change":"2023-09-23T22:39:09.972071+0000","last_active":"2023-09-23T22:39:35.245553+0000","last_peered":"2023-09-23T22:39:35.245553+0000","last_clean":"2023-09-23T22:39:35.245553+0000","last_became_active":"2023-09-23T22:39:06.318194+0000","last_became_peered":"2023-09-23T22:39:06.318194+0000","last_unstale":"2023-09-23T22:39:35.245553+0000","last_undegraded":"2023-09-23T22:39:35.245553+0000","last_fullsized":"2023-09-23T22:39:35.245553+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7479,"log_dups_size":0,"ondisk_log_size":7479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:31.028708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264962048,"num_objects":4043,"num_object_clones":0,"num_object_copies":12129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4043,"num_objects_unfound":0,"num_objects_dirty":4043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7479,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6634,"num_bytes_recovered":336199680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":4043}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":794,"reported_epoch":528,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:29.295442+0000","last_change":"2023-09-23T22:39:05.295156+0000","last_active":"2023-09-23T22:35:19.273163+0000","last_peered":"2023-09-23T22:39:29.295442+0000","last_clean":"2023-09-23T22:35:16.262487+0000","last_became_active":"2023-09-23T22:34:36.010161+0000","last_became_peered":"2023-09-23T22:39:05.295156+0000","last_unstale":"2023-09-23T22:39:29.295442+0000","last_undegraded":"2023-09-23T22:39:29.295442+0000","last_fullsized":"2023-09-23T22:39:05.292995+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":966,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.291056+0000","last_change":"2023-09-23T22:37:42.433834+0000","last_active":"2023-09-23T22:39:28.291056+0000","last_peered":"2023-09-23T22:39:28.291056+0000","last_clean":"2023-09-23T22:39:28.291056+0000","last_became_active":"2023-09-23T22:37:42.432400+0000","last_became_peered":"2023-09-23T22:37:42.432400+0000","last_unstale":"2023-09-23T22:39:28.291056+0000","last_undegraded":"2023-09-23T22:39:28.291056+0000","last_fullsized":"2023-09-23T22:39:28.291056+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:17.145403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":567,"reported_epoch":527,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:28.286643+0000","last_change":"2023-09-23T22:39:05.287940+0000","last_active":"2023-09-23T22:38:54.405816+0000","last_peered":"2023-09-23T22:39:28.286643+0000","last_clean":"2023-09-23T22:38:48.357982+0000","last_became_active":"2023-09-23T22:36:36.407307+0000","last_became_peered":"2023-09-23T22:39:05.287940+0000","last_unstale":"2023-09-23T22:39:28.286643+0000","last_undegraded":"2023-09-23T22:39:28.286643+0000","last_fullsized":"2023-09-23T22:39:05.287229+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:59.639645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":244,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.290197+0000","last_change":"2023-09-23T22:37:42.444779+0000","last_active":"2023-09-23T22:39:28.290197+0000","last_peered":"2023-09-23T22:39:28.290197+0000","last_clean":"2023-09-23T22:39:28.290197+0000","last_became_active":"2023-09-23T22:37:42.437935+0000","last_became_peered":"2023-09-23T22:37:42.437935+0000","last_unstale":"2023-09-23T22:39:28.290197+0000","last_undegraded":"2023-09-23T22:39:28.290197+0000","last_fullsized":"2023-09-23T22:39:28.290197+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:18.444049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"529'7462","reported_seq":7863,"reported_epoch":529,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:34.919614+0000","last_change":"2023-09-23T22:39:27.364387+0000","last_active":"2023-09-23T22:39:34.919614+0000","last_peered":"2023-09-23T22:39:34.919614+0000","last_clean":"2023-09-23T22:36:35.270317+0000","last_became_active":"2023-09-23T22:39:27.364387+0000","last_became_peered":"2023-09-23T22:39:27.364387+0000","last_unstale":"2023-09-23T22:39:34.919614+0000","last_undegraded":"2023-09-23T22:39:27.349064+0000","last_fullsized":"2023-09-23T22:39:27.348946+0000","mapping_epoch":517,"log_start":"458'200","ondisk_log_start":"458'200","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7262,"log_dups_size":0,"ondisk_log_size":7262,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134938624,"num_objects":2059,"num_object_clones":0,"num_object_copies":6177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2059,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3790,"num_write_kb":187168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72,"num_bytes_recovered":4751360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":2059}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":908,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.286302+0000","last_change":"2023-09-23T22:36:35.383077+0000","last_active":"2023-09-23T22:39:28.286302+0000","last_peered":"2023-09-23T22:39:28.286302+0000","last_clean":"2023-09-23T22:39:28.286302+0000","last_became_active":"2023-09-23T22:36:22.300226+0000","last_became_peered":"2023-09-23T22:36:22.300226+0000","last_unstale":"2023-09-23T22:39:28.286302+0000","last_undegraded":"2023-09-23T22:39:28.286302+0000","last_fullsized":"2023-09-23T22:39:28.286302+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1431,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:29.296465+0000","last_change":"2023-09-23T22:38:55.389270+0000","last_active":"2023-09-23T22:39:29.296465+0000","last_peered":"2023-09-23T22:39:29.296465+0000","last_clean":"2023-09-23T22:38:48.353781+0000","last_became_active":"2023-09-23T22:38:55.389270+0000","last_became_peered":"2023-09-23T22:38:55.389270+0000","last_unstale":"2023-09-23T22:39:29.296465+0000","last_undegraded":"2023-09-23T22:38:55.382069+0000","last_fullsized":"2023-09-23T22:38:55.381954+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:28.932588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":472,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289677+0000","last_change":"2023-09-23T22:38:55.397986+0000","last_active":"2023-09-23T22:39:28.289677+0000","last_peered":"2023-09-23T22:39:28.289677+0000","last_clean":"2023-09-23T22:35:23.289972+0000","last_became_active":"2023-09-23T22:38:55.397986+0000","last_became_peered":"2023-09-23T22:38:55.397986+0000","last_unstale":"2023-09-23T22:39:28.289677+0000","last_undegraded":"2023-09-23T22:39:28.289677+0000","last_fullsized":"2023-09-23T22:38:55.379810+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:58.839783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":155,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.287961+0000","last_change":"2023-09-23T22:39:05.291228+0000","last_active":"2023-09-23T22:39:28.287961+0000","last_peered":"2023-09-23T22:39:28.287961+0000","last_clean":"2023-09-23T22:39:01.247643+0000","last_became_active":"2023-09-23T22:39:05.291228+0000","last_became_peered":"2023-09-23T22:39:05.291228+0000","last_unstale":"2023-09-23T22:39:28.287961+0000","last_undegraded":"2023-09-23T22:39:28.287961+0000","last_fullsized":"2023-09-23T22:39:05.285277+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:47.583378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"528'7704","reported_seq":8028,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:30.625261+0000","last_change":"2023-09-23T22:39:06.557390+0000","last_active":"2023-09-23T22:39:30.625261+0000","last_peered":"2023-09-23T22:39:30.625261+0000","last_clean":"2023-09-23T22:36:35.350237+0000","last_became_active":"2023-09-23T22:39:06.291946+0000","last_became_peered":"2023-09-23T22:39:06.291946+0000","last_unstale":"2023-09-23T22:39:30.625261+0000","last_undegraded":"2023-09-23T22:39:06.286510+0000","last_fullsized":"2023-09-23T22:39:06.285916+0000","mapping_epoch":518,"log_start":"462'2558","ondisk_log_start":"462'2558","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5146,"log_dups_size":0,"ondisk_log_size":5146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:34.797366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289931264,"num_objects":4424,"num_object_clones":0,"num_object_copies":13272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4424,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7704,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":4424}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":904,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T22:39:29.295826+0000","last_change":"2023-09-23T22:36:22.310802+0000","last_active":"2023-09-23T22:39:29.295826+0000","last_peered":"2023-09-23T22:39:29.295826+0000","last_clean":"2023-09-23T22:39:29.295826+0000","last_became_active":"2023-09-23T22:36:22.310477+0000","last_became_peered":"2023-09-23T22:36:22.310477+0000","last_unstale":"2023-09-23T22:39:29.295826+0000","last_undegraded":"2023-09-23T22:39:29.295826+0000","last_fullsized":"2023-09-23T22:39:29.295826+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":201,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.286419+0000","last_change":"2023-09-23T22:39:05.309059+0000","last_active":"2023-09-23T22:39:28.286419+0000","last_peered":"2023-09-23T22:39:28.286419+0000","last_clean":"2023-09-23T22:39:01.248920+0000","last_became_active":"2023-09-23T22:39:05.309059+0000","last_became_peered":"2023-09-23T22:39:05.309059+0000","last_unstale":"2023-09-23T22:39:28.286419+0000","last_undegraded":"2023-09-23T22:39:28.286419+0000","last_fullsized":"2023-09-23T22:39:05.286315+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:26.012539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":146,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.287174+0000","last_change":"2023-09-23T22:38:55.382188+0000","last_active":"2023-09-23T22:39:28.287174+0000","last_peered":"2023-09-23T22:39:28.287174+0000","last_clean":"2023-09-23T22:38:48.445651+0000","last_became_active":"2023-09-23T22:38:55.382188+0000","last_became_peered":"2023-09-23T22:38:55.382188+0000","last_unstale":"2023-09-23T22:39:28.287174+0000","last_undegraded":"2023-09-23T22:39:28.287174+0000","last_fullsized":"2023-09-23T22:38:55.378777+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:42.378244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":168,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.290653+0000","last_change":"2023-09-23T22:37:42.437482+0000","last_active":"2023-09-23T22:39:28.290653+0000","last_peered":"2023-09-23T22:39:28.290653+0000","last_clean":"2023-09-23T22:39:28.290653+0000","last_became_active":"2023-09-23T22:37:42.433915+0000","last_became_peered":"2023-09-23T22:37:42.433915+0000","last_unstale":"2023-09-23T22:39:28.290653+0000","last_undegraded":"2023-09-23T22:39:28.290653+0000","last_fullsized":"2023-09-23T22:39:28.290653+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.522090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"528'7674","reported_seq":7866,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T22:39:30.690460+0000","last_change":"2023-09-23T22:39:27.361365+0000","last_active":"2023-09-23T22:39:30.690460+0000","last_peered":"2023-09-23T22:39:30.690460+0000","last_clean":"2023-09-23T22:39:30.690460+0000","last_became_active":"2023-09-23T22:39:27.353251+0000","last_became_peered":"2023-09-23T22:39:27.353251+0000","last_unstale":"2023-09-23T22:39:30.690460+0000","last_undegraded":"2023-09-23T22:39:30.690460+0000","last_fullsized":"2023-09-23T22:39:30.690460+0000","mapping_epoch":495,"log_start":"461'1700","ondisk_log_start":"461'1700","created":451,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142540800,"num_objects":2175,"num_object_clones":0,"num_object_copies":6525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3858,"num_write_kb":193056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1407,"reported_epoch":527,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:28.286387+0000","last_change":"2023-09-23T22:39:05.299849+0000","last_active":"2023-09-23T22:39:28.286387+0000","last_peered":"2023-09-23T22:39:28.286387+0000","last_clean":"2023-09-23T22:39:01.247439+0000","last_became_active":"2023-09-23T22:39:05.299849+0000","last_became_peered":"2023-09-23T22:39:05.299849+0000","last_unstale":"2023-09-23T22:39:28.286387+0000","last_undegraded":"2023-09-23T22:39:05.286114+0000","last_fullsized":"2023-09-23T22:39:05.286006+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:43.518374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":751,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.294509+0000","last_change":"2023-09-23T22:38:55.391540+0000","last_active":"2023-09-23T22:39:29.294509+0000","last_peered":"2023-09-23T22:39:29.294509+0000","last_clean":"2023-09-23T22:35:16.261819+0000","last_became_active":"2023-09-23T22:38:55.391540+0000","last_became_peered":"2023-09-23T22:38:55.391540+0000","last_unstale":"2023-09-23T22:39:29.294509+0000","last_undegraded":"2023-09-23T22:39:29.294509+0000","last_fullsized":"2023-09-23T22:38:55.384846+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":276,"reported_epoch":527,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:28.289065+0000","last_change":"2023-09-23T22:39:05.288671+0000","last_active":"2023-09-23T22:38:54.401344+0000","last_peered":"2023-09-23T22:39:28.289065+0000","last_clean":"2023-09-23T22:38:48.357699+0000","last_became_active":"2023-09-23T22:36:36.407197+0000","last_became_peered":"2023-09-23T22:39:05.288671+0000","last_unstale":"2023-09-23T22:39:28.289065+0000","last_undegraded":"2023-09-23T22:39:28.289065+0000","last_fullsized":"2023-09-23T22:39:05.287235+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.518505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":236,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289402+0000","last_change":"2023-09-23T22:38:55.387206+0000","last_active":"2023-09-23T22:39:28.289402+0000","last_peered":"2023-09-23T22:39:28.289402+0000","last_clean":"2023-09-23T22:35:23.291202+0000","last_became_active":"2023-09-23T22:38:55.387206+0000","last_became_peered":"2023-09-23T22:38:55.387206+0000","last_unstale":"2023-09-23T22:39:28.289402+0000","last_undegraded":"2023-09-23T22:39:28.289402+0000","last_fullsized":"2023-09-23T22:38:55.380731+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:29.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"529'7765","reported_seq":7857,"reported_epoch":529,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:35.244005+0000","last_change":"2023-09-23T22:38:55.398058+0000","last_active":"2023-09-23T22:39:35.244005+0000","last_peered":"2023-09-23T22:39:35.244005+0000","last_clean":"2023-09-23T22:38:54.300575+0000","last_became_active":"2023-09-23T22:38:55.398058+0000","last_became_peered":"2023-09-23T22:38:55.398058+0000","last_unstale":"2023-09-23T22:39:35.244005+0000","last_undegraded":"2023-09-23T22:38:55.382246+0000","last_fullsized":"2023-09-23T22:38:55.382121+0000","mapping_epoch":513,"log_start":"461'1400","ondisk_log_start":"461'1400","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6365,"log_dups_size":0,"ondisk_log_size":6365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:50.649091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274530304,"num_objects":4189,"num_object_clones":0,"num_object_copies":12567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4189,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7765,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":4189}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":574,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289419+0000","last_change":"2023-09-23T22:38:55.397258+0000","last_active":"2023-09-23T22:39:28.289419+0000","last_peered":"2023-09-23T22:39:28.289419+0000","last_clean":"2023-09-23T22:28:58.571704+0000","last_became_active":"2023-09-23T22:38:55.397258+0000","last_became_peered":"2023-09-23T22:38:55.397258+0000","last_unstale":"2023-09-23T22:39:28.289419+0000","last_undegraded":"2023-09-23T22:39:28.289419+0000","last_fullsized":"2023-09-23T22:38:55.379621+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":933,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.289534+0000","last_change":"2023-09-23T22:37:42.457358+0000","last_active":"2023-09-23T22:39:28.289534+0000","last_peered":"2023-09-23T22:39:28.289534+0000","last_clean":"2023-09-23T22:39:28.289534+0000","last_became_active":"2023-09-23T22:37:42.437821+0000","last_became_peered":"2023-09-23T22:37:42.437821+0000","last_unstale":"2023-09-23T22:39:28.289534+0000","last_undegraded":"2023-09-23T22:39:28.289534+0000","last_fullsized":"2023-09-23T22:39:28.289534+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:17.445534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":287,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289013+0000","last_change":"2023-09-23T22:39:05.307919+0000","last_active":"2023-09-23T22:39:28.289013+0000","last_peered":"2023-09-23T22:39:28.289013+0000","last_clean":"2023-09-23T22:39:00.405520+0000","last_became_active":"2023-09-23T22:39:05.307919+0000","last_became_peered":"2023-09-23T22:39:05.307919+0000","last_unstale":"2023-09-23T22:39:28.289013+0000","last_undegraded":"2023-09-23T22:39:28.289013+0000","last_fullsized":"2023-09-23T22:39:05.287534+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:26.565338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":402,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.289926+0000","last_change":"2023-09-23T22:35:52.675187+0000","last_active":"2023-09-23T22:39:28.289926+0000","last_peered":"2023-09-23T22:39:28.289926+0000","last_clean":"2023-09-23T22:39:28.289926+0000","last_became_active":"2023-09-23T22:35:52.674444+0000","last_became_peered":"2023-09-23T22:35:52.674444+0000","last_unstale":"2023-09-23T22:39:28.289926+0000","last_undegraded":"2023-09-23T22:39:28.289926+0000","last_fullsized":"2023-09-23T22:39:28.289926+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:17.802244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.e","version":"528'7623","reported_seq":12978,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T22:39:32.024155+0000","last_change":"2023-09-23T22:37:46.112536+0000","last_active":"2023-09-23T22:39:32.024155+0000","last_peered":"2023-09-23T22:39:32.024155+0000","last_clean":"2023-09-23T22:39:32.024155+0000","last_became_active":"2023-09-23T22:37:42.470866+0000","last_became_peered":"2023-09-23T22:37:42.470866+0000","last_unstale":"2023-09-23T22:39:32.024155+0000","last_undegraded":"2023-09-23T22:39:32.024155+0000","last_fullsized":"2023-09-23T22:39:32.024155+0000","mapping_epoch":495,"log_start":"461'1915","ondisk_log_start":"461'1915","created":451,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5708,"log_dups_size":0,"ondisk_log_size":5708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:22.071126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280559616,"num_objects":4281,"num_object_clones":0,"num_object_copies":12843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7623,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2609,"num_bytes_recovered":166002688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":693,"reported_epoch":528,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:29.299468+0000","last_change":"2023-09-23T22:39:05.302767+0000","last_active":"2023-09-23T22:39:04.293769+0000","last_peered":"2023-09-23T22:39:29.299468+0000","last_clean":"2023-09-23T22:28:58.573726+0000","last_became_active":"2023-09-23T22:29:02.583400+0000","last_became_peered":"2023-09-23T22:39:05.302767+0000","last_unstale":"2023-09-23T22:39:29.299468+0000","last_undegraded":"2023-09-23T22:39:29.299468+0000","last_fullsized":"2023-09-23T22:39:05.302023+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":121,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T22:39:29.296397+0000","last_change":"2023-09-23T22:35:52.688113+0000","last_active":"2023-09-23T22:39:29.296397+0000","last_peered":"2023-09-23T22:39:29.296397+0000","last_clean":"2023-09-23T22:39:29.296397+0000","last_became_active":"2023-09-23T22:35:52.687708+0000","last_became_peered":"2023-09-23T22:35:52.687708+0000","last_unstale":"2023-09-23T22:39:29.296397+0000","last_undegraded":"2023-09-23T22:39:29.296397+0000","last_fullsized":"2023-09-23T22:39:29.296397+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:07.029149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":149,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.288086+0000","last_change":"2023-09-23T22:37:42.424518+0000","last_active":"2023-09-23T22:39:28.288086+0000","last_peered":"2023-09-23T22:39:28.288086+0000","last_clean":"2023-09-23T22:39:28.288086+0000","last_became_active":"2023-09-23T22:37:42.424144+0000","last_became_peered":"2023-09-23T22:37:42.424144+0000","last_unstale":"2023-09-23T22:39:28.288086+0000","last_undegraded":"2023-09-23T22:39:28.288086+0000","last_fullsized":"2023-09-23T22:39:28.288086+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:52:51.255509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":333,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.290233+0000","last_change":"2023-09-23T22:37:42.432892+0000","last_active":"2023-09-23T22:39:28.290233+0000","last_peered":"2023-09-23T22:39:28.290233+0000","last_clean":"2023-09-23T22:39:28.290233+0000","last_became_active":"2023-09-23T22:37:42.432356+0000","last_became_peered":"2023-09-23T22:37:42.432356+0000","last_unstale":"2023-09-23T22:39:28.290233+0000","last_undegraded":"2023-09-23T22:39:28.290233+0000","last_fullsized":"2023-09-23T22:39:28.290233+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:14.288445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.d","version":"529'7866","reported_seq":13318,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T22:39:34.893547+0000","last_change":"2023-09-23T22:37:42.930274+0000","last_active":"2023-09-23T22:39:34.893547+0000","last_peered":"2023-09-23T22:39:34.893547+0000","last_clean":"2023-09-23T22:39:34.893547+0000","last_became_active":"2023-09-23T22:37:42.448382+0000","last_became_peered":"2023-09-23T22:37:42.448382+0000","last_unstale":"2023-09-23T22:39:34.893547+0000","last_undegraded":"2023-09-23T22:39:34.893547+0000","last_fullsized":"2023-09-23T22:39:34.893547+0000","mapping_epoch":495,"log_start":"462'2607","ondisk_log_start":"462'2607","created":451,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5259,"log_dups_size":0,"ondisk_log_size":5259,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:20.785382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278331416,"num_objects":4248,"num_object_clones":0,"num_object_copies":12744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4248,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7866,"num_write_kb":387585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2659,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":898,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289654+0000","last_change":"2023-09-23T22:38:55.383034+0000","last_active":"2023-09-23T22:39:28.289654+0000","last_peered":"2023-09-23T22:39:28.289654+0000","last_clean":"2023-09-23T22:38:48.354625+0000","last_became_active":"2023-09-23T22:38:55.383034+0000","last_became_peered":"2023-09-23T22:38:55.383034+0000","last_unstale":"2023-09-23T22:39:28.289654+0000","last_undegraded":"2023-09-23T22:39:28.289654+0000","last_fullsized":"2023-09-23T22:38:55.380898+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":166,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:29.296671+0000","last_change":"2023-09-23T22:39:05.312608+0000","last_active":"2023-09-23T22:39:29.296671+0000","last_peered":"2023-09-23T22:39:29.296671+0000","last_clean":"2023-09-23T22:35:17.625026+0000","last_became_active":"2023-09-23T22:39:05.312608+0000","last_became_peered":"2023-09-23T22:39:05.312608+0000","last_unstale":"2023-09-23T22:39:29.296671+0000","last_undegraded":"2023-09-23T22:39:05.293584+0000","last_fullsized":"2023-09-23T22:39:05.293517+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:47.778707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":1029,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.290638+0000","last_change":"2023-09-23T22:38:55.388290+0000","last_active":"2023-09-23T22:39:28.290638+0000","last_peered":"2023-09-23T22:39:28.290638+0000","last_clean":"2023-09-23T22:38:48.410284+0000","last_became_active":"2023-09-23T22:38:55.388290+0000","last_became_peered":"2023-09-23T22:38:55.388290+0000","last_unstale":"2023-09-23T22:39:28.290638+0000","last_undegraded":"2023-09-23T22:39:28.290638+0000","last_fullsized":"2023-09-23T22:38:55.384357+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:40.082337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":51,"reported_epoch":528,"state":"active+undersized","last_fresh":"2023-09-23T22:39:29.294678+0000","last_change":"2023-09-23T22:39:05.312578+0000","last_active":"2023-09-23T22:39:29.294678+0000","last_peered":"2023-09-23T22:39:29.294678+0000","last_clean":"2023-09-23T22:31:26.131914+0000","last_became_active":"2023-09-23T22:39:05.312578+0000","last_became_peered":"2023-09-23T22:39:05.312578+0000","last_unstale":"2023-09-23T22:39:29.294678+0000","last_undegraded":"2023-09-23T22:39:29.294678+0000","last_fullsized":"2023-09-23T22:39:05.291104+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:52.614783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"528'7671","reported_seq":10642,"reported_epoch":528,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T22:39:32.010882+0000","last_change":"2023-09-23T22:39:18.546591+0000","last_active":"2023-09-23T22:39:32.010882+0000","last_peered":"2023-09-23T22:39:32.010882+0000","last_clean":"2023-09-23T22:35:27.348348+0000","last_became_active":"2023-09-23T22:39:06.327496+0000","last_became_peered":"2023-09-23T22:39:06.327496+0000","last_unstale":"2023-09-23T22:39:32.010882+0000","last_undegraded":"2023-09-23T22:39:06.297790+0000","last_fullsized":"2023-09-23T22:39:32.010882+0000","mapping_epoch":518,"log_start":"461'1968","ondisk_log_start":"461'1968","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:42.584301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279511040,"num_objects":4265,"num_object_clones":0,"num_object_copies":12795,"num_objects_missing_on_primary":1369,"num_objects_missing":1369,"num_objects_degraded":4265,"num_objects_misplaced":1369,"num_objects_unfound":0,"num_objects_dirty":4265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7671,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1129,"num_bytes_recovered":73990144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,0],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,4,7","objects":2894},{"shards":"0,7","objects":1369}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":646,"reported_epoch":528,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:29.296025+0000","last_change":"2023-09-23T22:39:05.295060+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:39:29.296025+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.295060+0000","last_unstale":"2023-09-23T22:39:29.296025+0000","last_undegraded":"2023-09-23T22:39:29.296025+0000","last_fullsized":"2023-09-23T22:39:05.292715+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":151,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.288321+0000","last_change":"2023-09-23T22:38:55.388004+0000","last_active":"2023-09-23T22:39:28.288321+0000","last_peered":"2023-09-23T22:39:28.288321+0000","last_clean":"2023-09-23T22:33:41.182644+0000","last_became_active":"2023-09-23T22:38:55.388004+0000","last_became_peered":"2023-09-23T22:38:55.388004+0000","last_unstale":"2023-09-23T22:39:28.288321+0000","last_undegraded":"2023-09-23T22:39:28.288321+0000","last_fullsized":"2023-09-23T22:38:55.379533+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:15.982785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":267,"reported_epoch":527,"state":"active+clean","last_fresh":"2023-09-23T22:39:28.291033+0000","last_change":"2023-09-23T22:37:42.460308+0000","last_active":"2023-09-23T22:39:28.291033+0000","last_peered":"2023-09-23T22:39:28.291033+0000","last_clean":"2023-09-23T22:39:28.291033+0000","last_became_active":"2023-09-23T22:37:42.438206+0000","last_became_peered":"2023-09-23T22:37:42.438206+0000","last_unstale":"2023-09-23T22:39:28.291033+0000","last_undegraded":"2023-09-23T22:39:28.291033+0000","last_fullsized":"2023-09-23T22:39:28.291033+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:54.493150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":406,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289572+0000","last_change":"2023-09-23T22:38:55.382690+0000","last_active":"2023-09-23T22:39:28.289572+0000","last_peered":"2023-09-23T22:39:28.289572+0000","last_clean":"2023-09-23T22:35:23.293559+0000","last_became_active":"2023-09-23T22:38:55.382690+0000","last_became_peered":"2023-09-23T22:38:55.382690+0000","last_unstale":"2023-09-23T22:39:28.289572+0000","last_undegraded":"2023-09-23T22:39:28.289572+0000","last_fullsized":"2023-09-23T22:38:55.380032+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:58.155649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.11","version":"529'7278","reported_seq":20797,"reported_epoch":529,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:39:34.794187+0000","last_change":"2023-09-23T22:39:29.299428+0000","last_active":"2023-09-23T22:39:34.794187+0000","last_peered":"2023-09-23T22:39:34.794187+0000","last_clean":"2023-09-23T22:35:27.126327+0000","last_became_active":"2023-09-23T22:39:29.299428+0000","last_became_peered":"2023-09-23T22:39:29.299428+0000","last_unstale":"2023-09-23T22:39:34.794187+0000","last_undegraded":"2023-09-23T22:39:34.794187+0000","last_fullsized":"2023-09-23T22:39:29.294272+0000","mapping_epoch":527,"log_start":"461'1200","ondisk_log_start":"461'1200","created":525,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6078,"log_dups_size":0,"ondisk_log_size":6078,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133070848,"num_objects":2031,"num_object_clones":0,"num_object_copies":6093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4062,"num_objects_unfound":0,"num_objects_dirty":2031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3698,"num_write_kb":183296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3344,"num_bytes_recovered":183861248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2031}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":184,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T22:39:29.300034+0000","last_change":"2023-09-23T22:35:52.675772+0000","last_active":"2023-09-23T22:39:29.300034+0000","last_peered":"2023-09-23T22:39:29.300034+0000","last_clean":"2023-09-23T22:39:29.300034+0000","last_became_active":"2023-09-23T22:35:52.675279+0000","last_became_peered":"2023-09-23T22:35:52.675279+0000","last_unstale":"2023-09-23T22:39:29.300034+0000","last_undegraded":"2023-09-23T22:39:29.300034+0000","last_fullsized":"2023-09-23T22:39:29.300034+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.513213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":155,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.288163+0000","last_change":"2023-09-23T22:39:05.291652+0000","last_active":"2023-09-23T22:39:28.288163+0000","last_peered":"2023-09-23T22:39:28.288163+0000","last_clean":"2023-09-23T22:39:01.247800+0000","last_became_active":"2023-09-23T22:39:05.291652+0000","last_became_peered":"2023-09-23T22:39:05.291652+0000","last_unstale":"2023-09-23T22:39:28.288163+0000","last_undegraded":"2023-09-23T22:39:28.288163+0000","last_fullsized":"2023-09-23T22:39:05.287811+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:03.849150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"528'7426","reported_seq":7691,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:32.311448+0000","last_change":"2023-09-23T22:39:27.381437+0000","last_active":"2023-09-23T22:39:32.311448+0000","last_peered":"2023-09-23T22:39:32.311448+0000","last_clean":"2023-09-23T22:37:14.247564+0000","last_became_active":"2023-09-23T22:39:27.381437+0000","last_became_peered":"2023-09-23T22:39:27.381437+0000","last_unstale":"2023-09-23T22:39:32.311448+0000","last_undegraded":"2023-09-23T22:39:27.350981+0000","last_fullsized":"2023-09-23T22:39:27.346140+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7426,"log_dups_size":0,"ondisk_log_size":7426,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139100160,"num_objects":2123,"num_object_clones":0,"num_object_copies":6369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2123,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3752,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":2123}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":553,"reported_epoch":528,"state":"active+clean","last_fresh":"2023-09-23T22:39:29.296509+0000","last_change":"2023-09-23T22:37:42.419112+0000","last_active":"2023-09-23T22:39:29.296509+0000","last_peered":"2023-09-23T22:39:29.296509+0000","last_clean":"2023-09-23T22:39:29.296509+0000","last_became_active":"2023-09-23T22:37:42.418593+0000","last_became_peered":"2023-09-23T22:37:42.418593+0000","last_unstale":"2023-09-23T22:39:29.296509+0000","last_undegraded":"2023-09-23T22:39:29.296509+0000","last_fullsized":"2023-09-23T22:39:29.296509+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:46.031051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":516,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289318+0000","last_change":"2023-09-23T22:39:05.290461+0000","last_active":"2023-09-23T22:39:28.289318+0000","last_peered":"2023-09-23T22:39:28.289318+0000","last_clean":"2023-09-23T22:37:07.670799+0000","last_became_active":"2023-09-23T22:39:05.290461+0000","last_became_peered":"2023-09-23T22:39:05.290461+0000","last_unstale":"2023-09-23T22:39:28.289318+0000","last_undegraded":"2023-09-23T22:39:28.289318+0000","last_fullsized":"2023-09-23T22:39:05.285092+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:31.927928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.17","version":"528'7542","reported_seq":8687,"reported_epoch":528,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:39:32.346885+0000","last_change":"2023-09-23T22:39:29.314622+0000","last_active":"2023-09-23T22:39:32.346885+0000","last_peered":"2023-09-23T22:39:32.346885+0000","last_clean":"2023-09-23T22:36:35.361199+0000","last_became_active":"2023-09-23T22:39:29.302388+0000","last_became_peered":"2023-09-23T22:39:29.302388+0000","last_unstale":"2023-09-23T22:39:32.346885+0000","last_undegraded":"2023-09-23T22:39:29.295427+0000","last_fullsized":"2023-09-23T22:39:32.346885+0000","mapping_epoch":527,"log_start":"461'893","ondisk_log_start":"461'893","created":525,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6649,"log_dups_size":0,"ondisk_log_size":6649,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139165696,"num_objects":2124,"num_object_clones":0,"num_object_copies":6372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":272,"num_objects_misplaced":1852,"num_objects_unfound":0,"num_objects_dirty":2124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3816,"num_write_kb":190048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":254,"num_bytes_recovered":16613376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,1],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"1,2,7","objects":2124}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":568,"reported_epoch":528,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:29.296547+0000","last_change":"2023-09-23T22:38:55.382794+0000","last_active":"2023-09-23T22:39:29.296547+0000","last_peered":"2023-09-23T22:39:29.296547+0000","last_clean":"2023-09-23T22:38:48.353999+0000","last_became_active":"2023-09-23T22:38:55.382794+0000","last_became_peered":"2023-09-23T22:38:55.382794+0000","last_unstale":"2023-09-23T22:39:29.296547+0000","last_undegraded":"2023-09-23T22:38:55.381634+0000","last_fullsized":"2023-09-23T22:38:55.381551+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:52.591257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":293,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.289542+0000","last_change":"2023-09-23T22:39:05.307911+0000","last_active":"2023-09-23T22:39:28.289542+0000","last_peered":"2023-09-23T22:39:28.289542+0000","last_clean":"2023-09-23T22:35:23.286304+0000","last_became_active":"2023-09-23T22:39:05.307911+0000","last_became_peered":"2023-09-23T22:39:05.307911+0000","last_unstale":"2023-09-23T22:39:28.289542+0000","last_undegraded":"2023-09-23T22:39:28.289542+0000","last_fullsized":"2023-09-23T22:39:05.287595+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:03.410439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"528'7520","reported_seq":12421,"reported_epoch":528,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:39:30.732564+0000","last_change":"2023-09-23T22:39:29.338780+0000","last_active":"2023-09-23T22:39:30.732564+0000","last_peered":"2023-09-23T22:39:30.732564+0000","last_clean":"2023-09-23T22:39:30.732564+0000","last_became_active":"2023-09-23T22:39:29.325920+0000","last_became_peered":"2023-09-23T22:39:29.325920+0000","last_unstale":"2023-09-23T22:39:30.732564+0000","last_undegraded":"2023-09-23T22:39:30.732564+0000","last_fullsized":"2023-09-23T22:39:30.732564+0000","mapping_epoch":527,"log_start":"461'1500","ondisk_log_start":"461'1500","created":525,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":6020,"log_dups_size":0,"ondisk_log_size":6020,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142213120,"num_objects":2170,"num_object_clones":0,"num_object_copies":6510,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2170,"num_objects_unfound":0,"num_objects_dirty":2170,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3779,"num_write_kb":190368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1257,"num_bytes_recovered":79364096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":2170}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":126,"reported_epoch":527,"state":"undersized+peered","last_fresh":"2023-09-23T22:39:28.286814+0000","last_change":"2023-09-23T22:39:05.288020+0000","last_active":"2023-09-23T22:33:45.291092+0000","last_peered":"2023-09-23T22:39:28.286814+0000","last_clean":"2023-09-23T22:33:41.181859+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:39:05.288020+0000","last_unstale":"2023-09-23T22:39:28.286814+0000","last_undegraded":"2023-09-23T22:39:28.286814+0000","last_fullsized":"2023-09-23T22:39:05.287481+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.969554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":99,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.286565+0000","last_change":"2023-09-23T22:39:05.309675+0000","last_active":"2023-09-23T22:39:28.286565+0000","last_peered":"2023-09-23T22:39:28.286565+0000","last_clean":"2023-09-23T22:34:30.962054+0000","last_became_active":"2023-09-23T22:39:05.309675+0000","last_became_peered":"2023-09-23T22:39:05.309675+0000","last_unstale":"2023-09-23T22:39:28.286565+0000","last_undegraded":"2023-09-23T22:39:28.286565+0000","last_fullsized":"2023-09-23T22:39:05.286965+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:26.850825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"529'7515","reported_seq":7778,"reported_epoch":529,"state":"active+clean","last_fresh":"2023-09-23T22:39:35.175898+0000","last_change":"2023-09-23T22:39:27.419613+0000","last_active":"2023-09-23T22:39:35.175898+0000","last_peered":"2023-09-23T22:39:35.175898+0000","last_clean":"2023-09-23T22:39:35.175898+0000","last_became_active":"2023-09-23T22:39:27.409135+0000","last_became_peered":"2023-09-23T22:39:27.409135+0000","last_unstale":"2023-09-23T22:39:35.175898+0000","last_undegraded":"2023-09-23T22:39:35.175898+0000","last_fullsized":"2023-09-23T22:39:35.175898+0000","mapping_epoch":525,"log_start":"458'200","ondisk_log_start":"458'200","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7315,"log_dups_size":0,"ondisk_log_size":7315,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134578176,"num_objects":2054,"num_object_clones":0,"num_object_copies":6162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3818,"num_write_kb":187872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":478,"reported_epoch":527,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:39:28.288482+0000","last_change":"2023-09-23T22:38:55.394101+0000","last_active":"2023-09-23T22:39:28.288482+0000","last_peered":"2023-09-23T22:39:28.288482+0000","last_clean":"2023-09-23T22:38:48.446310+0000","last_became_active":"2023-09-23T22:38:55.394101+0000","last_became_peered":"2023-09-23T22:38:55.394101+0000","last_unstale":"2023-09-23T22:39:28.288482+0000","last_undegraded":"2023-09-23T22:38:55.380019+0000","last_fullsized":"2023-09-23T22:38:55.379907+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:57.436984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":611,"reported_epoch":527,"state":"active+undersized","last_fresh":"2023-09-23T22:39:28.288711+0000","last_change":"2023-09-23T22:38:55.384159+0000","last_active":"2023-09-23T22:39:28.288711+0000","last_peered":"2023-09-23T22:39:28.288711+0000","last_clean":"2023-09-23T22:38:48.354354+0000","last_became_active":"2023-09-23T22:38:55.384159+0000","last_became_peered":"2023-09-23T22:38:55.384159+0000","last_unstale":"2023-09-23T22:39:28.288711+0000","last_undegraded":"2023-09-23T22:39:28.288711+0000","last_fullsized":"2023-09-23T22:38:55.378819+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:01.489638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":66,"acting":66,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":31744,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48807,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":66,"acting":66,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1211762,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":38,"acting":39,"num_store_stats":8},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":4435214360,"num_objects":67677,"num_object_clones":0,"num_object_copies":203031,"num_objects_missing_on_primary":1450,"num_objects_missing":1450,"num_objects_degraded":26123,"num_objects_misplaced":30066,"num_objects_unfound":0,"num_objects_dirty":67677,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":121777,"num_write_kb":6062465,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25431,"num_bytes_recovered":1381564416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14105063424,"data_stored":14105051208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":165252,"ondisk_log_size":165252,"up":56,"acting":68,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":314,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":19,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234422,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3417708,"kb_used_data":2951332,"kb_used_omap":0,"kb_used_meta":466368,"kb_avail":90327444,"statfs":{"total":95995035648,"available":92495302656,"internally_reserved":0,"allocated":3022163968,"data_stored":3020392812,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":477560832},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:33 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.4870000000000001,"15min":3.7999999999999998},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":2.2629999999999999,"5min":49.024999999999999,"15min":220.66},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.4990000000000001,"15min":4.0609999999999999},"min":{"1min":0.24199999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.6379999999999999,"5min":48.901000000000003,"15min":220.59899999999999},"last":0.26600000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":3.4700000000000002,"5min":3.4700000000000002,"15min":3.4700000000000002},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.9500000000000002,"5min":2.9500000000000002,"15min":2.9500000000000002},"last":1.2230000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:39:10 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.6870000000000001,"15min":1.5980000000000001},"min":{"1min":0.28199999999999997,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":3.9889999999999999,"5min":52.622,"15min":70.183999999999997},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.62,"5min":2.0219999999999998,"15min":1.835},"min":{"1min":0.307,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.0110000000000001,"5min":52.744,"15min":70.140000000000001},"last":1.2689999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.2969999999999999,"15min":2.5390000000000001},"min":{"1min":0.35999999999999999,"5min":0.316,"15min":0.26900000000000002},"max":{"1min":3.5710000000000002,"5min":9.8989999999999991,"15min":219.90700000000001},"last":0.435},{"interface":"front","average":{"1min":0.82699999999999996,"5min":2.101,"15min":2.9540000000000002},"min":{"1min":0.38,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":3.3719999999999999,"5min":48.866999999999997,"15min":219.976},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.42699999999999999,"5min":0.65100000000000002,"15min":0.72299999999999998},"min":{"1min":0.27900000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.68000000000000005,"5min":2.8090000000000002,"15min":2.8090000000000002},"last":1.2509999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":1.8680000000000001,"15min":1.8220000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":2.2509999999999999,"5min":48.960000000000001,"15min":48.960000000000001},"last":0.34399999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:39:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.8420000000000001,"15min":1.756},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":1.794,"5min":53.720999999999997,"15min":125.083},"last":1.284},{"interface":"front","average":{"1min":0.503,"5min":1.95,"15min":1.7969999999999999},"min":{"1min":0.311,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.716,"5min":53.218000000000004,"15min":122.351},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 22:38:22 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":2.5499999999999998,"15min":1.389},"min":{"1min":0.32000000000000001,"5min":0.311,"15min":0.29499999999999998},"max":{"1min":0.77500000000000002,"5min":88.284999999999997,"15min":88.284999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.623,"5min":2.4009999999999998,"15min":1.4339999999999999},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.33700000000000002},"max":{"1min":2.3050000000000002,"5min":88.355000000000004,"15min":88.355000000000004},"last":0.54900000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496049,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2171296,"kb_used_data":1893144,"kb_used_omap":0,"kb_used_meta":278144,"kb_avail":91573856,"statfs":{"total":95995035648,"available":93771628544,"internally_reserved":0,"allocated":1938579456,"data_stored":1936876687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":284819456},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":4.9400000000000004,"15min":4.5979999999999999},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":0.623,"5min":173.291,"15min":213.23699999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":5.2850000000000001,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.22600000000000001},"max":{"1min":0.59799999999999998,"5min":174.232,"15min":213.304},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.4570000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:38:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":3.8730000000000002,"15min":2.5680000000000001},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.2770000000000001,"5min":103.548,"15min":103.548},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":3.8069999999999999,"15min":2.657},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.583,"5min":105.99299999999999,"15min":105.99299999999999},"last":2.5049999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:37:58 2023","interfaces":[{"interface":"back","average":{"1min":4.3259999999999996,"5min":2.2690000000000001,"15min":2.9209999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":82.584000000000003,"5min":82.584000000000003,"15min":125.125},"last":0.58399999999999996},{"interface":"front","average":{"1min":4.6360000000000001,"5min":2.2040000000000002,"15min":2.5070000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":82.316000000000003,"5min":82.316000000000003,"15min":82.316000000000003},"last":2.3290000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:37:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.99099999999999999,"15min":1.042},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.5099999999999998,"5min":7.5099999999999998,"15min":7.5099999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.4299999999999999,"15min":1.5589999999999999},"min":{"1min":0.309,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0569999999999999,"5min":6.3490000000000002,"15min":6.3490000000000002},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 22:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.7110000000000001,"15min":1.6779999999999999},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.4359999999999999,"5min":15.032999999999999,"15min":88.194999999999993},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.1279999999999999,"15min":1.359},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69499999999999995,"5min":7.0839999999999996,"15min":88.025000000000006},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.117,"15min":1.353},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.314},"max":{"1min":0.71199999999999997,"5min":7.5670000000000002,"15min":82.801000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.2649999999999999,"15min":1.508},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.29699999999999999},"max":{"1min":0.65400000000000003,"5min":13.465999999999999,"15min":83.052999999999997},"last":0.65800000000000003}]}]},{"osd":1,"up_from":495,"seq":2126008811543,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3557244,"kb_used_data":3392372,"kb_used_omap":12,"kb_used_meta":164851,"kb_avail":90187908,"statfs":{"total":95995035648,"available":92352417792,"internally_reserved":0,"allocated":3473788928,"data_stored":3471053125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":168808110},"hb_peers":[0,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,5,1],"upper_bound":4096},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":2.2189999999999999,"5min":2.2189999999999999,"15min":2.2189999999999999},"last":0.23000000000000001},{"interface":"front","average":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":0.90100000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.5860000000000003,"5min":5.5860000000000003,"15min":5.5860000000000003},"last":0.24199999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.8370000000000002,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":0.26600000000000001},{"interface":"front","average":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":2.1840000000000002,"5min":2.1840000000000002,"15min":2.1840000000000002},"last":0.28399999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.6390000000000002,"5min":5.6390000000000002,"15min":5.6390000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.032,"15min":1.032},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.5739999999999998,"5min":4.5739999999999998,"15min":4.5739999999999998},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.90600000000000003,"5min":0.90600000000000003,"15min":0.90600000000000003},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.0009999999999999,"15min":1.0009999999999999},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":5.6740000000000004,"5min":5.6740000000000004,"15min":5.6740000000000004},"last":0.33500000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":0.85499999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.23,"5min":2.23,"15min":2.23},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.087,"5min":1.087,"15min":1.087},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.503},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":5.7050000000000001,"5min":5.7050000000000001,"15min":5.7050000000000001},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.444,"5min":2.444,"15min":2.444},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.111,"5min":1.111,"15min":1.111},"last":0.38}]}]},{"osd":0,"up_from":93,"seq":399431958859,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2497152,"kb_used_data":1609464,"kb_used_omap":8,"kb_used_meta":887671,"kb_avail":91248000,"statfs":{"total":95995035648,"available":93437952000,"internally_reserved":0,"allocated":1648091136,"data_stored":1645297662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":908975978},"hb_peers":[1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":3.8599999999999999,"5min":3.8599999999999999,"15min":3.8599999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.6539999999999999,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 23 22:39:15 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":3.972,"15min":3.004},"min":{"1min":0.23499999999999999,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":2.5569999999999999,"5min":175.89400000000001,"15min":175.89400000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":3.7879999999999998,"15min":2.2669999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.827,"5min":175.66300000000001,"15min":175.66300000000001},"last":0.34999999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.024,"15min":2.4399999999999999},"min":{"1min":0.28799999999999998,"5min":0.23100000000000001,"15min":0.22500000000000001},"max":{"1min":1.698,"5min":7.0579999999999998,"15min":146.87299999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.149,"15min":2.516},"min":{"1min":0.311,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.1459999999999999,"5min":10.026,"15min":145.48099999999999},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":1.2010000000000001,"15min":1.262},"min":{"1min":0.32700000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.1899999999999999,"5min":9.968,"15min":9.968},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.2430000000000001,"15min":1.2989999999999999},"min":{"1min":0.378,"5min":0.373,"15min":0.373},"max":{"1min":3.25,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":0.44900000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":3.351,"15min":2.5129999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1459999999999999,"5min":126.872,"15min":126.872},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.501,"5min":1.8959999999999999,"15min":1.6910000000000001},"min":{"1min":0.33000000000000002,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":1.4099999999999999,"5min":126.521,"15min":126.521},"last":0.79900000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:38:23 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":2.25,"15min":2.3490000000000002},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.308},"max":{"1min":4.1280000000000001,"5min":74.355000000000004,"15min":74.355000000000004},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.8320000000000001,"15min":1.7529999999999999},"min":{"1min":0.39100000000000001,"5min":0.38400000000000001,"15min":0.35099999999999998},"max":{"1min":1.7330000000000001,"5min":73.150000000000006,"15min":73.150000000000006},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1659999999999999,"15min":1.4890000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.29999999999999999},"max":{"1min":3.218,"5min":6.2320000000000002,"15min":22.338999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.367,"15min":1.786},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.307},"max":{"1min":4.0979999999999999,"5min":6.8710000000000004,"15min":10.901},"last":0.58099999999999996}]}]},{"osd":2,"up_from":349,"seq":1498943586422,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1599796,"kb_used_data":1363756,"kb_used_omap":15,"kb_used_meta":236016,"kb_avail":92145356,"statfs":{"total":95995035648,"available":94356844544,"internally_reserved":0,"allocated":1396486144,"data_stored":1393813576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":241681142},"hb_peers":[0,1,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":2.0800000000000001,"15min":10.228999999999999},"min":{"1min":0.23499999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":1.5649999999999999,"5min":86.620000000000005,"15min":194.06299999999999},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.73199999999999998,"5min":2.464,"15min":11.512},"min":{"1min":0.22,"5min":0.22,"15min":0.215},"max":{"1min":3.9609999999999999,"5min":86.355000000000004,"15min":196.947},"last":1.8280000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.4350000000000001,"5min":3.4350000000000001,"15min":3.4350000000000001},"last":0.72699999999999998},{"interface":"front","average":{"1min":1.097,"5min":1.097,"15min":1.097},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":4.9829999999999997,"5min":4.9829999999999997,"15min":4.9829999999999997},"last":0.504}]},{"osd":3,"last update":"Sat Sep 23 22:38:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.6870000000000001,"15min":1.5680000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.0309999999999997,"5min":15.585000000000001,"15min":15.585000000000001},"last":0.745},{"interface":"front","average":{"1min":0.502,"5min":1.3320000000000001,"15min":1.0289999999999999},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0609999999999999,"5min":14.128,"15min":14.128},"last":0.629}]},{"osd":4,"last update":"Sat Sep 23 22:38:58 2023","interfaces":[{"interface":"back","average":{"1min":1.163,"5min":1.7649999999999999,"15min":1.917},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.4130000000000003,"5min":11.669,"15min":11.669},"last":0.629},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.073,"15min":1.173},"min":{"1min":0.36799999999999999,"5min":0.311,"15min":0.311},"max":{"1min":2.661,"5min":5.4969999999999999,"15min":5.4969999999999999},"last":0.748}]},{"osd":5,"last update":"Sat Sep 23 22:39:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":4.2830000000000004,"15min":3.3860000000000001},"min":{"1min":0.40300000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.4560000000000004,"5min":163.56399999999999,"15min":163.56399999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.57199999999999995,"5min":3.7200000000000002,"15min":2.9049999999999998},"min":{"1min":0.39000000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.9159999999999999,"5min":163.541,"15min":163.541},"last":0.70199999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":2.851,"15min":1.958},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":2.0979999999999999,"5min":169.798,"15min":169.798},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.169,"5min":3.7669999999999999,"15min":2.411},"min":{"1min":0.42399999999999999,"5min":0.373,"15min":0.373},"max":{"1min":7.1130000000000004,"5min":168.44200000000001,"15min":168.44200000000001},"last":1.49}]},{"osd":7,"last update":"Sat Sep 23 22:39:09 2023","interfaces":[{"interface":"back","average":{"1min":0.84099999999999997,"5min":2.2890000000000001,"15min":1.6719999999999999},"min":{"1min":0.40999999999999998,"5min":0.308,"15min":0.308},"max":{"1min":3.996,"5min":108.113,"15min":108.113},"last":0.437},{"interface":"front","average":{"1min":1.218,"5min":3.48,"15min":2.0329999999999999},"min":{"1min":0.42399999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":4.5039999999999996,"5min":108.093,"15min":108.093},"last":0.60199999999999998}]}]},{"osd":3,"up_from":108,"seq":463856468266,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1136452,"kb_used_data":582012,"kb_used_omap":4,"kb_used_meta":554427,"kb_avail":92608700,"statfs":{"total":95995035648,"available":94831308800,"internally_reserved":0,"allocated":595980288,"data_stored":593258702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":567733805},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.2639999999999998,"15min":3.7320000000000002},"min":{"1min":0.218,"5min":0.192,"15min":0.192},"max":{"1min":31.917000000000002,"5min":84.239000000000004,"15min":124.282},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.9399999999999999,"5min":2.9689999999999999,"15min":4.0309999999999997},"min":{"1min":0.28499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":31.984000000000002,"5min":84.188000000000002,"15min":124.30500000000001},"last":0.89400000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.4650000000000001,"5min":1.4650000000000001,"15min":1.4650000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.9570000000000001,"5min":1.9570000000000001,"15min":1.9570000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":31.856999999999999,"5min":31.856999999999999,"15min":31.856999999999999},"last":0.84899999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:39:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":3.048,"15min":2.3900000000000001},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":31.968,"5min":157.571,"15min":157.571},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.1269999999999998,"5min":3.6549999999999998,"15min":2.423},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.223},"max":{"1min":31.899999999999999,"5min":158.33000000000001,"15min":158.33000000000001},"last":0.96199999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.4059999999999999,"15min":1.3580000000000001},"min":{"1min":0.34999999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":32.015000000000001,"5min":32.015000000000001,"15min":32.015000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.04,"5min":1.3680000000000001,"15min":1.2929999999999999},"min":{"1min":0.34999999999999998,"5min":0.31,"15min":0.31},"max":{"1min":32.094999999999999,"5min":32.094999999999999,"15min":32.094999999999999},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":2.097,"15min":2.2320000000000002},"min":{"1min":0.32500000000000001,"5min":0.318,"15min":0.315},"max":{"1min":51.286999999999999,"5min":51.286999999999999,"15min":51.286999999999999},"last":0.92800000000000005},{"interface":"front","average":{"1min":2.9670000000000001,"5min":2.077,"15min":2.2679999999999998},"min":{"1min":0.39000000000000001,"5min":0.36299999999999999,"15min":0.29099999999999998},"max":{"1min":51.270000000000003,"5min":51.270000000000003,"15min":51.270000000000003},"last":0.81699999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:37:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.21,"15min":1.9670000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34200000000000003},"max":{"1min":7.5140000000000002,"5min":24.957000000000001,"15min":24.957000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.0379999999999998,"15min":2.206},"min":{"1min":0.376,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.4509999999999996,"5min":18.387,"15min":75.564999999999998},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.694,"15min":2.1110000000000002},"min":{"1min":0.436,"5min":0.38600000000000001,"15min":0.32400000000000001},"max":{"1min":32.040999999999997,"5min":32.040999999999997,"15min":75.525000000000006},"last":0.77700000000000002},{"interface":"front","average":{"1min":2.012,"5min":1.5600000000000001,"15min":1.804},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.379},"max":{"1min":32.058,"5min":32.058,"15min":32.058},"last":0.49199999999999999}]}]},{"osd":4,"up_from":465,"seq":1997159792685,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1373924,"kb_used_data":1295068,"kb_used_omap":12,"kb_used_meta":78835,"kb_avail":92371228,"statfs":{"total":95995035648,"available":94588137472,"internally_reserved":0,"allocated":1326149632,"data_stored":1324378996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":80727665},"hb_peers":[0,1,2,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.34799999999999998,"5min":0.98699999999999999,"15min":1.2549999999999999},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":0.56100000000000005,"5min":4.9909999999999997,"15min":4.9909999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.006,"15min":1.0780000000000001},"min":{"1min":0.27000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.3940000000000001,"5min":6.7839999999999998,"15min":6.7839999999999998},"last":0.253}]},{"osd":1,"last update":"Sat Sep 23 22:38:41 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":2.7549999999999999,"5min":2.7549999999999999,"15min":2.7549999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":3.6059999999999999,"5min":3.6059999999999999,"15min":3.6059999999999999},"last":0.378}]},{"osd":2,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.2050000000000001,"15min":1.3939999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.0019999999999998,"5min":4.8540000000000001,"15min":4.8540000000000001},"last":0.314},{"interface":"front","average":{"1min":0.70399999999999996,"5min":1.5760000000000001,"15min":1.798},"min":{"1min":0.33000000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.968,"5min":15.632999999999999,"15min":15.632999999999999},"last":0.36299999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.1339999999999999,"15min":1.2869999999999999},"min":{"1min":0.34699999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.9489999999999998,"5min":4.766,"15min":4.766},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.8839999999999999,"15min":2.407},"min":{"1min":0.36099999999999999,"5min":0.247,"15min":0.247},"max":{"1min":2.327,"5min":8.141,"15min":8.141},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.135,"15min":1.4359999999999999},"min":{"1min":0.28399999999999997,"5min":0.248,"15min":0.248},"max":{"1min":0.97499999999999998,"5min":5.5140000000000002,"15min":5.5140000000000002},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.1319999999999999,"15min":1.3720000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":2.9740000000000002,"5min":7.5999999999999996,"15min":7.5999999999999996},"last":0.29599999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:37:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":4.944,"15min":5.6680000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.343,"5min":104.521,"15min":104.521},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.58199999999999996,"5min":4.9610000000000003,"15min":5.6909999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.7969999999999999,"5min":101.419,"15min":101.419},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.333,"15min":1.649},"min":{"1min":0.32100000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.4690000000000003,"5min":8.1950000000000003,"15min":8.1950000000000003},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.6990000000000001,"15min":2.181},"min":{"1min":0.35699999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.4820000000000002,"5min":8.6140000000000008,"15min":8.6140000000000008},"last":0.34999999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757695,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1283028,"kb_used_data":803148,"kb_used_omap":0,"kb_used_meta":479872,"kb_avail":92462124,"statfs":{"total":95995035648,"available":94681214976,"internally_reserved":0,"allocated":822423552,"data_stored":820672340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":491388928},"hb_peers":[0,1,2,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":3.6869999999999998,"15min":5.4969999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.66700000000000004,"5min":145.46600000000001,"15min":163.54499999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":4.2610000000000001,"15min":5.3380000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":2.9550000000000001,"5min":144.429,"15min":162.81800000000001},"last":0.29599999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":3.0099999999999998,"5min":3.0099999999999998,"15min":3.0099999999999998},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.696,"5min":1.696,"15min":1.696},"last":0.26500000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:39:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1060000000000001,"5min":2.9470000000000001,"15min":2.419},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.8650000000000002,"5min":70.153000000000006,"15min":70.153000000000006},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.498,"5min":2.7250000000000001,"15min":2.4340000000000002},"min":{"1min":0.32000000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.0680000000000001,"5min":70.078999999999994,"15min":70.078999999999994},"last":1.278}]},{"osd":3,"last update":"Sat Sep 23 22:38:47 2023","interfaces":[{"interface":"back","average":{"1min":13.23,"5min":4.5650000000000004,"15min":4.0369999999999999},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":98.882000000000005,"5min":98.882000000000005,"15min":132.59},"last":0.39400000000000002},{"interface":"front","average":{"1min":13.026999999999999,"5min":4.0209999999999999,"15min":3.8980000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29299999999999998},"max":{"1min":98.856999999999999,"5min":98.856999999999999,"15min":129.381},"last":0.30599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:38:57 2023","interfaces":[{"interface":"back","average":{"1min":0.40899999999999997,"5min":2.6619999999999999,"15min":3.6619999999999999},"min":{"1min":0.26700000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.032,"5min":61.173999999999999,"15min":61.173999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.48999999999999999,"5min":2.7290000000000001,"15min":3.7229999999999999},"min":{"1min":0.376,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":1.0589999999999999,"5min":61.203000000000003,"15min":61.203000000000003},"last":0.379}]},{"osd":6,"last update":"Sat Sep 23 22:38:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.9319999999999999,"15min":1.4990000000000001},"min":{"1min":0.27000000000000002,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.375,"5min":56.667000000000002,"15min":70.581999999999994},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.71,"15min":1.478},"min":{"1min":0.35399999999999998,"5min":0.29099999999999998,"15min":0.248},"max":{"1min":1.1240000000000001,"5min":57.185000000000002,"15min":71.093000000000004},"last":0.53400000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.3540000000000001,"15min":1.256},"min":{"1min":0.31,"5min":0.31,"15min":0.28000000000000003},"max":{"1min":1.0880000000000001,"5min":59.101999999999997,"15min":59.101999999999997},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.51200000000000001,"5min":1.4670000000000001,"15min":1.3959999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0739999999999998,"5min":59.151000000000003,"15min":59.151000000000003},"last":0.35699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1631387648,"data_stored":1631387648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3451916288,"data_stored":3451912216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1380384768,"data_stored":1380384768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":595787776,"data_stored":595787776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1307709440,"data_stored":1307705368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":810811392,"data_stored":810811392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1928134656,"data_stored":1928134656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2998931456,"data_stored":2998927384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:39:38.911 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T22:39:38.912 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:38.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:38.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:39.016 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:39.023 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:39.063 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.158 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:39.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:39.263 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:39.293 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.345 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:39.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:39 smithi026 ceph-mon[223014]: pgmap v1373: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 25436/198126 objects degraded (12.838%); 29381/198126 objects misplaced (14.829%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:39:39.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:39.431 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:39.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:39 smithi026 ceph-mon[215775]: pgmap v1373: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 25436/198126 objects degraded (12.838%); 29381/198126 objects misplaced (14.829%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:39:39.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:39.432 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:39.453 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:39.490 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.549 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:39.556 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:39 smithi070 ceph-mon[192652]: pgmap v1373: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 560 op/s; 25436/198126 objects degraded (12.838%); 29381/198126 objects misplaced (14.829%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T22:39:39.556 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:39.557 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:39.657 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:39.708 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:39:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:39:39] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-09-23T22:39:39.716 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:39.834 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:39.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:39.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:39.938 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:39.966 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:40.009 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:40.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:40.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:40.120 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:40.151 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:40.395 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:39:40.399 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:40.423 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:40.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4261247707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:40.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:40 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:40.426 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4261247707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:40.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:40 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:40.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:40.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:40.451 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:40.528 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:40.569 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:40.626 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:40.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:40.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:40.732 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:40.763 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:40.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4261247707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:40.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:40 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:39:41.231 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:41.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:41.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:41.334 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:41.364 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:41.419 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:41.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:41.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:41.483 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[215775]: pgmap v1374: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 25436/198126 objects degraded (12.838%); 29381/198126 objects misplaced (14.829%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T22:39:41.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:41.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[215775]: failed to encode map e530 with expected crc 2023-09-23T22:39:41.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[215775]: failed to encode map e530 with expected crc 2023-09-23T22:39:41.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[215775]: failed to encode map e530 with expected crc 2023-09-23T22:39:41.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:39:41.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[215775]: osdmap e530: 8 total, 6 up, 8 in 2023-09-23T22:39:41.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[223014]: pgmap v1374: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 25436/198126 objects degraded (12.838%); 29381/198126 objects misplaced (14.829%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T22:39:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[223014]: failed to encode map e530 with expected crc 2023-09-23T22:39:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[223014]: failed to encode map e530 with expected crc 2023-09-23T22:39:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[223014]: failed to encode map e530 with expected crc 2023-09-23T22:39:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:39:41.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:41 smithi026 ceph-mon[223014]: osdmap e530: 8 total, 6 up, 8 in 2023-09-23T22:39:41.526 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:41.563 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:41.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:41 smithi070 ceph-mon[192652]: pgmap v1374: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 25436/198126 objects degraded (12.838%); 29381/198126 objects misplaced (14.829%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T22:39:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:41 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:41 smithi070 ceph-mon[192652]: failed to encode map e530 with expected crc 2023-09-23T22:39:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:41 smithi070 ceph-mon[192652]: failed to encode map e530 with expected crc 2023-09-23T22:39:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:41 smithi070 ceph-mon[192652]: failed to encode map e530 with expected crc 2023-09-23T22:39:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:41 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:39:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:41 smithi070 ceph-mon[192652]: osdmap e530: 8 total, 6 up, 8 in 2023-09-23T22:39:42.097 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:39:42.139 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:39:42.171 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:39:42.203 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:39:42.226 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:39:42.252 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:39:42.253 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:39:42.253 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:39:42.253 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:39:42.253 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:39:42.253 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:39:42.253 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:39:42.253 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:39:42.253 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:39:42.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:42 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1636951533' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:42.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:42 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1636951533' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:42.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:42 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1636951533' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:42.992 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:42.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:42.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:43.096 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:43.134 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:43.182 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:43.183 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:43.338 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:43.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:43.443 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:43.461 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:39:43.468 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:43.517 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:43.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:43.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:43.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:43.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:43 smithi026 ceph-mon[223014]: pgmap v1376: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 24838/192669 objects degraded (12.892%); 28483/192669 objects misplaced (14.783%); 871 KiB/s, 13 objects/s recovering 2023-09-23T22:39:43.686 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 24838/192669 objects degraded (12.892%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:43.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:43 smithi026 ceph-mon[215775]: pgmap v1376: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 24838/192669 objects degraded (12.892%); 28483/192669 objects misplaced (14.783%); 871 KiB/s, 13 objects/s recovering 2023-09-23T22:39:43.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 24838/192669 objects degraded (12.892%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:43.690 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:43 smithi070 ceph-mon[192652]: pgmap v1376: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 569 op/s; 24838/192669 objects degraded (12.892%); 28483/192669 objects misplaced (14.783%); 871 KiB/s, 13 objects/s recovering 2023-09-23T22:39:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 24838/192669 objects degraded (12.892%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:43.935 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:43.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:43.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:44.042 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:44.094 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:44.103 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:44.175 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:44.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:44.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:44.281 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:44.314 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:44.383 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:44.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:44.429 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pgp_num to 21 2023-09-23T22:39:44.496 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:44.527 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:44.576 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:44.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:44.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:44.581 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:44 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/914990469' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T22:39:44.583 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:44 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/914990469' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T22:39:44.583 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:44 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/914990469' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T22:39:44.602 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:44.602 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:44.681 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:44.709 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:44.757 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:44.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:44.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:44.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:44.895 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:44.940 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:44.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:44.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:45.045 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:45.082 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:45.184 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:39:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[215775]: pgmap v1377: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 24517/190794 objects degraded (12.850%); 28168/190794 objects misplaced (14.764%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:39:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[215775]: failed to encode map e531 with expected crc 2023-09-23T22:39:45.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[215775]: failed to encode map e531 with expected crc 2023-09-23T22:39:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/914990469' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T22:39:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[215775]: osdmap e531: 8 total, 6 up, 8 in 2023-09-23T22:39:45.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[215775]: failed to encode map e531 with expected crc 2023-09-23T22:39:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[223014]: pgmap v1377: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 24517/190794 objects degraded (12.850%); 28168/190794 objects misplaced (14.764%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:39:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[223014]: failed to encode map e531 with expected crc 2023-09-23T22:39:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[223014]: failed to encode map e531 with expected crc 2023-09-23T22:39:45.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/914990469' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T22:39:45.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[223014]: osdmap e531: 8 total, 6 up, 8 in 2023-09-23T22:39:45.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:45 smithi026 ceph-mon[223014]: failed to encode map e531 with expected crc 2023-09-23T22:39:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:45 smithi070 ceph-mon[192652]: pgmap v1377: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 24517/190794 objects degraded (12.850%); 28168/190794 objects misplaced (14.764%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T22:39:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:45 smithi070 ceph-mon[192652]: failed to encode map e531 with expected crc 2023-09-23T22:39:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:45 smithi070 ceph-mon[192652]: failed to encode map e531 with expected crc 2023-09-23T22:39:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/914990469' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T22:39:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:45 smithi070 ceph-mon[192652]: osdmap e531: 8 total, 6 up, 8 in 2023-09-23T22:39:45.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:45 smithi070 ceph-mon[192652]: failed to encode map e531 with expected crc 2023-09-23T22:39:46.473 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:46.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:46.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:46.578 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:46.606 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:46.670 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:46.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:46.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:46.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:39:46.775 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:46.807 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:46.845 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:46.859 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:46.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:46.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:46.965 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:46.993 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:47.044 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:47.151 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:47.180 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:47.238 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:47.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:47.348 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:47.378 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:47.584 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:47.586 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:47 smithi026 ceph-mon[215775]: pgmap v1379: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 24191/187299 objects degraded (12.916%); 27471/187299 objects misplaced (14.667%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:39:47.586 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:47 smithi026 ceph-mon[223014]: pgmap v1379: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 24191/187299 objects degraded (12.916%); 27471/187299 objects misplaced (14.667%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:39:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:47 smithi070 ceph-mon[192652]: pgmap v1379: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 24191/187299 objects degraded (12.916%); 27471/187299 objects misplaced (14.667%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T22:39:47.688 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:47.717 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:47.762 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:47.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:47.866 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:47.902 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:47.966 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:47.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:47.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:48.082 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:48.112 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:48.166 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:48.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:48.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:48.205 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:48.273 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:48.301 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:48.348 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:48.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:48.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:48.420 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:39:48.454 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:48.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3682555618' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:48.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:48 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:48.459 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 24191/187299 objects degraded (12.916%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:48.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3682555618' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:48.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:48 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:48.460 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 24191/187299 objects degraded (12.916%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:48.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3682555618' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:48.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:48 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:39:48.480 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 24191/187299 objects degraded (12.916%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:48.483 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:48.528 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:48.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:48.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:48.633 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:48.665 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:49.167 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:49.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:49.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:49.274 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:49.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:49.369 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:49.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:49.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:49.473 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:49.484 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:39:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[215775]: pgmap v1380: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 23482/182631 objects degraded (12.858%); 26796/182631 objects misplaced (14.672%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:39:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:39:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[215775]: failed to encode map e532 with expected crc 2023-09-23T22:39:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[215775]: failed to encode map e532 with expected crc 2023-09-23T22:39:49.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:49.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[215775]: osdmap e532: 8 total, 6 up, 8 in 2023-09-23T22:39:49.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[223014]: pgmap v1380: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 23482/182631 objects degraded (12.858%); 26796/182631 objects misplaced (14.672%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:39:49.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:39:49.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[223014]: failed to encode map e532 with expected crc 2023-09-23T22:39:49.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[223014]: failed to encode map e532 with expected crc 2023-09-23T22:39:49.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:49.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:49 smithi026 ceph-mon[223014]: osdmap e532: 8 total, 6 up, 8 in 2023-09-23T22:39:49.519 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:49.521 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:39:49.551 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:39:49.589 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:39:49.616 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:39:49.646 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:39:49.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:39:49.646 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:39:49.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:39:49.647 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:39:49.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:39:49.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:39:49.647 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:39:49.647 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:39:49.651 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:49.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:49.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:49.756 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:49.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:49 smithi070 ceph-mon[192652]: pgmap v1380: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 23482/182631 objects degraded (12.858%); 26796/182631 objects misplaced (14.672%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:39:49.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:49 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:39:49.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:49 smithi070 ceph-mon[192652]: failed to encode map e532 with expected crc 2023-09-23T22:39:49.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:49 smithi070 ceph-mon[192652]: failed to encode map e532 with expected crc 2023-09-23T22:39:49.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:49 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:39:49.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:49 smithi070 ceph-mon[192652]: osdmap e532: 8 total, 6 up, 8 in 2023-09-23T22:39:49.796 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:49.846 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:49.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:49.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:49.951 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:49.979 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:39:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:39:49] "GET /metrics HTTP/1.1" 200 36504 "" "Prometheus/2.43.0" 2023-09-23T22:39:50.034 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:50.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:50.142 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:50.177 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:50.231 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:50.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:50.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:50.336 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:50.365 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:50.618 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/712905065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:50.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/712905065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:50.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/712905065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:50.718 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:50.719 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:50.731 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:50.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:50.739 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:50.840 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:50.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:50.945 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:50.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:50.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:51.049 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:51.087 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:51.168 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:51.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:51.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:51.184 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:39:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:39:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:39:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:39:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151868 8.f 8.3587fcdf (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:39:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.193100+0000", 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 365.98703089600002, 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0010652879999999, 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151868 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193100+0000", 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193100+0000", 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193098+0000", 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193104+0000", 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9959999999999999e-06 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193112+0000", 2023-09-23T22:39:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8630000000000008e-06 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193120+0000", 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8820000000000001e-06 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.205704+0000", 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012583321999999999 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.205719+0000", 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5328999999999998e-05 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.194140+0000", 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98842133099999996 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.194165+0000", 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4830000000000001e-05 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188384 8.e 8:7bd73a78:::benchmark_data_smithi026_497658_object89492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.107077+0000", 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 308.07305339499999, 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0056152540000001, 2023-09-23T22:39:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188384 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107077+0000", 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107077+0000", 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107076+0000", 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107082+0000", 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9000000000000003e-06 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107094+0000", 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2679e-05 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107102+0000", 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3640000000000001e-06 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107151+0000", 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9509999999999999e-05 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107162+0000", 2023-09-23T22:39:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0845999999999999e-05 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.104169+0000", 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99700736599999995 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.104250+0000", 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0839000000000003e-05 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.104359+0000", 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109159 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112412+0000", 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080528330000000006 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112558+0000", 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014534199999999999 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112597+0000", 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9882000000000002e-05 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112651+0000", 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.363e-05 2023-09-23T22:39:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112693+0000", 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1485e-05 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188399 8.e 8:77b1158e:::benchmark_data_smithi026_497658_object89507:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.133354+0000", 2023-09-23T22:39:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 308.04677687200001, 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97957391500000002, 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188399 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133354+0000", 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133354+0000", 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133352+0000", 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133359+0000", 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.576e-06 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133371+0000", 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1505000000000001e-05 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133380+0000", 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0850000000000003e-06 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133426+0000", 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6149000000000001e-05 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.133436+0000", 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6930000000000002e-06 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.106563+0000", 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.973127305 2023-09-23T22:39:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.106691+0000", 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000127429 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.106810+0000", 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011979799999999999 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112773+0000", 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059625190000000003 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112784+0000", 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1277999999999999e-05 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112859+0000", 2023-09-23T22:39:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5185999999999995e-05 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112891+0000", 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1646000000000002e-05 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112928+0000", 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6625999999999998e-05 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188411 8.e 8:767a8bc9:::benchmark_data_smithi026_497658_object89519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.143178+0000", 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 308.03695295799997, 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97762692500000004, 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188411 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143178+0000", 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143178+0000", 2023-09-23T22:39:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143176+0000", 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143184+0000", 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7600000000000002e-06 2023-09-23T22:39:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143197+0000", 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3035e-05 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143206+0000", 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7339999999999997e-06 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143334+0000", 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000128713 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.143345+0000", 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0165e-05 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.108581+0000", 2023-09-23T22:39:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96523684499999995 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.108722+0000", 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014076299999999999 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.108854+0000", 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013138000000000001 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112790+0000", 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039360590000000004 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112821+0000", 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.15e-05 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120715+0000", 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007893466 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120762+0000", 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7178000000000001e-05 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120805+0000", 2023-09-23T22:39:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2877000000000003e-05 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188418 8.e 8:72a1b165:::benchmark_data_smithi026_497658_object89526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.147012+0000", 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 308.03311829400002, 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98431729099999998, 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188418 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147012+0000", 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147012+0000", 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147011+0000", 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147016+0000", 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2360000000000004e-06 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147028+0000", 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1131999999999999e-05 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147035+0000", 2023-09-23T22:39:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0099999999999998e-06 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147095+0000", 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0294000000000002e-05 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147104+0000", 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4949999999999997e-06 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.110529+0000", 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96342510100000001 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.110634+0000", 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104362 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.110768+0000", 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133886 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112828+0000", 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002059988 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112973+0000", 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145816 2023-09-23T22:39:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131241+0000", 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018267965000000001 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131290+0000", 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8979e-05 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131330+0000", 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9229e-05 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188423 8.e 8:79446d78:::benchmark_data_smithi026_497658_object89531:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.155111+0000", 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 308.02501919500003, 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98102083900000003, 2023-09-23T22:39:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188423 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155111+0000", 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155111+0000", 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155111+0000", 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155114+0000", 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.382e-06 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155122+0000", 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7270000000000007e-06 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155128+0000", 2023-09-23T22:39:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0360000000000003e-06 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155201+0000", 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2506e-05 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155209+0000", 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8100000000000004e-06 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.113014+0000", 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95780411099999996 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.113100+0000", 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6584999999999999e-05 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.113176+0000", 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6187999999999996e-05 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120640+0000", 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074639049999999998 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.123359+0000", 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027189860000000001 2023-09-23T22:39:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136058+0000", 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012698521000000001 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136105+0000", 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7261e-05 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136132+0000", 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7339e-05 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188425 8.e 8:7aac6c1a:::benchmark_data_smithi026_497658_object89533:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.155148+0000", 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 308.02498279700001, 2023-09-23T22:39:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98108856700000002, 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188425 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155148+0000", 2023-09-23T22:39:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155148+0000", 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155147+0000", 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155150+0000", 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0929999999999999e-06 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155158+0000", 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7659999999999998e-06 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155164+0000", 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8130000000000001e-06 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155232+0000", 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8145999999999994e-05 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155238+0000", 2023-09-23T22:39:51.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9680000000000003e-06 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.115278+0000", 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96004005299999995 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.115389+0000", 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000110332 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.115491+0000", 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010268999999999999 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120676+0000", 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051844309999999998 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.125970+0000", 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052943809999999999 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136166+0000", 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010195533 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136200+0000", 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4057999999999998e-05 2023-09-23T22:39:51.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136236+0000", 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6789999999999998e-05 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188426 8.e 8:7e42bbf5:::benchmark_data_smithi026_497658_object89534:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.240 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 308.02496161800002, 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98118840799999996, 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188426 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:39:51.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155171+0000", 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6089999999999999e-06 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155179+0000", 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9230000000000002e-06 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155185+0000", 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.468e-06 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155258+0000", 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3825999999999994e-05 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155277+0000", 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8732999999999998e-05 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.117481+0000", 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96220415800000003 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.117631+0000", 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149794 2023-09-23T22:39:51.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.117792+0000", 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016125699999999999 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120845+0000", 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030528819999999998 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126058+0000", 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052129309999999996 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136274+0000", 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010215504 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136323+0000", 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9487000000000002e-05 2023-09-23T22:39:51.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136357+0000", 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4270999999999998e-05 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188468 8.e 8:707983ca:::benchmark_data_smithi026_497658_object89576:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.203282+0000", 2023-09-23T22:39:51.248 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 307.976848227, 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93560747200000005, 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188468 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203282+0000", 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203282+0000", 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203281+0000", 2023-09-23T22:39:51.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203287+0000", 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.558e-06 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203301+0000", 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4270000000000001e-05 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203309+0000", 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7570000000000004e-06 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203404+0000", 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5044999999999995e-05 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.203414+0000", 2023-09-23T22:39:51.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.009e-05 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120892+0000", 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91747810299999999 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.121025+0000", 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000132965 2023-09-23T22:39:51.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.121149+0000", 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012415200000000001 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126021+0000", 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048717930000000001 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126064+0000", 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2577999999999998e-05 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138822+0000", 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012758472999999999 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138853+0000", 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0626000000000001e-05 2023-09-23T22:39:51.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138890+0000", 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7178999999999998e-05 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188494 8.e 8:77384446:::benchmark_data_smithi026_497658_object89602:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.256 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.231123+0000", 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 307.94900755700002, 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.907887483, 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188494 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231123+0000", 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231123+0000", 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231122+0000", 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231129+0000", 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.849e-06 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231147+0000", 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8558000000000001e-05 2023-09-23T22:39:51.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231154+0000", 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5889999999999999e-06 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231190+0000", 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5886999999999997e-05 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.231199+0000", 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6150000000000003e-06 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.123384+0000", 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89218471700000002 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.123494+0000", 2023-09-23T22:39:51.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011040900000000001 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.123601+0000", 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106308 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126048+0000", 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024475690000000001 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138928+0000", 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012879639 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138939+0000", 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.064e-05 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138977+0000", 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8488000000000001e-05 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139011+0000", 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3550999999999998e-05 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.264 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188560 8.e 8:7d1dd26f:::benchmark_data_smithi026_497658_object89668:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.299766+0000", 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 307.88036502199998, 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.84735557699999997, 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188560 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299766+0000", 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299766+0000", 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299764+0000", 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299770+0000", 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.356e-06 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299783+0000", 2023-09-23T22:39:51.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3609e-05 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299789+0000", 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7710000000000003e-06 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299829+0000", 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0308000000000001e-05 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.299837+0000", 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2529999999999998e-06 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126098+0000", 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.82626175000000002 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126228+0000", 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012984800000000001 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126361+0000", 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013277800000000001 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131369+0000", 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0050074830000000001 2023-09-23T22:39:51.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139019+0000", 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0076508490000000004 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147050+0000", 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080308989999999993 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147088+0000", 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7812999999999999e-05 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.147121+0000", 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3114999999999997e-05 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188594 8.e 8:72b9e1b0:::benchmark_data_smithi026_497658_object89702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.341081+0000", 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 307.839049359, 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96898526399999996, 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188594 2023-09-23T22:39:51.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341081+0000", 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341081+0000", 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341080+0000", 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341086+0000", 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0730000000000001e-06 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341100+0000", 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3634e-05 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341109+0000", 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1039999999999996e-06 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341162+0000", 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2799000000000001e-05 2023-09-23T22:39:51.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.341172+0000", 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0117e-05 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131392+0000", 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.79022011400000003 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131504+0000", 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011189800000000001 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131612+0000", 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000107944 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136037+0000", 2023-09-23T22:39:51.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004425544 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139029+0000", 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029920530000000002 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.309970+0000", 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.170940913 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310027+0000", 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6858999999999999e-05 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310067+0000", 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9443999999999997e-05 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188614 8.e 8:7bd220a9:::benchmark_data_smithi026_497658_object89722:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.369895+0000", 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 307.810236084, 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94027976199999996, 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188614 2023-09-23T22:39:51.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369895+0000", 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369895+0000", 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369893+0000", 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369898+0000", 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7949999999999998e-06 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369907+0000", 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5560000000000008e-06 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369912+0000", 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1200000000000001e-06 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369951+0000", 2023-09-23T22:39:51.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9601000000000003e-05 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.369962+0000", 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0267e-05 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.133818+0000", 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.76385622200000003 2023-09-23T22:39:51.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.133886+0000", 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7848000000000005e-05 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.133963+0000", 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7553999999999996e-05 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.138783+0000", 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004819416 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.139033+0000", 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00025017000000000003 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310106+0000", 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.17107334900000001 2023-09-23T22:39:51.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310144+0000", 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7375e-05 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.310174+0000", 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0691999999999997e-05 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.288 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104685 9.2 9:46fec5fe:::benchmark_data_smithi026_555954_object9956:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.292511+0000", 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 156.88761941000001, 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0057272559999999, 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:51.289 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104685 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292511+0000", 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292511+0000", 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292509+0000", 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292518+0000", 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4230000000000005e-06 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292537+0000", 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9038999999999998e-05 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292550+0000", 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3446e-05 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292613+0000", 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2655000000000002e-05 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292625+0000", 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1747e-05 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298147+0000", 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0055220330000001 2023-09-23T22:39:51.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298238+0000", 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1748000000000003e-05 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104686 9.2 9:4bc296ca:::benchmark_data_smithi026_555954_object9957:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:39:51.294 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 156.88756078099999, 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.032939702, 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104686 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292569+0000", 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:51.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292573+0000", 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6500000000000002e-06 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292581+0000", 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8080000000000005e-06 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292589+0000", 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4209999999999995e-06 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292656+0000", 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7243999999999995e-05 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292666+0000", 2023-09-23T22:39:51.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3850000000000008e-06 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298265+0000", 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.005598754 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298394+0000", 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012897599999999999 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298532+0000", 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001381 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.308571+0000", 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010039792000000001 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.308872+0000", 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030035000000000001 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325328+0000", 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016456529000000001 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325391+0000", 2023-09-23T22:39:51.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2385999999999995e-05 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325510+0000", 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118841 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104695 9.6 9:6983d852:::benchmark_data_smithi026_555954_object9966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:39:51.302 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.320260+0000", 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 156.85987114299999, 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.000856502, 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104695 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320260+0000", 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320260+0000", 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320258+0000", 2023-09-23T22:39:51.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320267+0000", 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6309999999999992e-06 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320286+0000", 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.893e-05 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320294+0000", 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1999999999999994e-06 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320349+0000", 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5631999999999999e-05 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320362+0000", 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3037000000000001e-05 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.309531+0000", 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98916838299999998 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.309666+0000", 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135649 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6", 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.309801+0000", 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134764 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.317002+0000", 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072011929999999998 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.320998+0000", 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039957480000000004 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.321034+0000", 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5407999999999997e-05 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.321116+0000", 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2433000000000002e-05 2023-09-23T22:39:51.309 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112814 9.6 9:6d8bc28c:::benchmark_data_smithi026_555954_object18085:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:38:54.442835+0000", 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 56.737295650999997, 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98434545699999998, 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.310 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112814 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442835+0000", 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442835+0000", 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442834+0000", 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442840+0000", 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0310000000000004e-06 2023-09-23T22:39:51.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442852+0000", 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2702e-05 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442860+0000", 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2420000000000003e-06 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442901+0000", 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1551000000000002e-05 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442909+0000", 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3880000000000002e-06 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.408301+0000", 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.965391954 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.408440+0000", 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013930600000000001 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1", 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.408565+0000", 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012514199999999999 2023-09-23T22:39:51.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.416305+0000", 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077403649999999999 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.427051+0000", 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010745265 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.427084+0000", 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3386999999999997e-05 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.427180+0000", 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6440999999999994e-05 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112966 9.7 9:e8a30d92:::benchmark_data_smithi026_555954_object18237:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328682+0000", 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.851448196, 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.030748977, 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112966 2023-09-23T22:39:51.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328682+0000", 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328682+0000", 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328680+0000", 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328690+0000", 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7580000000000004e-06 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328733+0000", 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3245999999999999e-05 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328745+0000", 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1802e-05 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328817+0000", 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2485999999999997e-05 2023-09-23T22:39:51.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328829+0000", 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1195999999999999e-05 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.340011+0000", 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.011182614 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.340147+0000", 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013621 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.340232+0000", 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4806000000000001e-05 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346112+0000", 2023-09-23T22:39:51.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005879843 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346121+0000", 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9819999999999997e-06 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359213+0000", 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013091635000000001 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359279+0000", 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5773999999999997e-05 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359431+0000", 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001529 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112980 9.7 9:e8e4b400:::benchmark_data_smithi026_555954_object18251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.348335+0000", 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.831796046999997, 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.011279043, 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112980 2023-09-23T22:39:51.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348335+0000", 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348335+0000", 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348333+0000", 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348339+0000", 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4689999999999996e-06 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348351+0000", 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2146e-05 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348356+0000", 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4840000000000003e-06 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348409+0000", 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2997000000000001e-05 2023-09-23T22:39:51.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348419+0000", 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3209999999999994e-06 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346148+0000", 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99772916899999997 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346286+0000", 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013786899999999999 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346413+0000", 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012667900000000001 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.331 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.354459+0000", 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080463510000000002 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.357736+0000", 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003277355 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359472+0000", 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001736118 2023-09-23T22:39:51.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359503+0000", 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0645000000000002e-05 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359614+0000", 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011066899999999999 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.333 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112988 9.7 9:e725ecc4:::benchmark_data_smithi026_555954_object18259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.354932+0000", 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.825198626000002, 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0114405769999999, 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:51.334 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112988 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354932+0000", 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354932+0000", 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:39:51.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354931+0000", 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354937+0000", 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5400000000000001e-06 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354955+0000", 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7791999999999998e-05 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354964+0000", 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6969999999999999e-06 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.355023+0000", 2023-09-23T22:39:51.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9036999999999997e-05 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.355035+0000", 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2679e-05 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.350800+0000", 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99576413200000002 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.350941+0000", 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014160400000000001 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.351070+0000", 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012856799999999999 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:39:51.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.357793+0000", 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067230629999999996 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359698+0000", 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00190493 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.363020+0000", 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033221280000000001 2023-09-23T22:39:51.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.363075+0000", 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5172e-05 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.366373+0000", 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032975359999999998 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.341 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.342 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:39:51.342 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:39:51.342 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:39:51.342 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:51.342 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:39:51.343 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:39:51.343 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T22:39:51.343 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:39:51.343 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:39:51.343 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T22:39:51.377 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:51.381 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:39:51.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:51 smithi026 ceph-mon[223014]: pgmap v1382: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 23482/182631 objects degraded (12.858%); 26796/182631 objects misplaced (14.672%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:39:51.442 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:51.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:51.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:51.549 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:51.585 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:51.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:51 smithi070 ceph-mon[192652]: pgmap v1382: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 23482/182631 objects degraded (12.858%); 26796/182631 objects misplaced (14.672%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:39:51.669 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:51.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:51.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:51.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:51 smithi026 ceph-mon[215775]: pgmap v1382: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 23482/182631 objects degraded (12.858%); 26796/182631 objects misplaced (14.672%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:39:51.774 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:51.813 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:51.871 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:51.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:51.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:51.977 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:52.012 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.027 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:39:52.088 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:52.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:52.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:52.135 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:52.189 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:52.195 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:52.230 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.295 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:52.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:52.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:52.403 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:52.448 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.498 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:52.560 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:52.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:52.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:52.665 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:52.692 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.719 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:39:52.751 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:52.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:52.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:52.890 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:52.993 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:52.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:52.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:53.098 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:53.125 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:53.172 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:53.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:53.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:53.276 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:53.305 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:53.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:53 smithi026 ceph-mon[223014]: pgmap v1383: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 22715/176025 objects degraded (12.904%); 25734/176025 objects misplaced (14.620%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:39:53.693 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 22715/176025 objects degraded (12.904%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:53.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:53 smithi026 ceph-mon[215775]: pgmap v1383: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 22715/176025 objects degraded (12.904%); 25734/176025 objects misplaced (14.620%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:39:53.693 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 22715/176025 objects degraded (12.904%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:53.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:53 smithi070 ceph-mon[192652]: pgmap v1383: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 22715/176025 objects degraded (12.904%); 25734/176025 objects misplaced (14.620%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T22:39:53.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 22715/176025 objects degraded (12.904%), 24 pgs degraded (PG_DEGRADED) 2023-09-23T22:39:54.242 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:54.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:54.346 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:54.416 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:54.511 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:54.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:54.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:54.621 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:54.642 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:39:54.671 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:54.796 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:39:54.797 INFO:teuthology.orchestra.run.smithi026.stdout: "success": "" 2023-09-23T22:39:54.797 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:39:54.923 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:54.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:54.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:55.028 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:55.074 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:55.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:55 smithi026 ceph-mon[223014]: pgmap v1384: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 22312/173613 objects degraded (12.852%); 25330/173613 objects misplaced (14.590%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:39:55.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:55.184 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:55 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:55.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:55 smithi026 ceph-mon[215775]: pgmap v1384: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 22312/173613 objects degraded (12.852%); 25330/173613 objects misplaced (14.590%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:39:55.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:55.186 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:55 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:55.236 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:55.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:55.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:55.347 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:39:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:55 smithi070 ceph-mon[192652]: pgmap v1384: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 22312/173613 objects degraded (12.852%); 25330/173613 objects misplaced (14.590%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T22:39:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:39:55.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:55 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:39:55.393 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:55.504 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:39:55.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:55.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:39:55.662 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:55.707 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:55.836 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:55.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:55.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:55.943 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:55.977 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:56.028 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:39:56.028 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:39:56.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.028 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135521 9.c 9:34583968:::benchmark_data_smithi026_555954_object40792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.028 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.823111+0000", 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.20280937700000001, 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.20284908199999999, 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135521 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.823111+0000", 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.823111+0000", 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.823110+0000", 2023-09-23T22:39:56.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.823113+0000", 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8890000000000002e-06 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.823120+0000", 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.438e-06 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.823124+0000", 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3429999999999998e-06 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.831884+0000", 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0087602519999999996 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.832929+0000", 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010443830000000001 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.856707+0000", 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.023778482 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.857529+0000", 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00082207099999999996 2023-09-23T22:39:56.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.022834+0000", 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.165305059 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.023633+0000", 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00079849999999999995 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.034 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.025391+0000", 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017583640000000001 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135560 9.c 9:3b99c9e8:::benchmark_data_smithi026_555954_object40831:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.925909+0000", 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.10001180899999999, 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.10015402, 2023-09-23T22:39:56.035 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135560 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.925909+0000", 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.925909+0000", 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.925907+0000", 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.925913+0000", 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3929999999999999e-06 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.925924+0000", 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0969e-05 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.925929+0000", 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0189999999999997e-06 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.927197+0000", 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0012682749999999999 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.927641+0000", 2023-09-23T22:39:56.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044383600000000002 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(9.c 532/518 [PushOp(9:3b99c9e8:::benchmark_data_smithi026_555954_object40831:head, version: 462'2600, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:3b99c9e8:::benchmark_data_smithi026_555954_object40831:head@462'2600, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.927235+0000", 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.098685637000000007, 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098886642999999996, 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T22:39:56.040 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.927235+0000", 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.927235+0000", 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.927234+0000", 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.927302+0000", 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7518e-05 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.927311+0000", 2023-09-23T22:39:56.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7159999999999998e-06 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.927318+0000", 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5889999999999999e-06 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:135594 9.14 e532/527 9:2c8aa669:::benchmark_data_smithi026_555954_object40865:head v 532'8000, mlcod=532'7997)", 2023-09-23T22:39:56.043 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:56.005116+0000", 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.020804319000000002, 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021036348999999999, 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.005116+0000", 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.005116+0000", 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.005115+0000", 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.005122+0000", 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3779999999999998e-06 2023-09-23T22:39:56.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.005134+0000", 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2415999999999999e-05 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.005140+0000", 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.519e-06 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.008666+0000", 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003526108 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.008693+0000", 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7282999999999999e-05 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135597 9.c 9:37d0121f:::benchmark_data_smithi026_555954_object40868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:56.006717+0000", 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.019203373999999999, 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019475010000000001, 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135597 2023-09-23T22:39:56.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.006717+0000", 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.006717+0000", 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.006716+0000", 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.006722+0000", 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5599999999999999e-06 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.050 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.006734+0000", 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1252e-05 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.006740+0000", 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8079999999999999e-06 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.010431+0000", 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003690923 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.011616+0000", 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001184832 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop(client.36252.0:135598 9.4 e532/525 9:20677670:::benchmark_data_smithi026_555954_object40869:head v 532'7981, mlcod=532'7978)", 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:56.007942+0000", 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.017977988, 2023-09-23T22:39:56.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018291422000000002, 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.007942+0000", 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.007942+0000", 2023-09-23T22:39:56.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.007942+0000", 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.007952+0000", 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0029e-05 2023-09-23T22:39:56.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.007965+0000", 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3618000000000001e-05 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.007971+0000", 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.682e-06 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.009038+0000", 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010670390000000001 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.009054+0000", 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5509999999999999e-05 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.056 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135536 9.c 9:31d78e92:::benchmark_data_smithi026_555954_object40807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.841930+0000", 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.18399014899999999, 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.18434594200000001, 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135536 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.841930+0000", 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.841930+0000", 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.058 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.841929+0000", 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.841933+0000", 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8600000000000003e-06 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.841954+0000", 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0579000000000001e-05 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.841959+0000", 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3410000000000002e-06 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.860521+0000", 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018562471000000001 2023-09-23T22:39:56.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.861586+0000", 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010641470000000001 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.868300+0000", 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067141370000000002 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:39:56.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.869186+0000", 2023-09-23T22:39:56.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00088637199999999996 2023-09-23T22:39:56.062 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.062 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.062 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.062 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135566 9.c 9:331adbc0:::benchmark_data_smithi026_555954_object40837:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.937407+0000", 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.088513003000000007, 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.088917568000000002, 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135566 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.937407+0000", 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.937407+0000", 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.937406+0000", 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.937411+0000", 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4449999999999998e-06 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.937421+0000", 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0139000000000001e-05 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.937425+0000", 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3579999999999996e-06 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.950794+0000", 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013368639999999999 2023-09-23T22:39:56.066 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.951791+0000", 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000997506 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.965201+0000", 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013409328999999999 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.966038+0000", 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000837044 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(9.c 532/518 [PushOp(9:37d0121f:::benchmark_data_smithi026_555954_object40868:head, version: 462'2603, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:37d0121f:::benchmark_data_smithi026_555954_object40868:head@462'2603, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:56.012002+0000", 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.013918170000000001, 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014382038999999999, 2023-09-23T22:39:56.068 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.012002+0000", 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.012002+0000", 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.012001+0000", 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.012062+0000", 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1272000000000006e-05 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.012079+0000", 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6266999999999999e-05 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.012089+0000", 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0431999999999999e-05 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135606 9.c 9.3a79ca6c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:56.016405+0000", 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0095156849999999994, 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010010797, 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135606 2023-09-23T22:39:56.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.016405+0000", 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.016405+0000", 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.016403+0000", 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.016411+0000", 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3040000000000005e-06 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.074 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.016423+0000", 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2177e-05 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.016429+0000", 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6370000000000004e-06 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135607 9.c 9.b0a1d38c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:56.017261+0000", 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0086595739999999997, 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0091901989999999996, 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135607 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.076 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.017261+0000", 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.017261+0000", 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.017260+0000", 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.017264+0000", 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.206e-06 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.017276+0000", 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1755e-05 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.017281+0000", 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9590000000000003e-06 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.079 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135569 9.c 9:3dd666e4:::benchmark_data_smithi026_555954_object40840:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.947634+0000", 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.078286395999999994, 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.078852432, 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135569 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.080 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.947634+0000", 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.947634+0000", 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.947633+0000", 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.947640+0000", 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4719999999999998e-06 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.082 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.947653+0000", 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2719e-05 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.947661+0000", 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9580000000000007e-06 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.951959+0000", 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.004298044 2023-09-23T22:39:56.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.953055+0000", 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010965370000000001 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.971099+0000", 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018043921000000001 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.971999+0000", 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00089967400000000005 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.36252.0:135508 9.c e532/518)", 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:56.019275+0000", 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0066448360000000003, 2023-09-23T22:39:56.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072533600000000004, 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019275+0000", 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019275+0000", 2023-09-23T22:39:56.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019275+0000", 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019280+0000", 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8600000000000001e-06 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019292+0000", 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2068e-05 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019296+0000", 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.848e-06 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135545 9.c 9:3dec2912:::benchmark_data_smithi026_555954_object40816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.853623+0000", 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.17229683500000001, 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.172937488, 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135545 2023-09-23T22:39:56.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.853623+0000", 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.853623+0000", 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.853622+0000", 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.853628+0000", 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3899999999999998e-06 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.853643+0000", 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4217e-05 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.853651+0000", 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3469999999999999e-06 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.861978+0000", 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0083271060000000008 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.862983+0000", 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001005466 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.874048+0000", 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011064391999999999 2023-09-23T22:39:56.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.874877+0000", 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00082866500000000002 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_repop_reply(client.36252.0:135508 9.c e532/518)", 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:56.019895+0000", 2023-09-23T22:39:56.094 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.0060251089999999998, 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067119650000000003, 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "queued for pg", 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019895+0000", 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019895+0000", 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019895+0000", 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019899+0000", 2023-09-23T22:39:56.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8959999999999996e-06 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019908+0000", 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7739999999999999e-06 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.019913+0000", 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7280000000000003e-06 2023-09-23T22:39:56.097 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135546 9.c 9:311e5d5b:::benchmark_data_smithi026_555954_object40817:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.855352+0000", 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.17056844500000001, 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.171288567, 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T22:39:56.098 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135546 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.855352+0000", 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.855352+0000", 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.855351+0000", 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.855359+0000", 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7500000000000003e-06 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.855371+0000", 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2088e-05 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.855379+0000", 2023-09-23T22:39:56.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9489999999999996e-06 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.863108+0000", 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077295380000000002 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.863865+0000", 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00075630100000000002 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.880128+0000", 2023-09-23T22:39:56.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016262977000000001 2023-09-23T22:39:56.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:39:56.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.880940+0000", 2023-09-23T22:39:56.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00081229200000000005 2023-09-23T22:39:56.105 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.105 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.105 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135549 9.c 9:314a0bde:::benchmark_data_smithi026_555954_object40820:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.864292+0000", 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.16162791100000001, 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.162393382, 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135549 2023-09-23T22:39:56.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.864292+0000", 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.864292+0000", 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.107 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.864292+0000", 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.864295+0000", 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0680000000000001e-06 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.864301+0000", 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.699e-06 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.864304+0000", 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4750000000000002e-06 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.881273+0000", 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016969227999999999 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.882048+0000", 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00077437600000000001 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.894385+0000", 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0123372 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.895259+0000", 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000873656 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135550 9.c 9:3766e45a:::benchmark_data_smithi026_555954_object40821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.865495+0000", 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.160425599, 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.161239783, 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.111 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "delayed", 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135550 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.865495+0000", 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.865495+0000", 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.865494+0000", 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.865498+0000", 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4939999999999997e-06 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.865511+0000", 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2319e-05 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.865516+0000", 2023-09-23T22:39:56.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.48e-06 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.882153+0000", 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016637174000000001 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.882965+0000", 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00081135699999999996 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.899678+0000", 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01671278 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.900537+0000", 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00085917800000000005 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.116 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:135508 9.c 9:3bd87281:::benchmark_data_smithi026_555954_object40779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e532)", 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:55.810846+0000", 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.21507397, 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.21593638900000001, 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 135508 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:39:56.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.810846+0000", 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.810846+0000", 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:39:56.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.810846+0000", 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.810849+0000", 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7170000000000002e-06 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.810857+0000", 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9789999999999993e-06 2023-09-23T22:39:56.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.810862+0000", 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2619999999999998e-06 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.829318+0000", 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018455606999999999 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.830354+0000", 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010359639999999999 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.851298+0000", 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020944183000000002 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:55.852153+0000", 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00085454300000000003 2023-09-23T22:39:56.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.014337+0000", 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.16218438900000001 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.015149+0000", 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00081203799999999995 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:39:56.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:39:56.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:56.016669+0000", 2023-09-23T22:39:56.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015196529999999999 2023-09-23T22:39:56.123 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.123 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:39:56.123 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.123 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:39:56.123 INFO:teuthology.orchestra.run.smithi070.stdout: ], 2023-09-23T22:39:56.123 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 19 2023-09-23T22:39:56.123 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:39:56.124 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1330256390' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:56.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1330256390' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:56.124 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:56 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:56.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:56.125 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:56.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:56.126 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:39:56.227 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:56.265 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:56.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1330256390' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:56.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:56 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:56.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:56 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:39:56.840 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:56.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:56.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:56.952 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:56.970 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:39:57.017 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:57.110 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:57.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:57.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:57.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:57.245 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:57 smithi070 ceph-mon[192652]: pgmap v1385: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 21826/168699 objects degraded (12.938%); 24400/168699 objects misplaced (14.464%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:39:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:57 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 15 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:39:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:57 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:57 smithi070 ceph-mon[192652]: failed to encode map e533 with expected crc 2023-09-23T22:39:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:57 smithi070 ceph-mon[192652]: failed to encode map e533 with expected crc 2023-09-23T22:39:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:57 smithi070 ceph-mon[192652]: failed to encode map e533 with expected crc 2023-09-23T22:39:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:57.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:57 smithi070 ceph-mon[192652]: osdmap e533: 8 total, 6 up, 8 in 2023-09-23T22:39:57.460 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:39:57.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[215775]: pgmap v1385: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 21826/168699 objects degraded (12.938%); 24400/168699 objects misplaced (14.464%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:39:57.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 15 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:39:57.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[215775]: failed to encode map e533 with expected crc 2023-09-23T22:39:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[215775]: failed to encode map e533 with expected crc 2023-09-23T22:39:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[215775]: failed to encode map e533 with expected crc 2023-09-23T22:39:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:57.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[215775]: osdmap e533: 8 total, 6 up, 8 in 2023-09-23T22:39:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[223014]: pgmap v1385: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 21826/168699 objects degraded (12.938%); 24400/168699 objects misplaced (14.464%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:39:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 15 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:39:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:39:57.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[223014]: failed to encode map e533 with expected crc 2023-09-23T22:39:57.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[223014]: failed to encode map e533 with expected crc 2023-09-23T22:39:57.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[223014]: failed to encode map e533 with expected crc 2023-09-23T22:39:57.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:39:57.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:57 smithi026 ceph-mon[223014]: osdmap e533: 8 total, 6 up, 8 in 2023-09-23T22:39:57.674 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:57.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:57.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:57.780 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:39:57.835 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:57.972 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:39:57.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:57.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:39:58.076 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:39:58.142 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:58.169 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:39:58.202 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:39:58.206 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:39:58.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:58.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:39:58.243 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:39:58.273 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:39:58.303 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:39:58.309 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:58.336 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:39:58.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:39:58.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:39:58.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:39:58.337 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:39:58.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:39:58.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:39:58.337 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:39:58.337 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:39:58.357 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:58.520 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:58.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:58.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:58.624 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:39:58.668 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:58.925 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:39:58.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:58.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:39:59.034 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:59.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:59.270 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:59 smithi070 ceph-mon[192652]: pgmap v1387: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 20788/161910 objects degraded (12.839%); 23426/161910 objects misplaced (14.469%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T22:39:59.270 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:59 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1586940701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:59.270 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:39:59 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 20788/161910 objects degraded (12.839%), 24 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T22:39:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:39:59.319 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:39:59.319 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:59.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:59.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:59.429 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:39:59.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:59 smithi026 ceph-mon[223014]: pgmap v1387: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 20788/161910 objects degraded (12.839%); 23426/161910 objects misplaced (14.469%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T22:39:59.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:59 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1586940701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:59.484 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:39:59 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 20788/161910 objects degraded (12.839%), 24 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T22:39:59.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:59 smithi026 ceph-mon[215775]: pgmap v1387: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 20788/161910 objects degraded (12.839%); 23426/161910 objects misplaced (14.469%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T22:39:59.484 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:59 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1586940701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:39:59.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:39:59 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 20788/161910 objects degraded (12.839%), 24 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T22:39:59.495 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:59.563 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:39:59.563 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:39:59.595 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:39:59.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:59.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:39:59.704 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:39:59.737 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:39:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:39:59] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-09-23T22:39:59.752 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:39:59.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:39:59.851 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:39:59.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:39:59.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:39:59.962 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:00.017 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:00.110 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:00.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:00.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:00.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: Health detail: HEALTH_WARN noscrub flag(s) set; 2 osds down; Reduced data availability: 15 pgs inactive; Degraded data redundancy: 20788/161910 objects degraded (12.839%), 24 pgs degraded, 28 pgs undersized 2023-09-23T22:40:00.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T22:40:00.140 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: [WRN] OSD_DOWN: 2 osds down 2023-09-23T22:40:00.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: osd.3 (root=default,host=smithi026) is down 2023-09-23T22:40:00.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: osd.6 (root=default,host=smithi070) is down 2023-09-23T22:40:00.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: [WRN] PG_AVAILABILITY: Reduced data availability: 15 pgs inactive 2023-09-23T22:40:00.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.4 is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.7 is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.c is stuck inactive for 4m, current state undersized+degraded+peered, last acting [7] 2023-09-23T22:40:00.141 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.f is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.217 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:00.282 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:00.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: Health detail: HEALTH_WARN noscrub flag(s) set; 2 osds down; Reduced data availability: 15 pgs inactive; Degraded data redundancy: 20788/161910 objects degraded (12.839%), 24 pgs degraded, 28 pgs undersized 2023-09-23T22:40:00.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T22:40:00.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: [WRN] OSD_DOWN: 2 osds down 2023-09-23T22:40:00.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: osd.3 (root=default,host=smithi026) is down 2023-09-23T22:40:00.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: osd.6 (root=default,host=smithi070) is down 2023-09-23T22:40:00.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: [WRN] PG_AVAILABILITY: Reduced data availability: 15 pgs inactive 2023-09-23T22:40:00.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.4 is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.7 is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.c is stuck inactive for 4m, current state undersized+degraded+peered, last acting [7] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.f is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.14 is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.2 is stuck inactive for 63s, current state undersized+peered, last acting [4] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.5 is stuck inactive for 63s, current state undersized+peered, last acting [2] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.c is stuck inactive for 63s, current state undersized+peered, last acting [1] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.d is stuck inactive for 4m, current state undersized+degraded+peered, last acting [7] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.17 is stuck inactive for 63s, current state undersized+peered, last acting [0] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.1e is stuck inactive for 6m, current state undersized+peered, last acting [2] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.c is stuck inactive for 63s, current state undersized+peered, last acting [4] 2023-09-23T22:40:00.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.10 is stuck inactive for 63s, current state undersized+peered, last acting [1] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.11 is stuck inactive for 63s, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.15 is stuck inactive for 63s, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: [WRN] PG_DEGRADED: Degraded data redundancy: 20788/161910 objects degraded (12.839%), 24 pgs degraded, 28 pgs undersized 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.0 is stuck undersized for 62s, current state active+undersized, last acting [7,0] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.3 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.5 is stuck undersized for 62s, current state active+undersized, last acting [4,2] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.a is stuck undersized for 62s, current state active+undersized, last acting [5,2] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.b is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.c is undersized+degraded+peered, acting [7] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.d is stuck undersized for 62s, current state active+undersized, last acting [4,2] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.10 is stuck undersized for 62s, current state active+undersized, last acting [7,0] 2023-09-23T22:40:00.356 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 1.13 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 2.2 is stuck undersized for 62s, current state active+undersized+degraded, last acting [5,1] 2023-09-23T22:40:00.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 2.3 is active+undersized+degraded, acting [2,7] 2023-09-23T22:40:00.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 2.6 is stuck undersized for 62s, current state active+undersized, last acting [1,4] 2023-09-23T22:40:00.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 2.7 is stuck undersized for 62s, current state active+undersized, last acting [2,7] 2023-09-23T22:40:00.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.0 is stuck undersized for 62s, current state active+undersized, last acting [2,1] 2023-09-23T22:40:00.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.3 is stuck undersized for 62s, current state active+undersized, last acting [4,0] 2023-09-23T22:40:00.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.7 is active+undersized+degraded, acting [7,0] 2023-09-23T22:40:00.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.a is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.357 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.b is active+undersized+degraded, acting [0,4] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.d is undersized+degraded+peered, acting [7] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.10 is undersized+degraded+peered, acting [0] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.11 is stuck undersized for 62s, current state active+undersized, last acting [4,7] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.14 is stuck undersized for 62s, current state active+undersized+degraded, last acting [4,7] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.15 is active+undersized+degraded, acting [7,4] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.1d is stuck undersized for 62s, current state active+undersized+degraded, last acting [5,4] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 3.1f is stuck undersized for 62s, current state active+undersized+degraded, last acting [2,0] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.1 is stuck undersized for 62s, current state active+undersized, last acting [4,0] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.5 is stuck undersized for 62s, current state active+undersized, last acting [4,7] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.8 is stuck undersized for 62s, current state active+undersized, last acting [7,4] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.a is stuck undersized for 62s, current state active+undersized, last acting [1,7] 2023-09-23T22:40:00.358 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.e is stuck undersized for 62s, current state active+undersized, last acting [1,4] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.12 is stuck undersized for 62s, current state active+undersized, last acting [1,2] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.13 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.18 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 4.1f is stuck undersized for 62s, current state active+undersized, last acting [5,1] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.0 is active+undersized+degraded, acting [1,2] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.2 is active+recovery_wait+degraded+remapped, acting [1,2,7] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.6 is active+undersized+degraded, acting [2,1] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.7 is active+recovery_wait+degraded+remapped, acting [2,7,1] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.9 is active+undersized+degraded, acting [0,7] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.a is active+undersized+degraded, acting [1,7] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.b is stuck undersized for 62s, current state active+undersized+degraded, last acting [5,0] 2023-09-23T22:40:00.359 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.c is active+recovering+degraded+remapped, acting [4,7,0] 2023-09-23T22:40:00.360 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.10 is active+undersized+degraded, acting [1,2] 2023-09-23T22:40:00.360 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.12 is active+recovery_wait+degraded+remapped, acting [1,2,7] 2023-09-23T22:40:00.360 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.16 is active+undersized+degraded, acting [2,1] 2023-09-23T22:40:00.360 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.17 is active+recovery_wait+degraded+remapped, acting [2,7,1] 2023-09-23T22:40:00.360 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:00 smithi070 ceph-mon[192652]: pg 9.19 is active+undersized+degraded, acting [0,7] 2023-09-23T22:40:00.387 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:00.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:00.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: Health detail: HEALTH_WARN noscrub flag(s) set; 2 osds down; Reduced data availability: 15 pgs inactive; Degraded data redundancy: 20788/161910 objects degraded (12.839%), 24 pgs degraded, 28 pgs undersized 2023-09-23T22:40:00.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T22:40:00.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: [WRN] OSD_DOWN: 2 osds down 2023-09-23T22:40:00.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: osd.3 (root=default,host=smithi026) is down 2023-09-23T22:40:00.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: osd.6 (root=default,host=smithi070) is down 2023-09-23T22:40:00.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: [WRN] PG_AVAILABILITY: Reduced data availability: 15 pgs inactive 2023-09-23T22:40:00.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.4 is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.7 is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.c is stuck inactive for 4m, current state undersized+degraded+peered, last acting [7] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.f is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.14 is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.2 is stuck inactive for 63s, current state undersized+peered, last acting [4] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.5 is stuck inactive for 63s, current state undersized+peered, last acting [2] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.c is stuck inactive for 63s, current state undersized+peered, last acting [1] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.d is stuck inactive for 4m, current state undersized+degraded+peered, last acting [7] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.17 is stuck inactive for 63s, current state undersized+peered, last acting [0] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.1e is stuck inactive for 6m, current state undersized+peered, last acting [2] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.c is stuck inactive for 63s, current state undersized+peered, last acting [4] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.10 is stuck inactive for 63s, current state undersized+peered, last acting [1] 2023-09-23T22:40:00.396 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.11 is stuck inactive for 63s, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.15 is stuck inactive for 63s, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: [WRN] PG_DEGRADED: Degraded data redundancy: 20788/161910 objects degraded (12.839%), 24 pgs degraded, 28 pgs undersized 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.0 is stuck undersized for 62s, current state active+undersized, last acting [7,0] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.3 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.5 is stuck undersized for 62s, current state active+undersized, last acting [4,2] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.a is stuck undersized for 62s, current state active+undersized, last acting [5,2] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.b is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.c is undersized+degraded+peered, acting [7] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.d is stuck undersized for 62s, current state active+undersized, last acting [4,2] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.10 is stuck undersized for 62s, current state active+undersized, last acting [7,0] 2023-09-23T22:40:00.397 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 1.13 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.14 is stuck inactive for 4m, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.2 is stuck inactive for 63s, current state undersized+peered, last acting [4] 2023-09-23T22:40:00.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.5 is stuck inactive for 63s, current state undersized+peered, last acting [2] 2023-09-23T22:40:00.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.c is stuck inactive for 63s, current state undersized+peered, last acting [1] 2023-09-23T22:40:00.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.d is stuck inactive for 4m, current state undersized+degraded+peered, last acting [7] 2023-09-23T22:40:00.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.17 is stuck inactive for 63s, current state undersized+peered, last acting [0] 2023-09-23T22:40:00.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.1e is stuck inactive for 6m, current state undersized+peered, last acting [2] 2023-09-23T22:40:00.398 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.c is stuck inactive for 63s, current state undersized+peered, last acting [4] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.10 is stuck inactive for 63s, current state undersized+peered, last acting [1] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.11 is stuck inactive for 63s, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.15 is stuck inactive for 63s, current state undersized+peered, last acting [7] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: [WRN] PG_DEGRADED: Degraded data redundancy: 20788/161910 objects degraded (12.839%), 24 pgs degraded, 28 pgs undersized 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.0 is stuck undersized for 62s, current state active+undersized, last acting [7,0] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.3 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.5 is stuck undersized for 62s, current state active+undersized, last acting [4,2] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.a is stuck undersized for 62s, current state active+undersized, last acting [5,2] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.b is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.c is undersized+degraded+peered, acting [7] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.d is stuck undersized for 62s, current state active+undersized, last acting [4,2] 2023-09-23T22:40:00.399 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.10 is stuck undersized for 62s, current state active+undersized, last acting [7,0] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 1.13 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 2.2 is stuck undersized for 62s, current state active+undersized+degraded, last acting [5,1] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 2.3 is active+undersized+degraded, acting [2,7] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 2.6 is stuck undersized for 62s, current state active+undersized, last acting [1,4] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 2.7 is stuck undersized for 62s, current state active+undersized, last acting [2,7] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.0 is stuck undersized for 62s, current state active+undersized, last acting [2,1] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.3 is stuck undersized for 62s, current state active+undersized, last acting [4,0] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.7 is active+undersized+degraded, acting [7,0] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.a is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.b is active+undersized+degraded, acting [0,4] 2023-09-23T22:40:00.400 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.d is undersized+degraded+peered, acting [7] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 2.2 is stuck undersized for 62s, current state active+undersized+degraded, last acting [5,1] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 2.3 is active+undersized+degraded, acting [2,7] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 2.6 is stuck undersized for 62s, current state active+undersized, last acting [1,4] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 2.7 is stuck undersized for 62s, current state active+undersized, last acting [2,7] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.0 is stuck undersized for 62s, current state active+undersized, last acting [2,1] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.3 is stuck undersized for 62s, current state active+undersized, last acting [4,0] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.7 is active+undersized+degraded, acting [7,0] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.a is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.b is active+undersized+degraded, acting [0,4] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.d is undersized+degraded+peered, acting [7] 2023-09-23T22:40:00.401 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.10 is undersized+degraded+peered, acting [0] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.11 is stuck undersized for 62s, current state active+undersized, last acting [4,7] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.14 is stuck undersized for 62s, current state active+undersized+degraded, last acting [4,7] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.15 is active+undersized+degraded, acting [7,4] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.1d is stuck undersized for 62s, current state active+undersized+degraded, last acting [5,4] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 3.1f is stuck undersized for 62s, current state active+undersized+degraded, last acting [2,0] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.1 is stuck undersized for 62s, current state active+undersized, last acting [4,0] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.5 is stuck undersized for 62s, current state active+undersized, last acting [4,7] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.8 is stuck undersized for 62s, current state active+undersized, last acting [7,4] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.a is stuck undersized for 62s, current state active+undersized, last acting [1,7] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.e is stuck undersized for 62s, current state active+undersized, last acting [1,4] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.12 is stuck undersized for 62s, current state active+undersized, last acting [1,2] 2023-09-23T22:40:00.402 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.13 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.18 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 4.1f is stuck undersized for 62s, current state active+undersized, last acting [5,1] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.0 is active+undersized+degraded, acting [1,2] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.2 is active+recovery_wait+degraded+remapped, acting [1,2,7] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.6 is active+undersized+degraded, acting [2,1] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.7 is active+recovery_wait+degraded+remapped, acting [2,7,1] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.9 is active+undersized+degraded, acting [0,7] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.a is active+undersized+degraded, acting [1,7] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.b is stuck undersized for 62s, current state active+undersized+degraded, last acting [5,0] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.c is active+recovering+degraded+remapped, acting [4,7,0] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.10 is active+undersized+degraded, acting [1,2] 2023-09-23T22:40:00.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.12 is active+recovery_wait+degraded+remapped, acting [1,2,7] 2023-09-23T22:40:00.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.16 is active+undersized+degraded, acting [2,1] 2023-09-23T22:40:00.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.17 is active+recovery_wait+degraded+remapped, acting [2,7,1] 2023-09-23T22:40:00.405 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[223014]: pg 9.19 is active+undersized+degraded, acting [0,7] 2023-09-23T22:40:00.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.10 is undersized+degraded+peered, acting [0] 2023-09-23T22:40:00.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.11 is stuck undersized for 62s, current state active+undersized, last acting [4,7] 2023-09-23T22:40:00.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.14 is stuck undersized for 62s, current state active+undersized+degraded, last acting [4,7] 2023-09-23T22:40:00.405 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.15 is active+undersized+degraded, acting [7,4] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.1d is stuck undersized for 62s, current state active+undersized+degraded, last acting [5,4] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 3.1f is stuck undersized for 62s, current state active+undersized+degraded, last acting [2,0] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.1 is stuck undersized for 62s, current state active+undersized, last acting [4,0] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.5 is stuck undersized for 62s, current state active+undersized, last acting [4,7] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.8 is stuck undersized for 62s, current state active+undersized, last acting [7,4] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.a is stuck undersized for 62s, current state active+undersized, last acting [1,7] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.e is stuck undersized for 62s, current state active+undersized, last acting [1,4] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.12 is stuck undersized for 62s, current state active+undersized, last acting [1,2] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.13 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.18 is stuck undersized for 62s, current state active+undersized, last acting [4,1] 2023-09-23T22:40:00.406 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 4.1f is stuck undersized for 62s, current state active+undersized, last acting [5,1] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.0 is active+undersized+degraded, acting [1,2] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.2 is active+recovery_wait+degraded+remapped, acting [1,2,7] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.6 is active+undersized+degraded, acting [2,1] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.7 is active+recovery_wait+degraded+remapped, acting [2,7,1] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.9 is active+undersized+degraded, acting [0,7] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.a is active+undersized+degraded, acting [1,7] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.b is stuck undersized for 62s, current state active+undersized+degraded, last acting [5,0] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.c is active+recovering+degraded+remapped, acting [4,7,0] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.10 is active+undersized+degraded, acting [1,2] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.12 is active+recovery_wait+degraded+remapped, acting [1,2,7] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.16 is active+undersized+degraded, acting [2,1] 2023-09-23T22:40:00.407 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.17 is active+recovery_wait+degraded+remapped, acting [2,7,1] 2023-09-23T22:40:00.408 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:00 smithi026 ceph-mon[215775]: pg 9.19 is active+undersized+degraded, acting [0,7] 2023-09-23T22:40:00.474 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:00.495 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:00.514 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:40:00.548 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:00.674 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:00.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:00.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:00.784 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:00.834 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:00.920 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:00.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:00.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:01.029 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:01.087 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:01.166 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:01.173 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:01.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:01.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:01.289 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:01.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:40:01.331 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:01 smithi026 ceph-mon[223014]: pgmap v1388: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 20788/161910 objects degraded (12.839%); 23426/161910 objects misplaced (14.469%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:40:01.332 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:01 smithi026 ceph-mon[215775]: pgmap v1388: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 20788/161910 objects degraded (12.839%); 23426/161910 objects misplaced (14.469%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:40:01.336 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:01.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:01 smithi070 ceph-mon[192652]: pgmap v1388: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 20788/161910 objects degraded (12.839%); 23426/161910 objects misplaced (14.469%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:40:01.438 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:01.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:01.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:01.542 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:01.569 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:01.617 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:01.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:01.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:01.720 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:01.769 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:01.869 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:01.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:01.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:01.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:02.022 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:40:02.022 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:40:02.022 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T22:40:02.022 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:40:02.022 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:40:02.023 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T22:40:02.023 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:02.115 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:02.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:02.218 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:02.223 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:02.258 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:02.411 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:02.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:02.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:02.516 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:02.581 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:02.707 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:02.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:02.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:02.812 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:02.823 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:02.882 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:03.011 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:03.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:03.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:03.117 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:03.167 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:03.172 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:40:03.261 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:03.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:03.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:03.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:03 smithi070 ceph-mon[192652]: pgmap v1389: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 19580/150675 objects degraded (12.995%); 21569/150675 objects misplaced (14.315%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:40:03.268 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2569571916' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:03.361 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:03.362 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:03.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:03.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:03 smithi026 ceph-mon[223014]: pgmap v1389: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 19580/150675 objects degraded (12.995%); 21569/150675 objects misplaced (14.315%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:40:03.420 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2569571916' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:03.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:03 smithi026 ceph-mon[215775]: pgmap v1389: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 19580/150675 objects degraded (12.995%); 21569/150675 objects misplaced (14.315%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T22:40:03.420 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2569571916' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:03.427 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:04.020 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:04.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:04.124 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:04.153 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:04.199 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:04.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:04.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:04.304 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:04.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:04 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:04.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2569571916' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:04.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:04 smithi070 ceph-mon[192652]: osdmap e534: 8 total, 6 up, 8 in 2023-09-23T22:40:04.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:04 smithi070 ceph-mon[192652]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:04 smithi070 ceph-mon[192652]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:04 smithi070 ceph-mon[192652]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.343 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:04 smithi070 ceph-mon[192652]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.344 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:04 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 19580/150675 objects degraded (12.995%), 24 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:04.350 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:04.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:04.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2569571916' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:04.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[223014]: osdmap e534: 8 total, 6 up, 8 in 2023-09-23T22:40:04.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[223014]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[223014]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[223014]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[223014]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 19580/150675 objects degraded (12.995%), 24 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:04.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:04.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2569571916' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:04.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[215775]: osdmap e534: 8 total, 6 up, 8 in 2023-09-23T22:40:04.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[215775]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[215775]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[215775]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[215775]: failed to encode map e534 with expected crc 2023-09-23T22:40:04.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:04 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 19580/150675 objects degraded (12.995%), 24 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:04.937 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:40:04.983 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:04.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:04.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:05.095 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:05.211 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:05.301 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:05 smithi026 ceph-mon[223014]: pgmap v1391: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 918 op/s; 18917/146640 objects degraded (12.900%); 20905/146640 objects misplaced (14.256%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:40:05.324 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:05.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:05.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:05.391 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:40:05.434 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:05.455 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:40:05.488 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:05.489 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:40:05.521 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:40:05.521 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "" 2023-09-23T22:40:05.521 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:40:05.542 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:40:05.550 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:05 smithi070 ceph-mon[192652]: pgmap v1391: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 918 op/s; 18917/146640 objects degraded (12.900%); 20905/146640 objects misplaced (14.256%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:40:05.561 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:05.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:05.579 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:40:05.642 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:40:05.642 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.643 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.643 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.643 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:40:05.643 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.643 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.643 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:40:05.643 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:40:05.655 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:05.657 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:05 smithi026 ceph-mon[215775]: pgmap v1391: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 918 op/s; 18917/146640 objects degraded (12.900%); 20905/146640 objects misplaced (14.256%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:40:05.666 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:05.714 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:05.783 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:05.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:05.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:05.890 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:05.925 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:05.966 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:05.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:05.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:06.070 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:06.109 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:06.316 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/451161155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:06.393 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:06.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:06.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:06.497 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:06.556 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:06.630 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:06.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:06.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:06.715 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:40:06.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/451161155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:06.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/451161155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:06.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:06.790 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:06.864 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:06.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:06.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:06.974 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:06.997 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:06.998 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:07.007 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:07.073 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:07.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:07.182 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:07.264 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:07.320 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:07 smithi026 ceph-mon[215775]: pgmap v1392: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 18221/139428 objects degraded (13.068%); 19568/139428 objects misplaced (14.034%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T22:40:07.320 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:07 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 18 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:40:07.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:07 smithi070 ceph-mon[192652]: pgmap v1392: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 18221/139428 objects degraded (13.068%); 19568/139428 objects misplaced (14.034%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T22:40:07.321 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:07 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 18 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:40:07.505 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:07.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:07.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:07.615 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:07.646 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:07 smithi026 ceph-mon[223014]: pgmap v1392: 119 pgs: 5 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 28 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 18221/139428 objects degraded (13.068%); 19568/139428 objects misplaced (14.034%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T22:40:07.647 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:07 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 18 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:40:07.649 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:07.731 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:07.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:07.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:07.836 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:07.884 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:07.963 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:07.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:07.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:08.069 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:08.121 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:08.314 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:08.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:08.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:08.422 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:08.423 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:08.460 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:08.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 18221/139428 objects degraded (13.068%), 24 pgs degraded, 72 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:08.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 18221/139428 objects degraded (13.068%), 24 pgs degraded, 72 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:08.517 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:08.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:08.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:08.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 18221/139428 objects degraded (13.068%), 24 pgs degraded, 72 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:08.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:08.669 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:08.729 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:08.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:08.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:08.838 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:08.878 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:08.935 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:08.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:08.939 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:08.999 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:40:09.040 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:09.064 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:40:09.064 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:40:09.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.065 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:149932 9.10 9:0d32ae77:::benchmark_data_smithi026_555954_object55203:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e534)", 2023-09-23T22:40:09.065 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:09.049630+0000", 2023-09-23T22:40:09.065 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0091004640000000008, 2023-09-23T22:40:09.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0091409009999999999, 2023-09-23T22:40:09.065 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:09.065 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:40:09.065 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:09.065 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:09.065 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:09.066 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 149932 2023-09-23T22:40:09.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.066 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:09.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:09.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049630+0000", 2023-09-23T22:40:09.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049630+0000", 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049629+0000", 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049635+0000", 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4980000000000003e-06 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049646+0000", 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.039e-05 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049654+0000", 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7559999999999999e-06 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049733+0000", 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9153999999999995e-05 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049844+0000", 2023-09-23T22:40:09.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011168899999999999 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2", 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050124+0000", 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000279716 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055994+0000", 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058697480000000002 2023-09-23T22:40:09.070 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:149936 9.12 9:49fe531c:::benchmark_data_smithi026_555954_object55207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e534)", 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:09.049660+0000", 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0090710180000000001, 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0092070829999999996, 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:09.071 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 149936 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049660+0000", 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049660+0000", 2023-09-23T22:40:09.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049659+0000", 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049662+0000", 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6769999999999999e-06 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049668+0000", 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.533e-06 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049674+0000", 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3120000000000001e-06 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049739+0000", 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5640000000000002e-05 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049861+0000", 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000121564 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050155+0000", 2023-09-23T22:40:09.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000294352 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.054417+0000", 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042613069999999998 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.054799+0000", 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038202199999999998 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.076 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.36252.0:149937 9.6 e534/525 9:65f867a4:::benchmark_data_smithi026_555954_object55208:head v 534'8313, mlcod=534'8311)", 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:09.050047+0000", 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0086837070000000006, 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0089198400000000001, 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.077 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050047+0000", 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050047+0000", 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050046+0000", 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050053+0000", 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0949999999999998e-06 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050069+0000", 2023-09-23T22:40:09.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5968999999999999e-05 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050077+0000", 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8110000000000003e-06 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050150+0000", 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2381999999999999e-05 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.050173+0000", 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3521999999999999e-05 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.36252.0:149934 9.e e534/495 9:78eecda3:::benchmark_data_smithi026_555954_object55205:head v 534'9481, mlcod=534'9479)", 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:09.055253+0000", 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0034777789999999999, 2023-09-23T22:40:09.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037864000000000001, 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055253+0000", 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055253+0000", 2023-09-23T22:40:09.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055252+0000", 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055258+0000", 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7749999999999998e-06 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055268+0000", 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0359e-05 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055275+0000", 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3289999999999999e-06 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055335+0000", 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0588000000000003e-05 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055356+0000", 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0605e-05 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.36252.0:149939 9.f e534/518 9:f61026e4:::benchmark_data_smithi026_555954_object55210:head v 534'9131, mlcod=534'9129)", 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:09.055281+0000", 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0034492569999999998, 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038356190000000002, 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055281+0000", 2023-09-23T22:40:09.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055281+0000", 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055281+0000", 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055287+0000", 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9880000000000001e-06 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055299+0000", 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.223e-05 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055305+0000", 2023-09-23T22:40:09.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1720000000000004e-06 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055365+0000", 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9790000000000002e-05 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.055396+0000", 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1146999999999998e-05 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.059101+0000", 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037047099999999999 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:149919 9.4 9:259d9ada:::benchmark_data_smithi026_555954_object55190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e534)", 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:09.036817+0000", 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.021913607000000002, 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022366898, 2023-09-23T22:40:09.090 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:09.091 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:40:09.091 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:09.091 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:09.091 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:09.091 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 149919 2023-09-23T22:40:09.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.091 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:09.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.036817+0000", 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.036817+0000", 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.036817+0000", 2023-09-23T22:40:09.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:09.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:09.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.036819+0000", 2023-09-23T22:40:09.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6709999999999999e-06 2023-09-23T22:40:09.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:09.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.036825+0000", 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.105e-06 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.036832+0000", 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3539999999999996e-06 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.038952+0000", 2023-09-23T22:40:09.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002120542 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.039064+0000", 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111427 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.039389+0000", 2023-09-23T22:40:09.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00032500200000000002 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.046636+0000", 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072474779999999999 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.046705+0000", 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8546000000000004e-05 2023-09-23T22:40:09.096 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.097 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:09.097 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.097 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:149927 9.18 9:18628019:::benchmark_data_smithi026_555954_object55198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e534)", 2023-09-23T22:40:09.097 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:09.044348+0000", 2023-09-23T22:40:09.097 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.014382221000000001, 2023-09-23T22:40:09.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014901848000000001, 2023-09-23T22:40:09.097 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 149927 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:09.098 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044348+0000", 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044348+0000", 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044348+0000", 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044350+0000", 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3130000000000001e-06 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044356+0000", 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7150000000000003e-06 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:09.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044361+0000", 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7140000000000001e-06 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.047532+0000", 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00317162 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.047640+0000", 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010760300000000001 2023-09-23T22:40:09.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:40:09.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.047983+0000", 2023-09-23T22:40:09.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034295899999999999 2023-09-23T22:40:09.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:09.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.053147+0000", 2023-09-23T22:40:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051637489999999996 2023-09-23T22:40:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.058995+0000", 2023-09-23T22:40:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058483249999999997 2023-09-23T22:40:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.103 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:149928 9.4 9:24586d46:::benchmark_data_smithi026_555954_object55199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e534)", 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:09.044365+0000", 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.014366043, 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.014961403999999999, 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:09.104 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 149928 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044365+0000", 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044365+0000", 2023-09-23T22:40:09.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044364+0000", 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044366+0000", 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0329999999999998e-06 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044372+0000", 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2039999999999996e-06 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044376+0000", 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3209999999999998e-06 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044590+0000", 2023-09-23T22:40:09.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021450899999999999 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.044705+0000", 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011445799999999999 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.045035+0000", 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033022999999999999 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.046715+0000", 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016801209999999999 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:09.049073+0000", 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023572559999999998 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.109 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:09.110 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:40:09.110 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 8 2023-09-23T22:40:09.110 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:40:09.110 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:09.143 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:09.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:09.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:09.255 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:09.286 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:09.346 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:09.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:09.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:09.456 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:09.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:09 smithi026 ceph-mon[215775]: pgmap v1393: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 13638/130233 objects degraded (10.472%); 21603/130233 objects misplaced (16.588%); 2.8 MiB/s, 103 objects/s recovering 2023-09-23T22:40:09.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:09.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:09.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:09 smithi026 ceph-mon[223014]: pgmap v1393: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 13638/130233 objects degraded (10.472%); 21603/130233 objects misplaced (16.588%); 2.8 MiB/s, 103 objects/s recovering 2023-09-23T22:40:09.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:09.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:09.492 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:09.562 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:09.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:09 smithi070 ceph-mon[192652]: pgmap v1393: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 13638/130233 objects degraded (10.472%); 21603/130233 objects misplaced (16.588%); 2.8 MiB/s, 103 objects/s recovering 2023-09-23T22:40:09.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:09.573 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:09.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:09.675 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:09.740 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:09.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:40:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:40:09] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-09-23T22:40:10.197 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:10.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:10.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:10.303 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:10.332 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:11.274 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:11.281 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:40:11.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:11.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:11.388 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:11.402 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:11.417 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:11.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:11 smithi026 ceph-mon[223014]: pgmap v1394: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 13638/130233 objects degraded (10.472%); 21603/130233 objects misplaced (16.588%); 2.8 MiB/s, 103 objects/s recovering 2023-09-23T22:40:11.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:11 smithi026 ceph-mon[215775]: pgmap v1394: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 13638/130233 objects degraded (10.472%); 21603/130233 objects misplaced (16.588%); 2.8 MiB/s, 103 objects/s recovering 2023-09-23T22:40:11.490 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:11.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:11.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:11.532 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:40:11.532 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:40:11.532 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T22:40:11.533 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:40:11.598 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:11.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:11 smithi070 ceph-mon[192652]: pgmap v1394: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 13638/130233 objects degraded (10.472%); 21603/130233 objects misplaced (16.588%); 2.8 MiB/s, 103 objects/s recovering 2023-09-23T22:40:11.628 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:11.687 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:11.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:11.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:11.793 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:11.827 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:11.882 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:11.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:11.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:11.990 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:12.076 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:12.448 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:12.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:12.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:12.562 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:12.633 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:12.757 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:12.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:12.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:12.865 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:12.929 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:13.029 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:13.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:13.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:13.133 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:13.201 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:13.343 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:40:13.474 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:13.475 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:13.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:13.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:13.521 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:13.581 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:13.611 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:13 smithi070 ceph-mon[192652]: pgmap v1395: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 12170/116853 objects degraded (10.415%); 19348/116853 objects misplaced (16.558%); 2.8 MiB/s, 103 objects/s recovering 2023-09-23T22:40:13.612 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/333815405' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:13.612 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:13 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:13.612 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12170/116853 objects degraded (10.415%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:13.614 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:13.682 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:13.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:13.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:13.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:13 smithi026 ceph-mon[223014]: pgmap v1395: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 12170/116853 objects degraded (10.415%); 19348/116853 objects misplaced (16.558%); 2.8 MiB/s, 103 objects/s recovering 2023-09-23T22:40:13.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/333815405' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:13.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:13 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:13.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12170/116853 objects degraded (10.415%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:13.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:13 smithi026 ceph-mon[215775]: pgmap v1395: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 12170/116853 objects degraded (10.415%); 19348/116853 objects misplaced (16.558%); 2.8 MiB/s, 103 objects/s recovering 2023-09-23T22:40:13.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/333815405' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:13.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:13 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:13.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12170/116853 objects degraded (10.415%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:13.790 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:13.832 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:13.900 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:13.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:13.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:14.006 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:14.061 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:14.130 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:14.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:14.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:14.237 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:14.266 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:14.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:14 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:14.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:14 smithi070 ceph-mon[192652]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:14 smithi070 ceph-mon[192652]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.354 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:14 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:14.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:14 smithi070 ceph-mon[192652]: osdmap e535: 8 total, 6 up, 8 in 2023-09-23T22:40:14.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:14 smithi070 ceph-mon[192652]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:14 smithi070 ceph-mon[192652]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:14 smithi070 ceph-mon[192652]: 9.8 scrub starts 2023-09-23T22:40:14.355 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:14 smithi070 ceph-mon[192652]: 9.8 scrub ok 2023-09-23T22:40:14.648 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:14.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:14.658 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[215775]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[215775]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:14.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[215775]: osdmap e535: 8 total, 6 up, 8 in 2023-09-23T22:40:14.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[215775]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[215775]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[215775]: 9.8 scrub starts 2023-09-23T22:40:14.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[215775]: 9.8 scrub ok 2023-09-23T22:40:14.659 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:14.659 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[223014]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[223014]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:14.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[223014]: osdmap e535: 8 total, 6 up, 8 in 2023-09-23T22:40:14.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[223014]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[223014]: failed to encode map e535 with expected crc 2023-09-23T22:40:14.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[223014]: 9.8 scrub starts 2023-09-23T22:40:14.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:14 smithi026 ceph-mon[223014]: 9.8 scrub ok 2023-09-23T22:40:14.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:14.761 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:14.814 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:15.045 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:15.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:15.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:15.150 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:15.237 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:15.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:15 smithi026 ceph-mon[223014]: pgmap v1397: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 11782/112389 objects degraded (10.483%); 18622/112389 objects misplaced (16.569%); 576 KiB/s, 67 objects/s recovering 2023-09-23T22:40:15.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:15 smithi026 ceph-mon[215775]: pgmap v1397: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 11782/112389 objects degraded (10.483%); 18622/112389 objects misplaced (16.569%); 576 KiB/s, 67 objects/s recovering 2023-09-23T22:40:15.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:15 smithi070 ceph-mon[192652]: pgmap v1397: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s; 11782/112389 objects degraded (10.483%); 18622/112389 objects misplaced (16.569%); 576 KiB/s, 67 objects/s recovering 2023-09-23T22:40:16.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:40:16.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:40:16.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:40:16.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:40:16.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:40:16.166 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:40:16.167 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:40:16.167 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:40:16.167 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:40:16.167 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:40:16.167 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:40:16.167 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:40:16.167 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:40:16.167 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:40:16.391 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:16.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:16.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:16.495 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:16.521 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:16.561 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:16.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:16.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:16 smithi070 ceph-mon[192652]: 1.2 scrub starts 2023-09-23T22:40:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:16 smithi070 ceph-mon[192652]: 1.2 scrub ok 2023-09-23T22:40:16.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2155169448' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:16.668 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:16.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:16 smithi026 ceph-mon[215775]: 1.2 scrub starts 2023-09-23T22:40:16.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:16 smithi026 ceph-mon[215775]: 1.2 scrub ok 2023-09-23T22:40:16.719 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2155169448' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:16.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:16 smithi026 ceph-mon[223014]: 1.2 scrub starts 2023-09-23T22:40:16.719 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:16 smithi026 ceph-mon[223014]: 1.2 scrub ok 2023-09-23T22:40:16.720 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2155169448' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:16.733 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:16.852 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:40:17.397 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:17.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:17.403 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:17.504 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:17.541 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:17.581 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:17.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:17.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:17.690 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:17.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[215775]: pgmap v1398: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 11138/105543 objects degraded (10.553%); 17298/105543 objects misplaced (16.390%); 576 KiB/s, 67 objects/s recovering 2023-09-23T22:40:17.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[215775]: 1.1 scrub starts 2023-09-23T22:40:17.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[215775]: 1.1 scrub ok 2023-09-23T22:40:17.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[215775]: 9.5 scrub starts 2023-09-23T22:40:17.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[215775]: 9.5 scrub ok 2023-09-23T22:40:17.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[223014]: pgmap v1398: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 11138/105543 objects degraded (10.553%); 17298/105543 objects misplaced (16.390%); 576 KiB/s, 67 objects/s recovering 2023-09-23T22:40:17.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[223014]: 1.1 scrub starts 2023-09-23T22:40:17.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[223014]: 1.1 scrub ok 2023-09-23T22:40:17.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[223014]: 9.5 scrub starts 2023-09-23T22:40:17.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:17 smithi026 ceph-mon[223014]: 9.5 scrub ok 2023-09-23T22:40:17.772 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:17.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:17 smithi070 ceph-mon[192652]: pgmap v1398: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.13k op/s; 11138/105543 objects degraded (10.553%); 17298/105543 objects misplaced (16.390%); 576 KiB/s, 67 objects/s recovering 2023-09-23T22:40:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:17 smithi070 ceph-mon[192652]: 1.1 scrub starts 2023-09-23T22:40:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:17 smithi070 ceph-mon[192652]: 1.1 scrub ok 2023-09-23T22:40:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:17 smithi070 ceph-mon[192652]: 9.5 scrub starts 2023-09-23T22:40:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:17 smithi070 ceph-mon[192652]: 9.5 scrub ok 2023-09-23T22:40:18.004 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:18.010 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:18.034 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:18.035 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:18.111 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:18.148 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:18.226 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:18.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:18.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:18.335 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:18.370 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:18.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:18 smithi070 ceph-mon[192652]: 9.4 scrub starts 2023-09-23T22:40:18.395 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:18 smithi070 ceph-mon[192652]: 9.4 scrub ok 2023-09-23T22:40:18.427 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:18.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:18.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:18.536 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:18.590 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:18.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:18 smithi026 ceph-mon[223014]: 9.4 scrub starts 2023-09-23T22:40:18.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:18 smithi026 ceph-mon[223014]: 9.4 scrub ok 2023-09-23T22:40:18.596 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 11138/105543 objects degraded (10.553%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:18.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:18 smithi026 ceph-mon[215775]: 9.4 scrub starts 2023-09-23T22:40:18.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:18 smithi026 ceph-mon[215775]: 9.4 scrub ok 2023-09-23T22:40:18.598 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 11138/105543 objects degraded (10.553%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:18.617 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:18.743 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:18.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:18.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:18.756 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:18.756 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:18.857 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 11138/105543 objects degraded (10.553%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:18.955 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:19.137 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:40:19.146 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:40:19.619 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:19 smithi026 ceph-mon[215775]: pgmap v1399: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 10099/96015 objects degraded (10.518%); 15932/96015 objects misplaced (16.593%) 2023-09-23T22:40:19.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:19 smithi026 ceph-mon[223014]: pgmap v1399: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 10099/96015 objects degraded (10.518%); 15932/96015 objects misplaced (16.593%) 2023-09-23T22:40:19.723 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:19.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:19.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:19.836 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:19 smithi070 ceph-mon[192652]: pgmap v1399: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 10099/96015 objects degraded (10.518%); 15932/96015 objects misplaced (16.593%) 2023-09-23T22:40:19.990 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:40:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:40:19] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-09-23T22:40:20.036 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:40:20.042 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:20.383 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:20.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:20.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:20.491 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:20.548 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:40:20.569 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:20.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[223014]: 1.11 scrub starts 2023-09-23T22:40:20.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[223014]: 1.11 scrub ok 2023-09-23T22:40:20.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[223014]: from='client.56377 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:40:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[223014]: 9.15 scrub starts 2023-09-23T22:40:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[223014]: 9.15 scrub ok 2023-09-23T22:40:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[215775]: 1.11 scrub starts 2023-09-23T22:40:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[215775]: 1.11 scrub ok 2023-09-23T22:40:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[215775]: from='client.56377 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:40:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[215775]: 9.15 scrub starts 2023-09-23T22:40:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:20 smithi026 ceph-mon[215775]: 9.15 scrub ok 2023-09-23T22:40:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:20 smithi070 ceph-mon[192652]: 1.11 scrub starts 2023-09-23T22:40:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:20 smithi070 ceph-mon[192652]: 1.11 scrub ok 2023-09-23T22:40:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:20 smithi070 ceph-mon[192652]: from='client.56377 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:40:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:20 smithi070 ceph-mon[192652]: 9.15 scrub starts 2023-09-23T22:40:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:20 smithi070 ceph-mon[192652]: 9.15 scrub ok 2023-09-23T22:40:21.147 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:21.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:21.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:21.256 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:21.294 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:21.363 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:21.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:21.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:21.472 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:21.514 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:21.573 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:21.578 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:21 smithi070 ceph-mon[192652]: pgmap v1400: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 10099/96015 objects degraded (10.518%); 15932/96015 objects misplaced (16.593%) 2023-09-23T22:40:21.578 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:21 smithi070 ceph-mon[192652]: 9.7 scrub starts 2023-09-23T22:40:21.578 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:21 smithi070 ceph-mon[192652]: 9.7 scrub ok 2023-09-23T22:40:21.578 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:21 smithi070 ceph-mon[192652]: 9.2 scrub starts 2023-09-23T22:40:21.578 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:21 smithi070 ceph-mon[192652]: 9.2 scrub ok 2023-09-23T22:40:21.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:21.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:21.680 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:21.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[215775]: pgmap v1400: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 10099/96015 objects degraded (10.518%); 15932/96015 objects misplaced (16.593%) 2023-09-23T22:40:21.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[215775]: 9.7 scrub starts 2023-09-23T22:40:21.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[215775]: 9.7 scrub ok 2023-09-23T22:40:21.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[215775]: 9.2 scrub starts 2023-09-23T22:40:21.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[215775]: 9.2 scrub ok 2023-09-23T22:40:21.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[223014]: pgmap v1400: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 637 op/s; 10099/96015 objects degraded (10.518%); 15932/96015 objects misplaced (16.593%) 2023-09-23T22:40:21.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[223014]: 9.7 scrub starts 2023-09-23T22:40:21.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[223014]: 9.7 scrub ok 2023-09-23T22:40:21.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[223014]: 9.2 scrub starts 2023-09-23T22:40:21.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:21 smithi026 ceph-mon[223014]: 9.2 scrub ok 2023-09-23T22:40:21.747 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:22.378 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1399,"stamp":"2023-09-23T22:40:17.634977+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2100423025,"num_objects":32005,"num_object_clones":0,"num_object_copies":96015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10099,"num_objects_misplaced":15932,"num_objects_unfound":0,"num_objects_dirty":32005,"num_whiteouts":0,"num_read":6027,"num_read_kb":12268,"num_write":153379,"num_write_kb":5882354,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27761,"num_bytes_recovered":1493629393,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":195994,"ondisk_log_size":195994,"up":245,"acting":258,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":366,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11250160,"kb_used_data":7995440,"kb_used_omap":52,"kb_used_meta":3254475,"kb_avail":738711056,"statfs":{"total":767960285184,"available":756440121344,"internally_reserved":0,"allocated":8187330560,"data_stored":8169229118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53844,"internal_metadata":3332582828},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,5,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-747503616,"num_objects":-11406,"num_object_clones":0,"num_object_copies":-34218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3539,"num_objects_misplaced":-5671,"num_objects_unfound":0,"num_objects_dirty":-11406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6379,"num_write_kb":-193056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-41,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005934"},"pg_stats":[{"pgid":"9.10","version":"535'8533","reported_seq":8723,"reported_epoch":535,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:15.791053+0000","last_change":"2023-09-23T22:39:27.389970+0000","last_active":"2023-09-23T22:40:15.791053+0000","last_peered":"2023-09-23T22:40:15.791053+0000","last_clean":"2023-09-23T22:38:54.291969+0000","last_became_active":"2023-09-23T22:39:27.389970+0000","last_became_peered":"2023-09-23T22:39:27.389970+0000","last_unstale":"2023-09-23T22:40:15.791053+0000","last_undegraded":"2023-09-23T22:39:27.360689+0000","last_fullsized":"2023-09-23T22:39:27.346831+0000","mapping_epoch":525,"log_start":"461'1400","ondisk_log_start":"461'1400","created":525,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7133,"log_dups_size":0,"ondisk_log_size":7133,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66781184,"num_objects":1019,"num_object_clones":0,"num_object_copies":3057,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1019,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1019,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4850,"num_write_kb":187808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":1019}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":595,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.163095+0000","last_change":"2023-09-23T22:37:42.441352+0000","last_active":"2023-09-23T22:40:03.163095+0000","last_peered":"2023-09-23T22:40:03.163095+0000","last_clean":"2023-09-23T22:40:03.163095+0000","last_became_active":"2023-09-23T22:37:42.440853+0000","last_became_peered":"2023-09-23T22:37:42.440853+0000","last_unstale":"2023-09-23T22:40:03.163095+0000","last_undegraded":"2023-09-23T22:40:03.163095+0000","last_fullsized":"2023-09-23T22:40:03.163095+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:11.866578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":234,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.169193+0000","last_change":"2023-09-23T22:39:05.290466+0000","last_active":"2023-09-23T22:40:03.169193+0000","last_peered":"2023-09-23T22:40:03.169193+0000","last_clean":"2023-09-23T22:37:07.670854+0000","last_became_active":"2023-09-23T22:39:05.290466+0000","last_became_peered":"2023-09-23T22:39:05.290466+0000","last_unstale":"2023-09-23T22:40:03.169193+0000","last_undegraded":"2023-09-23T22:40:03.169193+0000","last_fullsized":"2023-09-23T22:39:05.285697+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:14.593594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.13","version":"535'8611","reported_seq":13056,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:14.878365+0000","last_change":"2023-09-23T22:39:29.317804+0000","last_active":"2023-09-23T22:40:14.878365+0000","last_peered":"2023-09-23T22:40:14.878365+0000","last_clean":"2023-09-23T22:40:14.878365+0000","last_became_active":"2023-09-23T22:39:29.299257+0000","last_became_peered":"2023-09-23T22:39:29.299257+0000","last_unstale":"2023-09-23T22:40:14.878365+0000","last_undegraded":"2023-09-23T22:40:14.878365+0000","last_fullsized":"2023-09-23T22:40:14.878365+0000","mapping_epoch":527,"log_start":"461'1487","ondisk_log_start":"461'1487","created":525,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7124,"log_dups_size":0,"ondisk_log_size":7124,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67534848,"num_objects":1031,"num_object_clones":0,"num_object_copies":3093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1031,"num_objects_unfound":0,"num_objects_dirty":1031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4892,"num_write_kb":189504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1071,"num_bytes_recovered":33652736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":1031}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":279,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:05.794608+0000","last_change":"2023-09-23T22:39:05.289963+0000","last_active":"2023-09-23T22:40:05.794608+0000","last_peered":"2023-09-23T22:40:05.794608+0000","last_clean":"2023-09-23T22:39:00.403392+0000","last_became_active":"2023-09-23T22:39:05.289963+0000","last_became_peered":"2023-09-23T22:39:05.289963+0000","last_unstale":"2023-09-23T22:40:05.794608+0000","last_undegraded":"2023-09-23T22:40:05.794608+0000","last_fullsized":"2023-09-23T22:39:05.284569+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.806799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":337,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.157934+0000","last_change":"2023-09-23T22:39:05.307660+0000","last_active":"2023-09-23T22:40:03.157934+0000","last_peered":"2023-09-23T22:40:03.157934+0000","last_clean":"2023-09-23T22:35:17.625896+0000","last_became_active":"2023-09-23T22:39:05.307660+0000","last_became_peered":"2023-09-23T22:39:05.307660+0000","last_unstale":"2023-09-23T22:40:03.157934+0000","last_undegraded":"2023-09-23T22:40:03.157934+0000","last_fullsized":"2023-09-23T22:39:05.302351+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:13.440047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.12","version":"535'8610","reported_seq":8832,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:15.763333+0000","last_change":"2023-09-23T22:40:05.049641+0000","last_active":"2023-09-23T22:40:15.763333+0000","last_peered":"2023-09-23T22:40:15.763333+0000","last_clean":"2023-09-23T22:40:15.763333+0000","last_became_active":"2023-09-23T22:39:29.298891+0000","last_became_peered":"2023-09-23T22:39:29.298891+0000","last_unstale":"2023-09-23T22:40:15.763333+0000","last_undegraded":"2023-09-23T22:40:15.763333+0000","last_fullsized":"2023-09-23T22:40:15.763333+0000","mapping_epoch":527,"log_start":"461'1481","ondisk_log_start":"461'1481","created":525,"last_epoch_clean":534,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7129,"log_dups_size":0,"ondisk_log_size":7129,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66584576,"num_objects":1016,"num_object_clones":0,"num_object_copies":3048,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1016,"num_objects_unfound":0,"num_objects_dirty":1016,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4898,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":1016}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":559,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.167899+0000","last_change":"2023-09-23T22:39:05.304373+0000","last_active":"2023-09-23T22:40:03.167899+0000","last_peered":"2023-09-23T22:40:03.167899+0000","last_clean":"2023-09-23T22:37:07.670321+0000","last_became_active":"2023-09-23T22:39:05.304373+0000","last_became_peered":"2023-09-23T22:39:05.304373+0000","last_unstale":"2023-09-23T22:40:03.167899+0000","last_undegraded":"2023-09-23T22:40:03.167899+0000","last_fullsized":"2023-09-23T22:39:05.284343+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:51.889095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":332,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.168688+0000","last_change":"2023-09-23T22:38:55.388832+0000","last_active":"2023-09-23T22:40:03.168688+0000","last_peered":"2023-09-23T22:40:03.168688+0000","last_clean":"2023-09-23T22:37:07.661100+0000","last_became_active":"2023-09-23T22:38:55.388832+0000","last_became_peered":"2023-09-23T22:38:55.388832+0000","last_unstale":"2023-09-23T22:40:03.168688+0000","last_undegraded":"2023-09-23T22:40:03.168688+0000","last_fullsized":"2023-09-23T22:38:55.382522+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:09.227034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":364,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.157762+0000","last_change":"2023-09-23T22:39:05.302151+0000","last_active":"2023-09-23T22:38:54.405779+0000","last_peered":"2023-09-23T22:40:03.157762+0000","last_clean":"2023-09-23T22:38:48.372435+0000","last_became_active":"2023-09-23T22:36:36.406686+0000","last_became_peered":"2023-09-23T22:39:05.302151+0000","last_unstale":"2023-09-23T22:40:03.157762+0000","last_undegraded":"2023-09-23T22:40:03.157762+0000","last_fullsized":"2023-09-23T22:39:05.301734+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:12.647131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":315,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.168068+0000","last_change":"2023-09-23T22:39:05.285935+0000","last_active":"2023-09-23T22:38:54.400276+0000","last_peered":"2023-09-23T22:40:03.168068+0000","last_clean":"2023-09-23T22:38:48.357505+0000","last_became_active":"2023-09-23T22:37:42.458398+0000","last_became_peered":"2023-09-23T22:39:05.285935+0000","last_unstale":"2023-09-23T22:40:03.168068+0000","last_undegraded":"2023-09-23T22:40:03.168068+0000","last_fullsized":"2023-09-23T22:39:05.284557+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:22.365837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":652,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.171272+0000","last_change":"2023-09-23T22:39:05.294537+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:40:03.171272+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.294537+0000","last_unstale":"2023-09-23T22:40:03.171272+0000","last_undegraded":"2023-09-23T22:40:03.171272+0000","last_fullsized":"2023-09-23T22:39:05.292289+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":553,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:05.794778+0000","last_change":"2023-09-23T22:39:05.294737+0000","last_active":"2023-09-23T22:40:05.794778+0000","last_peered":"2023-09-23T22:40:05.794778+0000","last_clean":"2023-09-23T22:39:00.404530+0000","last_became_active":"2023-09-23T22:39:05.294737+0000","last_became_peered":"2023-09-23T22:39:05.294737+0000","last_unstale":"2023-09-23T22:40:05.794778+0000","last_undegraded":"2023-09-23T22:40:05.794778+0000","last_fullsized":"2023-09-23T22:39:05.286342+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:12.866122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.172393+0000","last_change":"2023-09-23T22:39:05.298634+0000","last_active":"2023-09-23T22:38:54.377414+0000","last_peered":"2023-09-23T22:40:03.172393+0000","last_clean":"2023-09-23T22:38:48.357759+0000","last_became_active":"2023-09-23T22:36:36.521405+0000","last_became_peered":"2023-09-23T22:39:05.298634+0000","last_unstale":"2023-09-23T22:40:03.172393+0000","last_undegraded":"2023-09-23T22:40:03.172393+0000","last_fullsized":"2023-09-23T22:39:05.297707+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:35.327335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":80,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:03.171608+0000","last_change":"2023-09-23T22:39:05.295204+0000","last_active":"2023-09-23T22:40:03.171608+0000","last_peered":"2023-09-23T22:40:03.171608+0000","last_clean":"2023-09-23T22:36:31.355768+0000","last_became_active":"2023-09-23T22:39:05.295204+0000","last_became_peered":"2023-09-23T22:39:05.295204+0000","last_unstale":"2023-09-23T22:40:03.171608+0000","last_undegraded":"2023-09-23T22:39:05.291232+0000","last_fullsized":"2023-09-23T22:39:05.291128+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:25.561328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":176,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.169677+0000","last_change":"2023-09-23T22:38:55.383671+0000","last_active":"2023-09-23T22:40:03.169677+0000","last_peered":"2023-09-23T22:40:03.169677+0000","last_clean":"2023-09-23T22:38:48.410000+0000","last_became_active":"2023-09-23T22:38:55.383671+0000","last_became_peered":"2023-09-23T22:38:55.383671+0000","last_unstale":"2023-09-23T22:40:03.169677+0000","last_undegraded":"2023-09-23T22:40:03.169677+0000","last_fullsized":"2023-09-23T22:38:55.381036+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:27.813898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":524,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:03.163274+0000","last_change":"2023-09-23T22:38:55.387313+0000","last_active":"2023-09-23T22:40:03.163274+0000","last_peered":"2023-09-23T22:40:03.163274+0000","last_clean":"2023-09-23T22:33:24.367826+0000","last_became_active":"2023-09-23T22:38:55.387313+0000","last_became_peered":"2023-09-23T22:38:55.387313+0000","last_unstale":"2023-09-23T22:40:03.163274+0000","last_undegraded":"2023-09-23T22:38:55.380607+0000","last_fullsized":"2023-09-23T22:38:55.380500+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:05.769113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":28404,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":558,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.162761+0000","last_change":"2023-09-23T22:38:55.397043+0000","last_active":"2023-09-23T22:40:03.162761+0000","last_peered":"2023-09-23T22:40:03.162761+0000","last_clean":"2023-09-23T22:38:48.354081+0000","last_became_active":"2023-09-23T22:38:55.397043+0000","last_became_peered":"2023-09-23T22:38:55.397043+0000","last_unstale":"2023-09-23T22:40:03.162761+0000","last_undegraded":"2023-09-23T22:40:03.162761+0000","last_fullsized":"2023-09-23T22:38:55.379386+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:55:38.446121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.19","version":"535'8464","reported_seq":8871,"reported_epoch":535,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:15.197848+0000","last_change":"2023-09-23T22:39:27.374950+0000","last_active":"2023-09-23T22:40:15.197848+0000","last_peered":"2023-09-23T22:40:15.197848+0000","last_clean":"2023-09-23T22:36:35.270317+0000","last_became_active":"2023-09-23T22:39:27.374950+0000","last_became_peered":"2023-09-23T22:39:27.374950+0000","last_unstale":"2023-09-23T22:40:15.197848+0000","last_undegraded":"2023-09-23T22:39:27.361410+0000","last_fullsized":"2023-09-23T22:39:27.349419+0000","mapping_epoch":525,"log_start":"458'200","ondisk_log_start":"458'200","created":525,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":8264,"log_dups_size":0,"ondisk_log_size":8264,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69271552,"num_objects":1057,"num_object_clones":0,"num_object_copies":3171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1057,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4792,"num_write_kb":187168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":4751360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1057}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":914,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.175489+0000","last_change":"2023-09-23T22:36:36.374902+0000","last_active":"2023-09-23T22:40:03.175489+0000","last_peered":"2023-09-23T22:40:03.175489+0000","last_clean":"2023-09-23T22:40:03.175489+0000","last_became_active":"2023-09-23T22:36:22.309281+0000","last_became_peered":"2023-09-23T22:36:22.309281+0000","last_unstale":"2023-09-23T22:40:03.175489+0000","last_undegraded":"2023-09-23T22:40:03.175489+0000","last_fullsized":"2023-09-23T22:40:03.175489+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:38.712315+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:38.712315+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.170517793,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":168,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.175936+0000","last_change":"2023-09-23T22:35:52.674513+0000","last_active":"2023-09-23T22:40:03.175936+0000","last_peered":"2023-09-23T22:40:03.175936+0000","last_clean":"2023-09-23T22:40:03.175936+0000","last_became_active":"2023-09-23T22:35:52.674194+0000","last_became_peered":"2023-09-23T22:35:52.674194+0000","last_unstale":"2023-09-23T22:40:03.175936+0000","last_undegraded":"2023-09-23T22:40:03.175936+0000","last_fullsized":"2023-09-23T22:40:03.175936+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:54.383524+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":360,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.168837+0000","last_change":"2023-09-23T22:39:05.294773+0000","last_active":"2023-09-23T22:40:03.168837+0000","last_peered":"2023-09-23T22:40:03.168837+0000","last_clean":"2023-09-23T22:37:07.671236+0000","last_became_active":"2023-09-23T22:39:05.294773+0000","last_became_peered":"2023-09-23T22:39:05.294773+0000","last_unstale":"2023-09-23T22:40:03.168837+0000","last_undegraded":"2023-09-23T22:40:03.168837+0000","last_fullsized":"2023-09-23T22:39:05.285024+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:47.212090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.18","version":"535'8893","reported_seq":9092,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T22:40:15.738265+0000","last_change":"2023-09-23T22:39:27.401907+0000","last_active":"2023-09-23T22:40:15.738265+0000","last_peered":"2023-09-23T22:40:15.738265+0000","last_clean":"2023-09-23T22:40:15.738265+0000","last_became_active":"2023-09-23T22:39:27.390503+0000","last_became_peered":"2023-09-23T22:39:27.390503+0000","last_unstale":"2023-09-23T22:40:15.738265+0000","last_undegraded":"2023-09-23T22:40:15.738265+0000","last_fullsized":"2023-09-23T22:40:15.738265+0000","mapping_epoch":525,"log_start":"461'1700","ondisk_log_start":"461'1700","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7193,"log_dups_size":0,"ondisk_log_size":7193,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62652416,"num_objects":956,"num_object_clones":0,"num_object_copies":2868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5077,"num_write_kb":193056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":757,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.170186+0000","last_change":"2023-09-23T22:38:55.391529+0000","last_active":"2023-09-23T22:40:03.170186+0000","last_peered":"2023-09-23T22:40:03.170186+0000","last_clean":"2023-09-23T22:35:16.261819+0000","last_became_active":"2023-09-23T22:38:55.391529+0000","last_became_peered":"2023-09-23T22:38:55.391529+0000","last_unstale":"2023-09-23T22:40:03.170186+0000","last_undegraded":"2023-09-23T22:40:03.170186+0000","last_fullsized":"2023-09-23T22:38:55.384595+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":395,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.157395+0000","last_change":"2023-09-23T22:39:05.307619+0000","last_active":"2023-09-23T22:40:03.157395+0000","last_peered":"2023-09-23T22:40:03.157395+0000","last_clean":"2023-09-23T22:36:31.476578+0000","last_became_active":"2023-09-23T22:39:05.307619+0000","last_became_peered":"2023-09-23T22:39:05.307619+0000","last_unstale":"2023-09-23T22:40:03.157395+0000","last_undegraded":"2023-09-23T22:40:03.157395+0000","last_fullsized":"2023-09-23T22:39:05.284898+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:50.408737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":445,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.171243+0000","last_change":"2023-09-23T22:39:05.294555+0000","last_active":"2023-09-23T22:38:54.405186+0000","last_peered":"2023-09-23T22:40:03.171243+0000","last_clean":"2023-09-23T22:38:48.357907+0000","last_became_active":"2023-09-23T22:36:36.406789+0000","last_became_peered":"2023-09-23T22:39:05.294555+0000","last_unstale":"2023-09-23T22:40:03.171243+0000","last_undegraded":"2023-09-23T22:40:03.171243+0000","last_fullsized":"2023-09-23T22:39:05.293080+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:45.131265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":581,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.162490+0000","last_change":"2023-09-23T22:38:55.396680+0000","last_active":"2023-09-23T22:40:03.162490+0000","last_peered":"2023-09-23T22:40:03.162490+0000","last_clean":"2023-09-23T22:28:58.571704+0000","last_became_active":"2023-09-23T22:38:55.396680+0000","last_became_peered":"2023-09-23T22:38:55.396680+0000","last_unstale":"2023-09-23T22:40:03.162490+0000","last_undegraded":"2023-09-23T22:40:03.162490+0000","last_fullsized":"2023-09-23T22:38:55.378990+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":498,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.162993+0000","last_change":"2023-09-23T22:38:55.387267+0000","last_active":"2023-09-23T22:40:03.162993+0000","last_peered":"2023-09-23T22:40:03.162993+0000","last_clean":"2023-09-23T22:38:48.353526+0000","last_became_active":"2023-09-23T22:38:55.387267+0000","last_became_peered":"2023-09-23T22:38:55.387267+0000","last_unstale":"2023-09-23T22:40:03.162993+0000","last_undegraded":"2023-09-23T22:40:03.162993+0000","last_fullsized":"2023-09-23T22:38:55.379616+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:39.130605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":381,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.158146+0000","last_change":"2023-09-23T22:39:05.307686+0000","last_active":"2023-09-23T22:40:03.158146+0000","last_peered":"2023-09-23T22:40:03.158146+0000","last_clean":"2023-09-23T22:35:17.626249+0000","last_became_active":"2023-09-23T22:39:05.307686+0000","last_became_peered":"2023-09-23T22:39:05.307686+0000","last_unstale":"2023-09-23T22:40:03.158146+0000","last_undegraded":"2023-09-23T22:40:03.158146+0000","last_fullsized":"2023-09-23T22:39:05.302436+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:24.866314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":909,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.170796+0000","last_change":"2023-09-23T22:36:22.311443+0000","last_active":"2023-09-23T22:40:03.170796+0000","last_peered":"2023-09-23T22:40:03.170796+0000","last_clean":"2023-09-23T22:40:03.170796+0000","last_became_active":"2023-09-23T22:36:22.311137+0000","last_became_peered":"2023-09-23T22:36:22.311137+0000","last_unstale":"2023-09-23T22:40:03.170796+0000","last_undegraded":"2023-09-23T22:40:03.170796+0000","last_fullsized":"2023-09-23T22:40:03.170796+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":409,"reported_epoch":534,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:40:03.157692+0000","last_change":"2023-09-23T22:39:05.302168+0000","last_active":"2023-09-23T22:39:04.291704+0000","last_peered":"2023-09-23T22:40:03.157692+0000","last_clean":"2023-09-23T22:28:58.571564+0000","last_became_active":"2023-09-23T22:28:05.154059+0000","last_became_peered":"2023-09-23T22:39:05.302168+0000","last_unstale":"2023-09-23T22:40:03.157692+0000","last_undegraded":"2023-09-23T22:39:05.286285+0000","last_fullsized":"2023-09-23T22:39:05.286213+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:03.826266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":77,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.170372+0000","last_change":"2023-09-23T22:39:05.295415+0000","last_active":"2023-09-23T22:40:03.170372+0000","last_peered":"2023-09-23T22:40:03.170372+0000","last_clean":"2023-09-23T22:37:07.674004+0000","last_became_active":"2023-09-23T22:39:05.295415+0000","last_became_peered":"2023-09-23T22:39:05.295415+0000","last_unstale":"2023-09-23T22:40:03.170372+0000","last_undegraded":"2023-09-23T22:40:03.170372+0000","last_fullsized":"2023-09-23T22:39:05.293230+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:45.148758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":901,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.164084+0000","last_change":"2023-09-23T22:38:55.382695+0000","last_active":"2023-09-23T22:40:03.164084+0000","last_peered":"2023-09-23T22:40:03.164084+0000","last_clean":"2023-09-23T22:38:53.587260+0000","last_became_active":"2023-09-23T22:38:55.382695+0000","last_became_peered":"2023-09-23T22:38:55.382695+0000","last_unstale":"2023-09-23T22:40:03.164084+0000","last_undegraded":"2023-09-23T22:40:03.164084+0000","last_fullsized":"2023-09-23T22:38:55.379918+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.5","version":"535'8505","reported_seq":8775,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T22:40:15.196641+0000","last_change":"2023-09-23T22:39:27.379732+0000","last_active":"2023-09-23T22:40:15.196641+0000","last_peered":"2023-09-23T22:40:15.196641+0000","last_clean":"2023-09-23T22:40:15.196641+0000","last_became_active":"2023-09-23T22:39:27.359019+0000","last_became_peered":"2023-09-23T22:39:27.359019+0000","last_unstale":"2023-09-23T22:40:15.196641+0000","last_undegraded":"2023-09-23T22:40:15.196641+0000","last_fullsized":"2023-09-23T22:40:15.196641+0000","mapping_epoch":495,"log_start":"458'200","ondisk_log_start":"458'200","created":451,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":8305,"log_dups_size":0,"ondisk_log_size":8305,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69697536,"num_objects":1063,"num_object_clones":0,"num_object_copies":3189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1063,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4807,"num_write_kb":187872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":561,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.163577+0000","last_change":"2023-09-23T22:35:52.675241+0000","last_active":"2023-09-23T22:40:03.163577+0000","last_peered":"2023-09-23T22:40:03.163577+0000","last_clean":"2023-09-23T22:40:03.163577+0000","last_became_active":"2023-09-23T22:35:52.674769+0000","last_became_peered":"2023-09-23T22:35:52.674769+0000","last_unstale":"2023-09-23T22:40:03.163577+0000","last_undegraded":"2023-09-23T22:40:03.163577+0000","last_fullsized":"2023-09-23T22:40:03.163577+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:43.611334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":471,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.170676+0000","last_change":"2023-09-23T22:38:55.387756+0000","last_active":"2023-09-23T22:40:03.170676+0000","last_peered":"2023-09-23T22:40:03.170676+0000","last_clean":"2023-09-23T22:35:23.290223+0000","last_became_active":"2023-09-23T22:38:55.387756+0000","last_became_peered":"2023-09-23T22:38:55.387756+0000","last_unstale":"2023-09-23T22:40:03.170676+0000","last_undegraded":"2023-09-23T22:40:03.170676+0000","last_fullsized":"2023-09-23T22:38:55.384253+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:13.075495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":652,"reported_epoch":534,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:40:03.171404+0000","last_change":"2023-09-23T22:39:05.294786+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:40:03.171404+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.294786+0000","last_unstale":"2023-09-23T22:40:03.171404+0000","last_undegraded":"2023-09-23T22:39:05.292513+0000","last_fullsized":"2023-09-23T22:39:05.292435+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"535'8662","reported_seq":13570,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:15.793816+0000","last_change":"2023-09-23T22:39:27.388674+0000","last_active":"2023-09-23T22:40:15.793816+0000","last_peered":"2023-09-23T22:40:15.793816+0000","last_clean":"2023-09-23T22:40:15.793816+0000","last_became_active":"2023-09-23T22:39:27.358268+0000","last_became_peered":"2023-09-23T22:39:27.358268+0000","last_unstale":"2023-09-23T22:40:15.793816+0000","last_undegraded":"2023-09-23T22:40:15.793816+0000","last_fullsized":"2023-09-23T22:40:15.793816+0000","mapping_epoch":517,"log_start":"461'1500","ondisk_log_start":"461'1500","created":451,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7162,"log_dups_size":0,"ondisk_log_size":7162,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67371008,"num_objects":1028,"num_object_clones":0,"num_object_copies":3084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1028,"num_objects_unfound":0,"num_objects_dirty":1028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4921,"num_write_kb":190368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1257,"num_bytes_recovered":79364096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1028}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":377,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.169806+0000","last_change":"2023-09-23T22:37:42.438301+0000","last_active":"2023-09-23T22:40:03.169806+0000","last_peered":"2023-09-23T22:40:03.169806+0000","last_clean":"2023-09-23T22:40:03.169806+0000","last_became_active":"2023-09-23T22:37:42.437753+0000","last_became_peered":"2023-09-23T22:37:42.437753+0000","last_unstale":"2023-09-23T22:40:03.169806+0000","last_undegraded":"2023-09-23T22:40:03.169806+0000","last_fullsized":"2023-09-23T22:40:03.169806+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:21.146523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":509,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.167870+0000","last_change":"2023-09-23T22:39:05.289891+0000","last_active":"2023-09-23T22:40:03.167870+0000","last_peered":"2023-09-23T22:40:03.167870+0000","last_clean":"2023-09-23T22:39:00.403311+0000","last_became_active":"2023-09-23T22:39:05.289891+0000","last_became_peered":"2023-09-23T22:39:05.289891+0000","last_unstale":"2023-09-23T22:40:03.167870+0000","last_undegraded":"2023-09-23T22:40:03.167870+0000","last_fullsized":"2023-09-23T22:39:05.284229+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:50.581117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":800,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.170852+0000","last_change":"2023-09-23T22:39:05.294943+0000","last_active":"2023-09-23T22:35:19.273163+0000","last_peered":"2023-09-23T22:40:03.170852+0000","last_clean":"2023-09-23T22:35:16.262487+0000","last_became_active":"2023-09-23T22:34:36.010161+0000","last_became_peered":"2023-09-23T22:39:05.294943+0000","last_unstale":"2023-09-23T22:40:03.170852+0000","last_undegraded":"2023-09-23T22:40:03.170852+0000","last_fullsized":"2023-09-23T22:39:05.292735+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.7","version":"535'8686","reported_seq":10312,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:17.336497+0000","last_change":"2023-09-23T22:40:04.885022+0000","last_active":"2023-09-23T22:40:17.336497+0000","last_peered":"2023-09-23T22:40:17.336497+0000","last_clean":"2023-09-23T22:40:17.336497+0000","last_became_active":"2023-09-23T22:39:27.360907+0000","last_became_peered":"2023-09-23T22:39:27.360907+0000","last_unstale":"2023-09-23T22:40:17.336497+0000","last_undegraded":"2023-09-23T22:40:17.336497+0000","last_fullsized":"2023-09-23T22:40:17.336497+0000","mapping_epoch":518,"log_start":"461'893","ondisk_log_start":"461'893","created":451,"last_epoch_clean":534,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7793,"log_dups_size":0,"ondisk_log_size":7793,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64192512,"num_objects":979,"num_object_clones":0,"num_object_copies":2937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":979,"num_objects_unfound":0,"num_objects_dirty":979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4959,"num_write_kb":190048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":488,"num_bytes_recovered":16613376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,1],"avail_no_missing":["2","1","7"],"object_location_counts":[{"shards":"1,2,7","objects":979}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":337,"reported_epoch":534,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:40:03.171354+0000","last_change":"2023-09-23T22:39:05.294595+0000","last_active":"2023-09-23T22:35:19.265438+0000","last_peered":"2023-09-23T22:40:03.171354+0000","last_clean":"2023-09-23T22:35:16.262186+0000","last_became_active":"2023-09-23T22:34:32.963996+0000","last_became_peered":"2023-09-23T22:39:05.294595+0000","last_unstale":"2023-09-23T22:40:03.171354+0000","last_undegraded":"2023-09-23T22:39:05.293265+0000","last_fullsized":"2023-09-23T22:39:05.293222+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:57.686355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":235,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.168360+0000","last_change":"2023-09-23T22:38:55.397046+0000","last_active":"2023-09-23T22:40:03.168360+0000","last_peered":"2023-09-23T22:40:03.168360+0000","last_clean":"2023-09-23T22:37:07.662421+0000","last_became_active":"2023-09-23T22:38:55.397046+0000","last_became_peered":"2023-09-23T22:38:55.397046+0000","last_unstale":"2023-09-23T22:40:03.168360+0000","last_undegraded":"2023-09-23T22:40:03.168360+0000","last_fullsized":"2023-09-23T22:38:55.385491+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:27.922878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":699,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.157589+0000","last_change":"2023-09-23T22:39:05.302770+0000","last_active":"2023-09-23T22:39:04.293181+0000","last_peered":"2023-09-23T22:40:03.157589+0000","last_clean":"2023-09-23T22:28:58.573726+0000","last_became_active":"2023-09-23T22:29:02.583400+0000","last_became_peered":"2023-09-23T22:39:05.302770+0000","last_unstale":"2023-09-23T22:40:03.157589+0000","last_undegraded":"2023-09-23T22:40:03.157589+0000","last_fullsized":"2023-09-23T22:39:05.302009+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.6","version":"535'8604","reported_seq":8877,"reported_epoch":535,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:17.336692+0000","last_change":"2023-09-23T22:39:27.360835+0000","last_active":"2023-09-23T22:40:17.336692+0000","last_peered":"2023-09-23T22:40:17.336692+0000","last_clean":"2023-09-23T22:37:14.247564+0000","last_became_active":"2023-09-23T22:39:27.360835+0000","last_became_peered":"2023-09-23T22:39:27.360835+0000","last_unstale":"2023-09-23T22:40:17.336692+0000","last_undegraded":"2023-09-23T22:39:27.345966+0000","last_fullsized":"2023-09-23T22:39:27.345858+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":8604,"log_dups_size":0,"ondisk_log_size":8604,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61898752,"num_objects":944,"num_object_clones":0,"num_object_copies":2832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":944,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4929,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":944}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":379,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.169542+0000","last_change":"2023-09-23T22:39:05.286647+0000","last_active":"2023-09-23T22:38:54.408907+0000","last_peered":"2023-09-23T22:40:03.169542+0000","last_clean":"2023-09-23T22:38:53.017852+0000","last_became_active":"2023-09-23T22:37:42.458233+0000","last_became_peered":"2023-09-23T22:39:05.286647+0000","last_unstale":"2023-09-23T22:40:03.169542+0000","last_undegraded":"2023-09-23T22:40:03.169542+0000","last_fullsized":"2023-09-23T22:39:05.285527+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:29.003765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":300,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.168172+0000","last_change":"2023-09-23T22:37:42.432660+0000","last_active":"2023-09-23T22:40:03.168172+0000","last_peered":"2023-09-23T22:40:03.168172+0000","last_clean":"2023-09-23T22:40:03.168172+0000","last_became_active":"2023-09-23T22:37:42.431969+0000","last_became_peered":"2023-09-23T22:37:42.431969+0000","last_unstale":"2023-09-23T22:40:03.168172+0000","last_undegraded":"2023-09-23T22:40:03.168172+0000","last_fullsized":"2023-09-23T22:40:03.168172+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:54.642557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"535'8336","reported_seq":21860,"reported_epoch":535,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:40:14.858397+0000","last_change":"2023-09-23T22:39:27.348267+0000","last_active":"2023-09-23T22:40:14.858397+0000","last_peered":"2023-09-23T22:40:14.858397+0000","last_clean":"2023-09-23T22:35:27.126327+0000","last_became_active":"2023-09-23T22:39:27.348267+0000","last_became_peered":"2023-09-23T22:39:27.348267+0000","last_unstale":"2023-09-23T22:40:14.858397+0000","last_undegraded":"2023-09-23T22:40:14.858397+0000","last_fullsized":"2023-09-23T22:39:27.346428+0000","mapping_epoch":517,"log_start":"461'1200","ondisk_log_start":"461'1200","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7136,"log_dups_size":0,"ondisk_log_size":7136,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63733760,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1944,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4755,"num_write_kb":183296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3343,"num_bytes_recovered":183861248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":972}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":835,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.175803+0000","last_change":"2023-09-23T22:39:05.299735+0000","last_active":"2023-09-23T22:40:03.175803+0000","last_peered":"2023-09-23T22:40:03.175803+0000","last_clean":"2023-09-23T22:36:31.469549+0000","last_became_active":"2023-09-23T22:39:05.299735+0000","last_became_peered":"2023-09-23T22:39:05.299735+0000","last_unstale":"2023-09-23T22:40:03.175803+0000","last_undegraded":"2023-09-23T22:40:03.175803+0000","last_fullsized":"2023-09-23T22:39:05.286634+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:10.963206+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":350,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:03.160023+0000","last_change":"2023-09-23T22:39:05.304092+0000","last_active":"2023-09-23T22:40:03.160023+0000","last_peered":"2023-09-23T22:40:03.160023+0000","last_clean":"2023-09-23T22:36:31.477294+0000","last_became_active":"2023-09-23T22:39:05.304092+0000","last_became_peered":"2023-09-23T22:39:05.304092+0000","last_unstale":"2023-09-23T22:40:03.160023+0000","last_undegraded":"2023-09-23T22:39:05.285659+0000","last_fullsized":"2023-09-23T22:39:05.285557+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:15.273429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":443,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.163147+0000","last_change":"2023-09-23T22:39:05.290625+0000","last_active":"2023-09-23T22:38:54.402963+0000","last_peered":"2023-09-23T22:40:03.163147+0000","last_clean":"2023-09-23T22:38:53.017791+0000","last_became_active":"2023-09-23T22:36:36.405891+0000","last_became_peered":"2023-09-23T22:39:05.290625+0000","last_unstale":"2023-09-23T22:40:03.163147+0000","last_undegraded":"2023-09-23T22:40:03.163147+0000","last_fullsized":"2023-09-23T22:39:05.289355+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:02.717357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"535'8589","reported_seq":8781,"reported_epoch":535,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:15.808173+0000","last_change":"2023-09-23T22:39:27.347160+0000","last_active":"2023-09-23T22:40:15.808173+0000","last_peered":"2023-09-23T22:40:15.808173+0000","last_clean":"2023-09-23T22:38:54.291969+0000","last_became_active":"2023-09-23T22:39:27.347160+0000","last_became_peered":"2023-09-23T22:39:27.347160+0000","last_unstale":"2023-09-23T22:40:15.808173+0000","last_undegraded":"2023-09-23T22:39:27.342633+0000","last_fullsized":"2023-09-23T22:39:27.342514+0000","mapping_epoch":513,"log_start":"461'1400","ondisk_log_start":"461'1400","created":451,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7189,"log_dups_size":0,"ondisk_log_size":7189,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63111168,"num_objects":963,"num_object_clones":0,"num_object_copies":2889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":963,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4906,"num_write_kb":187808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":963}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":969,"reported_epoch":534,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:40:03.175491+0000","last_change":"2023-09-23T22:39:05.288550+0000","last_active":"2023-09-23T22:40:03.175491+0000","last_peered":"2023-09-23T22:40:03.175491+0000","last_clean":"2023-09-23T22:36:31.469347+0000","last_became_active":"2023-09-23T22:39:05.288550+0000","last_became_peered":"2023-09-23T22:39:05.288550+0000","last_unstale":"2023-09-23T22:40:03.175491+0000","last_undegraded":"2023-09-23T22:40:03.175491+0000","last_fullsized":"2023-09-23T22:39:05.284160+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:48.075424+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":303,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.163674+0000","last_change":"2023-09-23T22:38:55.396620+0000","last_active":"2023-09-23T22:40:03.163674+0000","last_peered":"2023-09-23T22:40:03.163674+0000","last_clean":"2023-09-23T22:37:07.662537+0000","last_became_active":"2023-09-23T22:38:55.396620+0000","last_became_peered":"2023-09-23T22:38:55.396620+0000","last_unstale":"2023-09-23T22:40:03.163674+0000","last_undegraded":"2023-09-23T22:40:03.163674+0000","last_fullsized":"2023-09-23T22:38:55.379490+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:47.749424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":162,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.182969+0000","last_change":"2023-09-23T22:37:42.423759+0000","last_active":"2023-09-23T22:40:03.182969+0000","last_peered":"2023-09-23T22:40:03.182969+0000","last_clean":"2023-09-23T22:40:03.182969+0000","last_became_active":"2023-09-23T22:37:42.423441+0000","last_became_peered":"2023-09-23T22:37:42.423441+0000","last_unstale":"2023-09-23T22:40:03.182969+0000","last_undegraded":"2023-09-23T22:40:03.182969+0000","last_fullsized":"2023-09-23T22:40:03.182969+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:04.848811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":581,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.162634+0000","last_change":"2023-09-23T22:38:55.396840+0000","last_active":"2023-09-23T22:40:03.162634+0000","last_peered":"2023-09-23T22:40:03.162634+0000","last_clean":"2023-09-23T22:28:58.571704+0000","last_became_active":"2023-09-23T22:38:55.396840+0000","last_became_peered":"2023-09-23T22:38:55.396840+0000","last_unstale":"2023-09-23T22:40:03.162634+0000","last_undegraded":"2023-09-23T22:40:03.162634+0000","last_fullsized":"2023-09-23T22:38:55.379140+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"535'8604","reported_seq":13048,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:14.891659+0000","last_change":"2023-09-23T22:39:27.367715+0000","last_active":"2023-09-23T22:40:14.891659+0000","last_peered":"2023-09-23T22:40:14.891659+0000","last_clean":"2023-09-23T22:40:14.891659+0000","last_became_active":"2023-09-23T22:39:27.357626+0000","last_became_peered":"2023-09-23T22:39:27.357626+0000","last_unstale":"2023-09-23T22:40:14.891659+0000","last_undegraded":"2023-09-23T22:40:14.891659+0000","last_fullsized":"2023-09-23T22:40:14.891659+0000","mapping_epoch":522,"log_start":"461'1487","ondisk_log_start":"461'1487","created":451,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7117,"log_dups_size":0,"ondisk_log_size":7117,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67993600,"num_objects":1037,"num_object_clones":0,"num_object_copies":3111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1037,"num_objects_unfound":0,"num_objects_dirty":1037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4884,"num_write_kb":189504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1070,"num_bytes_recovered":33652736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":1037}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":549,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.162662+0000","last_change":"2023-09-23T22:35:52.681932+0000","last_active":"2023-09-23T22:40:03.162662+0000","last_peered":"2023-09-23T22:40:03.162662+0000","last_clean":"2023-09-23T22:40:03.162662+0000","last_became_active":"2023-09-23T22:35:52.681575+0000","last_became_peered":"2023-09-23T22:35:52.681575+0000","last_unstale":"2023-09-23T22:40:03.162662+0000","last_undegraded":"2023-09-23T22:40:03.162662+0000","last_fullsized":"2023-09-23T22:40:03.162662+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:25.183658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":522,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.169569+0000","last_change":"2023-09-23T22:38:55.388007+0000","last_active":"2023-09-23T22:40:03.169569+0000","last_peered":"2023-09-23T22:40:03.169569+0000","last_clean":"2023-09-23T22:37:07.660692+0000","last_became_active":"2023-09-23T22:38:55.388007+0000","last_became_peered":"2023-09-23T22:38:55.388007+0000","last_unstale":"2023-09-23T22:40:03.169569+0000","last_undegraded":"2023-09-23T22:40:03.169569+0000","last_fullsized":"2023-09-23T22:38:55.384033+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:14.119720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.2","version":"535'8631","reported_seq":8859,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:15.772526+0000","last_change":"2023-09-23T22:40:03.240423+0000","last_active":"2023-09-23T22:40:15.772526+0000","last_peered":"2023-09-23T22:40:15.772526+0000","last_clean":"2023-09-23T22:40:15.772526+0000","last_became_active":"2023-09-23T22:39:27.352556+0000","last_became_peered":"2023-09-23T22:39:27.352556+0000","last_unstale":"2023-09-23T22:40:15.772526+0000","last_undegraded":"2023-09-23T22:40:15.772526+0000","last_fullsized":"2023-09-23T22:40:15.772526+0000","mapping_epoch":518,"log_start":"461'1481","ondisk_log_start":"461'1481","created":451,"last_epoch_clean":534,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7150,"log_dups_size":0,"ondisk_log_size":7150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65208320,"num_objects":995,"num_object_clones":0,"num_object_copies":2985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":995,"num_objects_unfound":0,"num_objects_dirty":995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4919,"num_write_kb":189248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":32768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":995}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":916,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.167742+0000","last_change":"2023-09-23T22:38:55.383101+0000","last_active":"2023-09-23T22:40:03.167742+0000","last_peered":"2023-09-23T22:40:03.167742+0000","last_clean":"2023-09-23T22:38:48.353691+0000","last_became_active":"2023-09-23T22:38:55.383101+0000","last_became_peered":"2023-09-23T22:38:55.383101+0000","last_unstale":"2023-09-23T22:40:03.167742+0000","last_undegraded":"2023-09-23T22:40:03.167742+0000","last_fullsized":"2023-09-23T22:38:55.381406+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:55.111338+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":253,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.168210+0000","last_change":"2023-09-23T22:39:05.297947+0000","last_active":"2023-09-23T22:40:03.168210+0000","last_peered":"2023-09-23T22:40:03.168210+0000","last_clean":"2023-09-23T22:37:07.670248+0000","last_became_active":"2023-09-23T22:39:05.297947+0000","last_became_peered":"2023-09-23T22:39:05.297947+0000","last_unstale":"2023-09-23T22:40:03.168210+0000","last_undegraded":"2023-09-23T22:40:03.168210+0000","last_fullsized":"2023-09-23T22:39:05.284753+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:18.514918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":297,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.169792+0000","last_change":"2023-09-23T22:39:05.294341+0000","last_active":"2023-09-23T22:40:03.169792+0000","last_peered":"2023-09-23T22:40:03.169792+0000","last_clean":"2023-09-23T22:39:00.403604+0000","last_became_active":"2023-09-23T22:39:05.294341+0000","last_became_peered":"2023-09-23T22:39:05.294341+0000","last_unstale":"2023-09-23T22:40:03.169792+0000","last_undegraded":"2023-09-23T22:40:03.169792+0000","last_fullsized":"2023-09-23T22:39:05.290023+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:21.644681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"535'9551","reported_seq":23228,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:15.223699+0000","last_change":"2023-09-23T22:39:09.972071+0000","last_active":"2023-09-23T22:40:15.223699+0000","last_peered":"2023-09-23T22:40:15.223699+0000","last_clean":"2023-09-23T22:40:15.223699+0000","last_became_active":"2023-09-23T22:39:06.318194+0000","last_became_peered":"2023-09-23T22:39:06.318194+0000","last_unstale":"2023-09-23T22:40:15.223699+0000","last_undegraded":"2023-09-23T22:40:15.223699+0000","last_fullsized":"2023-09-23T22:40:15.223699+0000","mapping_epoch":518,"log_start":"0'0","ondisk_log_start":"0'0","created":451,"last_epoch_clean":520,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":9551,"log_dups_size":0,"ondisk_log_size":9551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:31.028708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129171456,"num_objects":1971,"num_object_clones":0,"num_object_copies":5913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1971,"num_objects_unfound":0,"num_objects_dirty":1971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9551,"num_write_kb":368704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6634,"num_bytes_recovered":336199680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,2],"avail_no_missing":["5","1","2"],"object_location_counts":[{"shards":"1,2,5","objects":1971}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":800,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.170543+0000","last_change":"2023-09-23T22:39:05.295156+0000","last_active":"2023-09-23T22:35:19.273163+0000","last_peered":"2023-09-23T22:40:03.170543+0000","last_clean":"2023-09-23T22:35:16.262487+0000","last_became_active":"2023-09-23T22:34:36.010161+0000","last_became_peered":"2023-09-23T22:39:05.295156+0000","last_unstale":"2023-09-23T22:40:03.170543+0000","last_undegraded":"2023-09-23T22:40:03.170543+0000","last_fullsized":"2023-09-23T22:39:05.292995+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":974,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.169430+0000","last_change":"2023-09-23T22:37:42.433834+0000","last_active":"2023-09-23T22:40:03.169430+0000","last_peered":"2023-09-23T22:40:03.169430+0000","last_clean":"2023-09-23T22:40:03.169430+0000","last_became_active":"2023-09-23T22:37:42.432400+0000","last_became_peered":"2023-09-23T22:37:42.432400+0000","last_unstale":"2023-09-23T22:40:03.169430+0000","last_undegraded":"2023-09-23T22:40:03.169430+0000","last_fullsized":"2023-09-23T22:40:03.169430+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:17.145403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":574,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.182925+0000","last_change":"2023-09-23T22:39:05.287940+0000","last_active":"2023-09-23T22:38:54.405816+0000","last_peered":"2023-09-23T22:40:03.182925+0000","last_clean":"2023-09-23T22:38:48.357982+0000","last_became_active":"2023-09-23T22:36:36.407307+0000","last_became_peered":"2023-09-23T22:39:05.287940+0000","last_unstale":"2023-09-23T22:40:03.182925+0000","last_undegraded":"2023-09-23T22:40:03.182925+0000","last_fullsized":"2023-09-23T22:39:05.287229+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:59.639645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":252,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.168162+0000","last_change":"2023-09-23T22:37:42.444779+0000","last_active":"2023-09-23T22:40:03.168162+0000","last_peered":"2023-09-23T22:40:03.168162+0000","last_clean":"2023-09-23T22:40:03.168162+0000","last_became_active":"2023-09-23T22:37:42.437935+0000","last_became_peered":"2023-09-23T22:37:42.437935+0000","last_unstale":"2023-09-23T22:40:03.168162+0000","last_undegraded":"2023-09-23T22:40:03.168162+0000","last_fullsized":"2023-09-23T22:40:03.168162+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:18.444049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"535'8544","reported_seq":8951,"reported_epoch":535,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:15.197993+0000","last_change":"2023-09-23T22:39:27.364387+0000","last_active":"2023-09-23T22:40:15.197993+0000","last_peered":"2023-09-23T22:40:15.197993+0000","last_clean":"2023-09-23T22:36:35.270317+0000","last_became_active":"2023-09-23T22:39:27.364387+0000","last_became_peered":"2023-09-23T22:39:27.364387+0000","last_unstale":"2023-09-23T22:40:15.197993+0000","last_undegraded":"2023-09-23T22:39:27.349064+0000","last_fullsized":"2023-09-23T22:39:27.348946+0000","mapping_epoch":517,"log_start":"458'200","ondisk_log_start":"458'200","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":8344,"log_dups_size":0,"ondisk_log_size":8344,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64028672,"num_objects":977,"num_object_clones":0,"num_object_copies":2931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":977,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4872,"num_write_kb":187168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72,"num_bytes_recovered":4751360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":977}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":927,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T22:40:16.067424+0000","last_change":"2023-09-23T22:40:16.067531+0000","last_active":"2023-09-23T22:40:16.067424+0000","last_peered":"2023-09-23T22:40:16.067424+0000","last_clean":"2023-09-23T22:40:16.067424+0000","last_became_active":"2023-09-23T22:36:22.300226+0000","last_became_peered":"2023-09-23T22:36:22.300226+0000","last_unstale":"2023-09-23T22:40:16.067424+0000","last_undegraded":"2023-09-23T22:40:16.067424+0000","last_fullsized":"2023-09-23T22:40:16.067424+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:16.067396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:16.067396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.615276+0000","scrub_duration":0.0072300120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,4],"acting":[2,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1437,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:03.167747+0000","last_change":"2023-09-23T22:38:55.389270+0000","last_active":"2023-09-23T22:40:03.167747+0000","last_peered":"2023-09-23T22:40:03.167747+0000","last_clean":"2023-09-23T22:38:48.353781+0000","last_became_active":"2023-09-23T22:38:55.389270+0000","last_became_peered":"2023-09-23T22:38:55.389270+0000","last_unstale":"2023-09-23T22:40:03.167747+0000","last_undegraded":"2023-09-23T22:38:55.382069+0000","last_fullsized":"2023-09-23T22:38:55.381954+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:28.932588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":479,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.163291+0000","last_change":"2023-09-23T22:38:55.397986+0000","last_active":"2023-09-23T22:40:03.163291+0000","last_peered":"2023-09-23T22:40:03.163291+0000","last_clean":"2023-09-23T22:35:23.289972+0000","last_became_active":"2023-09-23T22:38:55.397986+0000","last_became_peered":"2023-09-23T22:38:55.397986+0000","last_unstale":"2023-09-23T22:40:03.163291+0000","last_undegraded":"2023-09-23T22:40:03.163291+0000","last_fullsized":"2023-09-23T22:38:55.379810+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:58.839783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":163,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.176038+0000","last_change":"2023-09-23T22:39:05.291228+0000","last_active":"2023-09-23T22:40:03.176038+0000","last_peered":"2023-09-23T22:40:03.176038+0000","last_clean":"2023-09-23T22:39:01.247643+0000","last_became_active":"2023-09-23T22:39:05.291228+0000","last_became_peered":"2023-09-23T22:39:05.291228+0000","last_unstale":"2023-09-23T22:40:03.176038+0000","last_undegraded":"2023-09-23T22:40:03.176038+0000","last_fullsized":"2023-09-23T22:39:05.285277+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:47.583378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.a","version":"535'10044","reported_seq":10376,"reported_epoch":535,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:15.803159+0000","last_change":"2023-09-23T22:39:06.557390+0000","last_active":"2023-09-23T22:40:15.803159+0000","last_peered":"2023-09-23T22:40:15.803159+0000","last_clean":"2023-09-23T22:36:35.350237+0000","last_became_active":"2023-09-23T22:39:06.291946+0000","last_became_peered":"2023-09-23T22:39:06.291946+0000","last_unstale":"2023-09-23T22:40:15.803159+0000","last_undegraded":"2023-09-23T22:39:06.286510+0000","last_fullsized":"2023-09-23T22:39:06.285916+0000","mapping_epoch":518,"log_start":"475'4958","ondisk_log_start":"475'4958","created":451,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:34.797366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136577024,"num_objects":2084,"num_object_clones":0,"num_object_copies":6252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2084,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10044,"num_write_kb":388096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":2084}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":910,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.173788+0000","last_change":"2023-09-23T22:36:22.310802+0000","last_active":"2023-09-23T22:40:03.173788+0000","last_peered":"2023-09-23T22:40:03.173788+0000","last_clean":"2023-09-23T22:40:03.173788+0000","last_became_active":"2023-09-23T22:36:22.310477+0000","last_became_peered":"2023-09-23T22:36:22.310477+0000","last_unstale":"2023-09-23T22:40:03.173788+0000","last_undegraded":"2023-09-23T22:40:03.173788+0000","last_fullsized":"2023-09-23T22:40:03.173788+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:44.599809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:44.599809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":209,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.175765+0000","last_change":"2023-09-23T22:39:05.309059+0000","last_active":"2023-09-23T22:40:03.175765+0000","last_peered":"2023-09-23T22:40:03.175765+0000","last_clean":"2023-09-23T22:39:01.248920+0000","last_became_active":"2023-09-23T22:39:05.309059+0000","last_became_peered":"2023-09-23T22:39:05.309059+0000","last_unstale":"2023-09-23T22:40:03.175765+0000","last_undegraded":"2023-09-23T22:40:03.175765+0000","last_fullsized":"2023-09-23T22:39:05.286315+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:26.012539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":154,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.175504+0000","last_change":"2023-09-23T22:38:55.382188+0000","last_active":"2023-09-23T22:40:03.175504+0000","last_peered":"2023-09-23T22:40:03.175504+0000","last_clean":"2023-09-23T22:38:48.445651+0000","last_became_active":"2023-09-23T22:38:55.382188+0000","last_became_peered":"2023-09-23T22:38:55.382188+0000","last_unstale":"2023-09-23T22:40:03.175504+0000","last_undegraded":"2023-09-23T22:40:03.175504+0000","last_fullsized":"2023-09-23T22:38:55.378777+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:42.378244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":176,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.169592+0000","last_change":"2023-09-23T22:37:42.437482+0000","last_active":"2023-09-23T22:40:03.169592+0000","last_peered":"2023-09-23T22:40:03.169592+0000","last_clean":"2023-09-23T22:40:03.169592+0000","last_became_active":"2023-09-23T22:37:42.433915+0000","last_became_peered":"2023-09-23T22:37:42.433915+0000","last_unstale":"2023-09-23T22:40:03.169592+0000","last_undegraded":"2023-09-23T22:40:03.169592+0000","last_fullsized":"2023-09-23T22:40:03.169592+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.522090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"535'8837","reported_seq":9071,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T22:40:15.798544+0000","last_change":"2023-09-23T22:40:13.891064+0000","last_active":"2023-09-23T22:40:15.798544+0000","last_peered":"2023-09-23T22:40:15.798544+0000","last_clean":"2023-09-23T22:40:15.798544+0000","last_became_active":"2023-09-23T22:39:27.353251+0000","last_became_peered":"2023-09-23T22:39:27.353251+0000","last_unstale":"2023-09-23T22:40:15.798544+0000","last_undegraded":"2023-09-23T22:40:15.798544+0000","last_fullsized":"2023-09-23T22:40:15.798544+0000","mapping_epoch":495,"log_start":"461'1700","ondisk_log_start":"461'1700","created":451,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":0,"last_scrub":"535'8779","last_scrub_stamp":"2023-09-23T22:40:13.890917+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:40:13.890917+0000","objects_scrubbed":1006,"log_size":7137,"log_dups_size":0,"ondisk_log_size":7137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:05.991033+0000","scrub_duration":0.34226520399999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61734912,"num_objects":942,"num_object_clones":0,"num_object_copies":2826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1414,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:03.175792+0000","last_change":"2023-09-23T22:39:05.299849+0000","last_active":"2023-09-23T22:40:03.175792+0000","last_peered":"2023-09-23T22:40:03.175792+0000","last_clean":"2023-09-23T22:39:01.247439+0000","last_became_active":"2023-09-23T22:39:05.299849+0000","last_became_peered":"2023-09-23T22:39:05.299849+0000","last_unstale":"2023-09-23T22:40:03.175792+0000","last_undegraded":"2023-09-23T22:39:05.286114+0000","last_fullsized":"2023-09-23T22:39:05.286006+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:43.518374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":757,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.170536+0000","last_change":"2023-09-23T22:38:55.391540+0000","last_active":"2023-09-23T22:40:03.170536+0000","last_peered":"2023-09-23T22:40:03.170536+0000","last_clean":"2023-09-23T22:35:16.261819+0000","last_became_active":"2023-09-23T22:38:55.391540+0000","last_became_peered":"2023-09-23T22:38:55.391540+0000","last_unstale":"2023-09-23T22:40:03.170536+0000","last_undegraded":"2023-09-23T22:40:03.170536+0000","last_fullsized":"2023-09-23T22:38:55.384846+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":283,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.163247+0000","last_change":"2023-09-23T22:39:05.288671+0000","last_active":"2023-09-23T22:38:54.401344+0000","last_peered":"2023-09-23T22:40:03.163247+0000","last_clean":"2023-09-23T22:38:48.357699+0000","last_became_active":"2023-09-23T22:36:36.407197+0000","last_became_peered":"2023-09-23T22:39:05.288671+0000","last_unstale":"2023-09-23T22:40:03.163247+0000","last_undegraded":"2023-09-23T22:40:03.163247+0000","last_fullsized":"2023-09-23T22:39:05.287235+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:43.518505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4],"avail_no_missing":["4"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":243,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.164337+0000","last_change":"2023-09-23T22:38:55.387206+0000","last_active":"2023-09-23T22:40:03.164337+0000","last_peered":"2023-09-23T22:40:03.164337+0000","last_clean":"2023-09-23T22:35:23.291202+0000","last_became_active":"2023-09-23T22:38:55.387206+0000","last_became_peered":"2023-09-23T22:38:55.387206+0000","last_unstale":"2023-09-23T22:40:03.164337+0000","last_undegraded":"2023-09-23T22:40:03.164337+0000","last_fullsized":"2023-09-23T22:38:55.380731+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:29.010557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.b","version":"535'9883","reported_seq":9981,"reported_epoch":535,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:15.263164+0000","last_change":"2023-09-23T22:38:55.398058+0000","last_active":"2023-09-23T22:40:15.263164+0000","last_peered":"2023-09-23T22:40:15.263164+0000","last_clean":"2023-09-23T22:38:54.300575+0000","last_became_active":"2023-09-23T22:38:55.398058+0000","last_became_peered":"2023-09-23T22:38:55.398058+0000","last_unstale":"2023-09-23T22:40:15.263164+0000","last_undegraded":"2023-09-23T22:38:55.382246+0000","last_fullsized":"2023-09-23T22:38:55.382121+0000","mapping_epoch":513,"log_start":"461'1400","ondisk_log_start":"461'1400","created":451,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":8483,"log_dups_size":0,"ondisk_log_size":8483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:50.649091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135725056,"num_objects":2071,"num_object_clones":0,"num_object_copies":6213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2071,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9883,"num_write_kb":382528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":2071}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":581,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.163029+0000","last_change":"2023-09-23T22:38:55.397258+0000","last_active":"2023-09-23T22:40:03.163029+0000","last_peered":"2023-09-23T22:40:03.163029+0000","last_clean":"2023-09-23T22:28:58.571704+0000","last_became_active":"2023-09-23T22:38:55.397258+0000","last_became_peered":"2023-09-23T22:38:55.397258+0000","last_unstale":"2023-09-23T22:40:03.163029+0000","last_undegraded":"2023-09-23T22:40:03.163029+0000","last_fullsized":"2023-09-23T22:38:55.379621+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:21.362563+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:21.362563+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":941,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.168532+0000","last_change":"2023-09-23T22:37:42.457358+0000","last_active":"2023-09-23T22:40:03.168532+0000","last_peered":"2023-09-23T22:40:03.168532+0000","last_clean":"2023-09-23T22:40:03.168532+0000","last_became_active":"2023-09-23T22:37:42.437821+0000","last_became_peered":"2023-09-23T22:37:42.437821+0000","last_unstale":"2023-09-23T22:40:03.168532+0000","last_undegraded":"2023-09-23T22:40:03.168532+0000","last_fullsized":"2023-09-23T22:40:03.168532+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:17.445534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":294,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.162758+0000","last_change":"2023-09-23T22:39:05.307919+0000","last_active":"2023-09-23T22:40:03.162758+0000","last_peered":"2023-09-23T22:40:03.162758+0000","last_clean":"2023-09-23T22:39:00.405520+0000","last_became_active":"2023-09-23T22:39:05.307919+0000","last_became_peered":"2023-09-23T22:39:05.307919+0000","last_unstale":"2023-09-23T22:40:03.162758+0000","last_undegraded":"2023-09-23T22:40:03.162758+0000","last_fullsized":"2023-09-23T22:39:05.287534+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:26.565338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":409,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.163549+0000","last_change":"2023-09-23T22:35:52.675187+0000","last_active":"2023-09-23T22:40:03.163549+0000","last_peered":"2023-09-23T22:40:03.163549+0000","last_clean":"2023-09-23T22:40:03.163549+0000","last_became_active":"2023-09-23T22:35:52.674444+0000","last_became_peered":"2023-09-23T22:35:52.674444+0000","last_unstale":"2023-09-23T22:40:03.163549+0000","last_undegraded":"2023-09-23T22:40:03.163549+0000","last_fullsized":"2023-09-23T22:40:03.163549+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:17.802244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.e","version":"535'10029","reported_seq":15392,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T22:40:17.063346+0000","last_change":"2023-09-23T22:37:46.112536+0000","last_active":"2023-09-23T22:40:17.063346+0000","last_peered":"2023-09-23T22:40:17.063346+0000","last_clean":"2023-09-23T22:40:17.063346+0000","last_became_active":"2023-09-23T22:37:42.470866+0000","last_became_peered":"2023-09-23T22:37:42.470866+0000","last_unstale":"2023-09-23T22:40:17.063346+0000","last_undegraded":"2023-09-23T22:40:17.063346+0000","last_fullsized":"2023-09-23T22:40:17.063346+0000","mapping_epoch":495,"log_start":"469'4315","ondisk_log_start":"469'4315","created":451,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5714,"log_dups_size":0,"ondisk_log_size":5714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:22.071126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122880000,"num_objects":1875,"num_object_clones":0,"num_object_copies":5625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10029,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2609,"num_bytes_recovered":166002688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":699,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.157737+0000","last_change":"2023-09-23T22:39:05.302767+0000","last_active":"2023-09-23T22:39:04.293769+0000","last_peered":"2023-09-23T22:40:03.157737+0000","last_clean":"2023-09-23T22:28:58.573726+0000","last_became_active":"2023-09-23T22:29:02.583400+0000","last_became_peered":"2023-09-23T22:39:05.302767+0000","last_unstale":"2023-09-23T22:40:03.157737+0000","last_undegraded":"2023-09-23T22:40:03.157737+0000","last_fullsized":"2023-09-23T22:39:05.302023+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":127,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.171724+0000","last_change":"2023-09-23T22:35:52.688113+0000","last_active":"2023-09-23T22:40:03.171724+0000","last_peered":"2023-09-23T22:40:03.171724+0000","last_clean":"2023-09-23T22:40:03.171724+0000","last_became_active":"2023-09-23T22:35:52.687708+0000","last_became_peered":"2023-09-23T22:35:52.687708+0000","last_unstale":"2023-09-23T22:40:03.171724+0000","last_undegraded":"2023-09-23T22:40:03.171724+0000","last_fullsized":"2023-09-23T22:40:03.171724+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:07.029149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":157,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.176105+0000","last_change":"2023-09-23T22:37:42.424518+0000","last_active":"2023-09-23T22:40:03.176105+0000","last_peered":"2023-09-23T22:40:03.176105+0000","last_clean":"2023-09-23T22:40:03.176105+0000","last_became_active":"2023-09-23T22:37:42.424144+0000","last_became_peered":"2023-09-23T22:37:42.424144+0000","last_unstale":"2023-09-23T22:40:03.176105+0000","last_undegraded":"2023-09-23T22:40:03.176105+0000","last_fullsized":"2023-09-23T22:40:03.176105+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:52:51.255509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":341,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.168784+0000","last_change":"2023-09-23T22:37:42.432892+0000","last_active":"2023-09-23T22:40:03.168784+0000","last_peered":"2023-09-23T22:40:03.168784+0000","last_clean":"2023-09-23T22:40:03.168784+0000","last_became_active":"2023-09-23T22:37:42.432356+0000","last_became_peered":"2023-09-23T22:37:42.432356+0000","last_unstale":"2023-09-23T22:40:03.168784+0000","last_undegraded":"2023-09-23T22:40:03.168784+0000","last_fullsized":"2023-09-23T22:40:03.168784+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:14.288445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.d","version":"535'10032","reported_seq":15490,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T22:40:14.904549+0000","last_change":"2023-09-23T22:37:42.930274+0000","last_active":"2023-09-23T22:40:14.904549+0000","last_peered":"2023-09-23T22:40:14.904549+0000","last_clean":"2023-09-23T22:40:14.904549+0000","last_became_active":"2023-09-23T22:37:42.448382+0000","last_became_peered":"2023-09-23T22:37:42.448382+0000","last_unstale":"2023-09-23T22:40:14.904549+0000","last_undegraded":"2023-09-23T22:40:14.904549+0000","last_fullsized":"2023-09-23T22:40:14.904549+0000","mapping_epoch":495,"log_start":"473'4807","ondisk_log_start":"473'4807","created":451,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5225,"log_dups_size":0,"ondisk_log_size":5225,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:20.785382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136380440,"num_objects":2082,"num_object_clones":0,"num_object_copies":6246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2082,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10032,"num_write_kb":387585,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2659,"num_bytes_recovered":168820736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":905,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.164561+0000","last_change":"2023-09-23T22:38:55.383034+0000","last_active":"2023-09-23T22:40:03.164561+0000","last_peered":"2023-09-23T22:40:03.164561+0000","last_clean":"2023-09-23T22:38:48.354625+0000","last_became_active":"2023-09-23T22:38:55.383034+0000","last_became_peered":"2023-09-23T22:38:55.383034+0000","last_unstale":"2023-09-23T22:40:03.164561+0000","last_undegraded":"2023-09-23T22:40:03.164561+0000","last_fullsized":"2023-09-23T22:38:55.380898+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:37.492514+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:37.492514+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0052039740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":172,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:03.170671+0000","last_change":"2023-09-23T22:39:05.312608+0000","last_active":"2023-09-23T22:40:03.170671+0000","last_peered":"2023-09-23T22:40:03.170671+0000","last_clean":"2023-09-23T22:35:17.625026+0000","last_became_active":"2023-09-23T22:39:05.312608+0000","last_became_peered":"2023-09-23T22:39:05.312608+0000","last_unstale":"2023-09-23T22:40:03.170671+0000","last_undegraded":"2023-09-23T22:39:05.293584+0000","last_fullsized":"2023-09-23T22:39:05.293517+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:47.778707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":1037,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.169665+0000","last_change":"2023-09-23T22:38:55.388290+0000","last_active":"2023-09-23T22:40:03.169665+0000","last_peered":"2023-09-23T22:40:03.169665+0000","last_clean":"2023-09-23T22:38:48.410284+0000","last_became_active":"2023-09-23T22:38:55.388290+0000","last_became_peered":"2023-09-23T22:38:55.388290+0000","last_unstale":"2023-09-23T22:40:03.169665+0000","last_undegraded":"2023-09-23T22:40:03.169665+0000","last_fullsized":"2023-09-23T22:38:55.384357+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:40.082337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":57,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.170809+0000","last_change":"2023-09-23T22:39:05.312578+0000","last_active":"2023-09-23T22:40:03.170809+0000","last_peered":"2023-09-23T22:40:03.170809+0000","last_clean":"2023-09-23T22:31:26.131914+0000","last_became_active":"2023-09-23T22:39:05.312578+0000","last_became_peered":"2023-09-23T22:39:05.312578+0000","last_unstale":"2023-09-23T22:40:03.170809+0000","last_undegraded":"2023-09-23T22:40:03.170809+0000","last_fullsized":"2023-09-23T22:39:05.291104+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:52.614783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"535'10003","reported_seq":15725,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:17.078597+0000","last_change":"2023-09-23T22:40:03.053230+0000","last_active":"2023-09-23T22:40:17.078597+0000","last_peered":"2023-09-23T22:40:17.078597+0000","last_clean":"2023-09-23T22:40:17.078597+0000","last_became_active":"2023-09-23T22:39:06.327496+0000","last_became_peered":"2023-09-23T22:39:06.327496+0000","last_unstale":"2023-09-23T22:40:17.078597+0000","last_undegraded":"2023-09-23T22:40:17.078597+0000","last_fullsized":"2023-09-23T22:40:17.078597+0000","mapping_epoch":518,"log_start":"469'4268","ondisk_log_start":"469'4268","created":451,"last_epoch_clean":533,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":5735,"log_dups_size":0,"ondisk_log_size":5735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:42.584301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126681088,"num_objects":1933,"num_object_clones":0,"num_object_copies":5799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1933,"num_objects_unfound":0,"num_objects_dirty":1933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10003,"num_write_kb":381952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2500,"num_bytes_recovered":163840000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,0],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4,7","objects":1933}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":652,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.171712+0000","last_change":"2023-09-23T22:39:05.295060+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:40:03.171712+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.295060+0000","last_unstale":"2023-09-23T22:40:03.171712+0000","last_undegraded":"2023-09-23T22:40:03.171712+0000","last_fullsized":"2023-09-23T22:39:05.292715+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":158,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.184113+0000","last_change":"2023-09-23T22:38:55.388004+0000","last_active":"2023-09-23T22:40:03.184113+0000","last_peered":"2023-09-23T22:40:03.184113+0000","last_clean":"2023-09-23T22:33:41.182644+0000","last_became_active":"2023-09-23T22:38:55.388004+0000","last_became_peered":"2023-09-23T22:38:55.388004+0000","last_unstale":"2023-09-23T22:40:03.184113+0000","last_undegraded":"2023-09-23T22:40:03.184113+0000","last_fullsized":"2023-09-23T22:38:55.379533+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:15.982785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":275,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.169861+0000","last_change":"2023-09-23T22:37:42.460308+0000","last_active":"2023-09-23T22:40:03.169861+0000","last_peered":"2023-09-23T22:40:03.169861+0000","last_clean":"2023-09-23T22:40:03.169861+0000","last_became_active":"2023-09-23T22:37:42.438206+0000","last_became_peered":"2023-09-23T22:37:42.438206+0000","last_unstale":"2023-09-23T22:40:03.169861+0000","last_undegraded":"2023-09-23T22:40:03.169861+0000","last_fullsized":"2023-09-23T22:40:03.169861+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:54.493150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":413,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.163253+0000","last_change":"2023-09-23T22:38:55.382690+0000","last_active":"2023-09-23T22:40:03.163253+0000","last_peered":"2023-09-23T22:40:03.163253+0000","last_clean":"2023-09-23T22:35:23.293559+0000","last_became_active":"2023-09-23T22:38:55.382690+0000","last_became_peered":"2023-09-23T22:38:55.382690+0000","last_unstale":"2023-09-23T22:40:03.163253+0000","last_undegraded":"2023-09-23T22:40:03.163253+0000","last_fullsized":"2023-09-23T22:38:55.380032+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:58.155649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.11","version":"535'8319","reported_seq":21844,"reported_epoch":535,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:40:14.936107+0000","last_change":"2023-09-23T22:39:29.299428+0000","last_active":"2023-09-23T22:40:14.936107+0000","last_peered":"2023-09-23T22:40:14.936107+0000","last_clean":"2023-09-23T22:35:27.126327+0000","last_became_active":"2023-09-23T22:39:29.299428+0000","last_became_peered":"2023-09-23T22:39:29.299428+0000","last_unstale":"2023-09-23T22:40:14.936107+0000","last_undegraded":"2023-09-23T22:40:14.936107+0000","last_fullsized":"2023-09-23T22:39:29.294272+0000","mapping_epoch":527,"log_start":"461'1200","ondisk_log_start":"461'1200","created":525,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7119,"log_dups_size":0,"ondisk_log_size":7119,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64847872,"num_objects":990,"num_object_clones":0,"num_object_copies":2970,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1980,"num_objects_unfound":0,"num_objects_dirty":990,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4739,"num_write_kb":183296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3344,"num_bytes_recovered":183861248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":990}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":190,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.158103+0000","last_change":"2023-09-23T22:35:52.675772+0000","last_active":"2023-09-23T22:40:03.158103+0000","last_peered":"2023-09-23T22:40:03.158103+0000","last_clean":"2023-09-23T22:40:03.158103+0000","last_became_active":"2023-09-23T22:35:52.675279+0000","last_became_peered":"2023-09-23T22:35:52.675279+0000","last_unstale":"2023-09-23T22:40:03.158103+0000","last_undegraded":"2023-09-23T22:40:03.158103+0000","last_fullsized":"2023-09-23T22:40:03.158103+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.513213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":162,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.176155+0000","last_change":"2023-09-23T22:39:05.291652+0000","last_active":"2023-09-23T22:40:03.176155+0000","last_peered":"2023-09-23T22:40:03.176155+0000","last_clean":"2023-09-23T22:39:01.247800+0000","last_became_active":"2023-09-23T22:39:05.291652+0000","last_became_peered":"2023-09-23T22:39:05.291652+0000","last_unstale":"2023-09-23T22:40:03.176155+0000","last_undegraded":"2023-09-23T22:40:03.176155+0000","last_fullsized":"2023-09-23T22:39:05.287811+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:03.849150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.16","version":"535'8591","reported_seq":8863,"reported_epoch":535,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:17.583692+0000","last_change":"2023-09-23T22:39:27.381437+0000","last_active":"2023-09-23T22:40:17.583692+0000","last_peered":"2023-09-23T22:40:17.583692+0000","last_clean":"2023-09-23T22:37:14.247564+0000","last_became_active":"2023-09-23T22:39:27.381437+0000","last_became_peered":"2023-09-23T22:39:27.381437+0000","last_unstale":"2023-09-23T22:40:17.583692+0000","last_undegraded":"2023-09-23T22:39:27.350981+0000","last_fullsized":"2023-09-23T22:39:27.346140+0000","mapping_epoch":525,"log_start":"0'0","ondisk_log_start":"0'0","created":525,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":8591,"log_dups_size":0,"ondisk_log_size":8591,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62750720,"num_objects":958,"num_object_clones":0,"num_object_copies":2874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":958,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4917,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[{"shards":"1,2","objects":958}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":559,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T22:40:03.167790+0000","last_change":"2023-09-23T22:37:42.419112+0000","last_active":"2023-09-23T22:40:03.167790+0000","last_peered":"2023-09-23T22:40:03.167790+0000","last_clean":"2023-09-23T22:40:03.167790+0000","last_became_active":"2023-09-23T22:37:42.418593+0000","last_became_peered":"2023-09-23T22:37:42.418593+0000","last_unstale":"2023-09-23T22:40:03.167790+0000","last_undegraded":"2023-09-23T22:40:03.167790+0000","last_fullsized":"2023-09-23T22:40:03.167790+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:46.031051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":523,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.168522+0000","last_change":"2023-09-23T22:39:05.290461+0000","last_active":"2023-09-23T22:40:03.168522+0000","last_peered":"2023-09-23T22:40:03.168522+0000","last_clean":"2023-09-23T22:37:07.670799+0000","last_became_active":"2023-09-23T22:39:05.290461+0000","last_became_peered":"2023-09-23T22:39:05.290461+0000","last_unstale":"2023-09-23T22:40:03.168522+0000","last_undegraded":"2023-09-23T22:40:03.168522+0000","last_fullsized":"2023-09-23T22:39:05.285092+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:25:31.927928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.17","version":"535'8686","reported_seq":10386,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:17.447884+0000","last_change":"2023-09-23T22:40:06.849164+0000","last_active":"2023-09-23T22:40:17.447884+0000","last_peered":"2023-09-23T22:40:17.447884+0000","last_clean":"2023-09-23T22:40:17.447884+0000","last_became_active":"2023-09-23T22:39:29.302388+0000","last_became_peered":"2023-09-23T22:39:29.302388+0000","last_unstale":"2023-09-23T22:40:17.447884+0000","last_undegraded":"2023-09-23T22:40:17.447884+0000","last_fullsized":"2023-09-23T22:40:17.447884+0000","mapping_epoch":527,"log_start":"461'893","ondisk_log_start":"461'893","created":525,"last_epoch_clean":534,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7793,"log_dups_size":0,"ondisk_log_size":7793,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64192512,"num_objects":980,"num_object_clones":0,"num_object_copies":2940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":980,"num_objects_unfound":0,"num_objects_dirty":980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4960,"num_write_kb":190048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":526,"num_bytes_recovered":16613376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,1],"avail_no_missing":["2","1","7"],"object_location_counts":[{"shards":"1,2,7","objects":980}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":574,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:03.167916+0000","last_change":"2023-09-23T22:38:55.382794+0000","last_active":"2023-09-23T22:40:03.167916+0000","last_peered":"2023-09-23T22:40:03.167916+0000","last_clean":"2023-09-23T22:38:48.353999+0000","last_became_active":"2023-09-23T22:38:55.382794+0000","last_became_peered":"2023-09-23T22:38:55.382794+0000","last_unstale":"2023-09-23T22:40:03.167916+0000","last_undegraded":"2023-09-23T22:38:55.381634+0000","last_fullsized":"2023-09-23T22:38:55.381551+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:52.591257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":300,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.163805+0000","last_change":"2023-09-23T22:39:05.307911+0000","last_active":"2023-09-23T22:40:03.163805+0000","last_peered":"2023-09-23T22:40:03.163805+0000","last_clean":"2023-09-23T22:35:23.286304+0000","last_became_active":"2023-09-23T22:39:05.307911+0000","last_became_peered":"2023-09-23T22:39:05.307911+0000","last_unstale":"2023-09-23T22:40:03.163805+0000","last_undegraded":"2023-09-23T22:40:03.163805+0000","last_fullsized":"2023-09-23T22:39:05.287595+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:03.410439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.14","version":"535'8654","reported_seq":13562,"reported_epoch":535,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:40:15.777847+0000","last_change":"2023-09-23T22:39:29.338780+0000","last_active":"2023-09-23T22:40:15.777847+0000","last_peered":"2023-09-23T22:40:15.777847+0000","last_clean":"2023-09-23T22:40:15.777847+0000","last_became_active":"2023-09-23T22:39:29.325920+0000","last_became_peered":"2023-09-23T22:39:29.325920+0000","last_unstale":"2023-09-23T22:40:15.777847+0000","last_undegraded":"2023-09-23T22:40:15.777847+0000","last_fullsized":"2023-09-23T22:40:15.777847+0000","mapping_epoch":527,"log_start":"461'1500","ondisk_log_start":"461'1500","created":525,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":7154,"log_dups_size":0,"ondisk_log_size":7154,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67895296,"num_objects":1036,"num_object_clones":0,"num_object_copies":3108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1036,"num_objects_unfound":0,"num_objects_dirty":1036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4913,"num_write_kb":190368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1257,"num_bytes_recovered":79364096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1036}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":133,"reported_epoch":534,"state":"undersized+peered","last_fresh":"2023-09-23T22:40:03.183319+0000","last_change":"2023-09-23T22:39:05.288020+0000","last_active":"2023-09-23T22:33:45.291092+0000","last_peered":"2023-09-23T22:40:03.183319+0000","last_clean":"2023-09-23T22:33:41.181859+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:39:05.288020+0000","last_unstale":"2023-09-23T22:40:03.183319+0000","last_undegraded":"2023-09-23T22:40:03.183319+0000","last_fullsized":"2023-09-23T22:39:05.287481+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.969554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":106,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.175910+0000","last_change":"2023-09-23T22:39:05.309675+0000","last_active":"2023-09-23T22:40:03.175910+0000","last_peered":"2023-09-23T22:40:03.175910+0000","last_clean":"2023-09-23T22:34:30.962054+0000","last_became_active":"2023-09-23T22:39:05.309675+0000","last_became_peered":"2023-09-23T22:39:05.309675+0000","last_unstale":"2023-09-23T22:40:03.175910+0000","last_undegraded":"2023-09-23T22:40:03.175910+0000","last_fullsized":"2023-09-23T22:39:05.286965+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:26.850825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.15","version":"535'8551","reported_seq":8820,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T22:40:15.222200+0000","last_change":"2023-09-23T22:39:27.419613+0000","last_active":"2023-09-23T22:40:15.222200+0000","last_peered":"2023-09-23T22:40:15.222200+0000","last_clean":"2023-09-23T22:40:15.222200+0000","last_became_active":"2023-09-23T22:39:27.409135+0000","last_became_peered":"2023-09-23T22:39:27.409135+0000","last_unstale":"2023-09-23T22:40:15.222200+0000","last_undegraded":"2023-09-23T22:40:15.222200+0000","last_fullsized":"2023-09-23T22:40:15.222200+0000","mapping_epoch":525,"log_start":"458'200","ondisk_log_start":"458'200","created":525,"last_epoch_clean":526,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:35:14.233163+0000","last_clean_scrub_stamp":"2023-09-23T22:35:14.233163+0000","objects_scrubbed":0,"log_size":8351,"log_dups_size":0,"ondisk_log_size":8351,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66682880,"num_objects":1018,"num_object_clones":0,"num_object_copies":3054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4854,"num_write_kb":187872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":485,"reported_epoch":534,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:40:03.184192+0000","last_change":"2023-09-23T22:38:55.394101+0000","last_active":"2023-09-23T22:40:03.184192+0000","last_peered":"2023-09-23T22:40:03.184192+0000","last_clean":"2023-09-23T22:38:48.446310+0000","last_became_active":"2023-09-23T22:38:55.394101+0000","last_became_peered":"2023-09-23T22:38:55.394101+0000","last_unstale":"2023-09-23T22:40:03.184192+0000","last_undegraded":"2023-09-23T22:38:55.380019+0000","last_fullsized":"2023-09-23T22:38:55.379907+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:57.436984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":618,"reported_epoch":534,"state":"active+undersized","last_fresh":"2023-09-23T22:40:03.162727+0000","last_change":"2023-09-23T22:38:55.384159+0000","last_active":"2023-09-23T22:40:03.162727+0000","last_peered":"2023-09-23T22:40:03.162727+0000","last_clean":"2023-09-23T22:38:48.354354+0000","last_became_active":"2023-09-23T22:38:55.384159+0000","last_became_peered":"2023-09-23T22:38:55.384159+0000","last_unstale":"2023-09-23T22:40:03.162727+0000","last_undegraded":"2023-09-23T22:40:03.162727+0000","last_fullsized":"2023-09-23T22:38:55.378819+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:01.489638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":66,"acting":66,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":93,"num_bytes_recovered":31744,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48807,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":66,"acting":66,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1211762,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":38,"acting":39,"num_store_stats":8},{"poolid":9,"num_pg":26,"stat_sum":{"num_bytes":2095579160,"num_objects":31977,"num_object_clones":0,"num_object_copies":95931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10073,"num_objects_misplaced":15930,"num_objects_unfound":0,"num_objects_dirty":31977,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152450,"num_write_kb":5869409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27349,"num_bytes_recovered":1471414272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8053600256,"data_stored":8053588040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":191582,"ondisk_log_size":191582,"up":56,"acting":68,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":314,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":19,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234430,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1975412,"kb_used_data":1497452,"kb_used_omap":0,"kb_used_meta":477952,"kb_avail":91769740,"statfs":{"total":95995035648,"available":93972213760,"internally_reserved":0,"allocated":1533390848,"data_stored":1531596778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":489422848},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:33 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.4870000000000001,"15min":3.7999999999999998},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":2.2629999999999999,"5min":49.024999999999999,"15min":220.66},"last":6.7279999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":1.4990000000000001,"15min":4.0609999999999999},"min":{"1min":0.24199999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.6379999999999999,"5min":48.901000000000003,"15min":220.59899999999999},"last":0.24399999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.52200000000000002,"15min":0.52800000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.2160000000000002,"5min":3.4700000000000002,"15min":3.4700000000000002},"last":1.504},{"interface":"front","average":{"1min":0.443,"5min":0.60499999999999998,"15min":0.63200000000000001},"min":{"1min":0.27000000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":1.2230000000000001,"5min":2.9500000000000002,"15min":2.9500000000000002},"last":0.25900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:40:10 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.27,"15min":1.6060000000000001},"min":{"1min":0.28000000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.335,"5min":7.335,"15min":70.183999999999997},"last":4.3280000000000003},{"interface":"front","average":{"1min":1.034,"5min":1.516,"15min":1.8169999999999999},"min":{"1min":0.33900000000000002,"5min":0.307,"15min":0.28899999999999998},"max":{"1min":6.8860000000000001,"5min":48.762,"15min":70.140000000000001},"last":4.5599999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.2969999999999999,"15min":2.5390000000000001},"min":{"1min":0.35999999999999999,"5min":0.316,"15min":0.26900000000000002},"max":{"1min":3.5710000000000002,"5min":9.8989999999999991,"15min":219.90700000000001},"last":0.435},{"interface":"front","average":{"1min":0.82699999999999996,"5min":2.101,"15min":2.9540000000000002},"min":{"1min":0.38,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":3.3719999999999999,"5min":48.866999999999997,"15min":219.976},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:39:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.60399999999999998,"15min":0.70799999999999996},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":2.6389999999999998,"5min":2.8090000000000002,"15min":2.8090000000000002},"last":0.30399999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":1.627,"15min":1.742},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":2.3050000000000002,"5min":48.960000000000001,"15min":48.960000000000001},"last":0.29399999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:39:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.8420000000000001,"15min":1.756},"min":{"1min":0.314,"5min":0.28499999999999998,"15min":0.249},"max":{"1min":1.794,"5min":53.720999999999997,"15min":125.083},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.503,"5min":1.95,"15min":1.7969999999999999},"min":{"1min":0.311,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.716,"5min":53.218000000000004,"15min":122.351},"last":0.33200000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:38:22 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":2.5499999999999998,"15min":1.389},"min":{"1min":0.32000000000000001,"5min":0.311,"15min":0.29499999999999998},"max":{"1min":0.77500000000000002,"5min":88.284999999999997,"15min":88.284999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.623,"5min":2.4009999999999998,"15min":1.4339999999999999},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.33700000000000002},"max":{"1min":2.3050000000000002,"5min":88.355000000000004,"15min":88.355000000000004},"last":0.54900000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496049,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2171296,"kb_used_data":1893144,"kb_used_omap":0,"kb_used_meta":278144,"kb_avail":91573856,"statfs":{"total":95995035648,"available":93771628544,"internally_reserved":0,"allocated":1938579456,"data_stored":1936876687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":284819456},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":4.9400000000000004,"15min":4.5979999999999999},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":0.623,"5min":173.291,"15min":213.23699999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":5.2850000000000001,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.22600000000000001},"max":{"1min":0.59799999999999998,"5min":174.232,"15min":213.304},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.4570000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:38:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":3.8730000000000002,"15min":2.5680000000000001},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.2770000000000001,"5min":103.548,"15min":103.548},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":3.8069999999999999,"15min":2.657},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.583,"5min":105.99299999999999,"15min":105.99299999999999},"last":2.5049999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:37:58 2023","interfaces":[{"interface":"back","average":{"1min":4.3259999999999996,"5min":2.2690000000000001,"15min":2.9209999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":82.584000000000003,"5min":82.584000000000003,"15min":125.125},"last":0.58399999999999996},{"interface":"front","average":{"1min":4.6360000000000001,"5min":2.2040000000000002,"15min":2.5070000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":82.316000000000003,"5min":82.316000000000003,"15min":82.316000000000003},"last":2.3290000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:37:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.99099999999999999,"15min":1.042},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.5099999999999998,"5min":7.5099999999999998,"15min":7.5099999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.4299999999999999,"15min":1.5589999999999999},"min":{"1min":0.309,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0569999999999999,"5min":6.3490000000000002,"15min":6.3490000000000002},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 22:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.7110000000000001,"15min":1.6779999999999999},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.4359999999999999,"5min":15.032999999999999,"15min":88.194999999999993},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.1279999999999999,"15min":1.359},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69499999999999995,"5min":7.0839999999999996,"15min":88.025000000000006},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.117,"15min":1.353},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.314},"max":{"1min":0.71199999999999997,"5min":7.5670000000000002,"15min":82.801000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.2649999999999999,"15min":1.508},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.29699999999999999},"max":{"1min":0.65400000000000003,"5min":13.465999999999999,"15min":83.052999999999997},"last":0.65800000000000003}]}]},{"osd":1,"up_from":495,"seq":2126008811552,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1775000,"kb_used_data":1569936,"kb_used_omap":12,"kb_used_meta":205043,"kb_avail":91970152,"statfs":{"total":95995035648,"available":94177435648,"internally_reserved":0,"allocated":1607614464,"data_stored":1604844652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":209964718},"hb_peers":[0,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:43 2023","interfaces":[{"interface":"back","average":{"1min":4.1769999999999996,"5min":1.2589999999999999,"15min":0.77300000000000002},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":64.507000000000005,"5min":64.507000000000005,"15min":64.507000000000005},"last":2.867},{"interface":"front","average":{"1min":4.25,"5min":1.5700000000000001,"15min":1.1240000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":64.352000000000004,"5min":64.352000000000004,"15min":64.352000000000004},"last":2.3090000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.64500000000000002,"15min":0.66300000000000003},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":1.946,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":4.3419999999999996},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.80200000000000005,"15min":0.78100000000000003},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.585,"5min":2.585,"15min":2.585},"last":1.2290000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.6390000000000002,"5min":5.6390000000000002,"15min":5.6390000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.032,"15min":1.032},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.5739999999999998,"5min":4.5739999999999998,"15min":4.5739999999999998},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.56000000000000005,"15min":0.55400000000000005},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":2.0009999999999999,"5min":2.0009999999999999,"15min":2.0009999999999999},"last":0.37},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.89600000000000002,"15min":0.96599999999999997},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.2470000000000001,"5min":5.6740000000000004,"15min":5.6740000000000004},"last":0.42399999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.85199999999999998,"15min":0.85399999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":2.673,"5min":2.673,"15min":2.673},"last":1.075},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.59999999999999998,"15min":0.60199999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.1819999999999999,"5min":2.1819999999999999,"15min":2.1819999999999999},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.503},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":5.7050000000000001,"5min":5.7050000000000001,"15min":5.7050000000000001},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.92000000000000004,"5min":0.878,"15min":0.871},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.698,"5min":2.698,"15min":2.698},"last":1.133},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.66600000000000004,"15min":0.65000000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.8899999999999997,"5min":4.8899999999999997,"15min":4.8899999999999997},"last":0.79600000000000004}]}]},{"osd":0,"up_from":93,"seq":399431958867,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1705232,"kb_used_data":799496,"kb_used_omap":8,"kb_used_meta":905719,"kb_avail":92039920,"statfs":{"total":95995035648,"available":94248878080,"internally_reserved":0,"allocated":818683904,"data_stored":815860240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":927457130},"hb_peers":[1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:39:45 2023","interfaces":[{"interface":"back","average":{"1min":0.317,"5min":0.66800000000000004,"15min":0.72599999999999998},"min":{"1min":0.16600000000000001,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":0.61799999999999999,"5min":3.8599999999999999,"15min":3.8599999999999999},"last":1.171},{"interface":"front","average":{"1min":0.35299999999999998,"5min":0.49399999999999999,"15min":0.51800000000000002},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.56100000000000005,"5min":1.6539999999999999,"15min":1.6539999999999999},"last":0.26500000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:39:15 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":3.972,"15min":3.004},"min":{"1min":0.23499999999999999,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":2.5569999999999999,"5min":175.89400000000001,"15min":175.89400000000001},"last":4.109},{"interface":"front","average":{"1min":0.58599999999999997,"5min":3.7879999999999998,"15min":2.2669999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":1.827,"5min":175.66300000000001,"15min":175.66300000000001},"last":0.442}]},{"osd":3,"last update":"Sat Sep 23 22:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.024,"15min":2.4399999999999999},"min":{"1min":0.28799999999999998,"5min":0.23100000000000001,"15min":0.22500000000000001},"max":{"1min":1.698,"5min":7.0579999999999998,"15min":146.87299999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.149,"15min":2.516},"min":{"1min":0.311,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.1459999999999999,"5min":10.026,"15min":145.48099999999999},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":1.0529999999999999,"15min":1.2130000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":3.1840000000000002,"5min":9.968,"15min":9.968},"last":4.1820000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.099,"15min":1.2509999999999999},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.2599999999999998,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":4.21}]},{"osd":5,"last update":"Sat Sep 23 22:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":3.351,"15min":2.5129999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.1459999999999999,"5min":126.872,"15min":126.872},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.501,"5min":1.8959999999999999,"15min":1.6910000000000001},"min":{"1min":0.33000000000000002,"5min":0.30599999999999999,"15min":0.27000000000000002},"max":{"1min":1.4099999999999999,"5min":126.521,"15min":126.521},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:38:23 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":2.25,"15min":2.3490000000000002},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.308},"max":{"1min":4.1280000000000001,"5min":74.355000000000004,"15min":74.355000000000004},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.8320000000000001,"15min":1.7529999999999999},"min":{"1min":0.39100000000000001,"5min":0.38400000000000001,"15min":0.35099999999999998},"max":{"1min":1.7330000000000001,"5min":73.150000000000006,"15min":73.150000000000006},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:39:28 2023","interfaces":[{"interface":"back","average":{"1min":0.90400000000000003,"5min":1.1659999999999999,"15min":1.4890000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.29999999999999999},"max":{"1min":3.218,"5min":6.2320000000000002,"15min":22.338999999999999},"last":4.0579999999999998},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.367,"15min":1.786},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.307},"max":{"1min":4.0979999999999999,"5min":6.8710000000000004,"15min":10.901},"last":0.90600000000000003}]}]},{"osd":2,"up_from":349,"seq":1498943586431,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":866716,"kb_used_data":622356,"kb_used_omap":15,"kb_used_meta":244208,"kb_avail":92878436,"statfs":{"total":95995035648,"available":95107518464,"internally_reserved":0,"allocated":637292544,"data_stored":634574498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":250069750},"hb_peers":[0,1,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:40:09 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.99099999999999999,"15min":9.0139999999999993},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":7.9400000000000004,"5min":9.8759999999999994,"15min":194.06299999999999},"last":80.126000000000005},{"interface":"front","average":{"1min":0.65300000000000002,"5min":1.3100000000000001,"15min":10.122999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":3.2349999999999999,"5min":14.843999999999999,"15min":196.947},"last":83.899000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:39:42 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":0.67200000000000004,"15min":0.65200000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.0050000000000008,"5min":8.0050000000000008,"15min":8.0050000000000008},"last":70.042000000000002},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.95799999999999996,"15min":1.05},"min":{"1min":0.184,"5min":0.184,"15min":0.184},"max":{"1min":1.2070000000000001,"5min":4.9829999999999997,"15min":4.9829999999999997},"last":70.067999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:38:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.6870000000000001,"15min":1.5680000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.0309999999999997,"5min":15.585000000000001,"15min":15.585000000000001},"last":0.745},{"interface":"front","average":{"1min":0.502,"5min":1.3320000000000001,"15min":1.0289999999999999},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0609999999999999,"5min":14.128,"15min":14.128},"last":0.629}]},{"osd":4,"last update":"Sat Sep 23 22:39:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5590000000000002,"5min":1.8779999999999999,"15min":1.954},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":45.576999999999998,"5min":45.576999999999998,"15min":45.576999999999998},"last":0.317},{"interface":"front","average":{"1min":2.4369999999999998,"5min":1.3149999999999999,"15min":1.254},"min":{"1min":0.313,"5min":0.311,"15min":0.311},"max":{"1min":45.420999999999999,"5min":45.420999999999999,"15min":45.420999999999999},"last":0.33900000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:40:09 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":3.9329999999999998,"15min":3.3340000000000001},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":2.835,"5min":163.56399999999999,"15min":163.56399999999999},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.60899999999999999,"5min":3.4729999999999999,"15min":2.8799999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.3599999999999999,"5min":163.541,"15min":163.541},"last":0.28599999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":2.851,"15min":1.958},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":2.0979999999999999,"5min":169.798,"15min":169.798},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.169,"5min":3.7669999999999999,"15min":2.411},"min":{"1min":0.42399999999999999,"5min":0.373,"15min":0.373},"max":{"1min":7.1130000000000004,"5min":168.44200000000001,"15min":168.44200000000001},"last":1.49}]},{"osd":7,"last update":"Sat Sep 23 22:40:09 2023","interfaces":[{"interface":"back","average":{"1min":3.9620000000000002,"5min":2.8370000000000002,"15min":1.849},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":75.513999999999996,"5min":108.113,"15min":108.113},"last":0.30299999999999999},{"interface":"front","average":{"1min":3.9929999999999999,"5min":3.073,"15min":2.2050000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":74.007999999999996,"5min":108.093,"15min":108.093},"last":0.35999999999999999}]}]},{"osd":3,"up_from":108,"seq":463856468266,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1136452,"kb_used_data":582012,"kb_used_omap":4,"kb_used_meta":554427,"kb_avail":92608700,"statfs":{"total":95995035648,"available":94831308800,"internally_reserved":0,"allocated":595980288,"data_stored":593258702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":567733805},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.2639999999999998,"15min":3.7320000000000002},"min":{"1min":0.218,"5min":0.192,"15min":0.192},"max":{"1min":31.917000000000002,"5min":84.239000000000004,"15min":124.282},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.9399999999999999,"5min":2.9689999999999999,"15min":4.0309999999999997},"min":{"1min":0.28499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":31.984000000000002,"5min":84.188000000000002,"15min":124.30500000000001},"last":0.89400000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.4650000000000001,"5min":1.4650000000000001,"15min":1.4650000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.9570000000000001,"5min":1.9570000000000001,"15min":1.9570000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":31.856999999999999,"5min":31.856999999999999,"15min":31.856999999999999},"last":0.84899999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:39:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":3.048,"15min":2.3900000000000001},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":31.968,"5min":157.571,"15min":157.571},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.1269999999999998,"5min":3.6549999999999998,"15min":2.423},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.223},"max":{"1min":31.899999999999999,"5min":158.33000000000001,"15min":158.33000000000001},"last":0.96199999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.4059999999999999,"15min":1.3580000000000001},"min":{"1min":0.34999999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":32.015000000000001,"5min":32.015000000000001,"15min":32.015000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.04,"5min":1.3680000000000001,"15min":1.2929999999999999},"min":{"1min":0.34999999999999998,"5min":0.31,"15min":0.31},"max":{"1min":32.094999999999999,"5min":32.094999999999999,"15min":32.094999999999999},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":2.097,"15min":2.2320000000000002},"min":{"1min":0.32500000000000001,"5min":0.318,"15min":0.315},"max":{"1min":51.286999999999999,"5min":51.286999999999999,"15min":51.286999999999999},"last":0.92800000000000005},{"interface":"front","average":{"1min":2.9670000000000001,"5min":2.077,"15min":2.2679999999999998},"min":{"1min":0.39000000000000001,"5min":0.36299999999999999,"15min":0.29099999999999998},"max":{"1min":51.270000000000003,"5min":51.270000000000003,"15min":51.270000000000003},"last":0.81699999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:37:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.21,"15min":1.9670000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34200000000000003},"max":{"1min":7.5140000000000002,"5min":24.957000000000001,"15min":24.957000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.0379999999999998,"15min":2.206},"min":{"1min":0.376,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.4509999999999996,"5min":18.387,"15min":75.564999999999998},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.694,"15min":2.1110000000000002},"min":{"1min":0.436,"5min":0.38600000000000001,"15min":0.32400000000000001},"max":{"1min":32.040999999999997,"5min":32.040999999999997,"15min":75.525000000000006},"last":0.77700000000000002},{"interface":"front","average":{"1min":2.012,"5min":1.5600000000000001,"15min":1.804},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.379},"max":{"1min":32.058,"5min":32.058,"15min":32.058},"last":0.49199999999999999}]}]},{"osd":4,"up_from":465,"seq":1997159792694,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":723448,"kb_used_data":630192,"kb_used_omap":12,"kb_used_meta":93235,"kb_avail":93021704,"statfs":{"total":95995035648,"available":95254224896,"internally_reserved":0,"allocated":645316608,"data_stored":643519239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":95473265},"hb_peers":[0,1,2,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.80100000000000005,"15min":1.1930000000000001},"min":{"1min":0.23300000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":1.4570000000000001,"5min":4.9909999999999997,"15min":4.9909999999999997},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.32300000000000001,"5min":0.84799999999999998,"15min":1.0249999999999999},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.47699999999999998,"5min":6.7839999999999998,"15min":6.7839999999999998},"last":2.165}]},{"osd":1,"last update":"Sat Sep 23 22:39:44 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.47599999999999998,"15min":0.48699999999999999},"min":{"1min":0.26400000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.5820000000000001,"5min":2.7549999999999999,"15min":2.7549999999999999},"last":1.1040000000000001},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.73099999999999998,"15min":0.76500000000000001},"min":{"1min":0.25800000000000001,"5min":0.25,"15min":0.25},"max":{"1min":1.78,"5min":3.6059999999999999,"15min":3.6059999999999999},"last":0.318}]},{"osd":2,"last update":"Sat Sep 23 22:39:56 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":1.0089999999999999,"15min":1.329},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.3740000000000001,"5min":4.8540000000000001,"15min":4.8540000000000001},"last":2.21},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.3049999999999999,"15min":1.708},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.514,"5min":15.632999999999999,"15min":15.632999999999999},"last":1.3460000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:38:56 2023","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":1.1339999999999999,"15min":1.2869999999999999},"min":{"1min":0.34699999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.9489999999999998,"5min":4.766,"15min":4.766},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.59599999999999997,"5min":1.8839999999999999,"15min":2.407},"min":{"1min":0.36099999999999999,"5min":0.247,"15min":0.247},"max":{"1min":2.327,"5min":8.141,"15min":8.141},"last":0.48799999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:39:56 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.91300000000000003,"15min":1.3620000000000001},"min":{"1min":0.26700000000000002,"5min":0.248,"15min":0.248},"max":{"1min":1.548,"5min":5.5140000000000002,"15min":5.5140000000000002},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.499,"5min":0.93300000000000005,"15min":1.306},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.8500000000000001,"5min":7.5999999999999996,"15min":7.5999999999999996},"last":2.0049999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:37:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":4.944,"15min":5.6680000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.343,"5min":104.521,"15min":104.521},"last":0.91400000000000003},{"interface":"front","average":{"1min":0.58199999999999996,"5min":4.9610000000000003,"15min":5.6909999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.7969999999999999,"5min":101.419,"15min":101.419},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:39:56 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.071,"15min":1.5609999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":2.1110000000000002,"5min":8.1950000000000003,"15min":8.1950000000000003},"last":2.1339999999999999},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.3340000000000001,"15min":2.0600000000000001},"min":{"1min":0.29999999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.1459999999999999,"5min":8.6140000000000008,"15min":8.6140000000000008},"last":0.29199999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757703,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":896604,"kb_used_data":400852,"kb_used_omap":0,"kb_used_meta":495744,"kb_avail":92848548,"statfs":{"total":95995035648,"available":95076913152,"internally_reserved":0,"allocated":410472448,"data_stored":408698322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":507641856},"hb_peers":[0,1,2,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":3.6869999999999998,"15min":5.4969999999999999},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.66700000000000004,"5min":145.46600000000001,"15min":163.54499999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.54500000000000004,"5min":4.2610000000000001,"15min":5.3380000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":2.9550000000000001,"5min":144.429,"15min":162.81800000000001},"last":0.42299999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:39:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.67000000000000004,"15min":0.68000000000000005},"min":{"1min":0.29699999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.04,"5min":3.0099999999999998,"15min":3.0099999999999998},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.54300000000000004,"15min":0.54000000000000004},"min":{"1min":0.252,"5min":0.249,"15min":0.249},"max":{"1min":2.2010000000000001,"5min":2.2010000000000001,"15min":2.2010000000000001},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:40:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":2.673,"15min":2.3410000000000002},"min":{"1min":0.32700000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.3200000000000003,"5min":70.153000000000006,"15min":70.153000000000006},"last":0.372},{"interface":"front","average":{"1min":1.2190000000000001,"5min":2.512,"15min":2.347},"min":{"1min":0.30499999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.6790000000000003,"5min":70.078999999999994,"15min":70.078999999999994},"last":0.36299999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:38:47 2023","interfaces":[{"interface":"back","average":{"1min":13.23,"5min":4.5650000000000004,"15min":4.0369999999999999},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":98.882000000000005,"5min":98.882000000000005,"15min":132.59},"last":0.39400000000000002},{"interface":"front","average":{"1min":13.026999999999999,"5min":4.0209999999999999,"15min":3.8980000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29299999999999998},"max":{"1min":98.856999999999999,"5min":98.856999999999999,"15min":129.381},"last":0.30599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:39:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.9450000000000001,"15min":3.423},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":2.1360000000000001,"5min":61.173999999999999,"15min":61.173999999999999},"last":0.309},{"interface":"front","average":{"1min":0.67400000000000004,"5min":2.02,"15min":3.4870000000000001},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.2349999999999999,"5min":61.203000000000003,"15min":61.203000000000003},"last":0.40999999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:38:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.9319999999999999,"15min":1.4990000000000001},"min":{"1min":0.27000000000000002,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.375,"5min":56.667000000000002,"15min":70.581999999999994},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.71,"15min":1.478},"min":{"1min":0.35399999999999998,"5min":0.29099999999999998,"15min":0.248},"max":{"1min":1.1240000000000001,"5min":57.185000000000002,"15min":71.093000000000004},"last":0.53400000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:39:34 2023","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":1.3540000000000001,"15min":1.256},"min":{"1min":0.31,"5min":0.31,"15min":0.28000000000000003},"max":{"1min":1.0880000000000001,"5min":59.101999999999997,"15min":59.101999999999997},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":1.4670000000000001,"15min":1.3959999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0739999999999998,"5min":59.151000000000003,"15min":59.151000000000003},"last":0.34399999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":810614784,"data_stored":810614784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1581256704,"data_stored":1581252632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":619970560,"data_stored":619970560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":595787776,"data_stored":595787776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":626397184,"data_stored":626393112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":399376384,"data_stored":399376384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1928134656,"data_stored":1928134656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1492062208,"data_stored":1492058136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:40:22.388 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 26 2023-09-23T22:40:22.522 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:22.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:22.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:22.618 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:22.634 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:22.671 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:22.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:22 smithi026 ceph-mon[215775]: 9.17 scrub starts 2023-09-23T22:40:22.742 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:22 smithi026 ceph-mon[215775]: 9.17 scrub ok 2023-09-23T22:40:22.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:22 smithi026 ceph-mon[223014]: 9.17 scrub starts 2023-09-23T22:40:22.742 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:22 smithi026 ceph-mon[223014]: 9.17 scrub ok 2023-09-23T22:40:22.774 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:22 smithi070 ceph-mon[192652]: 9.17 scrub starts 2023-09-23T22:40:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:22 smithi070 ceph-mon[192652]: 9.17 scrub ok 2023-09-23T22:40:23.049 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:23.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:23.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:23.153 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:23.185 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:23.266 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:23.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:23.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:23.374 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:23.413 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:23.474 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:23 smithi070 ceph-mon[192652]: pgmap v1401: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 8858/84549 objects degraded (10.477%); 14015/84549 objects misplaced (16.576%) 2023-09-23T22:40:23.475 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:40:23.475 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 8858/84549 objects degraded (10.477%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:23.500 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:23.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:23.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:23.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:23.650 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:23.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:23 smithi026 ceph-mon[223014]: pgmap v1401: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 8858/84549 objects degraded (10.477%); 14015/84549 objects misplaced (16.576%) 2023-09-23T22:40:23.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:40:23.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 8858/84549 objects degraded (10.477%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:23.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:23 smithi026 ceph-mon[215775]: pgmap v1401: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 8858/84549 objects degraded (10.477%); 14015/84549 objects misplaced (16.576%) 2023-09-23T22:40:23.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:40:23.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 8858/84549 objects degraded (10.477%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:23.778 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:23.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:23.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:23.888 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:23.917 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:24.003 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:24.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:24.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:24.116 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:24.222 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:24.632 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:24.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:24.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:24.740 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:24.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:24 smithi026 ceph-mon[223014]: pgmap v1402: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 8536/80766 objects degraded (10.569%); 13368/80766 objects misplaced (16.552%) 2023-09-23T22:40:24.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:24 smithi026 ceph-mon[223014]: 9.18 scrub starts 2023-09-23T22:40:24.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:24 smithi026 ceph-mon[223014]: 9.18 scrub ok 2023-09-23T22:40:24.747 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:24.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:24 smithi026 ceph-mon[215775]: pgmap v1402: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 8536/80766 objects degraded (10.569%); 13368/80766 objects misplaced (16.552%) 2023-09-23T22:40:24.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:24 smithi026 ceph-mon[215775]: 9.18 scrub starts 2023-09-23T22:40:24.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:24 smithi026 ceph-mon[215775]: 9.18 scrub ok 2023-09-23T22:40:24.749 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:24.817 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:24.870 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:24 smithi070 ceph-mon[192652]: pgmap v1402: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 8536/80766 objects degraded (10.569%); 13368/80766 objects misplaced (16.552%) 2023-09-23T22:40:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:24 smithi070 ceph-mon[192652]: 9.18 scrub starts 2023-09-23T22:40:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:24 smithi070 ceph-mon[192652]: 9.18 scrub ok 2023-09-23T22:40:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:25.453 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:25.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:25.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:25.494 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:40:25.557 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:25.587 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:25.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2940627237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:25.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:25 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:25.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2940627237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:25.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:25 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:25.760 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:25.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:25.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:25.770 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2940627237' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:25.770 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:25 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:25.869 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:25.938 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:26.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:26 smithi070 ceph-mon[192652]: 1.12 scrub starts 2023-09-23T22:40:26.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:26 smithi070 ceph-mon[192652]: 1.12 scrub ok 2023-09-23T22:40:26.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:26 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:40:26.876 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:26 smithi070 ceph-mon[192652]: failed to encode map e536 with expected crc 2023-09-23T22:40:26.877 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:26 smithi070 ceph-mon[192652]: failed to encode map e536 with expected crc 2023-09-23T22:40:26.877 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:26 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:26.877 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:26 smithi070 ceph-mon[192652]: osdmap e536: 8 total, 6 up, 8 in 2023-09-23T22:40:26.877 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:26 smithi070 ceph-mon[192652]: failed to encode map e536 with expected crc 2023-09-23T22:40:26.877 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:26 smithi070 ceph-mon[192652]: pgmap v1404: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 7952/74553 objects degraded (10.666%); 12182/74553 objects misplaced (16.340%) 2023-09-23T22:40:26.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[215775]: 1.12 scrub starts 2023-09-23T22:40:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[215775]: 1.12 scrub ok 2023-09-23T22:40:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:40:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[215775]: failed to encode map e536 with expected crc 2023-09-23T22:40:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[215775]: failed to encode map e536 with expected crc 2023-09-23T22:40:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[215775]: osdmap e536: 8 total, 6 up, 8 in 2023-09-23T22:40:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[215775]: failed to encode map e536 with expected crc 2023-09-23T22:40:26.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[215775]: pgmap v1404: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 7952/74553 objects degraded (10.666%); 12182/74553 objects misplaced (16.340%) 2023-09-23T22:40:26.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[223014]: 1.12 scrub starts 2023-09-23T22:40:26.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[223014]: 1.12 scrub ok 2023-09-23T22:40:26.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:40:26.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[223014]: failed to encode map e536 with expected crc 2023-09-23T22:40:26.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[223014]: failed to encode map e536 with expected crc 2023-09-23T22:40:26.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:26.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[223014]: osdmap e536: 8 total, 6 up, 8 in 2023-09-23T22:40:26.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[223014]: failed to encode map e536 with expected crc 2023-09-23T22:40:26.991 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:26 smithi026 ceph-mon[223014]: pgmap v1404: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 7952/74553 objects degraded (10.666%); 12182/74553 objects misplaced (16.340%) 2023-09-23T22:40:27.094 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:40:27.094 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:40:27.094 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:40:27.094 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:40:27.110 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:40:27.146 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:40:27.146 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:40:27.146 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:40:27.146 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:40:27.146 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:40:27.146 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:40:27.147 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:40:27.147 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:40:27.147 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:40:27.158 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:27.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:27.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:27.266 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:27.344 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:27.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:27 smithi026 ceph-mon[223014]: 9.12 scrub starts 2023-09-23T22:40:27.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:27 smithi026 ceph-mon[223014]: 9.12 scrub ok 2023-09-23T22:40:27.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1786064759' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:27 smithi026 ceph-mon[215775]: 9.12 scrub starts 2023-09-23T22:40:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:27 smithi026 ceph-mon[215775]: 9.12 scrub ok 2023-09-23T22:40:27.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1786064759' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:27 smithi070 ceph-mon[192652]: 9.12 scrub starts 2023-09-23T22:40:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:27 smithi070 ceph-mon[192652]: 9.12 scrub ok 2023-09-23T22:40:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1786064759' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:28.172 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:28.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:28.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:28.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:28.370 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:28.514 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:28.514 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:28.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:28 smithi070 ceph-mon[192652]: pgmap v1405: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 6985/66402 objects degraded (10.519%); 10958/66402 objects misplaced (16.503%) 2023-09-23T22:40:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:28 smithi070 ceph-mon[192652]: 9.14 scrub starts 2023-09-23T22:40:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:28 smithi070 ceph-mon[192652]: 9.14 scrub ok 2023-09-23T22:40:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7952/74553 objects degraded (10.666%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:28 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:28 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:28.912 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:40:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:40:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:40:28.912 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112962 9.a 9:52973282:::benchmark_data_smithi026_555954_object18233:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328512+0000", 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.556329640000001, 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98606011100000002, 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112962 2023-09-23T22:40:28.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328512+0000", 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328512+0000", 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328511+0000", 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328518+0000", 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5150000000000003e-06 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328529+0000", 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1212e-05 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328542+0000", 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2842e-05 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328594+0000", 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1993999999999997e-05 2023-09-23T22:40:28.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328604+0000", 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0642000000000001e-05 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.308385+0000", 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97978035900000005 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.308529+0000", 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014458099999999999 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.308664+0000", 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134287 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314379+0000", 2023-09-23T22:40:28.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057154340000000001 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314423+0000", 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4088000000000002e-05 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314474+0000", 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1251000000000001e-05 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314573+0000", 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8189000000000005e-05 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112963 9.a 9:52aabdda:::benchmark_data_smithi026_555954_object18234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328560+0000", 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.556281596999995, 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99365022599999997, 2023-09-23T22:40:28.920 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112963 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328560+0000", 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328560+0000", 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328560+0000", 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328563+0000", 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4599999999999999e-06 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328571+0000", 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7770000000000002e-06 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328578+0000", 2023-09-23T22:40:28.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4919999999999998e-06 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328682+0000", 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010416000000000001 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328691+0000", 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6969999999999999e-06 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.312186+0000", 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98349548499999995 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.312326+0000", 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014028200000000001 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.312467+0000", 2023-09-23T22:40:28.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140822 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322021+0000", 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095533800000000002 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322035+0000", 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4491000000000001e-05 2023-09-23T22:40:28.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322095+0000", 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9570000000000001e-05 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322211+0000", 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011613 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112964 9.a 9:5eeb1e94:::benchmark_data_smithi026_555954_object18235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328584+0000", 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.556258569999997, 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.004056794, 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.928 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112964 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328584+0000", 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328584+0000", 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328583+0000", 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328586+0000", 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8080000000000001e-06 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328593+0000", 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8360000000000003e-06 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328598+0000", 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2989999999999996e-06 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328724+0000", 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012606499999999999 2023-09-23T22:40:28.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328729+0000", 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9400000000000001e-06 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314618+0000", 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98588852199999999 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314723+0000", 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000105509 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314816+0000", 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3148999999999999e-05 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.327349+0000", 2023-09-23T22:40:28.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012532857999999999 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332500+0000", 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051508040000000001 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332541+0000", 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1054999999999999e-05 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332640+0000", 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9347999999999998e-05 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112967 9.a 9:571abd47:::benchmark_data_smithi026_555954_object18238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328729+0000", 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.556113167000007, 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0040623639999999, 2023-09-23T22:40:28.935 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112967 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328729+0000", 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328729+0000", 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328728+0000", 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328734+0000", 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7930000000000003e-06 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328744+0000", 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8709999999999992e-06 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328749+0000", 2023-09-23T22:40:28.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2889999999999997e-06 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328789+0000", 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0247000000000003e-05 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328797+0000", 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0679999999999997e-06 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.316738+0000", 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98794093299999997 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.316885+0000", 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014646900000000001 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.317035+0000", 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015076900000000001 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.327390+0000", 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010354531 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332650+0000", 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052601890000000002 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332697+0000", 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7135000000000002e-05 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332791+0000", 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3982999999999998e-05 2023-09-23T22:40:28.942 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112968 9.2 9:4d21db9d:::benchmark_data_smithi026_555954_object18239:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328773+0000", 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.556068840999998, 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.008910618, 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.943 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112968 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328773+0000", 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328773+0000", 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328773+0000", 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328776+0000", 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8070000000000001e-06 2023-09-23T22:40:28.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328784+0000", 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2840000000000001e-06 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328789+0000", 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7030000000000003e-06 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328815+0000", 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5333999999999998e-05 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328819+0000", 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2710000000000003e-06 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322249+0000", 2023-09-23T22:40:28.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99342982999999996 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322398+0000", 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014946299999999999 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322843+0000", 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044482300000000001 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.333772+0000", 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010928927999999999 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.334363+0000", 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059104700000000004 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.337515+0000", 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031519579999999998 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.337571+0000", 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5683000000000002e-05 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.337684+0000", 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113097 2023-09-23T22:40:28.950 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112969 9.3 9:caad47cb:::benchmark_data_smithi026_555954_object18240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.555862480000002, 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6657709080000001, 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.951 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112969 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328979+0000", 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328983+0000", 2023-09-23T22:40:28.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.693e-06 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328991+0000", 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2220000000000002e-06 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328999+0000", 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4870000000000002e-06 2023-09-23T22:40:28.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.329055+0000", 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6026999999999997e-05 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.329062+0000", 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.161e-06 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.976223+0000", 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6471613999999999 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.977503+0000", 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012796629999999999 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.979185+0000", 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001682149 2023-09-23T22:40:28.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.984402+0000", 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052169599999999997 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.991082+0000", 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066800590000000003 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.992088+0000", 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001005964 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.992639+0000", 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055137800000000005 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.994750+0000", 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021113310000000001 2023-09-23T22:40:28.958 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112986 9.a 9:56c0f53c:::benchmark_data_smithi026_555954_object18257:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.354912+0000", 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.529930497999999, 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97904756500000001, 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.959 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112986 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354912+0000", 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354912+0000", 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354911+0000", 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354916+0000", 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3979999999999998e-06 2023-09-23T22:40:28.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354926+0000", 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0438000000000001e-05 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354932+0000", 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.694e-06 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354986+0000", 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.427e-05 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354997+0000", 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0295999999999999e-05 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.319413+0000", 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96441663899999996 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.319544+0000", 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130939 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.319686+0000", 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014142899999999999 2023-09-23T22:40:28.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332804+0000", 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013118094 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.333812+0000", 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001008461 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.333865+0000", 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2939999999999998e-05 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.333959+0000", 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3908999999999998e-05 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112990 9.3 9:c43d78c1:::benchmark_data_smithi026_555954_object18261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.525775835000005, 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.640044252, 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112990 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.967 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359065+0000", 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359070+0000", 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9409999999999998e-06 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359083+0000", 2023-09-23T22:40:28.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2697999999999999e-05 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359089+0000", 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7200000000000003e-06 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359144+0000", 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5954e-05 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359157+0000", 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2853e-05 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.984626+0000", 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.625468431 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.986258+0000", 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016318319999999999 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.988082+0000", 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018244540000000001 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.995233+0000", 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071506429999999999 2023-09-23T22:40:28.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.995368+0000", 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135067 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.996086+0000", 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00071818499999999998 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.996659+0000", 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057308399999999996 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.999110+0000", 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024514530000000001 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.974 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112992 9.3 9:cb6055b3:::benchmark_data_smithi026_555954_object18263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.524317726999996, 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6580400470000001, 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112992 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:40:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:40:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360528+0000", 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4259999999999996e-06 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360540+0000", 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1824999999999999e-05 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360545+0000", 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5289999999999999e-06 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360597+0000", 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1746000000000002e-05 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360609+0000", 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1953000000000001e-05 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.999299+0000", 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.638690135 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.000875+0000", 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015752990000000001 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.002719+0000", 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018446700000000001 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015384+0000", 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012664969999999999 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015497+0000", 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011238 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015923+0000", 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042655200000000002 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.016447+0000", 2023-09-23T22:40:28.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052434900000000002 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.018564+0000", 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021169520000000001 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112993 9.3 9:cd412e89:::benchmark_data_smithi026_555954_object18264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.983 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.520870392999996, 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.658274966, 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112993 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363971+0000", 2023-09-23T22:40:28.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363975+0000", 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0169999999999998e-06 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363986+0000", 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0569e-05 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363991+0000", 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0599999999999998e-06 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364039+0000", 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8767999999999998e-05 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364049+0000", 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8279999999999998e-06 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.008138+0000", 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6440885409999999 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.009772+0000", 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016337540000000001 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.011820+0000", 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020481570000000001 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.018990+0000", 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071701450000000002 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.019117+0000", 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012743300000000001 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.019436+0000", 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000318239 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.020027+0000", 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059157899999999995 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.022247+0000", 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022195629999999999 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.991 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112994 9.a 9:50601d0b:::benchmark_data_smithi026_555954_object18265:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.364104+0000", 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 83.520738112999993, 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97083823000000002, 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112994 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.992 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364104+0000", 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364104+0000", 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364103+0000", 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364107+0000", 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7050000000000001e-06 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:28.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364116+0000", 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8449999999999992e-06 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364121+0000", 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7609999999999998e-06 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364170+0000", 2023-09-23T22:40:28.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9106e-05 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364181+0000", 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0849000000000001e-05 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322921+0000", 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95874016200000001 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.323054+0000", 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013371100000000001 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.323194+0000", 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013997700000000001 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:28.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.334022+0000", 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010827687000000001 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.334793+0000", 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00077074499999999998 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.334847+0000", 2023-09-23T22:40:28.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3964e-05 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.334942+0000", 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5328999999999995e-05 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:113004 9.3 9:c3236786:::benchmark_data_smithi026_555954_object18275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:06.355590+0000", 2023-09-23T22:40:28.999 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.529252001000003, 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.68852056699999997, 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113004 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:29.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355590+0000", 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355590+0000", 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355588+0000", 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355596+0000", 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9300000000000003e-06 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355609+0000", 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.307e-05 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:29.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355619+0000", 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0237e-05 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.022432+0000", 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.66681292299999995 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.024094+0000", 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001661956 2023-09-23T22:40:29.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.025943+0000", 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001848845 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.039846+0000", 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013902733 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.040269+0000", 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042350899999999997 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.041262+0000", 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00099225200000000011 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.041895+0000", 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00063350899999999998 2023-09-23T22:40:29.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.044111+0000", 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022155539999999998 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:113005 9.3 9:c5a60e56:::benchmark_data_smithi026_555954_object18276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:06.355627+0000", 2023-09-23T22:40:29.006 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.529214983000003, 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.69180688499999998, 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113005 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355627+0000", 2023-09-23T22:40:29.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355627+0000", 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355627+0000", 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:29.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355631+0000", 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9750000000000001e-06 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355637+0000", 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9550000000000003e-06 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.355643+0000", 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8270000000000003e-06 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.028813+0000", 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.67316945500000003 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.030193+0000", 2023-09-23T22:40:29.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013807170000000001 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.031937+0000", 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001743149 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.040581+0000", 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086445610000000003 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.040732+0000", 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015037899999999999 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.044547+0000", 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003815709 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:29.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.045218+0000", 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00067093199999999995 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.047434+0000", 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022157539999999999 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.013 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:113020 9.3 9:c5699343:::benchmark_data_smithi026_555954_object18291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:06.381880+0000", 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.502961576999994, 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.66941808700000005, 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113020 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.014 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.381880+0000", 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.381880+0000", 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.381880+0000", 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.381883+0000", 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0599999999999999e-06 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.381896+0000", 2023-09-23T22:40:29.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3373999999999999e-05 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.381902+0000", 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9209999999999997e-06 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.034460+0000", 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.65255726700000005 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.035727+0000", 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012672180000000001 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.037348+0000", 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001621625 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.040844+0000", 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003495059 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.047961+0000", 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071175129999999998 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.048418+0000", 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045651600000000002 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.049030+0000", 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00061198999999999997 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.051299+0000", 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002269025 2023-09-23T22:40:29.020 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:113032 9.3 9:cbf2a52b:::benchmark_data_smithi026_555954_object18303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:06.405449+0000", 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.479393516000002, 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.66423878700000005, 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:29.021 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113032 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.405449+0000", 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:29.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.405449+0000", 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.405447+0000", 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.405454+0000", 2023-09-23T22:40:29.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8410000000000002e-06 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.405468+0000", 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4538e-05 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.405476+0000", 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8460000000000008e-06 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.051490+0000", 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.64601400200000003 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.053113+0000", 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001622552 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.054925+0000", 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018121599999999999 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.066462+0000", 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011536506 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.066585+0000", 2023-09-23T22:40:29.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012325199999999999 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.066935+0000", 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034995300000000002 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.067441+0000", 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000506183 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.069687+0000", 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022464130000000001 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:113033 9.3 9:c0f4f9ad:::benchmark_data_smithi026_555954_object18304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:06.410304+0000", 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.474537596000005, 2023-09-23T22:40:29.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.668798165, 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113033 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.410304+0000", 2023-09-23T22:40:29.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.410304+0000", 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.410303+0000", 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:29.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.410311+0000", 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0949999999999996e-06 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.410325+0000", 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4066999999999999e-05 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.410334+0000", 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1740000000000006e-06 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.058863+0000", 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.64852906700000001 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.060490+0000", 2023-09-23T22:40:29.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001626583 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.062211+0000", 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001721027 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.075696+0000", 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013484766 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.076044+0000", 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034850700000000002 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.076390+0000", 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034627799999999999 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:29.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.076946+0000", 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055543699999999999 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.079103+0000", 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021567890000000001 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:113037 9.3 9:ce759d06:::benchmark_data_smithi026_555954_object18308:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:40:29.035 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:06.415507+0000", 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.469335071000003, 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.67358876300000003, 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113037 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.415507+0000", 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.415507+0000", 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.415505+0000", 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:29.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.415513+0000", 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5109999999999997e-06 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.415529+0000", 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5846e-05 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.415538+0000", 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3079999999999997e-06 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.069869+0000", 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.65433058499999996 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.071343+0000", 2023-09-23T22:40:29.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001474473 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.073132+0000", 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017884260000000001 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.079237+0000", 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.006104946 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.086169+0000", 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069323259999999999 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.086716+0000", 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000547278 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:29.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.087289+0000", 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057280299999999998 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.089096+0000", 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018067770000000001 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.042 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:113045 9.3 9:c576f6b9:::benchmark_data_smithi026_555954_object18316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:06.446358+0000", 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 82.438484513000006, 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.65200274899999999, 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 113045 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:29.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.446358+0000", 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.446358+0000", 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:29.044 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.446356+0000", 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.446363+0000", 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3110000000000002e-06 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.446375+0000", 2023-09-23T22:40:29.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2422000000000001e-05 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.446384+0000", 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5609999999999999e-06 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.079389+0000", 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.63300582100000002 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.080968+0000", 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001578872 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.082974+0000", 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020058390000000001 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:29.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.086299+0000", 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00332476 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.095634+0000", 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093345359999999992 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.095974+0000", 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034081999999999998 2023-09-23T22:40:29.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.096465+0000", 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049036700000000002 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.098360+0000", 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00189562 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.049 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.050 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.050 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119010 9.4 9:22eb0dc0:::benchmark_data_smithi026_555954_object24281:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:40:29.050 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.442344+0000", 2023-09-23T22:40:29.050 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 62.442497963999998, 2023-09-23T22:40:29.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94600146900000004, 2023-09-23T22:40:29.050 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:29.050 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119010 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.442344+0000", 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.442344+0000", 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.442343+0000", 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:29.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.442352+0000", 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2299999999999997e-06 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.478635+0000", 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.036283043000000001 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.478651+0000", 2023-09-23T22:40:29.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6211000000000001e-05 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.721540+0000", 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.242889146 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.721552+0000", 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1749999999999999e-05 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388250+0000", 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.66669757399999996 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388346+0000", 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6019000000000003e-05 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.056 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119026 9.4 9:266f93d4:::benchmark_data_smithi026_555954_object24297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.788638+0000", 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 62.096203780000003, 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.61358515499999999, 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119026 2023-09-23T22:40:29.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.788638+0000", 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.788638+0000", 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:29.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.788637+0000", 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.788645+0000", 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2400000000000007e-06 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.788657+0000", 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2098e-05 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.788666+0000", 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.884e-06 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.788717+0000", 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0955000000000003e-05 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.788726+0000", 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.064e-06 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388823+0000", 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60009734699999995 2023-09-23T22:40:29.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388936+0000", 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000112246 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.389212+0000", 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00027593799999999999 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.396060+0000", 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0068482020000000003 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.401877+0000", 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0058170000000000001 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:29.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.402079+0000", 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020245699999999999 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.402128+0000", 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8662999999999998e-05 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.402223+0000", 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5618999999999994e-05 2023-09-23T22:40:29.064 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.065 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.065 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.065 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:29.065 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:29.065 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:40:29.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[223014]: pgmap v1405: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 6985/66402 objects degraded (10.519%); 10958/66402 objects misplaced (16.503%) 2023-09-23T22:40:29.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[223014]: 9.14 scrub starts 2023-09-23T22:40:29.065 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[223014]: 9.14 scrub ok 2023-09-23T22:40:29.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7952/74553 objects degraded (10.666%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:29.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:29.066 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:29.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[215775]: pgmap v1405: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 6985/66402 objects degraded (10.519%); 10958/66402 objects misplaced (16.503%) 2023-09-23T22:40:29.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[215775]: 9.14 scrub starts 2023-09-23T22:40:29.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[215775]: 9.14 scrub ok 2023-09-23T22:40:29.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7952/74553 objects degraded (10.666%), 19 pgs degraded, 73 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:29.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:29.066 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:28 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:29.215 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:40:29.251 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:29.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:29.358 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:29.429 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:29.563 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:29.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:29.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:29.671 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:29.701 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:29.988 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:40:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:40:29] "GET /metrics HTTP/1.1" 200 36486 "" "Prometheus/2.43.0" 2023-09-23T22:40:30.050 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:30.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:30.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:30.155 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:30.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:40:30.230 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:30.439 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:30.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:30.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:30.491 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:30 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:30.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:30 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:30.493 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:30 smithi070 ceph-mon[192652]: pgmap v1406: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 6985/66402 objects degraded (10.519%); 10958/66402 objects misplaced (16.503%) 2023-09-23T22:40:30.515 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:40:30.545 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:30.592 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:30.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:30 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:30.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:30 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:30.607 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:30 smithi026 ceph-mon[215775]: pgmap v1406: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 6985/66402 objects degraded (10.519%); 10958/66402 objects misplaced (16.503%) 2023-09-23T22:40:30.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:30 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:30.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:30 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:30.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:30 smithi026 ceph-mon[223014]: pgmap v1406: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.4 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 6985/66402 objects degraded (10.519%); 10958/66402 objects misplaced (16.503%) 2023-09-23T22:40:30.646 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:30.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:30.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:30.749 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:30.856 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:30.922 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:30.998 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:31.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:31.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:31.105 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:31.141 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:31.197 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:31.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:31.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:31.304 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:31.340 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:31.344 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:31.404 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:31.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:31.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:31.508 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:31.554 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 9 pgp_num to 26 2023-09-23T22:40:31.569 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3055657606' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T22:40:31.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3055657606' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T22:40:31.576 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:31.580 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3055657606' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T22:40:32.844 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[223014]: failed to encode map e537 with expected crc 2023-09-23T22:40:32.844 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3055657606' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T22:40:32.844 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[223014]: osdmap e537: 8 total, 6 up, 8 in 2023-09-23T22:40:32.844 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[223014]: failed to encode map e537 with expected crc 2023-09-23T22:40:32.845 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[223014]: failed to encode map e537 with expected crc 2023-09-23T22:40:32.845 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[223014]: pgmap v1408: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 5732/55620 objects degraded (10.306%); 9206/55620 objects misplaced (16.552%) 2023-09-23T22:40:32.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[215775]: failed to encode map e537 with expected crc 2023-09-23T22:40:32.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3055657606' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T22:40:32.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[215775]: osdmap e537: 8 total, 6 up, 8 in 2023-09-23T22:40:32.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[215775]: failed to encode map e537 with expected crc 2023-09-23T22:40:32.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[215775]: failed to encode map e537 with expected crc 2023-09-23T22:40:32.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:32 smithi026 ceph-mon[215775]: pgmap v1408: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 5732/55620 objects degraded (10.306%); 9206/55620 objects misplaced (16.552%) 2023-09-23T22:40:32.846 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:32.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:32.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:32 smithi070 ceph-mon[192652]: failed to encode map e537 with expected crc 2023-09-23T22:40:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:32 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3055657606' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T22:40:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:32 smithi070 ceph-mon[192652]: osdmap e537: 8 total, 6 up, 8 in 2023-09-23T22:40:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:32 smithi070 ceph-mon[192652]: failed to encode map e537 with expected crc 2023-09-23T22:40:32.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:32 smithi070 ceph-mon[192652]: failed to encode map e537 with expected crc 2023-09-23T22:40:32.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:32 smithi070 ceph-mon[192652]: pgmap v1408: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 5732/55620 objects degraded (10.306%); 9206/55620 objects misplaced (16.552%) 2023-09-23T22:40:32.953 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:33.143 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:33.499 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:33.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:33.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:33.603 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:33.644 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:33 smithi070 ceph-mon[192652]: 9.3 scrub starts 2023-09-23T22:40:33.644 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:33 smithi070 ceph-mon[192652]: 9.3 scrub ok 2023-09-23T22:40:33.644 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 5732/55620 objects degraded (10.306%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:33.652 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:33.799 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:33.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:33.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:33.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:33 smithi026 ceph-mon[215775]: 9.3 scrub starts 2023-09-23T22:40:33.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:33 smithi026 ceph-mon[215775]: 9.3 scrub ok 2023-09-23T22:40:33.873 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 5732/55620 objects degraded (10.306%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:33.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:33 smithi026 ceph-mon[223014]: 9.3 scrub starts 2023-09-23T22:40:33.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:33 smithi026 ceph-mon[223014]: 9.3 scrub ok 2023-09-23T22:40:33.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 5732/55620 objects degraded (10.306%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:33.903 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:33.953 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:34.035 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:34.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:34.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:34.140 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:34.208 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:34.294 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:34.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:34.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:34.398 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:34.462 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:34.544 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:34.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:34.660 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:34.715 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:34.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:34 smithi070 ceph-mon[192652]: 9.13 deep-scrub starts 2023-09-23T22:40:34.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:34 smithi070 ceph-mon[192652]: pgmap v1409: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 5470/52206 objects degraded (10.478%); 8618/52206 objects misplaced (16.508%) 2023-09-23T22:40:34.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:34 smithi026 ceph-mon[223014]: 9.13 deep-scrub starts 2023-09-23T22:40:34.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:34 smithi026 ceph-mon[223014]: pgmap v1409: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 5470/52206 objects degraded (10.478%); 8618/52206 objects misplaced (16.508%) 2023-09-23T22:40:34.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:34 smithi026 ceph-mon[215775]: 9.13 deep-scrub starts 2023-09-23T22:40:34.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:34 smithi026 ceph-mon[215775]: pgmap v1409: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 1.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 5470/52206 objects degraded (10.478%); 8618/52206 objects misplaced (16.508%) 2023-09-23T22:40:35.282 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:35.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:35.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:35.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:35.468 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:35.630 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:40:35.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1158081768' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:35.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:35 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:35.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1158081768' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:35.741 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:35 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:35.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1158081768' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:35.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:35 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:40:35.965 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:35.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:35.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:36.072 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:36.102 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:36.684 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:36.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:36.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:36.795 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:36.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[223014]: 9.13 deep-scrub ok 2023-09-23T22:40:36.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:36.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[223014]: failed to encode map e538 with expected crc 2023-09-23T22:40:36.824 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:40:36.825 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[223014]: osdmap e538: 8 total, 6 up, 8 in 2023-09-23T22:40:36.825 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[223014]: failed to encode map e538 with expected crc 2023-09-23T22:40:36.825 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[223014]: failed to encode map e538 with expected crc 2023-09-23T22:40:36.825 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[223014]: pgmap v1411: 119 pgs: 1 active+clean+remapped+scrubbing+deep, 9 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 965 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 4882/46185 objects degraded (10.571%); 7497/46185 objects misplaced (16.233%) 2023-09-23T22:40:36.827 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[215775]: 9.13 deep-scrub ok 2023-09-23T22:40:36.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:36.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[215775]: failed to encode map e538 with expected crc 2023-09-23T22:40:36.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:40:36.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[215775]: osdmap e538: 8 total, 6 up, 8 in 2023-09-23T22:40:36.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[215775]: failed to encode map e538 with expected crc 2023-09-23T22:40:36.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[215775]: failed to encode map e538 with expected crc 2023-09-23T22:40:36.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:36 smithi026 ceph-mon[215775]: pgmap v1411: 119 pgs: 1 active+clean+remapped+scrubbing+deep, 9 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 965 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 4882/46185 objects degraded (10.571%); 7497/46185 objects misplaced (16.233%) 2023-09-23T22:40:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:36 smithi070 ceph-mon[192652]: 9.13 deep-scrub ok 2023-09-23T22:40:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:36 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:36 smithi070 ceph-mon[192652]: failed to encode map e538 with expected crc 2023-09-23T22:40:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:40:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:36 smithi070 ceph-mon[192652]: osdmap e538: 8 total, 6 up, 8 in 2023-09-23T22:40:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:36 smithi070 ceph-mon[192652]: failed to encode map e538 with expected crc 2023-09-23T22:40:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:36 smithi070 ceph-mon[192652]: failed to encode map e538 with expected crc 2023-09-23T22:40:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:36 smithi070 ceph-mon[192652]: pgmap v1411: 119 pgs: 1 active+clean+remapped+scrubbing+deep, 9 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 965 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 4882/46185 objects degraded (10.571%); 7497/46185 objects misplaced (16.233%) 2023-09-23T22:40:36.879 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:37.058 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.36252.0:177706 9.4 e538/525 9:25174516:::benchmark_data_smithi026_555954_object82977:head v 538'9304, mlcod=538'9302)", 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:37.049840+0000", 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.005693002, 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057262379999999998, 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049840+0000", 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049840+0000", 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:37.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049839+0000", 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049848+0000", 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.789e-06 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049855+0000", 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3679999999999996e-06 2023-09-23T22:40:37.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049860+0000", 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7879999999999997e-06 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.050015+0000", 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015443799999999999 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.050050+0000", 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5312999999999999e-05 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.36252.0:177709 9.14 e538/527 9:2ac72450:::benchmark_data_smithi026_555954_object82980:head v 538'9339, mlcod=538'9337)", 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:37.049873+0000", 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0056594719999999996, 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057662449999999997, 2023-09-23T22:40:37.063 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049873+0000", 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049873+0000", 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:37.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049873+0000", 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049877+0000", 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8600000000000003e-06 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049882+0000", 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3469999999999998e-06 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049886+0000", 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8539999999999999e-06 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.052785+0000", 2023-09-23T22:40:37.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002899301 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.052814+0000", 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9125999999999999e-05 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.36252.0:177702 9.c e538/518 9:3f2b983c:::benchmark_data_smithi026_555954_object82973:head v 538'11233, mlcod=538'11231)", 2023-09-23T22:40:37.067 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:37.049905+0000", 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0056281129999999997, 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005798531, 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049905+0000", 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049905+0000", 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049904+0000", 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049909+0000", 2023-09-23T22:40:37.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9679999999999997e-06 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049915+0000", 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.015e-06 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.049920+0000", 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2760000000000002e-06 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.052850+0000", 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0029303879999999999 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:37.052873+0000", 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3099e-05 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:37.071 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:40:37.072 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 3 2023-09-23T22:40:37.072 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:40:37.072 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:37.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:37.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:37.177 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:37.216 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:37.275 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:40:37.284 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:37.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:37.313 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:40:37.361 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:40:37.391 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:37.399 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:40:37.436 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:40:37.462 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:37.470 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:40:37.470 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:40:37.470 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:40:37.471 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:40:37.471 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:40:37.471 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:40:37.471 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:40:37.471 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:40:37.471 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:40:37.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:37 smithi026 ceph-mon[215775]: failed to encode map e538 with expected crc 2023-09-23T22:40:37.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:37.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:37 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:37.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:37 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3186207564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:37.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:37 smithi026 ceph-mon[223014]: failed to encode map e538 with expected crc 2023-09-23T22:40:37.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:37.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:37 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:37.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:37 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3186207564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:37 smithi070 ceph-mon[192652]: failed to encode map e538 with expected crc 2023-09-23T22:40:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:37 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:37 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3186207564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:38.637 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:38.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:38.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:38.642 INFO:tasks.thrashosds.thrasher:in_osds: [4, 5, 0, 1, 6, 2, 7, 3] out_osds: [] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:40:38.642 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:40:38.642 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [4, 5, 0, 1, 6, 2, 7, 3] 2023-09-23T22:40:38.642 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T22:40:38.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:38 smithi026 ceph-mon[223014]: pgmap v1412: 119 pgs: 1 active+clean+remapped+scrubbing+deep, 9 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 800 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 4037/38271 objects degraded (10.548%); 6290/38271 objects misplaced (16.435%) 2023-09-23T22:40:38.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 4882/46185 objects degraded (10.571%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:38.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:38 smithi026 ceph-mon[215775]: pgmap v1412: 119 pgs: 1 active+clean+remapped+scrubbing+deep, 9 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 800 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 4037/38271 objects degraded (10.548%); 6290/38271 objects misplaced (16.435%) 2023-09-23T22:40:38.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 4882/46185 objects degraded (10.571%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:38.735 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:40:38.743 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:38.868 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:40:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:38 smithi070 ceph-mon[192652]: pgmap v1412: 119 pgs: 1 active+clean+remapped+scrubbing+deep, 9 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 800 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 4037/38271 objects degraded (10.548%); 6290/38271 objects misplaced (16.435%) 2023-09-23T22:40:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 4882/46185 objects degraded (10.571%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:38.911 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:39.157 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:39.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:39.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:39.251 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:39.251 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:39.262 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:39.344 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:39.481 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:39.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:39.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:39.554 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:39.590 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:39.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:39.615 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:39.621 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:39.677 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:39.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:39.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:39.788 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:39.830 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:39.887 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:39.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:39.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:39.907 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:40:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:40:39] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-09-23T22:40:39.907 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:39.993 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:40.088 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:40.223 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:40.381 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:40.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:40.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:40.489 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:40.613 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:41.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:41 smithi026 ceph-mon[215775]: pgmap v1413: 119 pgs: 1 active+clean+remapped+scrubbing+deep, 9 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 800 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 4037/38271 objects degraded (10.548%); 6290/38271 objects misplaced (16.435%) 2023-09-23T22:40:41.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:41.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:41.252 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:40:41.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:41 smithi026 ceph-mon[223014]: pgmap v1413: 119 pgs: 1 active+clean+remapped+scrubbing+deep, 9 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 800 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 4037/38271 objects degraded (10.548%); 6290/38271 objects misplaced (16.435%) 2023-09-23T22:40:41.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:41.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:41.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:41 smithi070 ceph-mon[192652]: pgmap v1413: 119 pgs: 1 active+clean+remapped+scrubbing+deep, 9 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 800 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 4037/38271 objects degraded (10.548%); 6290/38271 objects misplaced (16.435%) 2023-09-23T22:40:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:41.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:41.824 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:41.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:41.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:41.928 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:41.977 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:42.402 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:42.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:42.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:42.520 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:42.560 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:42.638 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:42.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:42.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:42.754 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:42.828 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:43.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:43 smithi026 ceph-mon[215775]: pgmap v1414: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 588 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2884/28098 objects degraded (10.264%); 4656/28098 objects misplaced (16.571%) 2023-09-23T22:40:43.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:40:43.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:40:43.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:43.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:43 smithi026 ceph-mon[223014]: pgmap v1414: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 588 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2884/28098 objects degraded (10.264%); 4656/28098 objects misplaced (16.571%) 2023-09-23T22:40:43.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:40:43.490 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:40:43.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:43.547 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:43.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:43.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:43 smithi070 ceph-mon[192652]: pgmap v1414: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 588 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 2884/28098 objects degraded (10.264%); 4656/28098 objects misplaced (16.571%) 2023-09-23T22:40:43.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:40:43.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:40:43.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:40:43.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:43.747 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:44.002 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:44.038 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:44.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:44.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:44.142 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:40:44.236 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:44.301 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:44 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 2884/28098 objects degraded (10.264%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:44.302 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:44 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 2884/28098 objects degraded (10.264%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:44.437 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:40:44.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:44.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:40:44.548 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:44.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:44 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 2884/28098 objects degraded (10.264%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:44.630 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:44 smithi026 conmon[215751]: 2023-09-23T22:40:44.419+0000 7f4580f04700 -1 mon.a@0(leader).osd e538 definitely_dead 0 2023-09-23T22:40:44.640 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:44.652 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:44.652 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:44.815 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:44.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:44.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:44.922 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:44.960 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:45.029 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:45.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:45.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:45.134 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:45.180 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:45.254 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:45.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:45.335 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:40:45.347 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.4. 2023-09-23T22:40:45.361 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:45.389 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:45.440 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:45.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:45.546 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:45.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[223014]: pgmap v1415: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 514 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2585/24543 objects degraded (10.533%); 4054/24543 objects misplaced (16.518%) 2023-09-23T22:40:45.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1560304187' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T22:40:45.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T22:40:45.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[223014]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T22:40:45.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2864117513' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:45.577 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:45.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[215775]: pgmap v1415: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 514 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2585/24543 objects degraded (10.533%); 4054/24543 objects misplaced (16.518%) 2023-09-23T22:40:45.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1560304187' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T22:40:45.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T22:40:45.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[215775]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T22:40:45.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2864117513' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:45.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:45 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:45.578 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:45 smithi070 ceph-mon[192652]: pgmap v1415: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 514 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 2585/24543 objects degraded (10.533%); 4054/24543 objects misplaced (16.518%) 2023-09-23T22:40:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1560304187' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T22:40:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:45 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T22:40:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:45 smithi070 ceph-mon[192652]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T22:40:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2864117513' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:45 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:40:46.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:46.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[223014]: failed to encode map e539 with expected crc 2023-09-23T22:40:46.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[223014]: failed to encode map e539 with expected crc 2023-09-23T22:40:46.619 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T22:40:46.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:46.620 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[223014]: osdmap e539: 8 total, 6 up, 7 in 2023-09-23T22:40:46.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:46.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[215775]: failed to encode map e539 with expected crc 2023-09-23T22:40:46.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[215775]: failed to encode map e539 with expected crc 2023-09-23T22:40:46.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T22:40:46.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:46.620 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:46 smithi026 ceph-mon[215775]: osdmap e539: 8 total, 6 up, 7 in 2023-09-23T22:40:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:46 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:46.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:46 smithi070 ceph-mon[192652]: failed to encode map e539 with expected crc 2023-09-23T22:40:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:46 smithi070 ceph-mon[192652]: failed to encode map e539 with expected crc 2023-09-23T22:40:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:46 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T22:40:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:46 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:40:46.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:46 smithi070 ceph-mon[192652]: osdmap e539: 8 total, 6 up, 7 in 2023-09-23T22:40:47.152 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:47.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:47.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:40:47.258 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:47.323 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:47.560 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:47.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:47.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[215775]: pgmap v1417: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 385 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 1990/18369 objects degraded (10.833%); 2935/18369 objects misplaced (15.978%) 2023-09-23T22:40:47.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[215775]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[215775]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[215775]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[215775]: osdmap e540: 8 total, 6 up, 7 in 2023-09-23T22:40:47.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[215775]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[215775]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[215775]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[223014]: pgmap v1417: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 385 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 1990/18369 objects degraded (10.833%); 2935/18369 objects misplaced (15.978%) 2023-09-23T22:40:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[223014]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[223014]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[223014]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[223014]: osdmap e540: 8 total, 6 up, 7 in 2023-09-23T22:40:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[223014]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[223014]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:47 smithi026 ceph-mon[223014]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:47 smithi070 ceph-mon[192652]: pgmap v1417: 119 pgs: 10 active+clean+remapped, 15 undersized+peered, 44 active+undersized, 3 active+undersized+remapped, 3 undersized+degraded+peered, 16 active+undersized+degraded, 28 active+clean; 385 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 379 op/s; 1990/18369 objects degraded (10.833%); 2935/18369 objects misplaced (15.978%) 2023-09-23T22:40:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:47 smithi070 ceph-mon[192652]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:47 smithi070 ceph-mon[192652]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:47 smithi070 ceph-mon[192652]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:47 smithi070 ceph-mon[192652]: osdmap e540: 8 total, 6 up, 7 in 2023-09-23T22:40:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:47 smithi070 ceph-mon[192652]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:47 smithi070 ceph-mon[192652]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:47 smithi070 ceph-mon[192652]: failed to encode map e540 with expected crc 2023-09-23T22:40:47.676 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:47.709 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:47.763 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:40:47.764 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:47.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:47.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:47.814 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:40:47.854 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:40:47.870 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:47.899 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:47.904 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:47.904 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:40:47.948 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:40:47.970 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:47.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:47.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:47.985 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:40:47.986 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:40:47.986 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:40:47.986 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:40:47.986 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:40:47.986 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:40:47.986 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:40:47.986 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:40:47.986 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:40:48.078 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:48.138 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:48 smithi070 ceph-mon[192652]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:48 smithi070 ceph-mon[192652]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:48 smithi070 ceph-mon[192652]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:48 smithi070 ceph-mon[192652]: osdmap e541: 8 total, 6 up, 7 in 2023-09-23T22:40:48.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:48 smithi070 ceph-mon[192652]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:48 smithi070 ceph-mon[192652]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 538'11655 2023-09-23T22:40:48.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3844545917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:48.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 1990/18369 objects degraded (10.833%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:48.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:48 smithi070 ceph-mon[192652]: osdmap e542: 8 total, 6 up, 7 in 2023-09-23T22:40:48.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[215775]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[215775]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[215775]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[215775]: osdmap e541: 8 total, 6 up, 7 in 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[215775]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[215775]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 538'11655 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3844545917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 1990/18369 objects degraded (10.833%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[215775]: osdmap e542: 8 total, 6 up, 7 in 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[223014]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[223014]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[223014]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[223014]: osdmap e541: 8 total, 6 up, 7 in 2023-09-23T22:40:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[223014]: failed to encode map e541 with expected crc 2023-09-23T22:40:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[223014]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 538'11655 2023-09-23T22:40:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3844545917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 1990/18369 objects degraded (10.833%), 19 pgs degraded, 81 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:48.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:48 smithi026 ceph-mon[223014]: osdmap e542: 8 total, 6 up, 7 in 2023-09-23T22:40:49.153 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:49.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:49.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:49.208 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:40:49.266 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:49.282 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:49.282 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:49.294 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:49.359 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:49.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:49.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:49.463 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:49.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[215775]: pgmap v1420: 119 pgs: 2 remapped+peering, 13 peering, 10 unknown, 12 active+clean+remapped, 13 undersized+peered, 27 active+undersized, 7 active+undersized+remapped, 3 undersized+degraded+peered, 15 active+undersized+degraded, 17 active+clean; 276 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 831 op/s; 1302/13122 objects degraded (9.922%); 2008/13122 objects misplaced (15.303%); 788 B/s, 0 objects/s recovering 2023-09-23T22:40:49.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[215775]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[215775]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.487 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[215775]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[215775]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[215775]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 20 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[215775]: osdmap e543: 8 total, 6 up, 7 in 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[223014]: pgmap v1420: 119 pgs: 2 remapped+peering, 13 peering, 10 unknown, 12 active+clean+remapped, 13 undersized+peered, 27 active+undersized, 7 active+undersized+remapped, 3 undersized+degraded+peered, 15 active+undersized+degraded, 17 active+clean; 276 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 831 op/s; 1302/13122 objects degraded (9.922%); 2008/13122 objects misplaced (15.303%); 788 B/s, 0 objects/s recovering 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[223014]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[223014]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[223014]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[223014]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[223014]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 20 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:40:49.489 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:49 smithi026 ceph-mon[223014]: osdmap e543: 8 total, 6 up, 7 in 2023-09-23T22:40:49.502 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:49.559 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:49.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:49 smithi070 ceph-mon[192652]: pgmap v1420: 119 pgs: 2 remapped+peering, 13 peering, 10 unknown, 12 active+clean+remapped, 13 undersized+peered, 27 active+undersized, 7 active+undersized+remapped, 3 undersized+degraded+peered, 15 active+undersized+degraded, 17 active+clean; 276 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 831 op/s; 1302/13122 objects degraded (9.922%); 2008/13122 objects misplaced (15.303%); 788 B/s, 0 objects/s recovering 2023-09-23T22:40:49.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:49 smithi070 ceph-mon[192652]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.564 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:49 smithi070 ceph-mon[192652]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:49 smithi070 ceph-mon[192652]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:49 smithi070 ceph-mon[192652]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:49 smithi070 ceph-mon[192652]: failed to encode map e542 with expected crc 2023-09-23T22:40:49.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:49 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 20 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:40:49.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:49 smithi070 ceph-mon[192652]: osdmap e543: 8 total, 6 up, 7 in 2023-09-23T22:40:49.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:49.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:49.667 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:49.699 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:49.738 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:40:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:40:49] "GET /metrics HTTP/1.1" 200 36482 "" "Prometheus/2.43.0" 2023-09-23T22:40:49.774 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:49.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:49.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:49.882 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:49.911 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:50.335 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:50.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:50.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:50.445 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:50.472 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:50.524 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:50.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:50.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:50.559 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:50.632 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:50.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:50 smithi026 ceph-mon[223014]: failed to encode map e543 with expected crc 2023-09-23T22:40:50.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:50 smithi026 ceph-mon[223014]: failed to encode map e543 with expected crc 2023-09-23T22:40:50.673 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:50 smithi026 ceph-mon[223014]: failed to encode map e543 with expected crc 2023-09-23T22:40:50.673 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:50 smithi026 ceph-mon[215775]: failed to encode map e543 with expected crc 2023-09-23T22:40:50.673 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:50 smithi026 ceph-mon[215775]: failed to encode map e543 with expected crc 2023-09-23T22:40:50.673 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:50 smithi026 ceph-mon[215775]: failed to encode map e543 with expected crc 2023-09-23T22:40:50.677 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:50.777 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:50.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:50.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:50 smithi070 ceph-mon[192652]: failed to encode map e543 with expected crc 2023-09-23T22:40:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:50 smithi070 ceph-mon[192652]: failed to encode map e543 with expected crc 2023-09-23T22:40:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:50 smithi070 ceph-mon[192652]: failed to encode map e543 with expected crc 2023-09-23T22:40:50.881 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:50.909 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:51.059 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:51.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:51.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:51.080 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151873 8.6 8:675a56d2:::benchmark_data_smithi026_497658_object52981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 426.02936708099998, 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6652286869999999, 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:40:51.081 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151873 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048419+0000", 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7810000000000001e-06 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048428+0000", 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2649999999999996e-06 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048434+0000", 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2419999999999997e-06 2023-09-23T22:40:51.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048492+0000", 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7899999999999998e-05 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048608+0000", 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116111 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:40:51.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048734+0000", 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012566199999999999 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.057680+0000", 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008946173 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202205+0000", 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.144524878 2023-09-23T22:40:51.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202215+0000", 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0217000000000001e-05 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198539+0000", 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99632416099999999 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.087 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198553+0000", 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4236999999999999e-05 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713565+0000", 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5150114879999999 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713646+0000", 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1019000000000005e-05 2023-09-23T22:40:51.088 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151874 8.e 8:70ae8a39:::benchmark_data_smithi026_497658_object52982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 426.02301168499997, 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6486478249999998, 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.089 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151874 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054771+0000", 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054778+0000", 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7599999999999997e-06 2023-09-23T22:40:51.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054794+0000", 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6138999999999999e-05 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054803+0000", 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8610000000000004e-06 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054857+0000", 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3900000000000002e-05 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.055039+0000", 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018265100000000001 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.055175+0000", 2023-09-23T22:40:51.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013610799999999999 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.057826+0000", 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00265037 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.060056+0000", 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022297520000000002 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202153+0000", 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.142097219 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202169+0000", 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5707000000000001e-05 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198586+0000", 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99641781500000004 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198596+0000", 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.662e-06 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703316+0000", 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.504720442 2023-09-23T22:40:51.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703420+0000", 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010388800000000001 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151883 8.e 8:73a3339c:::benchmark_data_smithi026_497658_object52991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:40:51.097 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 425.77921697300002, 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4185350429999999, 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151883 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.098 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298566+0000", 2023-09-23T22:40:51.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298574+0000", 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6419999999999996e-06 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298592+0000", 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.749e-05 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298603+0000", 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1551e-05 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298672+0000", 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8804000000000001e-05 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298688+0000", 2023-09-23T22:40:51.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6004000000000001e-05 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198619+0000", 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89993140699999996 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198625+0000", 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0419999999999999e-06 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703671+0000", 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.505045454 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703821+0000", 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014994300000000001 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:40:51.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703937+0000", 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116397 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713345+0000", 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094075259999999994 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713414+0000", 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9202999999999995e-05 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713603+0000", 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000189373 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716935+0000", 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033319270000000002 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716988+0000", 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3121000000000002e-05 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.717102+0000", 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011384599999999999 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.106 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151885 8.e 8:7d6f345f:::benchmark_data_smithi026_497658_object52993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 425.778226211, 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4256615969999999, 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151885 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299557+0000", 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299563+0000", 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1279999999999996e-06 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299579+0000", 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5736e-05 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299589+0000", 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9720000000000004e-06 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299662+0000", 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3312000000000006e-05 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299675+0000", 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3414999999999999e-05 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198647+0000", 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89897187599999995 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198655+0000", 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1219999999999995e-06 2023-09-23T22:40:51.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706694+0000", 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5080385459999999 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706833+0000", 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013966099999999999 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706985+0000", 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151915 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713514+0000", 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065285760000000003 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713609+0000", 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5025000000000006e-05 2023-09-23T22:40:51.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716876+0000", 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032670960000000001 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725068+0000", 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0081923059999999999 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725134+0000", 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5544999999999997e-05 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725220+0000", 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5604000000000002e-05 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151886 8.e 8:74052c51:::benchmark_data_smithi026_497658_object52994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 425.77594465099997, 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4236035519999999, 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151886 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.116 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301838+0000", 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301844+0000", 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2099999999999998e-06 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301857+0000", 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2943e-05 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301867+0000", 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3810000000000005e-06 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301924+0000", 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7370000000000001e-05 2023-09-23T22:40:51.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301936+0000", 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2551999999999999e-05 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198678+0000", 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89674120599999996 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198686+0000", 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0330000000000008e-06 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.709941+0000", 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5112555969999999 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.710081+0000", 2023-09-23T22:40:51.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013977400000000001 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.710232+0000", 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015061799999999999 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713594+0000", 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033623500000000001 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716839+0000", 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032453930000000001 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716906+0000", 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7041999999999999e-05 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725274+0000", 2023-09-23T22:40:51.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083672850000000003 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725329+0000", 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5031000000000002e-05 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725443+0000", 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114305 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.124 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104711 9.c 9:31531dd3:::benchmark_data_smithi026_555954_object9982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.333837+0000", 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 215.74394713000001, 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9901661960000001, 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.125 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104711 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333837+0000", 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333837+0000", 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333835+0000", 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333844+0000", 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9309999999999997e-06 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333865+0000", 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0749000000000001e-05 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333875+0000", 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9089999999999995e-06 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333951+0000", 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5932000000000004e-05 2023-09-23T22:40:51.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.334444+0000", 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049377199999999996 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.997213+0000", 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6627689000000001 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.997330+0000", 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011688699999999999 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.997482+0000", 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015183899999999999 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.014559+0000", 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017076560000000001 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045008+0000", 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030448994 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045090+0000", 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2562e-05 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.320023+0000", 2023-09-23T22:40:51.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.27493322999999997 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.320097+0000", 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3244000000000005e-05 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.324003+0000", 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039067069999999997 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.132 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104722 9.c 9:3a91d401:::benchmark_data_smithi026_555954_object9993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.362114+0000", 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 215.71567063099999, 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9830620269999999, 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.133 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104722 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362114+0000", 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362114+0000", 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362112+0000", 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362119+0000", 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5490000000000003e-06 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362132+0000", 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3563e-05 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362140+0000", 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5209999999999997e-06 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362187+0000", 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7746e-05 2023-09-23T22:40:51.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362696+0000", 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050887800000000004 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.009860+0000", 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6471635419999999 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.009977+0000", 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011697300000000001 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.010139+0000", 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000161762 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.022087+0000", 2023-09-23T22:40:51.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011948652000000001 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045050+0000", 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022963063999999998 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045126+0000", 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5426000000000006e-05 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.342067+0000", 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.29694130099999999 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.342108+0000", 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1276999999999998e-05 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.345176+0000", 2023-09-23T22:40:51.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030672630000000002 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119013 9.9 9:97fb371b:::benchmark_data_smithi026_555954_object24284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.616004+0000", 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.461779863999993, 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.772435714, 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.141 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119013 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616004+0000", 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.142 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616004+0000", 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616002+0000", 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616016+0000", 2023-09-23T22:40:51.143 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4297999999999999e-05 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616050+0000", 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3912000000000003e-05 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616066+0000", 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5749000000000001e-05 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.636982+0000", 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020916489999999999 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.636996+0000", 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3326000000000001e-05 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.372328+0000", 2023-09-23T22:40:51.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.73533267700000005 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.372459+0000", 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013030600000000001 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.372900+0000", 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044181100000000002 2023-09-23T22:40:51.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388194+0000", 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015293958999999999 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388300+0000", 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106106 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388352+0000", 2023-09-23T22:40:51.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1922999999999997e-05 2023-09-23T22:40:51.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388440+0000", 2023-09-23T22:40:51.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7700000000000004e-05 2023-09-23T22:40:51.148 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119014 9.5 9:a7fed247:::benchmark_data_smithi026_555954_object24285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.622481+0000", 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.455302826999997, 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.78627524299999996, 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.149 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119014 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622481+0000", 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622481+0000", 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622479+0000", 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622492+0000", 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3371e-05 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622526+0000", 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3466000000000001e-05 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622542+0000", 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6580999999999999e-05 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622626+0000", 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3819999999999994e-05 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622642+0000", 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6061000000000001e-05 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.153 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.377283+0000", 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.75464055699999999 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.377429+0000", 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014568900000000001 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.377864+0000", 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043552899999999998 2023-09-23T22:40:51.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.379798+0000", 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001934285 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.387859+0000", 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080606210000000005 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.408640+0000", 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020780664000000001 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.408698+0000", 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8724000000000002e-05 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.408757+0000", 2023-09-23T22:40:51.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8183000000000002e-05 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119027 9.9 9:95b0ebe6:::benchmark_data_smithi026_555954_object24298:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.792678+0000", 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.285105818999995, 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60675180200000001, 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.157 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119027 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792678+0000", 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792678+0000", 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792677+0000", 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792683+0000", 2023-09-23T22:40:51.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9030000000000001e-06 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792693+0000", 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8879999999999992e-06 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792699+0000", 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7769999999999999e-06 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792745+0000", 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6686e-05 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792752+0000", 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3999999999999997e-06 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.386094+0000", 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.59334233000000003 2023-09-23T22:40:51.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.386227+0000", 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013239799999999999 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.386635+0000", 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040815300000000002 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388625+0000", 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001989925 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.399291+0000", 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010666375000000001 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.399345+0000", 2023-09-23T22:40:51.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4129000000000003e-05 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.399430+0000", 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5032999999999997e-05 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119032 9.15 9:abacdbfd:::benchmark_data_smithi026_555954_object24303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.811691+0000", 2023-09-23T22:40:51.164 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 84.266093463999994, 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.61184655600000004, 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119032 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811691+0000", 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811691+0000", 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811689+0000", 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:40:51.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811700+0000", 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1749999999999999e-05 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811732+0000", 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1522000000000001e-05 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811738+0000", 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2349999999999996e-06 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811780+0000", 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1488e-05 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811786+0000", 2023-09-23T22:40:51.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1310000000000003e-06 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.417633+0000", 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60584729699999995 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.417795+0000", 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016233099999999999 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.169 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.418138+0000", 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034242300000000001 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.421355+0000", 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032175089999999999 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.423411+0000", 2023-09-23T22:40:51.170 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020557570000000001 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.423454+0000", 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2836000000000001e-05 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.423501+0000", 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6959000000000002e-05 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.423537+0000", 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6437999999999999e-05 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:169105 9.15 9:ae0dab03:::benchmark_data_smithi026_555954_object74376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:27.661466+0000", 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 23.416317798000001, 2023-09-23T22:40:51.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20122447700000001, 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 169105 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661466+0000", 2023-09-23T22:40:51.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661466+0000", 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661466+0000", 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661469+0000", 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9699999999999999e-06 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661475+0000", 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3400000000000003e-06 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661481+0000", 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6459999999999998e-06 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.664517+0000", 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030358999999999998 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.664639+0000", 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012202299999999999 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.664976+0000", 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033671900000000003 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.668612+0000", 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036362550000000001 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.669990+0000", 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013781310000000001 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.862600+0000", 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.192610429 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.862651+0000", 2023-09-23T22:40:51.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0909999999999999e-05 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.862691+0000", 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9598999999999998e-05 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:169106 9.15 9:a81cf775:::benchmark_data_smithi026_555954_object74377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:27.661513+0000", 2023-09-23T22:40:51.179 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 23.416271639000001, 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20626381099999999, 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 169106 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661513+0000", 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661513+0000", 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661512+0000", 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661515+0000", 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2179999999999999e-06 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661523+0000", 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.238e-06 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661529+0000", 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0929999999999999e-06 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.666957+0000", 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00542824 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.667040+0000", 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2954999999999994e-05 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.667225+0000", 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018559199999999999 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.668701+0000", 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014756389999999999 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.670095+0000", 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013935480000000001 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.867671+0000", 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.19757656800000001 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.867736+0000", 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4695000000000003e-05 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.867776+0000", 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0519999999999998e-05 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:170912 9.5 9:a0cf2275:::benchmark_data_smithi026_555954_object76183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:29.540307+0000", 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 21.537477197000001, 2023-09-23T22:40:51.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.17543514399999999, 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 170912 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540307+0000", 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540307+0000", 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540304+0000", 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999971 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540315+0000", 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0679999999999999e-05 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540339+0000", 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3620000000000001e-05 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540351+0000", 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2225000000000001e-05 2023-09-23T22:40:51.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540432+0000", 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0841999999999996e-05 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540559+0000", 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012773099999999999 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.541084+0000", 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052477899999999996 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.550067+0000", 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0089831589999999992 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.550076+0000", 2023-09-23T22:40:51.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0299999999999999e-06 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.715642+0000", 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.165565661 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.715695+0000", 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3397999999999999e-05 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.192 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.715742+0000", 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6686e-05 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:170915 9.15 9:aa637052:::benchmark_data_smithi026_555954_object76186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:29.540441+0000", 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 21.53734296, 2023-09-23T22:40:51.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.180261118, 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 170915 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540441+0000", 2023-09-23T22:40:51.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540441+0000", 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540441+0000", 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540445+0000", 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2980000000000002e-06 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540454+0000", 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1970000000000002e-06 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540460+0000", 2023-09-23T22:40:51.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1650000000000003e-06 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.543789+0000", 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033291839999999998 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.543919+0000", 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012918900000000001 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.544335+0000", 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000415857 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.550035+0000", 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057006499999999998 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.550081+0000", 2023-09-23T22:40:51.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6091000000000003e-05 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.720629+0000", 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.170548014 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.720662+0000", 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2610000000000001e-05 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.720702+0000", 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0460000000000002e-05 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:170919 9.15 9:a8567740:::benchmark_data_smithi026_555954_object76190:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:29.540491+0000", 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 21.537292892, 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.18042668100000001, 2023-09-23T22:40:51.200 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 170919 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540491+0000", 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540491+0000", 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540491+0000", 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540494+0000", 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9179999999999998e-06 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540500+0000", 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6490000000000002e-06 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540507+0000", 2023-09-23T22:40:51.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3899999999999998e-06 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.546781+0000", 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062743649999999996 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.546897+0000", 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116272 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.547303+0000", 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000405807 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.550346+0000", 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003043184 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.552140+0000", 2023-09-23T22:40:51.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017932689999999999 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.720820+0000", 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.168679942 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.720866+0000", 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6657999999999999e-05 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.720918+0000", 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1671e-05 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:170922 9.5 9:a405daf9:::benchmark_data_smithi026_555954_object76193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:29.540540+0000", 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 21.53724373, 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.18062175499999999, 2023-09-23T22:40:51.207 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 170922 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540540+0000", 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540540+0000", 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540540+0000", 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540543+0000", 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5919999999999999e-06 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540549+0000", 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8249999999999999e-06 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540555+0000", 2023-09-23T22:40:51.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7830000000000004e-06 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.546762+0000", 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062069550000000001 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.546881+0000", 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011924 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.547331+0000", 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044946599999999998 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.550352+0000", 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030210749999999998 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.552181+0000", 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001829027 2023-09-23T22:40:51.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.721082+0000", 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.168900772 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.721126+0000", 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4659000000000001e-05 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.721162+0000", 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5778999999999997e-05 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:186574 9.5 9:a2b7e1ed:::benchmark_data_smithi026_555954_object91845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:48.576335+0000", 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.5014496189999997, 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26174019599999998, 2023-09-23T22:40:51.214 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 186574 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576335+0000", 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576335+0000", 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576333+0000", 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576340+0000", 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6750000000000002e-06 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576359+0000", 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8476999999999999e-05 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576368+0000", 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7099999999999996e-06 2023-09-23T22:40:51.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576428+0000", 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0409999999999999e-05 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576560+0000", 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013158700000000001 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.577032+0000", 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000471881 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.582645+0000", 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056133629999999997 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.582675+0000", 2023-09-23T22:40:51.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9960000000000001e-05 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.837976+0000", 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25530079500000002 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838033+0000", 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7073000000000001e-05 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838075+0000", 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2055999999999998e-05 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:186575 9.5 9:a1f249a0:::benchmark_data_smithi026_555954_object91846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:48.576557+0000", 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 2.5012270679999999, 2023-09-23T22:40:51.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26162006599999998, 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 186575 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576557+0000", 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576557+0000", 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576556+0000", 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576562+0000", 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6659999999999996e-06 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576574+0000", 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2204e-05 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576582+0000", 2023-09-23T22:40:51.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0770000000000008e-06 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.579664+0000", 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030818740000000001 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.579795+0000", 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013058899999999999 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.580252+0000", 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045709100000000002 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.583584+0000", 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003332645 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.584967+0000", 2023-09-23T22:40:51.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001382538 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838112+0000", 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.253144753 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838147+0000", 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5052000000000003e-05 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838177+0000", 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0528000000000003e-05 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:187640 9.15 9:af3445ae:::benchmark_data_smithi026_555954_object92911:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:49.804222+0000", 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 1.273561725, 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.227025541, 2023-09-23T22:40:51.228 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 187640 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804222+0000", 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804222+0000", 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804221+0000", 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:40:51.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804227+0000", 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5969999999999999e-06 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804236+0000", 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4879999999999996e-06 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804247+0000", 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0560999999999999e-05 2023-09-23T22:40:51.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804296+0000", 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9296999999999999e-05 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804421+0000", 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124919 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804874+0000", 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045281999999999999 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.806885+0000", 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020113520000000001 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.807344+0000", 2023-09-23T22:40:51.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045847100000000001 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.031156+0000", 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22381212 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.031210+0000", 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3794e-05 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:40:51.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:40:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:40:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.031248+0000", 2023-09-23T22:40:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8448999999999997e-05 2023-09-23T22:40:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:40:51.235 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:40:51.235 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:40:51.235 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:51.265 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:51.282 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:40:51.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:51 smithi026 ceph-mon[215775]: pgmap v1423: 119 pgs: 2 remapped+peering, 13 peering, 10 unknown, 12 active+clean+remapped, 13 undersized+peered, 27 active+undersized, 7 active+undersized+remapped, 3 undersized+degraded+peered, 15 active+undersized+degraded, 17 active+clean; 276 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 1302/13122 objects degraded (9.922%); 2008/13122 objects misplaced (15.303%); 1.2 KiB/s, 0 objects/s recovering 2023-09-23T22:40:51.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:51 smithi026 ceph-mon[215775]: failed to encode map e544 with expected crc 2023-09-23T22:40:51.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:51 smithi026 ceph-mon[215775]: failed to encode map e544 with expected crc 2023-09-23T22:40:51.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:51 smithi026 ceph-mon[215775]: osdmap e544: 8 total, 6 up, 7 in 2023-09-23T22:40:51.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:51 smithi026 ceph-mon[223014]: pgmap v1423: 119 pgs: 2 remapped+peering, 13 peering, 10 unknown, 12 active+clean+remapped, 13 undersized+peered, 27 active+undersized, 7 active+undersized+remapped, 3 undersized+degraded+peered, 15 active+undersized+degraded, 17 active+clean; 276 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 1302/13122 objects degraded (9.922%); 2008/13122 objects misplaced (15.303%); 1.2 KiB/s, 0 objects/s recovering 2023-09-23T22:40:51.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:51 smithi026 ceph-mon[223014]: failed to encode map e544 with expected crc 2023-09-23T22:40:51.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:51 smithi026 ceph-mon[223014]: failed to encode map e544 with expected crc 2023-09-23T22:40:51.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:51 smithi026 ceph-mon[223014]: osdmap e544: 8 total, 6 up, 7 in 2023-09-23T22:40:51.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:51 smithi070 ceph-mon[192652]: pgmap v1423: 119 pgs: 2 remapped+peering, 13 peering, 10 unknown, 12 active+clean+remapped, 13 undersized+peered, 27 active+undersized, 7 active+undersized+remapped, 3 undersized+degraded+peered, 15 active+undersized+degraded, 17 active+clean; 276 MiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 437 op/s; 1302/13122 objects degraded (9.922%); 2008/13122 objects misplaced (15.303%); 1.2 KiB/s, 0 objects/s recovering 2023-09-23T22:40:51.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:51 smithi070 ceph-mon[192652]: failed to encode map e544 with expected crc 2023-09-23T22:40:51.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:51 smithi070 ceph-mon[192652]: failed to encode map e544 with expected crc 2023-09-23T22:40:51.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:51 smithi070 ceph-mon[192652]: osdmap e544: 8 total, 6 up, 7 in 2023-09-23T22:40:52.127 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:52.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:52.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:52.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:52.267 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:52.310 INFO:tasks.thrashosds.thrasher:in_osds: [5, 0, 1, 6, 2, 7, 3] out_osds: [4] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:40:52.311 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:40:52.311 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T22:40:52.311 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:40:52.311 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:40:52.311 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T22:40:52.542 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:40:52.587 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:52 smithi070 ceph-mon[192652]: failed to encode map e545 with expected crc 2023-09-23T22:40:52.587 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:52 smithi070 ceph-mon[192652]: failed to encode map e545 with expected crc 2023-09-23T22:40:52.587 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:52 smithi070 ceph-mon[192652]: osdmap e545: 8 total, 6 up, 7 in 2023-09-23T22:40:52.588 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:52 smithi070 ceph-mon[192652]: failed to encode map e545 with expected crc 2023-09-23T22:40:52.717 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 94726 objects 2023-09-23T22:40:52.718 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :237.314 2023-09-23T22:40:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:52 smithi026 ceph-mon[223014]: failed to encode map e545 with expected crc 2023-09-23T22:40:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:52 smithi026 ceph-mon[223014]: failed to encode map e545 with expected crc 2023-09-23T22:40:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:52 smithi026 ceph-mon[223014]: osdmap e545: 8 total, 6 up, 7 in 2023-09-23T22:40:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:52 smithi026 ceph-mon[223014]: failed to encode map e545 with expected crc 2023-09-23T22:40:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:52 smithi026 ceph-mon[215775]: failed to encode map e545 with expected crc 2023-09-23T22:40:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:52 smithi026 ceph-mon[215775]: failed to encode map e545 with expected crc 2023-09-23T22:40:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:52 smithi026 ceph-mon[215775]: osdmap e545: 8 total, 6 up, 7 in 2023-09-23T22:40:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:52 smithi026 ceph-mon[215775]: failed to encode map e545 with expected crc 2023-09-23T22:40:52.766 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:52.854 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:52.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:52.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:52.880 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:52.958 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:52.985 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:53.030 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:53.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:53.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:53.135 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:53.168 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:53.220 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:53.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:53.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:53.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:53.330 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:53.357 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:53 smithi070 ceph-mon[192652]: pgmap v1426: 119 pgs: 1 peering, 31 active+clean+remapped, 13 undersized+peered, 23 active+undersized, 17 active+undersized+remapped, 3 undersized+degraded+peered, 11 active+undersized+degraded, 20 active+clean; 117 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 610/5466 objects degraded (11.160%); 1072/5466 objects misplaced (19.612%); 2.8 MiB/s, 49 objects/s recovering 2023-09-23T22:40:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 610/5466 objects degraded (11.160%), 14 pgs degraded, 53 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:53.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:53 smithi026 ceph-mon[223014]: pgmap v1426: 119 pgs: 1 peering, 31 active+clean+remapped, 13 undersized+peered, 23 active+undersized, 17 active+undersized+remapped, 3 undersized+degraded+peered, 11 active+undersized+degraded, 20 active+clean; 117 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 610/5466 objects degraded (11.160%); 1072/5466 objects misplaced (19.612%); 2.8 MiB/s, 49 objects/s recovering 2023-09-23T22:40:53.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 610/5466 objects degraded (11.160%), 14 pgs degraded, 53 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:53.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:53 smithi026 ceph-mon[215775]: pgmap v1426: 119 pgs: 1 peering, 31 active+clean+remapped, 13 undersized+peered, 23 active+undersized, 17 active+undersized+remapped, 3 undersized+degraded+peered, 11 active+undersized+degraded, 20 active+clean; 117 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 610/5466 objects degraded (11.160%); 1072/5466 objects misplaced (19.612%); 2.8 MiB/s, 49 objects/s recovering 2023-09-23T22:40:53.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 610/5466 objects degraded (11.160%), 14 pgs degraded, 53 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:53.933 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T22:40:53.934 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T22:40:54.426 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:40:54.544 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:40:54.564 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:40:54.582 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:40:54.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2387339570' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:54.590 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:54.591 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2387339570' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:54.591 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:54.603 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:40:54.623 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:40:54.644 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:40:54.644 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.644 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.644 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.644 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:40:54.644 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.645 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.645 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:40:54.645 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:40:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2387339570' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:40:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:40:55.293 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:55.478 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:55.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:55.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:55.518 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:40:55.519 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:40:55.583 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:55.611 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:55.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[223014]: pgmap v1427: 119 pgs: 1 peering, 33 active+clean+remapped, 13 undersized+peered, 24 active+undersized, 13 active+undersized+remapped, 3 undersized+degraded+peered, 10 active+undersized+degraded, 22 active+clean; 86 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 375/4014 objects degraded (9.342%); 823/4014 objects misplaced (20.503%); 2.1 MiB/s, 37 objects/s recovering 2023-09-23T22:40:55.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:55.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[223014]: failed to encode map e546 with expected crc 2023-09-23T22:40:55.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[223014]: failed to encode map e546 with expected crc 2023-09-23T22:40:55.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[223014]: failed to encode map e546 with expected crc 2023-09-23T22:40:55.674 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2387339570' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:55.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[223014]: osdmap e546: 8 total, 6 up, 7 in 2023-09-23T22:40:55.675 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1995773505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:55.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[215775]: pgmap v1427: 119 pgs: 1 peering, 33 active+clean+remapped, 13 undersized+peered, 24 active+undersized, 13 active+undersized+remapped, 3 undersized+degraded+peered, 10 active+undersized+degraded, 22 active+clean; 86 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 375/4014 objects degraded (9.342%); 823/4014 objects misplaced (20.503%); 2.1 MiB/s, 37 objects/s recovering 2023-09-23T22:40:55.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:55.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[215775]: failed to encode map e546 with expected crc 2023-09-23T22:40:55.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[215775]: failed to encode map e546 with expected crc 2023-09-23T22:40:55.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[215775]: failed to encode map e546 with expected crc 2023-09-23T22:40:55.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2387339570' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:55.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[215775]: osdmap e546: 8 total, 6 up, 7 in 2023-09-23T22:40:55.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1995773505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:55.708 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:55 smithi070 ceph-mon[192652]: pgmap v1427: 119 pgs: 1 peering, 33 active+clean+remapped, 13 undersized+peered, 24 active+undersized, 13 active+undersized+remapped, 3 undersized+degraded+peered, 10 active+undersized+degraded, 22 active+clean; 86 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 573 op/s; 375/4014 objects degraded (9.342%); 823/4014 objects misplaced (20.503%); 2.1 MiB/s, 37 objects/s recovering 2023-09-23T22:40:55.708 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:55 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:40:55.708 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:55 smithi070 ceph-mon[192652]: failed to encode map e546 with expected crc 2023-09-23T22:40:55.708 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:55 smithi070 ceph-mon[192652]: failed to encode map e546 with expected crc 2023-09-23T22:40:55.708 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:55 smithi070 ceph-mon[192652]: failed to encode map e546 with expected crc 2023-09-23T22:40:55.708 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2387339570' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:40:55.709 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:55 smithi070 ceph-mon[192652]: osdmap e546: 8 total, 6 up, 7 in 2023-09-23T22:40:55.709 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1995773505' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:40:55.714 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:55.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:55.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:55.818 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:55.846 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:55.939 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:55.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:55.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:56.045 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:56.072 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:56.330 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:40:56.330 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "" 2023-09-23T22:40:56.330 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:40:56.929 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:56.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:56.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:57.032 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:57.063 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:57.111 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:57.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:57.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:57.214 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:57.243 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:57.320 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:40:57.437 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_4' removed 2023-09-23T22:40:57.487 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:57.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:40:57.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:57.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:57.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:57 smithi070 ceph-mon[192652]: pgmap v1429: 119 pgs: 1 peering, 33 active+clean+remapped, 13 undersized+peered, 25 active+undersized, 13 active+undersized+remapped, 3 undersized+degraded+peered, 9 active+undersized+degraded, 22 active+clean; 39 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 254/1746 objects degraded (14.548%); 177/1746 objects misplaced (10.137%); 1.9 MiB/s, 32 objects/s recovering 2023-09-23T22:40:57.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:57 smithi070 ceph-mon[192652]: 1.3 scrub starts 2023-09-23T22:40:57.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:57 smithi070 ceph-mon[192652]: 1.3 scrub ok 2023-09-23T22:40:57.621 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/484851820' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:40:57.622 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:40:57.630 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[215775]: pgmap v1429: 119 pgs: 1 peering, 33 active+clean+remapped, 13 undersized+peered, 25 active+undersized, 13 active+undersized+remapped, 3 undersized+degraded+peered, 9 active+undersized+degraded, 22 active+clean; 39 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 254/1746 objects degraded (14.548%); 177/1746 objects misplaced (10.137%); 1.9 MiB/s, 32 objects/s recovering 2023-09-23T22:40:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[215775]: 1.3 scrub starts 2023-09-23T22:40:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[215775]: 1.3 scrub ok 2023-09-23T22:40:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/484851820' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:40:57.631 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:40:57.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[223014]: pgmap v1429: 119 pgs: 1 peering, 33 active+clean+remapped, 13 undersized+peered, 25 active+undersized, 13 active+undersized+remapped, 3 undersized+degraded+peered, 9 active+undersized+degraded, 22 active+clean; 39 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 254/1746 objects degraded (14.548%); 177/1746 objects misplaced (10.137%); 1.9 MiB/s, 32 objects/s recovering 2023-09-23T22:40:57.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[223014]: 1.3 scrub starts 2023-09-23T22:40:57.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[223014]: 1.3 scrub ok 2023-09-23T22:40:57.631 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/484851820' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:40:57.632 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:40:57.722 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:57.752 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:57.752 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:40:57.752 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:40:57.793 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:57.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:57.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:57.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:40:57.928 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:58.021 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:40:58.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:58.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:40:58.099 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T22:40:58.100 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T22:40:58.100 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T22:40:58.100 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T22:40:58.125 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:58.185 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:58.185 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:58.250 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:58.297 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:40:58.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:58.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:40:58.402 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:40:58.430 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:58.495 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:40:58.524 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:40:58.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:58.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:40:58.631 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:58.668 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:58.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[223014]: failed to encode map e547 with expected crc 2023-09-23T22:40:58.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[223014]: failed to encode map e547 with expected crc 2023-09-23T22:40:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:40:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[223014]: osdmap e547: 8 total, 6 up, 7 in 2023-09-23T22:40:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[223014]: failed to encode map e547 with expected crc 2023-09-23T22:40:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[223014]: 1.13 scrub starts 2023-09-23T22:40:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[223014]: 1.13 scrub ok 2023-09-23T22:40:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 16 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:40:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 254/1746 objects degraded (14.548%), 12 pgs degraded, 53 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:58.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[215775]: failed to encode map e547 with expected crc 2023-09-23T22:40:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[215775]: failed to encode map e547 with expected crc 2023-09-23T22:40:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:40:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[215775]: osdmap e547: 8 total, 6 up, 7 in 2023-09-23T22:40:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[215775]: failed to encode map e547 with expected crc 2023-09-23T22:40:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[215775]: 1.13 scrub starts 2023-09-23T22:40:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[215775]: 1.13 scrub ok 2023-09-23T22:40:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 16 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:40:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 254/1746 objects degraded (14.548%), 12 pgs degraded, 53 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:58.860 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:40:58.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:58 smithi070 ceph-mon[192652]: failed to encode map e547 with expected crc 2023-09-23T22:40:58.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:58 smithi070 ceph-mon[192652]: failed to encode map e547 with expected crc 2023-09-23T22:40:58.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:40:58.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:58 smithi070 ceph-mon[192652]: osdmap e547: 8 total, 6 up, 7 in 2023-09-23T22:40:58.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:58 smithi070 ceph-mon[192652]: failed to encode map e547 with expected crc 2023-09-23T22:40:58.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:58 smithi070 ceph-mon[192652]: 1.13 scrub starts 2023-09-23T22:40:58.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:58 smithi070 ceph-mon[192652]: 1.13 scrub ok 2023-09-23T22:40:58.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:58 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 16 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:40:58.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 254/1746 objects degraded (14.548%), 12 pgs degraded, 53 pgs undersized (PG_DEGRADED) 2023-09-23T22:40:59.331 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:59.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:59.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:59.435 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:40:59.464 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:59.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:40:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:59.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:40:59.611 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:40:59.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:40:59 smithi026 ceph-mon[215775]: pgmap v1431: 93 pgs: 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:40:59.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:40:59 smithi026 ceph-mon[223014]: pgmap v1431: 93 pgs: 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:40:59.643 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:40:59.757 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:40:59.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:40:59.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:40:59.795 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:40:59.862 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:40:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:40:59 smithi070 ceph-mon[192652]: pgmap v1431: 93 pgs: 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:40:59.892 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:40:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:40:59] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-09-23T22:40:59.897 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:00.182 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:00.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:00.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:00.287 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:00.326 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:00.415 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:00.415 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:00.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[215775]: 1.5 scrub starts 2023-09-23T22:41:00.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[215775]: 1.5 scrub ok 2023-09-23T22:41:00.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[215775]: 1.b scrub starts 2023-09-23T22:41:00.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[215775]: 1.b scrub ok 2023-09-23T22:41:00.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2819730866' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:00.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:00.446 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:41:00.530 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:41:00.556 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:41:00.578 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:41:00.581 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:00.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:00.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:00.599 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:41:00.618 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:41:00.638 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:41:00.638 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:41:00.638 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:41:00.638 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:41:00.638 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:41:00.638 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:41:00.639 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:41:00.639 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:41:00.639 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:41:00.685 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:00.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:00 smithi070 ceph-mon[192652]: 1.5 scrub starts 2023-09-23T22:41:00.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:00 smithi070 ceph-mon[192652]: 1.5 scrub ok 2023-09-23T22:41:00.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:00 smithi070 ceph-mon[192652]: 1.b scrub starts 2023-09-23T22:41:00.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:00 smithi070 ceph-mon[192652]: 1.b scrub ok 2023-09-23T22:41:00.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:00 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2819730866' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:00.712 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:00 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:00.715 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:00.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[223014]: 1.5 scrub starts 2023-09-23T22:41:00.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[223014]: 1.5 scrub ok 2023-09-23T22:41:00.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[223014]: 1.b scrub starts 2023-09-23T22:41:00.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[223014]: 1.b scrub ok 2023-09-23T22:41:00.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2819730866' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:00.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:00 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:00.758 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:00.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:00.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:00.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:00.889 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:01.048 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:01.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:01.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:01.102 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:41:01.152 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:01.178 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:01.221 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:01.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:01.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:01.326 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:01.352 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:01.398 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:01.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:01.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:01.453 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:01.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:01.543 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:01.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:01 smithi070 ceph-mon[192652]: pgmap v1432: 93 pgs: 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:01 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:01 smithi070 ceph-mon[192652]: failed to encode map e548 with expected crc 2023-09-23T22:41:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:01 smithi070 ceph-mon[192652]: failed to encode map e548 with expected crc 2023-09-23T22:41:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:01 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:01 smithi070 ceph-mon[192652]: osdmap e548: 8 total, 6 up, 7 in 2023-09-23T22:41:01.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/690021947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:01.644 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:01.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:01.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[223014]: pgmap v1432: 93 pgs: 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:01.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:01.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[223014]: failed to encode map e548 with expected crc 2023-09-23T22:41:01.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[223014]: failed to encode map e548 with expected crc 2023-09-23T22:41:01.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:01.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[223014]: osdmap e548: 8 total, 6 up, 7 in 2023-09-23T22:41:01.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/690021947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:01.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[215775]: pgmap v1432: 93 pgs: 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:01.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:01.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[215775]: failed to encode map e548 with expected crc 2023-09-23T22:41:01.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[215775]: failed to encode map e548 with expected crc 2023-09-23T22:41:01.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:01.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[215775]: osdmap e548: 8 total, 6 up, 7 in 2023-09-23T22:41:01.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/690021947' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:01.748 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:01.776 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:01.819 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:01.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:01.867 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:41:01.867 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:41:01.922 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:01.949 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:02.026 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:02.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:02.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:02.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:02.158 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:02.200 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:02.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:02.304 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:02.331 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:02.469 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:02.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:02.472 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:02.573 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:02.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:02 smithi026 ceph-mon[223014]: 1.d scrub starts 2023-09-23T22:41:02.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:02 smithi026 ceph-mon[223014]: 1.d scrub ok 2023-09-23T22:41:02.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:02 smithi026 ceph-mon[223014]: failed to encode map e548 with expected crc 2023-09-23T22:41:02.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:02 smithi026 ceph-mon[215775]: 1.d scrub starts 2023-09-23T22:41:02.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:02 smithi026 ceph-mon[215775]: 1.d scrub ok 2023-09-23T22:41:02.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:02 smithi026 ceph-mon[215775]: failed to encode map e548 with expected crc 2023-09-23T22:41:02.603 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:02.623 INFO:tasks.thrashosds.thrasher:in_osds: [5, 0, 1, 6, 2, 7, 3] out_osds: [4] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:41:02.623 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:41:02.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:02 smithi070 ceph-mon[192652]: 1.d scrub starts 2023-09-23T22:41:02.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:02 smithi070 ceph-mon[192652]: 1.d scrub ok 2023-09-23T22:41:02.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:02 smithi070 ceph-mon[192652]: failed to encode map e548 with expected crc 2023-09-23T22:41:02.652 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:02.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:02.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:02.756 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:02.783 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:02.928 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:02.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:02.987 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:41:02.987 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:41:02.987 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:41:02.987 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:41:02.987 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.987 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179625 8.d 8:bd963f3c:::benchmark_data_smithi026_497658_object80733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T22:41:02.987 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.331387+0000", 2023-09-23T22:41:02.987 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.652682631, 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.046390226, 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179625 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:02.988 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331387+0000", 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331387+0000", 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331386+0000", 2023-09-23T22:41:02.989 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331389+0000", 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6369999999999999e-06 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331396+0000", 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.516e-06 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.990 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331400+0000", 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1180000000000002e-06 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331465+0000", 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5592000000000005e-05 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331589+0000", 2023-09-23T22:41:02.991 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000123306 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2,6", 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331711+0000", 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012258500000000001 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.340951+0000", 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0092401549999999999 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:02.992 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.340962+0000", 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0322999999999999e-05 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377694+0000", 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.036732049 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377777+0000", 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3311000000000002e-05 2023-09-23T22:41:02.993 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179634 8.d 8:b7c2e7fe:::benchmark_data_smithi026_497658_object80742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.331413+0000", 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.65265691500002, 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.046453466, 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:41:02.994 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179634 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331413+0000", 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.995 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331413+0000", 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331412+0000", 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331415+0000", 2023-09-23T22:41:02.996 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5009999999999999e-06 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331422+0000", 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2089999999999999e-06 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.331426+0000", 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.318e-06 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.997 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.335320+0000", 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038939280000000001 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.335421+0000", 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010108299999999999 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,2,6", 2023-09-23T22:41:02.998 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.335493+0000", 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1797000000000004e-05 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.340994+0000", 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055008339999999996 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.341019+0000", 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5037000000000001e-05 2023-09-23T22:41:02.999 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377815+0000", 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.036795911 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377866+0000", 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1248000000000001e-05 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.000 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179638 8.d 8:b6313638:::benchmark_data_smithi026_497658_object80746:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.353396+0000", 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.63067350799997, 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.061055393, 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179638 2023-09-23T22:41:03.001 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353396+0000", 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353396+0000", 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.002 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353396+0000", 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353398+0000", 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.323e-06 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.003 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353404+0000", 2023-09-23T22:41:03.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0750000000000003e-06 2023-09-23T22:41:03.004 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.004 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.004 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.004 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353408+0000", 2023-09-23T22:41:03.004 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8419999999999998e-06 2023-09-23T22:41:03.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.005 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353461+0000", 2023-09-23T22:41:03.005 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2778000000000002e-05 2023-09-23T22:41:03.005 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.005 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.005 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.353471+0000", 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0149000000000001e-05 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.377887+0000", 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0244162080000001 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.006 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.378011+0000", 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012360800000000001 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.378162+0000", 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015116000000000001 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395006+0000", 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016844624999999998 2023-09-23T22:41:03.007 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414290+0000", 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019283582000000001 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414387+0000", 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6844999999999999e-05 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.008 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414426+0000", 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9545999999999997e-05 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414451+0000", 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5034000000000001e-05 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.009 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179642 8.d 8:b7bd1a79:::benchmark_data_smithi026_497658_object80750:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.358239+0000", 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.62583008000001, 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0563148179999999, 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179642 2023-09-23T22:41:03.010 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358239+0000", 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358239+0000", 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.011 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358239+0000", 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358242+0000", 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4400000000000001e-06 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.012 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358251+0000", 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5439999999999999e-06 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358255+0000", 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4710000000000001e-06 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358302+0000", 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6727999999999997e-05 2023-09-23T22:41:03.013 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.358312+0000", 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9790000000000005e-06 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.381332+0000", 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0230205020000001 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.014 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.381469+0000", 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013704400000000001 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.381594+0000", 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012458900000000001 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.015 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395011+0000", 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013417247 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414343+0000", 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019331883000000001 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414489+0000", 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014642799999999999 2023-09-23T22:41:03.016 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414520+0000", 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0605999999999998e-05 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414554+0000", 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4215999999999999e-05 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.017 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179657 8.d 8:bab24576:::benchmark_data_smithi026_497658_object80765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.371809+0000", 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.61226006300001, 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.263966353, 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.018 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179657 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371809+0000", 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371809+0000", 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.019 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371809+0000", 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371811+0000", 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.249e-06 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.020 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371818+0000", 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2480000000000001e-06 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371823+0000", 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2809999999999999e-06 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371869+0000", 2023-09-23T22:41:03.021 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6317999999999999e-05 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.371877+0000", 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8299999999999996e-06 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.385058+0000", 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0131806569999999 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.022 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.385156+0000", 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8345999999999997e-05 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.385288+0000", 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013212900000000001 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.023 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400220+0000", 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014931542000000001 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414579+0000", 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014359764000000001 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635719+0000", 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.221139108 2023-09-23T22:41:03.024 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635749+0000", 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0750000000000002e-05 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635776+0000", 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6500999999999999e-05 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.025 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179686 8.d 8:b2459d19:::benchmark_data_smithi026_497658_object80794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.423416+0000", 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.56065300500001, 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2124665779999999, 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.026 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179686 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423416+0000", 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423416+0000", 2023-09-23T22:41:03.027 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423415+0000", 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423422+0000", 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6440000000000003e-06 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.028 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423433+0000", 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1664999999999999e-05 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.423440+0000", 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8889999999999996e-06 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.029 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.424713+0000", 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001272537 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.424720+0000", 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8380000000000002e-06 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.388740+0000", 2023-09-23T22:41:03.030 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.96401941300000005 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.388878+0000", 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013780300000000001 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.389037+0000", 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015977600000000001 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.031 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400224+0000", 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011186125999999999 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414667+0000", 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014443754 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.032 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635827+0000", 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22116000899999999 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635860+0000", 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2710999999999999e-05 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635883+0000", 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3011999999999999e-05 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.033 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179691 8.d 8:b7906ebe:::benchmark_data_smithi026_497658_object80799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.437137+0000", 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.54693234400003, 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.198820869, 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.034 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179691 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437137+0000", 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437137+0000", 2023-09-23T22:41:03.035 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437135+0000", 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437145+0000", 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3409999999999992e-06 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.036 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437163+0000", 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7986e-05 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437171+0000", 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5490000000000006e-06 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437226+0000", 2023-09-23T22:41:03.037 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4580999999999998e-05 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.437235+0000", 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.3279999999999995e-06 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.391971+0000", 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.95473568900000005 2023-09-23T22:41:03.038 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.392111+0000", 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014051100000000001 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.392262+0000", 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015037500000000001 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.039 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402309+0000", 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010047512999999999 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414716+0000", 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012406531 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635907+0000", 2023-09-23T22:41:03.040 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.221191161 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635933+0000", 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5809e-05 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635958+0000", 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5369999999999999e-05 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.041 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179708 8.d 8:baab2a80:::benchmark_data_smithi026_497658_object80816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.457089+0000", 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.526980288, 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1789815729999999, 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.042 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179708 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457089+0000", 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457089+0000", 2023-09-23T22:41:03.043 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457087+0000", 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457095+0000", 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.148e-06 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.044 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457108+0000", 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3293e-05 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457114+0000", 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.596e-06 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457169+0000", 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4818999999999998e-05 2023-09-23T22:41:03.045 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.457179+0000", 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8709999999999992e-06 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395053+0000", 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93787340500000005 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.046 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395194+0000", 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014127500000000001 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.395338+0000", 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014388000000000001 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.047 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402314+0000", 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069762490000000003 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414739+0000", 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012424663000000001 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.635996+0000", 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22125688900000001 2023-09-23T22:41:03.048 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.636041+0000", 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5510000000000003e-05 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.636071+0000", 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9723999999999999e-05 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.049 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179715 8.d 8:bf536315:::benchmark_data_smithi026_497658_object80823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.470395+0000", 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.51367483199999, 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.170438106, 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179715 2023-09-23T22:41:03.050 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470395+0000", 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470395+0000", 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.051 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470393+0000", 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470401+0000", 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1629999999999996e-06 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470414+0000", 2023-09-23T22:41:03.052 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2549e-05 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470420+0000", 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3879999999999997e-06 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470473+0000", 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2553000000000003e-05 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.053 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.470482+0000", 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1449999999999997e-06 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.397798+0000", 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92731558999999997 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.054 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.397927+0000", 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128928 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.398079+0000", 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000152218 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.408311+0000", 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010232196000000001 2023-09-23T22:41:03.055 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414784+0000", 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0064727860000000003 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640770+0000", 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22598658799999999 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.056 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640804+0000", 2023-09-23T22:41:03.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4106999999999997e-05 2023-09-23T22:41:03.057 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.057 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.057 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.057 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640833+0000", 2023-09-23T22:41:03.057 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8300999999999999e-05 2023-09-23T22:41:03.057 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179746 8.d 8:bb9d95f1:::benchmark_data_smithi026_497658_object80854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.513114+0000", 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.47095541800002, 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1278254560000001, 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.058 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179746 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513114+0000", 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.059 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513114+0000", 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513113+0000", 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513119+0000", 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5379999999999999e-06 2023-09-23T22:41:03.060 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513133+0000", 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3495e-05 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513140+0000", 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8700000000000003e-06 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.061 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513255+0000", 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115285 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.513261+0000", 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7089999999999999e-06 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400241+0000", 2023-09-23T22:41:03.062 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.886980027 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400312+0000", 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1092000000000003e-05 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.400404+0000", 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2526000000000001e-05 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.063 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.413998+0000", 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013593684999999999 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414805+0000", 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00080711400000000005 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640877+0000", 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.22607189499999999 2023-09-23T22:41:03.064 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640910+0000", 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3482999999999998e-05 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640940+0000", 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9099e-05 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.065 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179749 8.d 8:b1665d54:::benchmark_data_smithi026_497658_object80857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.517186+0000", 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.46688319399999, 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.123864177, 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.066 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179749 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517186+0000", 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517186+0000", 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.067 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517185+0000", 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517191+0000", 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8329999999999999e-06 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.068 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517205+0000", 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3961000000000001e-05 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517210+0000", 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1139999999999997e-06 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517267+0000", 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7213999999999998e-05 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.069 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.517277+0000", 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9899999999999992e-06 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402354+0000", 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.88507623999999996 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.070 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402479+0000", 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000125093 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.402556+0000", 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7556000000000001e-05 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414007+0000", 2023-09-23T22:41:03.071 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011450514 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414840+0000", 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00083372199999999996 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.640988+0000", 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.226147288 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.072 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.641029+0000", 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1393999999999997e-05 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.641050+0000", 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1396e-05 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.073 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179761 8.d 8:b0ca34a6:::benchmark_data_smithi026_497658_object80869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.535617+0000", 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.44845215999999, 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.12096347, 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.074 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179761 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535617+0000", 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535617+0000", 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.075 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535617+0000", 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535620+0000", 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7050000000000001e-06 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.076 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535633+0000", 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3096e-05 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535640+0000", 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2020000000000001e-06 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.077 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535706+0000", 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6019999999999995e-05 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.535716+0000", 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0825000000000001e-05 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.404445+0000", 2023-09-23T22:41:03.078 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.86872851699999998 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.404517+0000", 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2002999999999995e-05 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.404606+0000", 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8734999999999997e-05 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.079 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414015+0000", 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094092859999999993 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414876+0000", 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00086061600000000003 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656491+0000", 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.24161501499999999 2023-09-23T22:41:03.080 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656537+0000", 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5915999999999999e-05 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656581+0000", 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4224000000000002e-05 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.081 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.45842.0:179767 8.d 8:ba13a7eb:::benchmark_data_smithi026_497658_object80875:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e426)", 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:34:25.545488+0000", 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 397.43858184700002, 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1111883810000001, 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.45842", 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:41:03.082 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 179767 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545488+0000", 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545488+0000", 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.083 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545486+0000", 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545492+0000", 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3219999999999998e-06 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545507+0000", 2023-09-23T22:41:03.084 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4817000000000001e-05 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545512+0000", 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4859999999999999e-06 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545551+0000", 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8937999999999999e-05 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.085 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:25.545556+0000", 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4580000000000001e-06 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.406356+0000", 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.86079989899999998 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.086 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.406432+0000", 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5601999999999999e-05 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.406530+0000", 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8529999999999993e-05 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414042+0000", 2023-09-23T22:41:03.087 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0075122269999999998 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.414912+0000", 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00086938100000000002 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656633+0000", 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.24172130999999999 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.088 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656656+0000", 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2836e-05 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:34:26.656676+0000", 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0058999999999999e-05 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.089 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104682 9.f 9:f58eaa2e:::benchmark_data_smithi026_555954_object9953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.265869+0000", 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 228.71820059800001, 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1940709429999998, 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104682 2023-09-23T22:41:03.090 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265869+0000", 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265869+0000", 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.091 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265868+0000", 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265875+0000", 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.52e-06 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265888+0000", 2023-09-23T22:41:03.092 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2493e-05 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265892+0000", 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6199999999999998e-06 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265936+0000", 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3751999999999997e-05 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.093 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266080+0000", 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014391999999999999 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266249+0000", 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000168729 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.269497+0000", 2023-09-23T22:41:03.094 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003247757 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292030+0000", 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022533972999999999 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292040+0000", 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6260000000000007e-06 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.095 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276089+0000", 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98404894300000001 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276099+0000", 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8539999999999992e-06 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.096 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425242+0000", 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1491429769999999 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425390+0000", 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000148601 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.459827+0000", 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034436178999999997 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.097 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.459940+0000", 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011315900000000001 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104687 9.f 9:fb4465c7:::benchmark_data_smithi026_555954_object9958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.279126+0000", 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 228.70494317199999, 2023-09-23T22:41:03.098 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1856558540000002, 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104687 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279126+0000", 2023-09-23T22:41:03.099 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279126+0000", 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279125+0000", 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.100 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279132+0000", 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.968e-06 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279147+0000", 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5252000000000001e-05 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279155+0000", 2023-09-23T22:41:03.101 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4139999999999997e-06 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279205+0000", 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0002e-05 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279383+0000", 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017872 2023-09-23T22:41:03.102 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279550+0000", 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000166343 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292069+0000", 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012519417 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.103 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292074+0000", 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1499999999999998e-06 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276124+0000", 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98404924299999996 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276129+0000", 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6949999999999997e-06 2023-09-23T22:41:03.104 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425447+0000", 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1493180839999999 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425628+0000", 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018074899999999999 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.105 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464649+0000", 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039020688999999997 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464782+0000", 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013333600000000001 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.106 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104699 9.f 9:fa6f98c4:::benchmark_data_smithi026_555954_object9970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.355308+0000", 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 228.628761905, 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1458075220000001, 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104699 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.107 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355308+0000", 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355308+0000", 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.108 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355306+0000", 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355314+0000", 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2239999999999995e-06 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355332+0000", 2023-09-23T22:41:03.109 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7728999999999999e-05 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355355+0000", 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3218999999999999e-05 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355394+0000", 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8661999999999997e-05 2023-09-23T22:41:03.110 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355401+0000", 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9550000000000003e-06 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276149+0000", 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92074822599999995 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.111 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276155+0000", 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5539999999999997e-06 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425668+0000", 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.149513765 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426016+0000", 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000347487 2023-09-23T22:41:03.112 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.445038+0000", 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0190222 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.445155+0000", 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011651599999999999 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.113 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.445326+0000", 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017155499999999999 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464769+0000", 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019442566000000001 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.114 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464804+0000", 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5648999999999997e-05 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.497550+0000", 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.032745511999999997 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.497594+0000", 2023-09-23T22:41:03.115 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4270000000000001e-05 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.501115+0000", 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035209569999999999 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104701 9.f 9:f9048e3a:::benchmark_data_smithi026_555954_object9972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:41:03.116 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.295854+0000", 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 227.68821528399999, 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.208795673, 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104701 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.117 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295854+0000", 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295854+0000", 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295853+0000", 2023-09-23T22:41:03.118 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295861+0000", 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3720000000000005e-06 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295877+0000", 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5875e-05 2023-09-23T22:41:03.119 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295885+0000", 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5109999999999997e-06 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295932+0000", 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6916999999999998e-05 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.120 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295940+0000", 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0020000000000006e-06 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426058+0000", 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1301181819999999 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426486+0000", 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00042859000000000001 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.121 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.452638+0000", 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026151917 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.452720+0000", 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1389000000000003e-05 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.122 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.452832+0000", 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112194 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464779+0000", 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011947107 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464856+0000", 2023-09-23T22:41:03.123 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6678e-05 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.501167+0000", 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036311450000000002 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.501211+0000", 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4246999999999998e-05 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.124 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.504650+0000", 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034385829999999998 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104709 9.f 9:fe76147f:::benchmark_data_smithi026_555954_object9980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.348800+0000", 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 227.635269951, 2023-09-23T22:41:03.125 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.175744452, 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104709 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348800+0000", 2023-09-23T22:41:03.126 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348800+0000", 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348798+0000", 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:03.127 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348807+0000", 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7070000000000004e-06 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348822+0000", 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4944e-05 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.128 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348831+0000", 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7280000000000001e-06 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.389358+0000", 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.040527173999999999 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.389371+0000", 2023-09-23T22:41:03.129 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2469e-05 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426523+0000", 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0371526820000001 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426916+0000", 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039232699999999999 2023-09-23T22:41:03.130 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.493074+0000", 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.066158162000000006 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.493164+0000", 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0309000000000006e-05 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.131 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.493254+0000", 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0066000000000001e-05 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.504667+0000", 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011412558999999999 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.504705+0000", 2023-09-23T22:41:03.132 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8373e-05 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.521152+0000", 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016447092999999999 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.521194+0000", 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1467999999999997e-05 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.133 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.524544+0000", 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033503230000000001 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:105990 9.f 9:fc198dae:::benchmark_data_smithi026_555954_object11261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:43.319569+0000", 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 199.66450072800001, 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4132751570000002, 2023-09-23T22:41:03.134 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 105990 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319569+0000", 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.135 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319569+0000", 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319566+0000", 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999971 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.136 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319582+0000", 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5855999999999999e-05 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319622+0000", 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0615999999999999e-05 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.137 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319637+0000", 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4922999999999999e-05 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319766+0000", 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012936299999999999 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319786+0000", 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9723000000000001e-05 2023-09-23T22:41:03.138 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.680894+0000", 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3611081509999998 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.681104+0000", 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020942399999999999 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.139 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.681225+0000", 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012102199999999999 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.729953+0000", 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048728018999999997 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.140 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730035+0000", 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2010999999999998e-05 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730114+0000", 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9276000000000005e-05 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730166+0000", 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1437000000000002e-05 2023-09-23T22:41:03.141 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.732844+0000", 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026783789999999998 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:105991 9.f 9:f5926516:::benchmark_data_smithi026_555954_object11262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:41:03.142 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 199.66440309399999, 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4159624549999998, 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 105991 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:03.143 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:41:03.144 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319671+0000", 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4060000000000004e-06 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319681+0000", 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0563e-05 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.145 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319688+0000", 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8000000000000001e-06 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319843+0000", 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015483200000000001 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:03.146 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319865+0000", 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1933000000000002e-05 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.684141+0000", 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.364276378 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.684269+0000", 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012795800000000001 2023-09-23T22:41:03.147 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.684369+0000", 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9584000000000001e-05 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730094+0000", 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045724787000000003 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:03.148 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.732880+0000", 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002786186 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.732981+0000", 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100999 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.733032+0000", 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0701000000000001e-05 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:03.149 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:03.150 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:03.150 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.735629+0000", 2023-09-23T22:41:03.150 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025971940000000002 2023-09-23T22:41:03.150 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.150 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.150 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.150 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:03.150 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:03.150 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:41:03.151 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:03.178 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:03.223 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:03.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:03.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:03.327 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:03.354 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:03.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:03 smithi026 ceph-mon[215775]: pgmap v1434: 93 pgs: 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:03.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2680129208' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T22:41:03.448 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 18/84 objects degraded (21.429%), 7 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:03.451 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_5' created 2023-09-23T22:41:03.579 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:41:03.594 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:41:03.598 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:03.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:03.704 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:03.727 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:03 smithi070 ceph-mon[192652]: pgmap v1434: 93 pgs: 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:03.727 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2680129208' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T22:41:03.728 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 18/84 objects degraded (21.429%), 7 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:03.730 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:03.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:03 smithi026 ceph-mon[223014]: pgmap v1434: 93 pgs: 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:03.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2680129208' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T22:41:03.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 18/84 objects degraded (21.429%), 7 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:03.773 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:03.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:03.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:03.868 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:41:03.881 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:03.908 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:03.950 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:03.953 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:03.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:03.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:04.060 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:04.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:04.218 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:04.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:04.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:04.226 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T22:41:04.298 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:04.325 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:04.325 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:04.358 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:04.453 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:04.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[215775]: failed to encode map e549 with expected crc 2023-09-23T22:41:04.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[215775]: failed to encode map e549 with expected crc 2023-09-23T22:41:04.457 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[215775]: failed to encode map e549 with expected crc 2023-09-23T22:41:04.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2680129208' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T22:41:04.458 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[215775]: osdmap e549: 8 total, 6 up, 7 in 2023-09-23T22:41:04.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:04.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:04.560 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:04.588 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:04.631 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:04.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:04.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:04.735 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[223014]: failed to encode map e549 with expected crc 2023-09-23T22:41:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[223014]: failed to encode map e549 with expected crc 2023-09-23T22:41:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[223014]: failed to encode map e549 with expected crc 2023-09-23T22:41:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2680129208' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T22:41:04.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:04 smithi026 ceph-mon[223014]: osdmap e549: 8 total, 6 up, 7 in 2023-09-23T22:41:04.759 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:04 smithi070 ceph-mon[192652]: failed to encode map e549 with expected crc 2023-09-23T22:41:04.759 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:04 smithi070 ceph-mon[192652]: failed to encode map e549 with expected crc 2023-09-23T22:41:04.759 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:04 smithi070 ceph-mon[192652]: failed to encode map e549 with expected crc 2023-09-23T22:41:04.759 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2680129208' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T22:41:04.759 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:04 smithi070 ceph-mon[192652]: osdmap e549: 8 total, 6 up, 7 in 2023-09-23T22:41:04.761 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:04.927 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:04.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:05.031 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:05.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:05.127 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:05.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:05.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:05.231 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:05.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:05.406 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:05.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:05.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:05.465 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:41:05.465 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:41:05.466 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:41:05.466 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:41:05.509 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:05.539 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:05.713 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:05 smithi070 ceph-mon[192652]: pgmap v1436: 109 pgs: 16 unknown, 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:05.713 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:05 smithi070 ceph-mon[192652]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.713 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:05 smithi070 ceph-mon[192652]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.713 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:05 smithi070 ceph-mon[192652]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.713 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:05 smithi070 ceph-mon[192652]: osdmap e550: 8 total, 6 up, 7 in 2023-09-23T22:41:05.713 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:05 smithi070 ceph-mon[192652]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.713 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:05 smithi070 ceph-mon[192652]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[215775]: pgmap v1436: 109 pgs: 16 unknown, 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:05.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[215775]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[215775]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[215775]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[215775]: osdmap e550: 8 total, 6 up, 7 in 2023-09-23T22:41:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[215775]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[215775]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[223014]: pgmap v1436: 109 pgs: 16 unknown, 22 active+clean+remapped, 13 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[223014]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[223014]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[223014]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[223014]: osdmap e550: 8 total, 6 up, 7 in 2023-09-23T22:41:05.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[223014]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:05 smithi026 ceph-mon[223014]: failed to encode map e550 with expected crc 2023-09-23T22:41:05.737 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:05.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:05.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:05.744 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:05.841 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:05.868 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:06.097 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:06.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:06.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:06.151 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:41:06.202 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:06.233 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:06.436 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:06.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:06.442 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:06.487 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:06.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:06.570 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:06.606 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:41:06.611 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:06.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:06.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:06.625 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:41:06.643 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:41:06.663 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:41:06.682 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:41:06.702 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:41:06.702 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:41:06.702 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:41:06.703 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:41:06.703 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:41:06.703 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:41:06.703 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:41:06.703 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:41:06.703 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:41:06.714 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:06.720 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:06 smithi070 ceph-mon[192652]: failed to encode map e551 with expected crc 2023-09-23T22:41:06.720 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:06 smithi070 ceph-mon[192652]: failed to encode map e551 with expected crc 2023-09-23T22:41:06.720 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:06 smithi070 ceph-mon[192652]: failed to encode map e551 with expected crc 2023-09-23T22:41:06.720 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:06 smithi070 ceph-mon[192652]: osdmap e551: 8 total, 6 up, 7 in 2023-09-23T22:41:06.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:06 smithi026 ceph-mon[223014]: failed to encode map e551 with expected crc 2023-09-23T22:41:06.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:06 smithi026 ceph-mon[223014]: failed to encode map e551 with expected crc 2023-09-23T22:41:06.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:06 smithi026 ceph-mon[223014]: failed to encode map e551 with expected crc 2023-09-23T22:41:06.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:06 smithi026 ceph-mon[223014]: osdmap e551: 8 total, 6 up, 7 in 2023-09-23T22:41:06.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:06 smithi026 ceph-mon[215775]: failed to encode map e551 with expected crc 2023-09-23T22:41:06.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:06 smithi026 ceph-mon[215775]: failed to encode map e551 with expected crc 2023-09-23T22:41:06.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:06 smithi026 ceph-mon[215775]: failed to encode map e551 with expected crc 2023-09-23T22:41:06.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:06 smithi026 ceph-mon[215775]: osdmap e551: 8 total, 6 up, 7 in 2023-09-23T22:41:06.744 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:07.479 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:41:07.515 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:07.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:07.613 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:41:07.614 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:41:07.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:07.648 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:07.691 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:07 smithi070 ceph-mon[192652]: pgmap v1439: 109 pgs: 12 unknown, 22 active+clean+remapped, 14 undersized+peered, 25 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:07.691 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2124340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:07.691 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2200965385' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:07.691 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:07 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:07.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:07 smithi026 ceph-mon[223014]: pgmap v1439: 109 pgs: 12 unknown, 22 active+clean+remapped, 14 undersized+peered, 25 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:07.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2124340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:07.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2200965385' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:07.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:07 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:07 smithi026 ceph-mon[215775]: pgmap v1439: 109 pgs: 12 unknown, 22 active+clean+remapped, 14 undersized+peered, 25 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 18 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2124340' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2200965385' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:07 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:07.856 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:07.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:07.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:07.872 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:07.872 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:07.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:07.988 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:08.436 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:08.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:08.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:08.460 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:41:08.540 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:08.569 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:08.616 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:08.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:08.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:08.720 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[223014]: 1.2 scrub starts 2023-09-23T22:41:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[223014]: 1.2 scrub ok 2023-09-23T22:41:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[223014]: failed to encode map e552 with expected crc 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[223014]: failed to encode map e552 with expected crc 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[223014]: failed to encode map e552 with expected crc 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[223014]: osdmap e552: 8 total, 6 up, 7 in 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[215775]: 1.2 scrub starts 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[215775]: 1.2 scrub ok 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[215775]: failed to encode map e552 with expected crc 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[215775]: failed to encode map e552 with expected crc 2023-09-23T22:41:08.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[215775]: failed to encode map e552 with expected crc 2023-09-23T22:41:08.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:41:08.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:08 smithi026 ceph-mon[215775]: osdmap e552: 8 total, 6 up, 7 in 2023-09-23T22:41:08.748 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:08.793 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:08.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:08.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:08.804 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:08 smithi070 ceph-mon[192652]: 1.2 scrub starts 2023-09-23T22:41:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:08 smithi070 ceph-mon[192652]: 1.2 scrub ok 2023-09-23T22:41:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:08 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:08 smithi070 ceph-mon[192652]: failed to encode map e552 with expected crc 2023-09-23T22:41:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:08 smithi070 ceph-mon[192652]: failed to encode map e552 with expected crc 2023-09-23T22:41:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:08 smithi070 ceph-mon[192652]: failed to encode map e552 with expected crc 2023-09-23T22:41:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:08 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:41:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:08 smithi070 ceph-mon[192652]: osdmap e552: 8 total, 6 up, 7 in 2023-09-23T22:41:08.897 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:08.924 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:09.070 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:09.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:09.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:09.174 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:09.205 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:09.253 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:09.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:09.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:09.357 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:09.385 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:09.430 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:09.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:09.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:09.485 INFO:teuthology.orchestra.run.smithi026.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T22:41:09.534 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:09.564 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:09.615 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:41:09.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:41:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:41:09] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-09-23T22:41:09.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:09 smithi026 ceph-mon[215775]: pgmap v1441: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 32 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 20 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:09 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3845415381' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:09 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:09.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:09.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:09 smithi026 ceph-mon[223014]: pgmap v1441: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 32 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 20 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:09.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:09 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3845415381' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:09.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:09 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:09.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:09.817 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:09 smithi070 ceph-mon[192652]: pgmap v1441: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 32 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 20 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:09.817 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:09 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3845415381' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:09.817 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:09 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:41:09.817 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:09.851 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:09.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:09.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:09.956 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:09.985 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:10.049 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:10.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:10.053 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:10.153 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:10.223 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:10.224 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:10.273 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:10.274 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:10.274 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:10.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:10.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:10.378 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:10.406 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:10.426 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:41:10.552 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:10.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:10.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:10.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:10 smithi026 ceph-mon[223014]: failed to encode map e553 with expected crc 2023-09-23T22:41:10.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:10 smithi026 ceph-mon[223014]: failed to encode map e553 with expected crc 2023-09-23T22:41:10.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:41:10.576 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:10 smithi026 ceph-mon[223014]: osdmap e553: 8 total, 6 up, 7 in 2023-09-23T22:41:10.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:10 smithi026 ceph-mon[215775]: failed to encode map e553 with expected crc 2023-09-23T22:41:10.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:10 smithi026 ceph-mon[215775]: failed to encode map e553 with expected crc 2023-09-23T22:41:10.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:41:10.576 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:10 smithi026 ceph-mon[215775]: osdmap e553: 8 total, 6 up, 7 in 2023-09-23T22:41:10.655 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:10.684 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:10 smithi070 ceph-mon[192652]: failed to encode map e553 with expected crc 2023-09-23T22:41:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:10 smithi070 ceph-mon[192652]: failed to encode map e553 with expected crc 2023-09-23T22:41:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T22:41:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:10 smithi070 ceph-mon[192652]: osdmap e553: 8 total, 6 up, 7 in 2023-09-23T22:41:10.917 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:41:11.134 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:11.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:11.155 DEBUG:teuthology.orchestra.run.smithi026:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T22:41:11.156 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T22:41:11.156 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:41:11.158 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T22:41:11.158 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T22:41:11.226 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:hints = 1 2023-09-23T22:41:11.226 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T22:41:11.226 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object prefix: benchmark_data_smithi026_640738 2023-09-23T22:41:11.227 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:41:11.227 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 0 0 0 0 0 0 - 0 2023-09-23T22:41:11.239 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:11.296 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:11.353 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:11.360 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:11.430 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:11.458 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:11.461 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:11.499 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:11.624 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:11.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:11.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:11.728 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:11.749 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:41:11.754 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:11 smithi070 ceph-mon[192652]: pgmap v1443: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 32 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 20 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:11.757 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:11.768 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:41:11.788 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:41:11.799 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:11.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:11.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:11.810 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:41:11.832 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:41:11.853 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:41:11.853 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:41:11.853 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:41:11.853 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:41:11.853 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:41:11.853 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:41:11.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:41:11.854 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:41:11.854 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:41:11.854 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:11 smithi026 ceph-mon[223014]: pgmap v1443: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 32 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 20 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:11.854 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:11 smithi026 ceph-mon[215775]: pgmap v1443: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 32 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 20 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:11.906 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:11.932 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:11.974 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:11.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:11.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:12.079 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:12.105 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:12.147 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:12.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:12.227 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 1 16 67 51 3.18719 3.1875 0.00118217 0.00470231 2023-09-23T22:41:12.254 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:12.279 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:12.321 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:12.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:12.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:12.427 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:12.457 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:12 smithi026 ceph-mon[223014]: pgmap v1444: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 32 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 20 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:12.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:12 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3592878623' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:12 smithi026 ceph-mon[215775]: pgmap v1444: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 32 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 20 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:12.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:12 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3592878623' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:12.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:12 smithi070 ceph-mon[192652]: pgmap v1444: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 32 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 4 active+undersized+degraded, 20 active+clean; 4.6 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 18/84 objects degraded (21.429%); 13/84 objects misplaced (15.476%) 2023-09-23T22:41:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:12 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3592878623' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:13.108 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:13.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:13.212 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:13.227 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 2 16 67 51 1.59357 0 - 0.00470231 2023-09-23T22:41:13.240 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:13.578 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:13.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:13.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:13.593 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:41:13.594 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:41:13.646 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:13.682 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:13.716 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:13.766 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:13.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:13.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:13.872 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:13.901 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:13.943 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:13.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:13.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:14.031 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:41:14.031 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:41:14.031 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:41:14.031 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:41:14.050 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:14.080 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:14.122 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:14.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:14.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:14.227 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 3 16 67 51 1.06239 0 - 0.00470231 2023-09-23T22:41:14.228 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:14.261 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:14.307 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:14.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:14.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:14.382 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:14 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 31/141 objects degraded (21.986%), 10 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:14.410 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:14.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:14.658 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:14 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 31/141 objects degraded (21.986%), 10 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:14.774 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:14.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:14.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:14.827 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:41:14.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:14 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 31/141 objects degraded (21.986%), 10 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:14.879 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:14.912 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:15.228 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 4 16 67 51 0.796729 0 - 0.00470231 2023-09-23T22:41:15.394 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:41:15.591 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:15.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:15.594 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:41:15.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:15 smithi026 ceph-mon[215775]: pgmap v1445: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 29 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 7 active+undersized+degraded, 20 active+clean; 5.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 152 KiB/s wr, 2 op/s; 31/141 objects degraded (21.986%); 13/141 objects misplaced (9.220%) 2023-09-23T22:41:15.659 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:15 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1086189023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:15.660 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:15 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:15.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:15 smithi026 ceph-mon[223014]: pgmap v1445: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 29 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 7 active+undersized+degraded, 20 active+clean; 5.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 152 KiB/s wr, 2 op/s; 31/141 objects degraded (21.986%); 13/141 objects misplaced (9.220%) 2023-09-23T22:41:15.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:15 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1086189023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:15.660 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:15 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:15.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:15.725 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:15 smithi070 ceph-mon[192652]: pgmap v1445: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 29 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 7 active+undersized+degraded, 20 active+clean; 5.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 152 KiB/s wr, 2 op/s; 31/141 objects degraded (21.986%); 13/141 objects misplaced (9.220%) 2023-09-23T22:41:15.725 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:15 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1086189023' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:15.726 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:15 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:15.728 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:15.775 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:15.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:15.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:15.881 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:15.908 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:16.079 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:16.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:16.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:16.184 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:16.215 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:16.228 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 5 16 67 51 0.637402 0 - 0.00470231 2023-09-23T22:41:16.231 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:16.381 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:16.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:16.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:16.423 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:16.486 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:16.518 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:16.568 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:16.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:16.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:16.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:16.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[215775]: failed to encode map e554 with expected crc 2023-09-23T22:41:16.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[215775]: failed to encode map e554 with expected crc 2023-09-23T22:41:16.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:16.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[215775]: osdmap e554: 8 total, 6 up, 7 in 2023-09-23T22:41:16.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[215775]: failed to encode map e554 with expected crc 2023-09-23T22:41:16.671 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:16.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[223014]: failed to encode map e554 with expected crc 2023-09-23T22:41:16.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[223014]: failed to encode map e554 with expected crc 2023-09-23T22:41:16.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:16.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[223014]: osdmap e554: 8 total, 6 up, 7 in 2023-09-23T22:41:16.672 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:16 smithi026 ceph-mon[223014]: failed to encode map e554 with expected crc 2023-09-23T22:41:16.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:16.709 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:16.743 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:41:16.746 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:41:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:16 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:16 smithi070 ceph-mon[192652]: failed to encode map e554 with expected crc 2023-09-23T22:41:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:16 smithi070 ceph-mon[192652]: failed to encode map e554 with expected crc 2023-09-23T22:41:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:16 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:16 smithi070 ceph-mon[192652]: osdmap e554: 8 total, 6 up, 7 in 2023-09-23T22:41:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:16 smithi070 ceph-mon[192652]: failed to encode map e554 with expected crc 2023-09-23T22:41:17.228 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 6 16 67 51 0.53117 0 - 0.00470231 2023-09-23T22:41:17.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:17 smithi026 ceph-mon[215775]: pgmap v1447: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 26 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 10 active+undersized+degraded, 20 active+clean; 6.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 248 KiB/s wr, 3 op/s; 43/177 objects degraded (24.294%); 13/177 objects misplaced (7.345%) 2023-09-23T22:41:17.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:17 smithi026 ceph-mon[223014]: pgmap v1447: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 26 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 10 active+undersized+degraded, 20 active+clean; 6.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 248 KiB/s wr, 3 op/s; 43/177 objects degraded (24.294%); 13/177 objects misplaced (7.345%) 2023-09-23T22:41:17.618 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:17.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:17.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:17.723 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:17.751 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:17.832 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1447,"stamp":"2023-09-23T22:41:15.669887+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6821811,"num_objects":59,"num_object_clones":0,"num_object_copies":177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":43,"num_objects_misplaced":13,"num_objects_unfound":0,"num_objects_dirty":59,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":938,"num_write_kb":14709,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":431,"num_bytes_recovered":22058957,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4443,"ondisk_log_size":4443,"up":207,"acting":240,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5787028,"kb_used_data":2522972,"kb_used_omap":40,"kb_used_meta":3263959,"kb_avail":650429036,"statfs":{"total":671965249536,"available":666039332864,"internally_reserved":0,"allocated":2583523328,"data_stored":2566777043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41157,"internal_metadata":3342294843},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2031616,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":31,"num_write_kb":1984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004315"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":481,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.120217+0000","last_change":"2023-09-23T22:40:46.378982+0000","last_active":"2023-09-23T22:41:08.120217+0000","last_peered":"2023-09-23T22:41:08.120217+0000","last_clean":"2023-09-23T22:41:08.120217+0000","last_became_active":"2023-09-23T22:40:46.378577+0000","last_became_peered":"2023-09-23T22:40:46.378577+0000","last_unstale":"2023-09-23T22:41:08.120217+0000","last_undegraded":"2023-09-23T22:41:08.120217+0000","last_fullsized":"2023-09-23T22:41:08.120217+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:11.038151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":259,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:08.628920+0000","last_change":"2023-09-23T22:40:47.370494+0000","last_active":"2023-09-23T22:41:08.628920+0000","last_peered":"2023-09-23T22:41:08.628920+0000","last_clean":"2023-09-23T22:37:07.670854+0000","last_became_active":"2023-09-23T22:40:47.370494+0000","last_became_peered":"2023-09-23T22:40:47.370494+0000","last_unstale":"2023-09-23T22:41:08.628920+0000","last_undegraded":"2023-09-23T22:41:08.628920+0000","last_fullsized":"2023-09-23T22:40:47.361278+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:58.704479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":304,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:08.628147+0000","last_change":"2023-09-23T22:40:47.360752+0000","last_active":"2023-09-23T22:41:08.628147+0000","last_peered":"2023-09-23T22:41:08.628147+0000","last_clean":"2023-09-23T22:39:00.403392+0000","last_became_active":"2023-09-23T22:40:47.360752+0000","last_became_peered":"2023-09-23T22:40:47.360752+0000","last_unstale":"2023-09-23T22:41:08.628147+0000","last_undegraded":"2023-09-23T22:41:08.628147+0000","last_fullsized":"2023-09-23T22:40:47.356075+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:36.777818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":356,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:09.129622+0000","last_change":"2023-09-23T22:39:05.307660+0000","last_active":"2023-09-23T22:41:09.129622+0000","last_peered":"2023-09-23T22:41:09.129622+0000","last_clean":"2023-09-23T22:35:17.625896+0000","last_became_active":"2023-09-23T22:39:05.307660+0000","last_became_peered":"2023-09-23T22:39:05.307660+0000","last_unstale":"2023-09-23T22:41:09.129622+0000","last_undegraded":"2023-09-23T22:41:09.129622+0000","last_fullsized":"2023-09-23T22:39:05.302351+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:13.440047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":577,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.628004+0000","last_change":"2023-09-23T22:39:05.304373+0000","last_active":"2023-09-23T22:41:08.628004+0000","last_peered":"2023-09-23T22:41:08.628004+0000","last_clean":"2023-09-23T22:37:07.670321+0000","last_became_active":"2023-09-23T22:39:05.304373+0000","last_became_peered":"2023-09-23T22:39:05.304373+0000","last_unstale":"2023-09-23T22:41:08.628004+0000","last_undegraded":"2023-09-23T22:41:08.628004+0000","last_fullsized":"2023-09-23T22:39:05.284343+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:51.889095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":350,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:07.477379+0000","last_change":"2023-09-23T22:38:55.388832+0000","last_active":"2023-09-23T22:41:07.477379+0000","last_peered":"2023-09-23T22:41:07.477379+0000","last_clean":"2023-09-23T22:37:07.661100+0000","last_became_active":"2023-09-23T22:38:55.388832+0000","last_became_peered":"2023-09-23T22:38:55.388832+0000","last_unstale":"2023-09-23T22:41:07.477379+0000","last_undegraded":"2023-09-23T22:41:07.477379+0000","last_fullsized":"2023-09-23T22:38:55.382522+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:09.227034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":382,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:09.129440+0000","last_change":"2023-09-23T22:39:05.302151+0000","last_active":"2023-09-23T22:38:54.405779+0000","last_peered":"2023-09-23T22:41:09.129440+0000","last_clean":"2023-09-23T22:38:48.372435+0000","last_became_active":"2023-09-23T22:36:36.406686+0000","last_became_peered":"2023-09-23T22:39:05.302151+0000","last_unstale":"2023-09-23T22:41:09.129440+0000","last_undegraded":"2023-09-23T22:41:09.129440+0000","last_fullsized":"2023-09-23T22:39:05.301734+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:12.647131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":333,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:08.628192+0000","last_change":"2023-09-23T22:39:05.285935+0000","last_active":"2023-09-23T22:38:54.400276+0000","last_peered":"2023-09-23T22:41:08.628192+0000","last_clean":"2023-09-23T22:38:48.357505+0000","last_became_active":"2023-09-23T22:37:42.458398+0000","last_became_peered":"2023-09-23T22:39:05.285935+0000","last_unstale":"2023-09-23T22:41:08.628192+0000","last_undegraded":"2023-09-23T22:41:08.628192+0000","last_fullsized":"2023-09-23T22:39:05.284557+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:22.365837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":670,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:07.478615+0000","last_change":"2023-09-23T22:39:05.294537+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:41:07.478615+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.294537+0000","last_unstale":"2023-09-23T22:41:07.478615+0000","last_undegraded":"2023-09-23T22:41:07.478615+0000","last_fullsized":"2023-09-23T22:39:05.292289+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":571,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.629495+0000","last_change":"2023-09-23T22:39:05.294737+0000","last_active":"2023-09-23T22:41:08.629495+0000","last_peered":"2023-09-23T22:41:08.629495+0000","last_clean":"2023-09-23T22:39:00.404530+0000","last_became_active":"2023-09-23T22:39:05.294737+0000","last_became_peered":"2023-09-23T22:39:05.294737+0000","last_unstale":"2023-09-23T22:41:08.629495+0000","last_undegraded":"2023-09-23T22:41:08.629495+0000","last_fullsized":"2023-09-23T22:39:05.286342+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:12.866122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":191,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:07.478131+0000","last_change":"2023-09-23T22:39:05.298634+0000","last_active":"2023-09-23T22:38:54.377414+0000","last_peered":"2023-09-23T22:41:07.478131+0000","last_clean":"2023-09-23T22:38:48.357759+0000","last_became_active":"2023-09-23T22:36:36.521405+0000","last_became_peered":"2023-09-23T22:39:05.298634+0000","last_unstale":"2023-09-23T22:41:07.478131+0000","last_undegraded":"2023-09-23T22:41:07.478131+0000","last_fullsized":"2023-09-23T22:39:05.297707+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:35.327335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":116,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:07.478863+0000","last_change":"2023-09-23T22:40:47.413463+0000","last_active":"2023-09-23T22:41:07.478863+0000","last_peered":"2023-09-23T22:41:07.478863+0000","last_clean":"2023-09-23T22:41:07.478863+0000","last_became_active":"2023-09-23T22:40:47.353194+0000","last_became_peered":"2023-09-23T22:40:47.353194+0000","last_unstale":"2023-09-23T22:41:07.478863+0000","last_undegraded":"2023-09-23T22:41:07.478863+0000","last_fullsized":"2023-09-23T22:41:07.478863+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:00:03.601726+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,4],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":194,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.628564+0000","last_change":"2023-09-23T22:38:55.383671+0000","last_active":"2023-09-23T22:41:08.628564+0000","last_peered":"2023-09-23T22:41:08.628564+0000","last_clean":"2023-09-23T22:38:48.410000+0000","last_became_active":"2023-09-23T22:38:55.383671+0000","last_became_peered":"2023-09-23T22:38:55.383671+0000","last_unstale":"2023-09-23T22:41:08.628564+0000","last_undegraded":"2023-09-23T22:41:08.628564+0000","last_fullsized":"2023-09-23T22:38:55.381036+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:27.813898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":35,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.119989+0000","last_change":"2023-09-23T22:40:47.447310+0000","last_active":"2023-09-23T22:41:08.119989+0000","last_peered":"2023-09-23T22:41:08.119989+0000","last_clean":"2023-09-23T22:41:08.119989+0000","last_became_active":"2023-09-23T22:40:47.355810+0000","last_became_peered":"2023-09-23T22:40:47.355810+0000","last_unstale":"2023-09-23T22:41:08.119989+0000","last_undegraded":"2023-09-23T22:41:08.119989+0000","last_fullsized":"2023-09-23T22:41:08.119989+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:23.327916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":33138,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4,7","objects":4}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":557,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.629122+0000","last_change":"2023-09-23T22:40:47.378100+0000","last_active":"2023-09-23T22:41:08.629122+0000","last_peered":"2023-09-23T22:41:08.629122+0000","last_clean":"2023-09-23T22:41:08.629122+0000","last_became_active":"2023-09-23T22:40:47.376294+0000","last_became_peered":"2023-09-23T22:40:47.376294+0000","last_unstale":"2023-09-23T22:41:08.629122+0000","last_undegraded":"2023-09-23T22:41:08.629122+0000","last_fullsized":"2023-09-23T22:41:08.629122+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:46.163326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5,4],"avail_no_missing":["1","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":962,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.120306+0000","last_change":"2023-09-23T22:40:49.348177+0000","last_active":"2023-09-23T22:41:08.120306+0000","last_peered":"2023-09-23T22:41:08.120306+0000","last_clean":"2023-09-23T22:41:08.120306+0000","last_became_active":"2023-09-23T22:40:49.347830+0000","last_became_peered":"2023-09-23T22:40:49.347830+0000","last_unstale":"2023-09-23T22:41:08.120306+0000","last_undegraded":"2023-09-23T22:41:08.120306+0000","last_fullsized":"2023-09-23T22:41:08.120306+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:19.077318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:19.077318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:06.899748+0000","scrub_duration":0.0080980949999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":197,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.120477+0000","last_change":"2023-09-23T22:40:47.355289+0000","last_active":"2023-09-23T22:41:08.120477+0000","last_peered":"2023-09-23T22:41:08.120477+0000","last_clean":"2023-09-23T22:41:08.120477+0000","last_became_active":"2023-09-23T22:40:47.354770+0000","last_became_peered":"2023-09-23T22:40:47.354770+0000","last_unstale":"2023-09-23T22:41:08.120477+0000","last_undegraded":"2023-09-23T22:41:08.120477+0000","last_fullsized":"2023-09-23T22:41:08.120477+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:16.286103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,4],"avail_no_missing":["2","4","7"],"object_location_counts":[{"shards":"2,4,7","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":378,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.628761+0000","last_change":"2023-09-23T22:39:05.294773+0000","last_active":"2023-09-23T22:41:08.628761+0000","last_peered":"2023-09-23T22:41:08.628761+0000","last_clean":"2023-09-23T22:37:07.671236+0000","last_became_active":"2023-09-23T22:39:05.294773+0000","last_became_peered":"2023-09-23T22:39:05.294773+0000","last_unstale":"2023-09-23T22:41:08.628761+0000","last_undegraded":"2023-09-23T22:41:08.628761+0000","last_fullsized":"2023-09-23T22:39:05.285024+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:47.212090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":775,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:07.477218+0000","last_change":"2023-09-23T22:38:55.391529+0000","last_active":"2023-09-23T22:41:07.477218+0000","last_peered":"2023-09-23T22:41:07.477218+0000","last_clean":"2023-09-23T22:35:16.261819+0000","last_became_active":"2023-09-23T22:38:55.391529+0000","last_became_peered":"2023-09-23T22:38:55.391529+0000","last_unstale":"2023-09-23T22:41:07.477218+0000","last_undegraded":"2023-09-23T22:41:07.477218+0000","last_fullsized":"2023-09-23T22:38:55.384595+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":413,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:09.129100+0000","last_change":"2023-09-23T22:39:05.307619+0000","last_active":"2023-09-23T22:41:09.129100+0000","last_peered":"2023-09-23T22:41:09.129100+0000","last_clean":"2023-09-23T22:36:31.476578+0000","last_became_active":"2023-09-23T22:39:05.307619+0000","last_became_peered":"2023-09-23T22:39:05.307619+0000","last_unstale":"2023-09-23T22:41:09.129100+0000","last_undegraded":"2023-09-23T22:41:09.129100+0000","last_fullsized":"2023-09-23T22:39:05.284898+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:50.408737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":463,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:07.478512+0000","last_change":"2023-09-23T22:39:05.294555+0000","last_active":"2023-09-23T22:38:54.405186+0000","last_peered":"2023-09-23T22:41:07.478512+0000","last_clean":"2023-09-23T22:38:48.357907+0000","last_became_active":"2023-09-23T22:36:36.406789+0000","last_became_peered":"2023-09-23T22:39:05.294555+0000","last_unstale":"2023-09-23T22:41:07.478512+0000","last_undegraded":"2023-09-23T22:41:07.478512+0000","last_fullsized":"2023-09-23T22:39:05.293080+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:45.131265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":591,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.628770+0000","last_change":"2023-09-23T22:40:57.922633+0000","last_active":"2023-09-23T22:41:08.628770+0000","last_peered":"2023-09-23T22:41:08.628770+0000","last_clean":"2023-09-23T22:41:08.628770+0000","last_became_active":"2023-09-23T22:40:49.352729+0000","last_became_peered":"2023-09-23T22:40:49.352729+0000","last_unstale":"2023-09-23T22:41:08.628770+0000","last_undegraded":"2023-09-23T22:41:08.628770+0000","last_fullsized":"2023-09-23T22:41:08.628770+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:57.922489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:57.922489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:23.246521+0000","scrub_duration":0.001544351,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":499,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:07.478246+0000","last_change":"2023-09-23T22:40:47.352876+0000","last_active":"2023-09-23T22:41:07.478246+0000","last_peered":"2023-09-23T22:41:07.478246+0000","last_clean":"2023-09-23T22:41:07.478246+0000","last_became_active":"2023-09-23T22:40:47.352110+0000","last_became_peered":"2023-09-23T22:40:47.352110+0000","last_unstale":"2023-09-23T22:41:07.478246+0000","last_undegraded":"2023-09-23T22:41:07.478246+0000","last_fullsized":"2023-09-23T22:41:07.478246+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:18.882858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":399,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:09.129678+0000","last_change":"2023-09-23T22:39:05.307686+0000","last_active":"2023-09-23T22:41:09.129678+0000","last_peered":"2023-09-23T22:41:09.129678+0000","last_clean":"2023-09-23T22:35:17.626249+0000","last_became_active":"2023-09-23T22:39:05.307686+0000","last_became_peered":"2023-09-23T22:39:05.307686+0000","last_unstale":"2023-09-23T22:41:09.129678+0000","last_undegraded":"2023-09-23T22:41:09.129678+0000","last_fullsized":"2023-09-23T22:39:05.302436+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:24.866314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":823,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.120066+0000","last_change":"2023-09-23T22:40:49.350372+0000","last_active":"2023-09-23T22:41:08.120066+0000","last_peered":"2023-09-23T22:41:08.120066+0000","last_clean":"2023-09-23T22:41:08.120066+0000","last_became_active":"2023-09-23T22:40:49.349764+0000","last_became_peered":"2023-09-23T22:40:49.349764+0000","last_unstale":"2023-09-23T22:41:08.120066+0000","last_undegraded":"2023-09-23T22:41:08.120066+0000","last_fullsized":"2023-09-23T22:41:08.120066+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:25.187461+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:25.187461+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051458980000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":427,"reported_epoch":552,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:41:09.128824+0000","last_change":"2023-09-23T22:39:05.302168+0000","last_active":"2023-09-23T22:39:04.291704+0000","last_peered":"2023-09-23T22:41:09.128824+0000","last_clean":"2023-09-23T22:28:58.571564+0000","last_became_active":"2023-09-23T22:28:05.154059+0000","last_became_peered":"2023-09-23T22:39:05.302168+0000","last_unstale":"2023-09-23T22:41:09.128824+0000","last_undegraded":"2023-09-23T22:39:05.286285+0000","last_fullsized":"2023-09-23T22:39:05.286213+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:03.826266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":95,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:07.478458+0000","last_change":"2023-09-23T22:39:05.295415+0000","last_active":"2023-09-23T22:41:07.478458+0000","last_peered":"2023-09-23T22:41:07.478458+0000","last_clean":"2023-09-23T22:37:07.674004+0000","last_became_active":"2023-09-23T22:39:05.295415+0000","last_became_peered":"2023-09-23T22:39:05.295415+0000","last_unstale":"2023-09-23T22:41:07.478458+0000","last_undegraded":"2023-09-23T22:41:07.478458+0000","last_fullsized":"2023-09-23T22:39:05.293230+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:45.148758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":10,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.629002+0000","last_change":"2023-09-23T22:41:04.471332+0000","last_active":"2023-09-23T22:41:08.629002+0000","last_peered":"2023-09-23T22:41:08.629002+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.471332+0000","last_became_peered":"2023-09-23T22:41:04.471332+0000","last_unstale":"2023-09-23T22:41:08.629002+0000","last_undegraded":"2023-09-23T22:41:08.629002+0000","last_fullsized":"2023-09-23T22:41:04.462160+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.187867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":831,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.120176+0000","last_change":"2023-09-23T22:41:01.280273+0000","last_active":"2023-09-23T22:41:08.120176+0000","last_peered":"2023-09-23T22:41:08.120176+0000","last_clean":"2023-09-23T22:41:08.120176+0000","last_became_active":"2023-09-23T22:40:49.352968+0000","last_became_peered":"2023-09-23T22:40:49.352968+0000","last_unstale":"2023-09-23T22:41:08.120176+0000","last_undegraded":"2023-09-23T22:41:08.120176+0000","last_fullsized":"2023-09-23T22:41:08.120176+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:01.280094+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:01.280094+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:52.143826+0000","scrub_duration":0.0026122960000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,4],"avail_no_missing":["2","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":28,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.628882+0000","last_change":"2023-09-23T22:40:46.385157+0000","last_active":"2023-09-23T22:41:08.628882+0000","last_peered":"2023-09-23T22:41:08.628882+0000","last_clean":"2023-09-23T22:41:08.628882+0000","last_became_active":"2023-09-23T22:40:46.380164+0000","last_became_peered":"2023-09-23T22:40:46.380164+0000","last_unstale":"2023-09-23T22:41:08.628882+0000","last_undegraded":"2023-09-23T22:41:08.628882+0000","last_fullsized":"2023-09-23T22:41:08.628882+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:22.139830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":498,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:07.477662+0000","last_change":"2023-09-23T22:40:47.350683+0000","last_active":"2023-09-23T22:41:07.477662+0000","last_peered":"2023-09-23T22:41:07.477662+0000","last_clean":"2023-09-23T22:35:23.290223+0000","last_became_active":"2023-09-23T22:40:47.350683+0000","last_became_peered":"2023-09-23T22:40:47.350683+0000","last_unstale":"2023-09-23T22:41:07.477662+0000","last_undegraded":"2023-09-23T22:41:07.477662+0000","last_fullsized":"2023-09-23T22:40:47.347498+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.418708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"0'0","reported_seq":9,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:08.120375+0000","last_change":"2023-09-23T22:41:04.459901+0000","last_active":"2023-09-23T22:41:03.442376+0000","last_peered":"2023-09-23T22:41:08.120375+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"0.000000","last_became_peered":"2023-09-23T22:41:04.459901+0000","last_unstale":"2023-09-23T22:41:08.120375+0000","last_undegraded":"2023-09-23T22:41:08.120375+0000","last_fullsized":"2023-09-23T22:41:04.455586+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:08.615998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":670,"reported_epoch":552,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:41:07.478744+0000","last_change":"2023-09-23T22:39:05.294786+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:41:07.478744+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.294786+0000","last_unstale":"2023-09-23T22:41:07.478744+0000","last_undegraded":"2023-09-23T22:39:05.292513+0000","last_fullsized":"2023-09-23T22:39:05.292435+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":408,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.629623+0000","last_change":"2023-09-23T22:40:46.386961+0000","last_active":"2023-09-23T22:41:08.629623+0000","last_peered":"2023-09-23T22:41:08.629623+0000","last_clean":"2023-09-23T22:41:08.629623+0000","last_became_active":"2023-09-23T22:40:46.382025+0000","last_became_peered":"2023-09-23T22:40:46.382025+0000","last_unstale":"2023-09-23T22:41:08.629623+0000","last_undegraded":"2023-09-23T22:41:08.629623+0000","last_fullsized":"2023-09-23T22:41:08.629623+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:03.936056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":534,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:08.628207+0000","last_change":"2023-09-23T22:40:47.361227+0000","last_active":"2023-09-23T22:41:08.628207+0000","last_peered":"2023-09-23T22:41:08.628207+0000","last_clean":"2023-09-23T22:39:00.403311+0000","last_became_active":"2023-09-23T22:40:47.361227+0000","last_became_peered":"2023-09-23T22:40:47.361227+0000","last_unstale":"2023-09-23T22:41:08.628207+0000","last_undegraded":"2023-09-23T22:41:08.628207+0000","last_fullsized":"2023-09-23T22:40:47.356420+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:39.270346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"553'4","reported_seq":14,"reported_epoch":553,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:11.242869+0000","last_change":"2023-09-23T22:41:11.241314+0000","last_active":"2023-09-23T22:41:11.242869+0000","last_peered":"2023-09-23T22:41:11.242869+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.463868+0000","last_became_peered":"2023-09-23T22:41:04.463868+0000","last_unstale":"2023-09-23T22:41:11.242869+0000","last_undegraded":"2023-09-23T22:41:11.232791+0000","last_fullsized":"2023-09-23T22:41:04.461216+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:12.840758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":818,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:07.478749+0000","last_change":"2023-09-23T22:39:05.294943+0000","last_active":"2023-09-23T22:35:19.273163+0000","last_peered":"2023-09-23T22:41:07.478749+0000","last_clean":"2023-09-23T22:35:16.262487+0000","last_became_active":"2023-09-23T22:34:36.010161+0000","last_became_peered":"2023-09-23T22:39:05.294943+0000","last_unstale":"2023-09-23T22:41:07.478749+0000","last_undegraded":"2023-09-23T22:41:07.478749+0000","last_fullsized":"2023-09-23T22:39:05.292735+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":355,"reported_epoch":552,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T22:41:07.478649+0000","last_change":"2023-09-23T22:39:05.294595+0000","last_active":"2023-09-23T22:35:19.265438+0000","last_peered":"2023-09-23T22:41:07.478649+0000","last_clean":"2023-09-23T22:35:16.262186+0000","last_became_active":"2023-09-23T22:34:32.963996+0000","last_became_peered":"2023-09-23T22:39:05.294595+0000","last_unstale":"2023-09-23T22:41:07.478649+0000","last_undegraded":"2023-09-23T22:39:05.293265+0000","last_fullsized":"2023-09-23T22:39:05.293222+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:57.686355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[{"shards":"7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":254,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.628427+0000","last_change":"2023-09-23T22:38:55.397046+0000","last_active":"2023-09-23T22:41:08.628427+0000","last_peered":"2023-09-23T22:41:08.628427+0000","last_clean":"2023-09-23T22:37:07.662421+0000","last_became_active":"2023-09-23T22:38:55.397046+0000","last_became_peered":"2023-09-23T22:38:55.397046+0000","last_unstale":"2023-09-23T22:41:08.628427+0000","last_undegraded":"2023-09-23T22:41:08.628427+0000","last_fullsized":"2023-09-23T22:38:55.385491+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:27.922878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.5","version":"553'3","reported_seq":13,"reported_epoch":553,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:11.241338+0000","last_change":"2023-09-23T22:41:11.241338+0000","last_active":"2023-09-23T22:41:11.241338+0000","last_peered":"2023-09-23T22:41:11.241338+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.467396+0000","last_became_peered":"2023-09-23T22:41:04.467396+0000","last_unstale":"2023-09-23T22:41:11.241338+0000","last_undegraded":"2023-09-23T22:41:11.235484+0000","last_fullsized":"2023-09-23T22:41:04.455972+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:34.304040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":3}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":717,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:09.129396+0000","last_change":"2023-09-23T22:39:05.302770+0000","last_active":"2023-09-23T22:39:04.293181+0000","last_peered":"2023-09-23T22:41:09.129396+0000","last_clean":"2023-09-23T22:28:58.573726+0000","last_became_active":"2023-09-23T22:29:02.583400+0000","last_became_peered":"2023-09-23T22:39:05.302770+0000","last_unstale":"2023-09-23T22:41:09.129396+0000","last_undegraded":"2023-09-23T22:41:09.129396+0000","last_fullsized":"2023-09-23T22:39:05.302009+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":397,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:08.629268+0000","last_change":"2023-09-23T22:39:05.286647+0000","last_active":"2023-09-23T22:38:54.408907+0000","last_peered":"2023-09-23T22:41:08.629268+0000","last_clean":"2023-09-23T22:38:53.017852+0000","last_became_active":"2023-09-23T22:37:42.458233+0000","last_became_peered":"2023-09-23T22:39:05.286647+0000","last_unstale":"2023-09-23T22:41:08.629268+0000","last_undegraded":"2023-09-23T22:41:08.629268+0000","last_fullsized":"2023-09-23T22:39:05.285527+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:29.003765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":328,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.628561+0000","last_change":"2023-09-23T22:40:47.379929+0000","last_active":"2023-09-23T22:41:08.628561+0000","last_peered":"2023-09-23T22:41:08.628561+0000","last_clean":"2023-09-23T22:41:08.628561+0000","last_became_active":"2023-09-23T22:40:47.377701+0000","last_became_peered":"2023-09-23T22:40:47.377701+0000","last_unstale":"2023-09-23T22:41:08.628561+0000","last_undegraded":"2023-09-23T22:41:08.628561+0000","last_fullsized":"2023-09-23T22:41:08.628561+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:39:56.004481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":10,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.628272+0000","last_change":"2023-09-23T22:41:04.472505+0000","last_active":"2023-09-23T22:41:08.628272+0000","last_peered":"2023-09-23T22:41:08.628272+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.472505+0000","last_became_peered":"2023-09-23T22:41:04.472505+0000","last_unstale":"2023-09-23T22:41:08.628272+0000","last_undegraded":"2023-09-23T22:41:08.628272+0000","last_fullsized":"2023-09-23T22:41:04.462986+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:47.586097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":853,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.119785+0000","last_change":"2023-09-23T22:39:05.299735+0000","last_active":"2023-09-23T22:41:08.119785+0000","last_peered":"2023-09-23T22:41:08.119785+0000","last_clean":"2023-09-23T22:36:31.469549+0000","last_became_active":"2023-09-23T22:39:05.299735+0000","last_became_peered":"2023-09-23T22:39:05.299735+0000","last_unstale":"2023-09-23T22:41:08.119785+0000","last_undegraded":"2023-09-23T22:41:08.119785+0000","last_fullsized":"2023-09-23T22:39:05.286634+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:51:10.963206+0000","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":382,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:09.129386+0000","last_change":"2023-09-23T22:40:47.484545+0000","last_active":"2023-09-23T22:41:09.129386+0000","last_peered":"2023-09-23T22:41:09.129386+0000","last_clean":"2023-09-23T22:41:09.129386+0000","last_became_active":"2023-09-23T22:40:47.379744+0000","last_became_peered":"2023-09-23T22:40:47.379744+0000","last_unstale":"2023-09-23T22:41:09.129386+0000","last_undegraded":"2023-09-23T22:41:09.129386+0000","last_fullsized":"2023-09-23T22:41:09.129386+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:19.188614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,4],"avail_no_missing":["0","1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":22,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:08.628699+0000","last_change":"2023-09-23T22:40:47.372748+0000","last_active":"2023-09-23T22:41:08.628699+0000","last_peered":"2023-09-23T22:41:08.628699+0000","last_clean":"2023-09-23T22:38:53.017791+0000","last_became_active":"2023-09-23T22:40:47.372748+0000","last_became_peered":"2023-09-23T22:40:47.372748+0000","last_unstale":"2023-09-23T22:41:08.628699+0000","last_undegraded":"2023-09-23T22:41:08.628699+0000","last_fullsized":"2023-09-23T22:40:47.364946+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:17.130062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.3","version":"553'3","reported_seq":13,"reported_epoch":553,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:11.235301+0000","last_change":"2023-09-23T22:41:11.235301+0000","last_active":"2023-09-23T22:41:11.235301+0000","last_peered":"2023-09-23T22:41:11.235301+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.468140+0000","last_became_peered":"2023-09-23T22:41:04.468140+0000","last_unstale":"2023-09-23T22:41:11.235301+0000","last_undegraded":"2023-09-23T22:41:11.235147+0000","last_fullsized":"2023-09-23T22:41:04.454963+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:40.150697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":3}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":988,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:08.119731+0000","last_change":"2023-09-23T22:39:05.288550+0000","last_active":"2023-09-23T22:41:08.119731+0000","last_peered":"2023-09-23T22:41:08.119731+0000","last_clean":"2023-09-23T22:36:31.469347+0000","last_became_active":"2023-09-23T22:39:05.288550+0000","last_became_peered":"2023-09-23T22:39:05.288550+0000","last_unstale":"2023-09-23T22:41:08.119731+0000","last_undegraded":"2023-09-23T22:41:08.119731+0000","last_fullsized":"2023-09-23T22:39:05.284160+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:41:48.075424+0000","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":299,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:08.628550+0000","last_change":"2023-09-23T22:40:47.381564+0000","last_active":"2023-09-23T22:41:08.628550+0000","last_peered":"2023-09-23T22:41:08.628550+0000","last_clean":"2023-09-23T22:37:07.662537+0000","last_became_active":"2023-09-23T22:40:47.381564+0000","last_became_peered":"2023-09-23T22:40:47.381564+0000","last_unstale":"2023-09-23T22:41:08.628550+0000","last_undegraded":"2023-09-23T22:41:08.628550+0000","last_fullsized":"2023-09-23T22:40:47.366160+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:17.611005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":191,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.120688+0000","last_change":"2023-09-23T22:40:47.371656+0000","last_active":"2023-09-23T22:41:08.120688+0000","last_peered":"2023-09-23T22:41:08.120688+0000","last_clean":"2023-09-23T22:41:08.120688+0000","last_became_active":"2023-09-23T22:40:47.371219+0000","last_became_peered":"2023-09-23T22:40:47.371219+0000","last_unstale":"2023-09-23T22:41:08.120688+0000","last_undegraded":"2023-09-23T22:41:08.120688+0000","last_fullsized":"2023-09-23T22:41:08.120688+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:28.796581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1,4],"avail_no_missing":["2","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":10,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.627991+0000","last_change":"2023-09-23T22:41:04.468830+0000","last_active":"2023-09-23T22:41:08.627991+0000","last_peered":"2023-09-23T22:41:08.627991+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.468830+0000","last_became_peered":"2023-09-23T22:41:04.468830+0000","last_unstale":"2023-09-23T22:41:08.627991+0000","last_undegraded":"2023-09-23T22:41:08.627991+0000","last_fullsized":"2023-09-23T22:41:04.460005+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:38.718835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":591,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.628958+0000","last_change":"2023-09-23T22:40:59.889390+0000","last_active":"2023-09-23T22:41:08.628958+0000","last_peered":"2023-09-23T22:41:08.628958+0000","last_clean":"2023-09-23T22:41:08.628958+0000","last_became_active":"2023-09-23T22:40:49.363061+0000","last_became_peered":"2023-09-23T22:40:49.363061+0000","last_unstale":"2023-09-23T22:41:08.628958+0000","last_undegraded":"2023-09-23T22:41:08.628958+0000","last_fullsized":"2023-09-23T22:41:08.628958+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:59.889215+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:59.889215+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:27.177097+0000","scrub_duration":0.0027720409999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":496,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.120101+0000","last_change":"2023-09-23T22:40:46.377796+0000","last_active":"2023-09-23T22:41:08.120101+0000","last_peered":"2023-09-23T22:41:08.120101+0000","last_clean":"2023-09-23T22:41:08.120101+0000","last_became_active":"2023-09-23T22:40:46.377471+0000","last_became_peered":"2023-09-23T22:40:46.377471+0000","last_unstale":"2023-09-23T22:41:08.120101+0000","last_undegraded":"2023-09-23T22:41:08.120101+0000","last_fullsized":"2023-09-23T22:41:08.120101+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:18.634395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":548,"reported_epoch":553,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:10.763391+0000","last_change":"2023-09-23T22:40:47.391036+0000","last_active":"2023-09-23T22:41:10.763391+0000","last_peered":"2023-09-23T22:41:10.763391+0000","last_clean":"2023-09-23T22:37:07.660692+0000","last_became_active":"2023-09-23T22:40:47.391036+0000","last_became_peered":"2023-09-23T22:40:47.391036+0000","last_unstale":"2023-09-23T22:41:10.763391+0000","last_undegraded":"2023-09-23T22:41:10.763391+0000","last_fullsized":"2023-09-23T22:40:47.357096+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:05.648321+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":13,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.628042+0000","last_change":"2023-09-23T22:41:04.472136+0000","last_active":"2023-09-23T22:41:08.628042+0000","last_peered":"2023-09-23T22:41:08.628042+0000","last_clean":"2023-09-23T22:41:08.628042+0000","last_became_active":"2023-09-23T22:41:04.471861+0000","last_became_peered":"2023-09-23T22:41:04.471861+0000","last_unstale":"2023-09-23T22:41:08.628042+0000","last_undegraded":"2023-09-23T22:41:08.628042+0000","last_fullsized":"2023-09-23T22:41:08.628042+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:31.788504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":934,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:07.477086+0000","last_change":"2023-09-23T22:38:55.383101+0000","last_active":"2023-09-23T22:41:07.477086+0000","last_peered":"2023-09-23T22:41:07.477086+0000","last_clean":"2023-09-23T22:38:48.353691+0000","last_became_active":"2023-09-23T22:38:55.383101+0000","last_became_peered":"2023-09-23T22:38:55.383101+0000","last_unstale":"2023-09-23T22:41:07.477086+0000","last_undegraded":"2023-09-23T22:41:07.477086+0000","last_fullsized":"2023-09-23T22:38:55.381406+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:55.111338+0000","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":271,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.628320+0000","last_change":"2023-09-23T22:39:05.297947+0000","last_active":"2023-09-23T22:41:08.628320+0000","last_peered":"2023-09-23T22:41:08.628320+0000","last_clean":"2023-09-23T22:37:07.670248+0000","last_became_active":"2023-09-23T22:39:05.297947+0000","last_became_peered":"2023-09-23T22:39:05.297947+0000","last_unstale":"2023-09-23T22:41:08.628320+0000","last_undegraded":"2023-09-23T22:41:08.628320+0000","last_fullsized":"2023-09-23T22:39:05.284753+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:18.514918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":325,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.628971+0000","last_change":"2023-09-23T22:40:47.382232+0000","last_active":"2023-09-23T22:41:08.628971+0000","last_peered":"2023-09-23T22:41:08.628971+0000","last_clean":"2023-09-23T22:41:08.628971+0000","last_became_active":"2023-09-23T22:40:47.381852+0000","last_became_peered":"2023-09-23T22:40:47.381852+0000","last_unstale":"2023-09-23T22:41:08.628971+0000","last_undegraded":"2023-09-23T22:41:08.628971+0000","last_fullsized":"2023-09-23T22:41:08.628971+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:02.731880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,4],"avail_no_missing":["1","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"553'5","reported_seq":14,"reported_epoch":553,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:11.246136+0000","last_change":"2023-09-23T22:41:11.240024+0000","last_active":"2023-09-23T22:41:11.246136+0000","last_peered":"2023-09-23T22:41:11.246136+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.472141+0000","last_became_peered":"2023-09-23T22:41:04.472141+0000","last_unstale":"2023-09-23T22:41:11.246136+0000","last_undegraded":"2023-09-23T22:41:11.235005+0000","last_fullsized":"2023-09-23T22:41:04.468548+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:18.029200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5,"num_write_kb":320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":5}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":818,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:07.478582+0000","last_change":"2023-09-23T22:39:05.295156+0000","last_active":"2023-09-23T22:35:19.273163+0000","last_peered":"2023-09-23T22:41:07.478582+0000","last_clean":"2023-09-23T22:35:16.262487+0000","last_became_active":"2023-09-23T22:34:36.010161+0000","last_became_peered":"2023-09-23T22:39:05.295156+0000","last_unstale":"2023-09-23T22:41:07.478582+0000","last_undegraded":"2023-09-23T22:41:07.478582+0000","last_fullsized":"2023-09-23T22:39:05.292995+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":992,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.628857+0000","last_change":"2023-09-23T22:37:42.433834+0000","last_active":"2023-09-23T22:41:08.628857+0000","last_peered":"2023-09-23T22:41:08.628857+0000","last_clean":"2023-09-23T22:41:08.628857+0000","last_became_active":"2023-09-23T22:37:42.432400+0000","last_became_peered":"2023-09-23T22:37:42.432400+0000","last_unstale":"2023-09-23T22:41:08.628857+0000","last_undegraded":"2023-09-23T22:41:08.628857+0000","last_fullsized":"2023-09-23T22:41:08.628857+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:17.145403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":45,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":593,"reported_epoch":553,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:09.821031+0000","last_change":"2023-09-23T22:39:05.287940+0000","last_active":"2023-09-23T22:38:54.405816+0000","last_peered":"2023-09-23T22:41:09.821031+0000","last_clean":"2023-09-23T22:38:48.357982+0000","last_became_active":"2023-09-23T22:36:36.407307+0000","last_became_peered":"2023-09-23T22:39:05.287940+0000","last_unstale":"2023-09-23T22:41:09.821031+0000","last_undegraded":"2023-09-23T22:41:09.821031+0000","last_fullsized":"2023-09-23T22:39:05.287229+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:59.639645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":279,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.628210+0000","last_change":"2023-09-23T22:40:46.377235+0000","last_active":"2023-09-23T22:41:08.628210+0000","last_peered":"2023-09-23T22:41:08.628210+0000","last_clean":"2023-09-23T22:41:08.628210+0000","last_became_active":"2023-09-23T22:40:46.376575+0000","last_became_peered":"2023-09-23T22:40:46.376575+0000","last_unstale":"2023-09-23T22:41:08.628210+0000","last_undegraded":"2023-09-23T22:41:08.628210+0000","last_fullsized":"2023-09-23T22:41:08.628210+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:13.988804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"553'3","reported_seq":12,"reported_epoch":553,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:11.241498+0000","last_change":"2023-09-23T22:41:11.241498+0000","last_active":"2023-09-23T22:41:11.241498+0000","last_peered":"2023-09-23T22:41:11.241498+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.471310+0000","last_became_peered":"2023-09-23T22:41:04.471310+0000","last_unstale":"2023-09-23T22:41:11.241498+0000","last_undegraded":"2023-09-23T22:41:11.241383+0000","last_fullsized":"2023-09-23T22:41:04.461633+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:51.324148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196608,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":3}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":963,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.120204+0000","last_change":"2023-09-23T22:40:49.348391+0000","last_active":"2023-09-23T22:41:08.120204+0000","last_peered":"2023-09-23T22:41:08.120204+0000","last_clean":"2023-09-23T22:41:08.120204+0000","last_became_active":"2023-09-23T22:40:49.347969+0000","last_became_peered":"2023-09-23T22:40:49.347969+0000","last_unstale":"2023-09-23T22:41:08.120204+0000","last_undegraded":"2023-09-23T22:41:08.120204+0000","last_fullsized":"2023-09-23T22:41:08.120204+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:16.067396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:16.067396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:22:12.722368+0000","scrub_duration":0.0072300120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1455,"reported_epoch":552,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:07.477123+0000","last_change":"2023-09-23T22:38:55.389270+0000","last_active":"2023-09-23T22:41:07.477123+0000","last_peered":"2023-09-23T22:41:07.477123+0000","last_clean":"2023-09-23T22:38:48.353781+0000","last_became_active":"2023-09-23T22:38:55.389270+0000","last_became_peered":"2023-09-23T22:38:55.389270+0000","last_unstale":"2023-09-23T22:41:07.477123+0000","last_undegraded":"2023-09-23T22:38:55.382069+0000","last_fullsized":"2023-09-23T22:38:55.381954+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:47:28.932588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":25,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.628587+0000","last_change":"2023-09-23T22:40:47.381776+0000","last_active":"2023-09-23T22:41:08.628587+0000","last_peered":"2023-09-23T22:41:08.628587+0000","last_clean":"2023-09-23T22:41:08.628587+0000","last_became_active":"2023-09-23T22:40:47.378458+0000","last_became_peered":"2023-09-23T22:40:47.378458+0000","last_unstale":"2023-09-23T22:41:08.628587+0000","last_undegraded":"2023-09-23T22:41:08.628587+0000","last_fullsized":"2023-09-23T22:41:08.628587+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:45:40.959955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":181,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.120336+0000","last_change":"2023-09-23T22:39:05.291228+0000","last_active":"2023-09-23T22:41:08.120336+0000","last_peered":"2023-09-23T22:41:08.120336+0000","last_clean":"2023-09-23T22:39:01.247643+0000","last_became_active":"2023-09-23T22:39:05.291228+0000","last_became_peered":"2023-09-23T22:39:05.291228+0000","last_unstale":"2023-09-23T22:41:08.120336+0000","last_undegraded":"2023-09-23T22:41:08.120336+0000","last_fullsized":"2023-09-23T22:39:05.285277+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:36:47.583378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"553'7","reported_seq":17,"reported_epoch":553,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:11.258888+0000","last_change":"2023-09-23T22:41:11.240037+0000","last_active":"2023-09-23T22:41:11.258888+0000","last_peered":"2023-09-23T22:41:11.258888+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.468356+0000","last_became_peered":"2023-09-23T22:41:04.468356+0000","last_unstale":"2023-09-23T22:41:11.258888+0000","last_undegraded":"2023-09-23T22:41:11.235070+0000","last_fullsized":"2023-09-23T22:41:04.454943+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:26.625786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":458752,"num_objects":7,"num_object_clones":0,"num_object_copies":21,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7,"num_write_kb":448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":7}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":835,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.119905+0000","last_change":"2023-09-23T22:41:07.291291+0000","last_active":"2023-09-23T22:41:08.119905+0000","last_peered":"2023-09-23T22:41:08.119905+0000","last_clean":"2023-09-23T22:41:08.119905+0000","last_became_active":"2023-09-23T22:40:49.350099+0000","last_became_peered":"2023-09-23T22:40:49.350099+0000","last_unstale":"2023-09-23T22:41:08.119905+0000","last_undegraded":"2023-09-23T22:41:08.119905+0000","last_fullsized":"2023-09-23T22:41:08.119905+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:07.291196+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:07.291196+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:15.341591+0000","scrub_duration":0.0016895390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":227,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.119950+0000","last_change":"2023-09-23T22:39:05.309059+0000","last_active":"2023-09-23T22:41:08.119950+0000","last_peered":"2023-09-23T22:41:08.119950+0000","last_clean":"2023-09-23T22:39:01.248920+0000","last_became_active":"2023-09-23T22:39:05.309059+0000","last_became_peered":"2023-09-23T22:39:05.309059+0000","last_unstale":"2023-09-23T22:41:08.119950+0000","last_undegraded":"2023-09-23T22:41:08.119950+0000","last_fullsized":"2023-09-23T22:39:05.286315+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:26.012539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":172,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.119812+0000","last_change":"2023-09-23T22:38:55.382188+0000","last_active":"2023-09-23T22:41:08.119812+0000","last_peered":"2023-09-23T22:41:08.119812+0000","last_clean":"2023-09-23T22:38:48.445651+0000","last_became_active":"2023-09-23T22:38:55.382188+0000","last_became_peered":"2023-09-23T22:38:55.382188+0000","last_unstale":"2023-09-23T22:41:08.119812+0000","last_undegraded":"2023-09-23T22:41:08.119812+0000","last_fullsized":"2023-09-23T22:38:55.378777+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:16:42.378244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":194,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.629188+0000","last_change":"2023-09-23T22:37:42.437482+0000","last_active":"2023-09-23T22:41:08.629188+0000","last_peered":"2023-09-23T22:41:08.629188+0000","last_clean":"2023-09-23T22:41:08.629188+0000","last_became_active":"2023-09-23T22:37:42.433915+0000","last_became_peered":"2023-09-23T22:37:42.433915+0000","last_unstale":"2023-09-23T22:41:08.629188+0000","last_undegraded":"2023-09-23T22:41:08.629188+0000","last_fullsized":"2023-09-23T22:41:08.629188+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:12.522090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"0'0","reported_seq":10,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.628729+0000","last_change":"2023-09-23T22:41:04.471347+0000","last_active":"2023-09-23T22:41:08.628729+0000","last_peered":"2023-09-23T22:41:08.628729+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:41:04.471347+0000","last_became_peered":"2023-09-23T22:41:04.471347+0000","last_unstale":"2023-09-23T22:41:08.628729+0000","last_undegraded":"2023-09-23T22:41:08.628729+0000","last_fullsized":"2023-09-23T22:41:04.461828+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:02.991776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1433,"reported_epoch":552,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:08.120352+0000","last_change":"2023-09-23T22:39:05.299849+0000","last_active":"2023-09-23T22:41:08.120352+0000","last_peered":"2023-09-23T22:41:08.120352+0000","last_clean":"2023-09-23T22:39:01.247439+0000","last_became_active":"2023-09-23T22:39:05.299849+0000","last_became_peered":"2023-09-23T22:39:05.299849+0000","last_unstale":"2023-09-23T22:41:08.120352+0000","last_undegraded":"2023-09-23T22:39:05.286114+0000","last_fullsized":"2023-09-23T22:39:05.286006+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":500,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:43.518374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":775,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:07.477534+0000","last_change":"2023-09-23T22:38:55.391540+0000","last_active":"2023-09-23T22:41:07.477534+0000","last_peered":"2023-09-23T22:41:07.477534+0000","last_clean":"2023-09-23T22:35:16.261819+0000","last_became_active":"2023-09-23T22:38:55.391540+0000","last_became_peered":"2023-09-23T22:38:55.391540+0000","last_unstale":"2023-09-23T22:41:07.477534+0000","last_undegraded":"2023-09-23T22:41:07.477534+0000","last_fullsized":"2023-09-23T22:38:55.384846+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":473,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.627488+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.627488+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:08.628131+0000","last_change":"2023-09-23T22:40:47.381236+0000","last_active":"2023-09-23T22:41:08.628131+0000","last_peered":"2023-09-23T22:41:08.628131+0000","last_clean":"2023-09-23T22:38:48.357699+0000","last_became_active":"2023-09-23T22:40:47.381236+0000","last_became_peered":"2023-09-23T22:40:47.381236+0000","last_unstale":"2023-09-23T22:41:08.628131+0000","last_undegraded":"2023-09-23T22:41:08.628131+0000","last_fullsized":"2023-09-23T22:40:47.365900+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:59.849480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":244,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:07.478767+0000","last_change":"2023-09-23T22:40:47.352742+0000","last_active":"2023-09-23T22:41:07.478767+0000","last_peered":"2023-09-23T22:41:07.478767+0000","last_clean":"2023-09-23T22:41:07.478767+0000","last_became_active":"2023-09-23T22:40:47.351870+0000","last_became_peered":"2023-09-23T22:40:47.351870+0000","last_unstale":"2023-09-23T22:41:07.478767+0000","last_undegraded":"2023-09-23T22:41:07.478767+0000","last_fullsized":"2023-09-23T22:41:07.478767+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:13.168148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.8","version":"553'6","reported_seq":19,"reported_epoch":553,"state":"active+clean","last_fresh":"2023-09-23T22:41:11.254393+0000","last_change":"2023-09-23T22:41:04.468790+0000","last_active":"2023-09-23T22:41:11.254393+0000","last_peered":"2023-09-23T22:41:11.254393+0000","last_clean":"2023-09-23T22:41:11.254393+0000","last_became_active":"2023-09-23T22:41:04.468091+0000","last_became_peered":"2023-09-23T22:41:04.468091+0000","last_unstale":"2023-09-23T22:41:11.254393+0000","last_undegraded":"2023-09-23T22:41:11.254393+0000","last_fullsized":"2023-09-23T22:41:11.254393+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":550,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:29.648265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393216,"num_objects":6,"num_object_clones":0,"num_object_copies":18,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,7],"acting":[2,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":591,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.629004+0000","last_change":"2023-09-23T22:40:56.928287+0000","last_active":"2023-09-23T22:41:08.629004+0000","last_peered":"2023-09-23T22:41:08.629004+0000","last_clean":"2023-09-23T22:41:08.629004+0000","last_became_active":"2023-09-23T22:40:49.363557+0000","last_became_peered":"2023-09-23T22:40:49.363557+0000","last_unstale":"2023-09-23T22:41:08.629004+0000","last_undegraded":"2023-09-23T22:41:08.629004+0000","last_fullsized":"2023-09-23T22:41:08.629004+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:56.928006+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:56.928006+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:21.139455+0000","scrub_duration":0.0018627959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":959,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.628246+0000","last_change":"2023-09-23T22:37:42.457358+0000","last_active":"2023-09-23T22:41:08.628246+0000","last_peered":"2023-09-23T22:41:08.628246+0000","last_clean":"2023-09-23T22:41:08.628246+0000","last_became_active":"2023-09-23T22:37:42.437821+0000","last_became_peered":"2023-09-23T22:37:42.437821+0000","last_unstale":"2023-09-23T22:41:08.628246+0000","last_undegraded":"2023-09-23T22:41:08.628246+0000","last_fullsized":"2023-09-23T22:41:08.628246+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:17.445534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":25,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:07.476943+0000","last_change":"2023-09-23T22:40:47.353806+0000","last_active":"2023-09-23T22:41:07.476943+0000","last_peered":"2023-09-23T22:41:07.476943+0000","last_clean":"2023-09-23T22:41:07.476943+0000","last_became_active":"2023-09-23T22:40:47.353358+0000","last_became_peered":"2023-09-23T22:40:47.353358+0000","last_unstale":"2023-09-23T22:41:07.476943+0000","last_undegraded":"2023-09-23T22:41:07.476943+0000","last_fullsized":"2023-09-23T22:41:07.476943+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:51.127835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0,4],"avail_no_missing":["5","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":356,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.120152+0000","last_change":"2023-09-23T22:40:46.378070+0000","last_active":"2023-09-23T22:41:08.120152+0000","last_peered":"2023-09-23T22:41:08.120152+0000","last_clean":"2023-09-23T22:41:08.120152+0000","last_became_active":"2023-09-23T22:40:46.377738+0000","last_became_peered":"2023-09-23T22:40:46.377738+0000","last_unstale":"2023-09-23T22:41:08.120152+0000","last_undegraded":"2023-09-23T22:41:08.120152+0000","last_fullsized":"2023-09-23T22:41:08.120152+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:41.151344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":9,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:09.129561+0000","last_change":"2023-09-23T22:41:04.466953+0000","last_active":"2023-09-23T22:41:03.442376+0000","last_peered":"2023-09-23T22:41:09.129561+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"0.000000","last_became_peered":"2023-09-23T22:41:04.466953+0000","last_unstale":"2023-09-23T22:41:09.129561+0000","last_undegraded":"2023-09-23T22:41:09.129561+0000","last_fullsized":"2023-09-23T22:41:04.465878+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:44:32.249849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":718,"reported_epoch":553,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:09.479367+0000","last_change":"2023-09-23T22:39:05.302767+0000","last_active":"2023-09-23T22:39:04.293769+0000","last_peered":"2023-09-23T22:41:09.479367+0000","last_clean":"2023-09-23T22:28:58.573726+0000","last_became_active":"2023-09-23T22:29:02.583400+0000","last_became_peered":"2023-09-23T22:39:05.302767+0000","last_unstale":"2023-09-23T22:41:09.479367+0000","last_undegraded":"2023-09-23T22:41:09.479367+0000","last_fullsized":"2023-09-23T22:39:05.302023+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":175,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:07.478908+0000","last_change":"2023-09-23T22:40:46.393258+0000","last_active":"2023-09-23T22:41:07.478908+0000","last_peered":"2023-09-23T22:41:07.478908+0000","last_clean":"2023-09-23T22:41:07.478908+0000","last_became_active":"2023-09-23T22:40:46.382839+0000","last_became_peered":"2023-09-23T22:40:46.382839+0000","last_unstale":"2023-09-23T22:41:07.478908+0000","last_undegraded":"2023-09-23T22:41:07.478908+0000","last_fullsized":"2023-09-23T22:41:07.478908+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:35.246162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":175,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.120169+0000","last_change":"2023-09-23T22:37:42.424518+0000","last_active":"2023-09-23T22:41:08.120169+0000","last_peered":"2023-09-23T22:41:08.120169+0000","last_clean":"2023-09-23T22:41:08.120169+0000","last_became_active":"2023-09-23T22:37:42.424144+0000","last_became_peered":"2023-09-23T22:37:42.424144+0000","last_unstale":"2023-09-23T22:41:08.120169+0000","last_undegraded":"2023-09-23T22:41:08.120169+0000","last_fullsized":"2023-09-23T22:41:08.120169+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:52:51.255509+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,0],"acting":[2,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":359,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.629181+0000","last_change":"2023-09-23T22:37:42.432892+0000","last_active":"2023-09-23T22:41:08.629181+0000","last_peered":"2023-09-23T22:41:08.629181+0000","last_clean":"2023-09-23T22:41:08.629181+0000","last_became_active":"2023-09-23T22:37:42.432356+0000","last_became_peered":"2023-09-23T22:37:42.432356+0000","last_unstale":"2023-09-23T22:41:08.629181+0000","last_undegraded":"2023-09-23T22:41:08.629181+0000","last_fullsized":"2023-09-23T22:41:08.629181+0000","mapping_epoch":495,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:14.288445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":9,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:08.120365+0000","last_change":"2023-09-23T22:41:04.457017+0000","last_active":"2023-09-23T22:41:03.442376+0000","last_peered":"2023-09-23T22:41:08.120365+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"0.000000","last_became_peered":"2023-09-23T22:41:04.457017+0000","last_unstale":"2023-09-23T22:41:08.120365+0000","last_undegraded":"2023-09-23T22:41:08.120365+0000","last_fullsized":"2023-09-23T22:41:04.455486+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:26.180982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":831,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.120648+0000","last_change":"2023-09-23T22:40:59.300859+0000","last_active":"2023-09-23T22:41:08.120648+0000","last_peered":"2023-09-23T22:41:08.120648+0000","last_clean":"2023-09-23T22:41:08.120648+0000","last_became_active":"2023-09-23T22:40:49.352991+0000","last_became_peered":"2023-09-23T22:40:49.352991+0000","last_unstale":"2023-09-23T22:41:08.120648+0000","last_undegraded":"2023-09-23T22:41:08.120648+0000","last_fullsized":"2023-09-23T22:41:08.120648+0000","mapping_epoch":542,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":543,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:59.300698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:59.300698+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:44.597966+0000","scrub_duration":0.0015585799999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,4],"avail_no_missing":["2","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":190,"reported_epoch":552,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:07.478615+0000","last_change":"2023-09-23T22:39:05.312608+0000","last_active":"2023-09-23T22:41:07.478615+0000","last_peered":"2023-09-23T22:41:07.478615+0000","last_clean":"2023-09-23T22:35:17.625026+0000","last_became_active":"2023-09-23T22:39:05.312608+0000","last_became_peered":"2023-09-23T22:39:05.312608+0000","last_unstale":"2023-09-23T22:41:07.478615+0000","last_undegraded":"2023-09-23T22:39:05.293584+0000","last_fullsized":"2023-09-23T22:39:05.293517+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:47.778707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":1062,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:08.629350+0000","last_change":"2023-09-23T22:40:47.391582+0000","last_active":"2023-09-23T22:41:08.629350+0000","last_peered":"2023-09-23T22:41:08.629350+0000","last_clean":"2023-09-23T22:38:48.410284+0000","last_became_active":"2023-09-23T22:40:47.391582+0000","last_became_peered":"2023-09-23T22:40:47.391582+0000","last_unstale":"2023-09-23T22:41:08.629350+0000","last_undegraded":"2023-09-23T22:41:08.629350+0000","last_fullsized":"2023-09-23T22:40:47.367122+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:32.098813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":75,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:07.477785+0000","last_change":"2023-09-23T22:39:05.312578+0000","last_active":"2023-09-23T22:41:07.477785+0000","last_peered":"2023-09-23T22:41:07.477785+0000","last_clean":"2023-09-23T22:31:26.131914+0000","last_became_active":"2023-09-23T22:39:05.312578+0000","last_became_peered":"2023-09-23T22:39:05.312578+0000","last_unstale":"2023-09-23T22:41:07.477785+0000","last_undegraded":"2023-09-23T22:41:07.477785+0000","last_fullsized":"2023-09-23T22:39:05.291104+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:52.614783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"0'0","reported_seq":9,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:08.120563+0000","last_change":"2023-09-23T22:41:04.460252+0000","last_active":"2023-09-23T22:41:03.442376+0000","last_peered":"2023-09-23T22:41:08.120563+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"0.000000","last_became_peered":"2023-09-23T22:41:04.460252+0000","last_unstale":"2023-09-23T22:41:08.120563+0000","last_undegraded":"2023-09-23T22:41:08.120563+0000","last_fullsized":"2023-09-23T22:41:04.455974+0000","mapping_epoch":549,"log_start":"0'0","ondisk_log_start":"0'0","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:16.365977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":670,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:07.479090+0000","last_change":"2023-09-23T22:39:05.295060+0000","last_active":"2023-09-23T22:35:19.269278+0000","last_peered":"2023-09-23T22:41:07.479090+0000","last_clean":"2023-09-23T22:35:16.261991+0000","last_became_active":"2023-09-23T22:34:36.002999+0000","last_became_peered":"2023-09-23T22:39:05.295060+0000","last_unstale":"2023-09-23T22:41:07.479090+0000","last_undegraded":"2023-09-23T22:41:07.479090+0000","last_fullsized":"2023-09-23T22:39:05.292715+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":481,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:34:49.465774+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:34:49.465774+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010319872000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":177,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.120510+0000","last_change":"2023-09-23T22:38:55.388004+0000","last_active":"2023-09-23T22:41:08.120510+0000","last_peered":"2023-09-23T22:41:08.120510+0000","last_clean":"2023-09-23T22:33:41.182644+0000","last_became_active":"2023-09-23T22:38:55.388004+0000","last_became_peered":"2023-09-23T22:38:55.388004+0000","last_unstale":"2023-09-23T22:41:08.120510+0000","last_undegraded":"2023-09-23T22:41:08.120510+0000","last_fullsized":"2023-09-23T22:38:55.379533+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":455,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:15.982785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":306,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.629725+0000","last_change":"2023-09-23T22:40:46.388690+0000","last_active":"2023-09-23T22:41:08.629725+0000","last_peered":"2023-09-23T22:41:08.629725+0000","last_clean":"2023-09-23T22:41:08.629725+0000","last_became_active":"2023-09-23T22:40:46.381984+0000","last_became_peered":"2023-09-23T22:40:46.381984+0000","last_unstale":"2023-09-23T22:41:08.629725+0000","last_undegraded":"2023-09-23T22:41:08.629725+0000","last_fullsized":"2023-09-23T22:41:08.629725+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:58.718242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":412,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:09.128753+0000","last_change":"2023-09-23T22:40:47.351265+0000","last_active":"2023-09-23T22:41:09.128753+0000","last_peered":"2023-09-23T22:41:09.128753+0000","last_clean":"2023-09-23T22:41:09.128753+0000","last_became_active":"2023-09-23T22:40:47.350860+0000","last_became_peered":"2023-09-23T22:40:47.350860+0000","last_unstale":"2023-09-23T22:41:09.128753+0000","last_undegraded":"2023-09-23T22:41:09.128753+0000","last_fullsized":"2023-09-23T22:41:09.128753+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:24.282928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":24,"reported_epoch":552,"state":"active+clean","last_fresh":"2023-09-23T22:41:08.629109+0000","last_change":"2023-09-23T22:40:46.383566+0000","last_active":"2023-09-23T22:41:08.629109+0000","last_peered":"2023-09-23T22:41:08.629109+0000","last_clean":"2023-09-23T22:41:08.629109+0000","last_became_active":"2023-09-23T22:40:46.383198+0000","last_became_peered":"2023-09-23T22:40:46.383198+0000","last_unstale":"2023-09-23T22:41:08.629109+0000","last_undegraded":"2023-09-23T22:41:08.629109+0000","last_fullsized":"2023-09-23T22:41:08.629109+0000","mapping_epoch":539,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":540,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:44.956360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":180,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.120428+0000","last_change":"2023-09-23T22:39:05.291652+0000","last_active":"2023-09-23T22:41:08.120428+0000","last_peered":"2023-09-23T22:41:08.120428+0000","last_clean":"2023-09-23T22:39:01.247800+0000","last_became_active":"2023-09-23T22:39:05.291652+0000","last_became_peered":"2023-09-23T22:39:05.291652+0000","last_unstale":"2023-09-23T22:41:08.120428+0000","last_undegraded":"2023-09-23T22:41:08.120428+0000","last_fullsized":"2023-09-23T22:39:05.287811+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:31:03.849150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":589,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:07.477218+0000","last_change":"2023-09-23T22:40:47.370896+0000","last_active":"2023-09-23T22:41:07.477218+0000","last_peered":"2023-09-23T22:41:07.477218+0000","last_clean":"2023-09-23T22:41:07.477218+0000","last_became_active":"2023-09-23T22:40:47.370472+0000","last_became_peered":"2023-09-23T22:40:47.370472+0000","last_unstale":"2023-09-23T22:41:07.477218+0000","last_undegraded":"2023-09-23T22:41:07.477218+0000","last_fullsized":"2023-09-23T22:41:07.477218+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:59.618515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":551,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.628861+0000","last_change":"2023-09-23T22:40:47.379410+0000","last_active":"2023-09-23T22:41:08.628861+0000","last_peered":"2023-09-23T22:41:08.628861+0000","last_clean":"2023-09-23T22:41:08.628861+0000","last_became_active":"2023-09-23T22:40:47.377291+0000","last_became_peered":"2023-09-23T22:40:47.377291+0000","last_unstale":"2023-09-23T22:41:08.628861+0000","last_undegraded":"2023-09-23T22:41:08.628861+0000","last_fullsized":"2023-09-23T22:41:08.628861+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:20.819140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2,4],"avail_no_missing":["1","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":608,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:07.477040+0000","last_change":"2023-09-23T22:40:47.464653+0000","last_active":"2023-09-23T22:41:07.477040+0000","last_peered":"2023-09-23T22:41:07.477040+0000","last_clean":"2023-09-23T22:41:07.477040+0000","last_became_active":"2023-09-23T22:40:47.370413+0000","last_became_peered":"2023-09-23T22:40:47.370413+0000","last_unstale":"2023-09-23T22:41:07.477040+0000","last_undegraded":"2023-09-23T22:41:07.477040+0000","last_fullsized":"2023-09-23T22:41:07.477040+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:05.989785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":552,"state":"active+clean+remapped","last_fresh":"2023-09-23T22:41:08.119871+0000","last_change":"2023-09-23T22:40:47.354656+0000","last_active":"2023-09-23T22:41:08.119871+0000","last_peered":"2023-09-23T22:41:08.119871+0000","last_clean":"2023-09-23T22:41:08.119871+0000","last_became_active":"2023-09-23T22:40:47.354227+0000","last_became_peered":"2023-09-23T22:40:47.354227+0000","last_unstale":"2023-09-23T22:41:08.119871+0000","last_undegraded":"2023-09-23T22:41:08.119871+0000","last_fullsized":"2023-09-23T22:41:08.119871+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":541,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:29.648715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,4],"avail_no_missing":["2","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":152,"reported_epoch":552,"state":"undersized+peered","last_fresh":"2023-09-23T22:41:08.120399+0000","last_change":"2023-09-23T22:39:05.288020+0000","last_active":"2023-09-23T22:33:45.291092+0000","last_peered":"2023-09-23T22:41:08.120399+0000","last_clean":"2023-09-23T22:33:41.181859+0000","last_became_active":"2023-09-23T22:31:28.125529+0000","last_became_peered":"2023-09-23T22:39:05.288020+0000","last_unstale":"2023-09-23T22:41:08.120399+0000","last_undegraded":"2023-09-23T22:41:08.120399+0000","last_fullsized":"2023-09-23T22:39:05.287481+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:35.969554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":552,"state":"active+undersized","last_fresh":"2023-09-23T22:41:08.119770+0000","last_change":"2023-09-23T22:39:05.309675+0000","last_active":"2023-09-23T22:41:08.119770+0000","last_peered":"2023-09-23T22:41:08.119770+0000","last_clean":"2023-09-23T22:34:30.962054+0000","last_became_active":"2023-09-23T22:39:05.309675+0000","last_became_peered":"2023-09-23T22:39:05.309675+0000","last_unstale":"2023-09-23T22:41:08.119770+0000","last_undegraded":"2023-09-23T22:41:08.119770+0000","last_fullsized":"2023-09-23T22:39:05.286965+0000","mapping_epoch":517,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:26.850825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":504,"reported_epoch":552,"state":"active+undersized+degraded","last_fresh":"2023-09-23T22:41:08.120701+0000","last_change":"2023-09-23T22:38:55.394101+0000","last_active":"2023-09-23T22:41:08.120701+0000","last_peered":"2023-09-23T22:41:08.120701+0000","last_clean":"2023-09-23T22:38:48.446310+0000","last_became_active":"2023-09-23T22:38:55.394101+0000","last_became_peered":"2023-09-23T22:38:55.394101+0000","last_unstale":"2023-09-23T22:41:08.120701+0000","last_undegraded":"2023-09-23T22:38:55.380019+0000","last_fullsized":"2023-09-23T22:38:55.379907+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":431,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:57.436984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":614,"reported_epoch":552,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:41:08.628351+0000","last_change":"2023-09-23T22:40:47.368661+0000","last_active":"2023-09-23T22:41:08.628351+0000","last_peered":"2023-09-23T22:41:08.628351+0000","last_clean":"2023-09-23T22:38:48.354354+0000","last_became_active":"2023-09-23T22:40:47.368661+0000","last_became_peered":"2023-09-23T22:40:47.368661+0000","last_unstale":"2023-09-23T22:41:08.628351+0000","last_undegraded":"2023-09-23T22:41:08.628351+0000","last_fullsized":"2023-09-23T22:40:47.352843+0000","mapping_epoch":540,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:34.396429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":59,"acting":73,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2031616,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":31,"num_write_kb":1984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5767168,"data_stored":5767168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":31,"ondisk_log_size":31,"up":30,"acting":30,"num_store_stats":5},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":11,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":36590,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":48807,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":62,"acting":74,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":38,"acting":44,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":323,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":18,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234442,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":523596,"kb_used_data":7876,"kb_used_omap":0,"kb_used_meta":515712,"kb_avail":93221556,"statfs":{"total":95995035648,"available":95458873344,"internally_reserved":0,"allocated":8065024,"data_stored":6213391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":528089088},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:40:38 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":1.6399999999999999,"15min":3.8740000000000001},"min":{"1min":0.217,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.7530000000000001,"5min":49.024999999999999,"15min":220.66},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.5109999999999999,"15min":4.0490000000000004},"min":{"1min":0.23499999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.734,"5min":48.901000000000003,"15min":220.59899999999999},"last":0.54700000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:40:44 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":0.86399999999999999,"15min":0.64200000000000002},"min":{"1min":0.27100000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.016,"5min":8.016,"15min":8.016},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.363,"5min":0.748,"15min":0.68000000000000005},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":5.085,"5min":5.085,"15min":5.085},"last":0.56399999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:41:10 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.3979999999999999,"15min":1.641},"min":{"1min":0.29999999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.5919999999999996,"5min":7.335,"15min":70.183999999999997},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.0600000000000001,"5min":1.5189999999999999,"15min":1.8009999999999999},"min":{"1min":0.32400000000000001,"5min":0.307,"15min":0.28899999999999998},"max":{"1min":4.5599999999999996,"5min":48.762,"15min":70.140000000000001},"last":0.624}]},{"osd":3,"last update":"Sat Sep 23 22:38:47 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.2969999999999999,"15min":2.5390000000000001},"min":{"1min":0.35999999999999999,"5min":0.316,"15min":0.26900000000000002},"max":{"1min":3.5710000000000002,"5min":9.8989999999999991,"15min":219.90700000000001},"last":0.435},{"interface":"front","average":{"1min":0.82699999999999996,"5min":2.101,"15min":2.9540000000000002},"min":{"1min":0.38,"5min":0.30099999999999999,"15min":0.28299999999999997},"max":{"1min":3.3719999999999999,"5min":48.866999999999997,"15min":219.976},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:40:58 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":0.54400000000000004,"15min":0.68799999999999994},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.9319999999999999,"5min":2.8090000000000002,"15min":2.8090000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.448,"15min":1.6819999999999999},"min":{"1min":0.247,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":3.0880000000000001,"5min":48.960000000000001,"15min":48.960000000000001},"last":0.38300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:40:23 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.847,"15min":1.756},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.249},"max":{"1min":5.8170000000000002,"5min":53.720999999999997,"15min":125.083},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.84299999999999997,"5min":1.8759999999999999,"15min":1.7410000000000001},"min":{"1min":0.307,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.472,"5min":53.218000000000004,"15min":122.351},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:38:22 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":2.5499999999999998,"15min":1.389},"min":{"1min":0.32000000000000001,"5min":0.311,"15min":0.29499999999999998},"max":{"1min":0.77500000000000002,"5min":88.284999999999997,"15min":88.284999999999997},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.623,"5min":2.4009999999999998,"15min":1.4339999999999999},"min":{"1min":0.39000000000000001,"5min":0.34699999999999998,"15min":0.33700000000000002},"max":{"1min":2.3050000000000002,"5min":88.355000000000004,"15min":88.355000000000004},"last":0.54900000000000004}]}]},{"osd":6,"up_from":46,"seq":197568496049,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2171296,"kb_used_data":1893144,"kb_used_omap":0,"kb_used_meta":278144,"kb_avail":91573856,"statfs":{"total":95995035648,"available":93771628544,"internally_reserved":0,"allocated":1938579456,"data_stored":1936876687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":284819456},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":4.9400000000000004,"15min":4.5979999999999999},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.22700000000000001},"max":{"1min":0.623,"5min":173.291,"15min":213.23699999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.41299999999999998,"5min":5.2850000000000001,"15min":5.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26500000000000001,"15min":0.22600000000000001},"max":{"1min":0.59799999999999998,"5min":174.232,"15min":213.304},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 22:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":1.627,"5min":1.627,"15min":1.627},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.2769999999999999,"5min":1.2769999999999999,"15min":1.2769999999999999},"last":1.4570000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:38:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":3.8730000000000002,"15min":2.5680000000000001},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.2770000000000001,"5min":103.548,"15min":103.548},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":3.8069999999999999,"15min":2.657},"min":{"1min":0.30499999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.583,"5min":105.99299999999999,"15min":105.99299999999999},"last":2.5049999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:37:58 2023","interfaces":[{"interface":"back","average":{"1min":4.3259999999999996,"5min":2.2690000000000001,"15min":2.9209999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":82.584000000000003,"5min":82.584000000000003,"15min":125.125},"last":0.58399999999999996},{"interface":"front","average":{"1min":4.6360000000000001,"5min":2.2040000000000002,"15min":2.5070000000000001},"min":{"1min":0.32400000000000001,"5min":0.313,"15min":0.313},"max":{"1min":82.316000000000003,"5min":82.316000000000003,"15min":82.316000000000003},"last":2.3290000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:37:58 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.99099999999999999,"15min":1.042},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.5099999999999998,"5min":7.5099999999999998,"15min":7.5099999999999998},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.4299999999999999,"15min":1.5589999999999999},"min":{"1min":0.309,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":3.0569999999999999,"5min":6.3490000000000002,"15min":6.3490000000000002},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 22:38:02 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.7110000000000001,"15min":1.6779999999999999},"min":{"1min":0.27300000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.4359999999999999,"5min":15.032999999999999,"15min":88.194999999999993},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":1.1279999999999999,"15min":1.359},"min":{"1min":0.29899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.69499999999999995,"5min":7.0839999999999996,"15min":88.025000000000006},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:36 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":1.117,"15min":1.353},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.314},"max":{"1min":0.71199999999999997,"5min":7.5670000000000002,"15min":82.801000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.2649999999999999,"15min":1.508},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.29699999999999999},"max":{"1min":0.65400000000000003,"5min":13.465999999999999,"15min":83.052999999999997},"last":0.65800000000000003}]}]},{"osd":1,"up_from":495,"seq":2126008811563,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":244180,"kb_used_data":11212,"kb_used_omap":12,"kb_used_meta":232947,"kb_avail":93500972,"statfs":{"total":95995035648,"available":95744995328,"internally_reserved":0,"allocated":11481088,"data_stored":8612872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":238538414},"hb_peers":[0,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:40:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":1.623,"15min":0.89400000000000002},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.4169999999999998,"5min":64.507000000000005,"15min":64.507000000000005},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.325,"5min":1.655,"15min":1.1519999999999999},"min":{"1min":0.216,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":4.867,"5min":64.352000000000004,"15min":64.352000000000004},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:40:44 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":0.83899999999999997,"15min":0.72699999999999998},"min":{"1min":0.23400000000000001,"5min":0.192,"15min":0.192},"max":{"1min":5.484,"5min":5.484,"15min":5.484},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.5149999999999999,"5min":0.95099999999999996,"15min":0.83099999999999996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":5.7320000000000002,"5min":5.7320000000000002,"15min":5.7320000000000002},"last":0.52200000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.6390000000000002,"5min":5.6390000000000002,"15min":5.6390000000000002},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.032,"15min":1.032},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.5739999999999998,"5min":4.5739999999999998,"15min":4.5739999999999998},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:40:44 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":0.68200000000000005,"15min":0.59399999999999997},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.2939999999999996,"5min":5.2939999999999996,"15min":5.2939999999999996},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.175,"5min":0.93000000000000005,"15min":0.97699999999999998},"min":{"1min":0.28999999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.4160000000000004,"5min":5.6740000000000004,"15min":5.6740000000000004},"last":0.495}]},{"osd":5,"last update":"Sat Sep 23 22:40:44 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":0.98299999999999998,"15min":0.89700000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":6.1429999999999998,"5min":6.1429999999999998,"15min":6.1429999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.228,"5min":0.72499999999999998,"15min":0.64300000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":5.0919999999999996,"5min":5.0919999999999996,"15min":5.0919999999999996},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.503},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":5.7050000000000001,"5min":5.7050000000000001,"15min":5.7050000000000001},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:40:44 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":1.0640000000000001,"15min":0.93300000000000005},"min":{"1min":0.34399999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.3289999999999997,"5min":5.3289999999999997,"15min":5.3289999999999997},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.2829999999999999,"5min":0.79400000000000004,"15min":0.69299999999999995},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":5.3319999999999999,"5min":5.3319999999999999,"15min":5.3319999999999999},"last":0.54300000000000004}]}]},{"osd":0,"up_from":93,"seq":399431958879,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":929564,"kb_used_data":9876,"kb_used_omap":8,"kb_used_meta":919671,"kb_avail":92815588,"statfs":{"total":95995035648,"available":95043162112,"internally_reserved":0,"allocated":10113024,"data_stored":7184068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":941743978},"hb_peers":[1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:40:49 2023","interfaces":[{"interface":"back","average":{"1min":2.976,"5min":1.1120000000000001,"15min":0.874},"min":{"1min":0.189,"5min":0.16600000000000001,"15min":0.16600000000000001},"max":{"1min":15.99,"5min":15.99,"15min":15.99},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.488,"5min":0.88600000000000001,"15min":0.64800000000000002},"min":{"1min":0.255,"5min":0.252,"15min":0.252},"max":{"1min":16.207999999999998,"5min":16.207999999999998,"15min":16.207999999999998},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:40:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.052,"15min":2.8879999999999999},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":4.6680000000000001,"5min":4.6680000000000001,"15min":175.89400000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.097,"5min":0.77400000000000002,"15min":2.2170000000000001},"min":{"1min":0.23100000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":5.3049999999999997,"5min":5.3049999999999997,"15min":175.66300000000001},"last":0.497}]},{"osd":3,"last update":"Sat Sep 23 22:38:50 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.024,"15min":2.4399999999999999},"min":{"1min":0.28799999999999998,"5min":0.23100000000000001,"15min":0.22500000000000001},"max":{"1min":1.698,"5min":7.0579999999999998,"15min":146.87299999999999},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.93799999999999994,"5min":1.149,"15min":2.516},"min":{"1min":0.311,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.1459999999999999,"5min":10.026,"15min":145.48099999999999},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:41:01 2023","interfaces":[{"interface":"back","average":{"1min":2.226,"5min":1.24,"15min":1.2749999999999999},"min":{"1min":0.376,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.4730000000000008,"5min":9.968,"15min":9.968},"last":0.55400000000000005},{"interface":"front","average":{"1min":2.2690000000000001,"5min":1.288,"15min":1.3140000000000001},"min":{"1min":0.38,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":9.5719999999999992,"5min":9.9949999999999992,"15min":9.9949999999999992},"last":0.59399999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:40:30 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.4630000000000001,"15min":2.5779999999999998},"min":{"1min":0.30099999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.6519999999999992,"5min":126.872,"15min":126.872},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.9390000000000001,"15min":1.698},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.27000000000000002},"max":{"1min":11.787000000000001,"5min":126.521,"15min":126.521},"last":0.57299999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:38:23 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":2.25,"15min":2.3490000000000002},"min":{"1min":0.34100000000000003,"5min":0.31,"15min":0.308},"max":{"1min":4.1280000000000001,"5min":74.355000000000004,"15min":74.355000000000004},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.8320000000000001,"15min":1.7529999999999999},"min":{"1min":0.39100000000000001,"5min":0.38400000000000001,"15min":0.35099999999999998},"max":{"1min":1.7330000000000001,"5min":73.150000000000006,"15min":73.150000000000006},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:40:30 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.2609999999999999,"15min":1.5329999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.29999999999999999},"max":{"1min":6.8179999999999996,"5min":6.8179999999999996,"15min":22.338999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":2.012,"5min":1.5109999999999999,"15min":1.8580000000000001},"min":{"1min":0.38400000000000001,"5min":0.36599999999999999,"15min":0.307},"max":{"1min":8.0579999999999998,"5min":8.0579999999999998,"15min":10.901},"last":0.72099999999999997}]}]},{"osd":2,"up_from":349,"seq":1498943586442,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":274940,"kb_used_data":8564,"kb_used_omap":15,"kb_used_meta":266352,"kb_avail":93470212,"statfs":{"total":95995035648,"available":95713497088,"internally_reserved":0,"allocated":8769536,"data_stored":5932172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":272745206},"hb_peers":[0,1,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:41:09 2023","interfaces":[{"interface":"back","average":{"1min":5.7060000000000004,"5min":1.972,"15min":8.1189999999999998},"min":{"1min":0.191,"5min":0.182,"15min":0.182},"max":{"1min":80.126000000000005,"5min":80.126000000000005,"15min":194.06299999999999},"last":0.434},{"interface":"front","average":{"1min":5.1319999999999997,"5min":2.0739999999999998,"15min":9.032},"min":{"1min":0.22,"5min":0.215,"15min":0.215},"max":{"1min":83.899000000000001,"5min":83.899000000000001,"15min":196.947},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:40:44 2023","interfaces":[{"interface":"back","average":{"1min":4.8840000000000003,"5min":1.52,"15min":0.93400000000000005},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":70.042000000000002,"5min":70.042000000000002,"15min":70.042000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":5.4829999999999997,"5min":1.8360000000000001,"15min":1.343},"min":{"1min":0.20899999999999999,"5min":0.184,"15min":0.184},"max":{"1min":70.067999999999998,"5min":70.067999999999998,"15min":70.067999999999998},"last":0.29899999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:38:08 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.6870000000000001,"15min":1.5680000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.0309999999999997,"5min":15.585000000000001,"15min":15.585000000000001},"last":0.745},{"interface":"front","average":{"1min":0.502,"5min":1.3320000000000001,"15min":1.0289999999999999},"min":{"1min":0.27400000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0609999999999999,"5min":14.128,"15min":14.128},"last":0.629}]},{"osd":4,"last update":"Sat Sep 23 22:41:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9530000000000001,"5min":1.8700000000000001,"15min":1.952},"min":{"1min":0.30099999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":7.1310000000000002,"5min":45.576999999999998,"15min":45.576999999999998},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.0640000000000001,"5min":1.4830000000000001,"15min":1.3100000000000001},"min":{"1min":0.33300000000000002,"5min":0.311,"15min":0.311},"max":{"1min":14.691000000000001,"5min":45.420999999999999,"15min":45.420999999999999},"last":0.56000000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:41:09 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":2.8690000000000002,"15min":3.3660000000000001},"min":{"1min":0.34499999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.7329999999999997,"5min":163.56399999999999,"15min":163.56399999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.048,"5min":2.4380000000000002,"15min":2.9510000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":14.895,"5min":163.541,"15min":163.541},"last":0.36799999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":2.851,"15min":1.958},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":2.0979999999999999,"5min":169.798,"15min":169.798},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.169,"5min":3.7669999999999999,"15min":2.411},"min":{"1min":0.42399999999999999,"5min":0.373,"15min":0.373},"max":{"1min":7.1130000000000004,"5min":168.44200000000001,"15min":168.44200000000001},"last":1.49}]},{"osd":7,"last update":"Sat Sep 23 22:41:09 2023","interfaces":[{"interface":"back","average":{"1min":1.2769999999999999,"5min":1.831,"15min":1.847},"min":{"1min":0.30299999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.375,"5min":75.513999999999996,"15min":108.113},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.0419999999999998,"5min":2.2370000000000001,"15min":2.246},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.2910000000000004,"5min":74.007999999999996,"15min":108.093},"last":0.498}]}]},{"osd":3,"up_from":108,"seq":463856468266,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1136452,"kb_used_data":582012,"kb_used_omap":4,"kb_used_meta":554427,"kb_avail":92608700,"statfs":{"total":95995035648,"available":94831308800,"internally_reserved":0,"allocated":595980288,"data_stored":593258702,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4563,"internal_metadata":567733805},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.2639999999999998,"15min":3.7320000000000002},"min":{"1min":0.218,"5min":0.192,"15min":0.192},"max":{"1min":31.917000000000002,"5min":84.239000000000004,"15min":124.282},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.9399999999999999,"5min":2.9689999999999999,"15min":4.0309999999999997},"min":{"1min":0.28499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":31.984000000000002,"5min":84.188000000000002,"15min":124.30500000000001},"last":0.89400000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:38:44 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.4650000000000001,"5min":1.4650000000000001,"15min":1.4650000000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.9570000000000001,"5min":1.9570000000000001,"15min":1.9570000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":31.856999999999999,"5min":31.856999999999999,"15min":31.856999999999999},"last":0.84899999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:39:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0369999999999999,"5min":3.048,"15min":2.3900000000000001},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":31.968,"5min":157.571,"15min":157.571},"last":0.56499999999999995},{"interface":"front","average":{"1min":2.1269999999999998,"5min":3.6549999999999998,"15min":2.423},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.223},"max":{"1min":31.899999999999999,"5min":158.33000000000001,"15min":158.33000000000001},"last":0.96199999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.012,"5min":1.4059999999999999,"15min":1.3580000000000001},"min":{"1min":0.34999999999999998,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":32.015000000000001,"5min":32.015000000000001,"15min":32.015000000000001},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.04,"5min":1.3680000000000001,"15min":1.2929999999999999},"min":{"1min":0.34999999999999998,"5min":0.31,"15min":0.31},"max":{"1min":32.094999999999999,"5min":32.094999999999999,"15min":32.094999999999999},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":2.097,"15min":2.2320000000000002},"min":{"1min":0.32500000000000001,"5min":0.318,"15min":0.315},"max":{"1min":51.286999999999999,"5min":51.286999999999999,"15min":51.286999999999999},"last":0.92800000000000005},{"interface":"front","average":{"1min":2.9670000000000001,"5min":2.077,"15min":2.2679999999999998},"min":{"1min":0.39000000000000001,"5min":0.36299999999999999,"15min":0.29099999999999998},"max":{"1min":51.270000000000003,"5min":51.270000000000003,"15min":51.270000000000003},"last":0.81699999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:37:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":2.21,"15min":1.9670000000000001},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34200000000000003},"max":{"1min":7.5140000000000002,"5min":24.957000000000001,"15min":24.957000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.6180000000000001,"5min":2.0379999999999998,"15min":2.206},"min":{"1min":0.376,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.4509999999999996,"5min":18.387,"15min":75.564999999999998},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:38:55 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.694,"15min":2.1110000000000002},"min":{"1min":0.436,"5min":0.38600000000000001,"15min":0.32400000000000001},"max":{"1min":32.040999999999997,"5min":32.040999999999997,"15min":75.525000000000006},"last":0.77700000000000002},{"interface":"front","average":{"1min":2.012,"5min":1.5600000000000001,"15min":1.804},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.379},"max":{"1min":32.058,"5min":32.058,"15min":32.058},"last":0.49199999999999999}]}]},{"osd":4,"up_from":0,"seq":1997159792705,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757715,"num_pgs":8,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":507000,"kb_used_data":10288,"kb_used_omap":0,"kb_used_meta":496704,"kb_avail":93238152,"statfs":{"total":95995035648,"available":95475867648,"internally_reserved":0,"allocated":10534912,"data_stored":8699151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":508624896},"hb_peers":[0,1,2,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:40:35 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":2.1150000000000002,"15min":5.5149999999999997},"min":{"1min":0.22700000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.4610000000000003,"5min":117.40000000000001,"15min":163.54499999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.216,"5min":2.7189999999999999,"15min":5.3150000000000004},"min":{"1min":0.251,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":4.7389999999999999,"5min":117.42400000000001,"15min":162.81800000000001},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:40:45 2023","interfaces":[{"interface":"back","average":{"1min":1.504,"5min":0.83399999999999996,"15min":0.73399999999999999},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.266,"5min":4.266,"15min":4.266},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.6339999999999999,"5min":0.76200000000000001,"15min":0.61299999999999999},"min":{"1min":0.25,"5min":0.249,"15min":0.249},"max":{"1min":10.351000000000001,"5min":10.351000000000001,"15min":10.351000000000001},"last":0.56699999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:41:10 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":2.0070000000000001,"15min":2.3090000000000002},"min":{"1min":0.33000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.6459999999999999,"5min":58.847999999999999,"15min":70.153000000000006},"last":0.624},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.8120000000000001,"15min":2.2829999999999999},"min":{"1min":0.314,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.6879999999999997,"5min":58.890000000000001,"15min":70.078999999999994},"last":0.495}]},{"osd":3,"last update":"Sat Sep 23 22:38:47 2023","interfaces":[{"interface":"back","average":{"1min":13.23,"5min":4.5650000000000004,"15min":4.0369999999999999},"min":{"1min":0.31900000000000001,"5min":0.316,"15min":0.28299999999999997},"max":{"1min":98.882000000000005,"5min":98.882000000000005,"15min":132.59},"last":0.39400000000000002},{"interface":"front","average":{"1min":13.026999999999999,"5min":4.0209999999999999,"15min":3.8980000000000001},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29299999999999998},"max":{"1min":98.856999999999999,"5min":98.856999999999999,"15min":129.381},"last":0.30599999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":1.27,"15min":3.198},"min":{"1min":0.255,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.0750000000000002,"5min":61.173999999999999,"15min":61.173999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.347,"15min":3.2629999999999999},"min":{"1min":0.35599999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.157,"5min":61.203000000000003,"15min":61.203000000000003},"last":0.66000000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:38:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.9319999999999999,"15min":1.4990000000000001},"min":{"1min":0.27000000000000002,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.375,"5min":56.667000000000002,"15min":70.581999999999994},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.71,"15min":1.478},"min":{"1min":0.35399999999999998,"5min":0.29099999999999998,"15min":0.248},"max":{"1min":1.1240000000000001,"5min":57.185000000000002,"15min":71.093000000000004},"last":0.53400000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:40:35 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":1.708,"15min":1.3480000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":31.042999999999999,"5min":59.101999999999997,"15min":59.101999999999997},"last":0.60699999999999998},{"interface":"front","average":{"1min":3.0990000000000002,"5min":1.972,"15min":1.55},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":30.882999999999999,"5min":59.151000000000003,"15min":59.151000000000003},"last":0.46200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3166,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2752512,"data_stored":2752512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1245184,"data_stored":1245184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":196608,"data_stored":196608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1572864,"data_stored":1572864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:41:17.839 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T22:41:17.839 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-09-23T22:41:17.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:17 smithi070 ceph-mon[192652]: pgmap v1447: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 26 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 10 active+undersized+degraded, 20 active+clean; 6.5 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 248 KiB/s wr, 3 op/s; 43/177 objects degraded (24.294%); 13/177 objects misplaced (7.345%) 2023-09-23T22:41:17.982 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:17.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:18.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:18.113 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:18.139 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:41:18.228 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 7 16 67 51 0.455291 0 - 0.00470231 2023-09-23T22:41:18.313 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:18.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:18.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:18.395 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:18 smithi026 ceph-mon[223014]: from='client.46652 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:41:18.421 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:18.453 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:18.500 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:18.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:18.508 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:18.608 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:18.643 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:18.651 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:18 smithi026 ceph-mon[215775]: from='client.46652 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:41:18.751 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:18.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:18.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:18.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:18.856 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:18 smithi070 ceph-mon[192652]: from='client.46652 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:41:18.889 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:18.952 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:41:18.952 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:41:18.952 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:41:18.952 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:41:18.952 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:41:18.960 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:18.996 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:18.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:18.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:19.100 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:19.133 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:19.173 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:41:19.181 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:19.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:19.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:19.192 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:41:19.210 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:41:19.228 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 8 16 67 51 0.398382 0 - 0.00470231 2023-09-23T22:41:19.231 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:41:19.250 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:41:19.269 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:41:19.269 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:41:19.269 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:41:19.270 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:41:19.270 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:41:19.270 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:41:19.270 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:41:19.270 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:41:19.270 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:41:19.286 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:19.322 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:19.377 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:19.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:19.382 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:19.483 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:19.511 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:19.555 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:19.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:19.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:19.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:19 smithi026 ceph-mon[215775]: pgmap v1448: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 398 KiB/s wr, 6 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:19.595 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2459613014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:19.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:19 smithi026 ceph-mon[223014]: pgmap v1448: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 398 KiB/s wr, 6 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:19.595 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2459613014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:19.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:19.691 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:19 smithi070 ceph-mon[192652]: pgmap v1448: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 398 KiB/s wr, 6 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2459613014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:19.916 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:41:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:41:19] "GET /metrics HTTP/1.1" 200 36459 "" "Prometheus/2.43.0" 2023-09-23T22:41:19.933 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:41:20.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 9 16 67 51 0.35412 0 - 0.00470231 2023-09-23T22:41:20.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:20.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:20.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:21.058 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:21.060 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:41:21.060 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:41:21.090 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:21.137 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:21.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:21.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:21.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 10 16 67 51 0.318697 0 - 0.00470231 2023-09-23T22:41:21.243 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:21.270 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:21.312 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:21.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:21.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:21.450 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:21.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:21 smithi026 ceph-mon[215775]: pgmap v1449: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 326 KiB/s wr, 5 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:21.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:21 smithi026 ceph-mon[223014]: pgmap v1449: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 326 KiB/s wr, 5 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:21 smithi070 ceph-mon[192652]: pgmap v1449: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 326 KiB/s wr, 5 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:21.924 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:21.954 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:21.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:21.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:22.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:22.090 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:22.141 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:22.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:22.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 11 16 67 51 0.289726 0 - 0.00470231 2023-09-23T22:41:22.246 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:22.279 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:22.328 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:22.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:22.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:22.410 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pg_num to 31 2023-09-23T22:41:22.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:22.465 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:22.570 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:22.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3876063652' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T22:41:22.571 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:22 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T22:41:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:22.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:22.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3876063652' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T22:41:22.573 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:22 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T22:41:22.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3876063652' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T22:41:22.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:22 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T22:41:22.674 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:22.703 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:22.747 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:22.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:22.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:22.854 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:22.880 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:22.922 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:22.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:22.926 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:23.026 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:23.052 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:23.062 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:41:23.093 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:23.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:23.199 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:23.224 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:23.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 12 16 67 51 0.265583 0 - 0.00470231 2023-09-23T22:41:23.265 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:23.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:23.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:23.401 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:23.417 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:41:23.677 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[215775]: pgmap v1450: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 326 KiB/s wr, 5 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:23.677 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[215775]: failed to encode map e555 with expected crc 2023-09-23T22:41:23.677 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[215775]: failed to encode map e555 with expected crc 2023-09-23T22:41:23.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[215775]: failed to encode map e555 with expected crc 2023-09-23T22:41:23.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T22:41:23.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[215775]: osdmap e555: 8 total, 6 up, 7 in 2023-09-23T22:41:23.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1001117518' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:23.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:23.678 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 59/237 objects degraded (24.895%), 17 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:23.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[223014]: pgmap v1450: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 326 KiB/s wr, 5 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:23.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[223014]: failed to encode map e555 with expected crc 2023-09-23T22:41:23.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[223014]: failed to encode map e555 with expected crc 2023-09-23T22:41:23.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[223014]: failed to encode map e555 with expected crc 2023-09-23T22:41:23.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T22:41:23.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[223014]: osdmap e555: 8 total, 6 up, 7 in 2023-09-23T22:41:23.679 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1001117518' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:23.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:23.680 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 59/237 objects degraded (24.895%), 17 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:23.765 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:23.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:23.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:23.869 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:23.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:23 smithi070 ceph-mon[192652]: pgmap v1450: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 326 KiB/s wr, 5 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:23 smithi070 ceph-mon[192652]: failed to encode map e555 with expected crc 2023-09-23T22:41:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:23 smithi070 ceph-mon[192652]: failed to encode map e555 with expected crc 2023-09-23T22:41:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:23 smithi070 ceph-mon[192652]: failed to encode map e555 with expected crc 2023-09-23T22:41:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:23 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T22:41:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:23 smithi070 ceph-mon[192652]: osdmap e555: 8 total, 6 up, 7 in 2023-09-23T22:41:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1001117518' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:23 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 59/237 objects degraded (24.895%), 17 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:23.899 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:24.145 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:24.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:24.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 13 16 67 51 0.245155 0 - 0.00470231 2023-09-23T22:41:24.250 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:24.282 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:24.332 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:24.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:24.394 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:24.437 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:24.462 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:24.462 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:24.464 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:24.507 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:24.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:24.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:24.612 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:24.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:24 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:24.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:24 smithi070 ceph-mon[192652]: failed to encode map e556 with expected crc 2023-09-23T22:41:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:24 smithi070 ceph-mon[192652]: failed to encode map e556 with expected crc 2023-09-23T22:41:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:24 smithi070 ceph-mon[192652]: failed to encode map e556 with expected crc 2023-09-23T22:41:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:24 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:24 smithi070 ceph-mon[192652]: osdmap e556: 8 total, 6 up, 7 in 2023-09-23T22:41:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T22:41:24.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:24.638 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:24.679 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:24.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:24.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:24.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:24.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[215775]: failed to encode map e556 with expected crc 2023-09-23T22:41:24.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[215775]: failed to encode map e556 with expected crc 2023-09-23T22:41:24.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[215775]: failed to encode map e556 with expected crc 2023-09-23T22:41:24.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:24.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[215775]: osdmap e556: 8 total, 6 up, 7 in 2023-09-23T22:41:24.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T22:41:24.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[223014]: failed to encode map e556 with expected crc 2023-09-23T22:41:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[223014]: failed to encode map e556 with expected crc 2023-09-23T22:41:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[223014]: failed to encode map e556 with expected crc 2023-09-23T22:41:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[223014]: osdmap e556: 8 total, 6 up, 7 in 2023-09-23T22:41:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T22:41:24.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:24.785 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:24.815 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:25.214 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:25.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:25.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:25.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 14 16 67 51 0.227645 0 - 0.00470231 2023-09-23T22:41:25.318 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:25.339 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:41:25.346 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:25.391 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:25.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:25.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:25.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[215775]: pgmap v1453: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 160 KiB/s wr, 2 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:25.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[215775]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[215775]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[215775]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T22:41:25.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[215775]: osdmap e557: 8 total, 6 up, 7 in 2023-09-23T22:41:25.429 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[215775]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.499 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:25.525 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:25.571 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:25.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:25.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:25 smithi070 ceph-mon[192652]: pgmap v1453: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 160 KiB/s wr, 2 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:25.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:25 smithi070 ceph-mon[192652]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:25 smithi070 ceph-mon[192652]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:25 smithi070 ceph-mon[192652]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T22:41:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:25 smithi070 ceph-mon[192652]: osdmap e557: 8 total, 6 up, 7 in 2023-09-23T22:41:25.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:25 smithi070 ceph-mon[192652]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.677 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:25.700 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:25.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[223014]: pgmap v1453: 109 pgs: 22 active+clean+remapped, 17 undersized+peered, 22 active+undersized, 11 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 160 KiB/s wr, 2 op/s; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:25.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[223014]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[223014]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[223014]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T22:41:25.703 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[223014]: osdmap e557: 8 total, 6 up, 7 in 2023-09-23T22:41:25.704 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:25 smithi026 ceph-mon[223014]: failed to encode map e557 with expected crc 2023-09-23T22:41:25.706 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:25.751 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:25.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:25.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:25.854 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:25.881 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.030 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:26.135 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:26.164 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 15 16 67 51 0.21247 0 - 0.00470231 2023-09-23T22:41:26.261 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:26.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:26.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:41:26.364 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:26.393 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.438 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:26.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:26.477 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:41:26.496 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:41:26.515 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:41:26.539 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:41:26.543 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:26.561 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:41:26.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:26 smithi070 ceph-mon[192652]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:26 smithi070 ceph-mon[192652]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:26 smithi070 ceph-mon[192652]: osdmap e558: 8 total, 6 up, 7 in 2023-09-23T22:41:26.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:26 smithi070 ceph-mon[192652]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:26 smithi070 ceph-mon[192652]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.568 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:26 smithi070 ceph-mon[192652]: osdmap e559: 8 total, 6 up, 7 in 2023-09-23T22:41:26.570 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.581 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:41:26.581 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:41:26.581 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:41:26.581 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:41:26.581 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:41:26.582 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:41:26.582 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:41:26.582 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:41:26.582 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:41:26.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[223014]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[223014]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[223014]: osdmap e558: 8 total, 6 up, 7 in 2023-09-23T22:41:26.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[223014]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[223014]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.599 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[223014]: osdmap e559: 8 total, 6 up, 7 in 2023-09-23T22:41:26.599 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[215775]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[215775]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[215775]: osdmap e558: 8 total, 6 up, 7 in 2023-09-23T22:41:26.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[215775]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[215775]: failed to encode map e558 with expected crc 2023-09-23T22:41:26.600 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:26 smithi026 ceph-mon[215775]: osdmap e559: 8 total, 6 up, 7 in 2023-09-23T22:41:26.748 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:26.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:26.852 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:26.882 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:26.927 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:26.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:26.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:27.031 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:27.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:27.106 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:27.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:27.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:27.210 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:27.224 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:41:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:41:27.224 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:41:27.224 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:41:27.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 16 16 67 51 0.199191 0 - 0.00470231 2023-09-23T22:41:27.239 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:27.352 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:27.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:27.381 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:27.402 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:41:27.403 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:41:27.456 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:27.483 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:27.528 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:27.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: pgmap v1456: 119 pgs: 5 peering, 10 unknown, 22 active+clean+remapped, 14 undersized+peered, 21 active+undersized, 11 active+undersized+remapped, 2 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.7 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 57/234 objects degraded (24.359%); 13/234 objects misplaced (5.556%) 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: 1.12 scrub starts 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: 1.12 scrub ok 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 18 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/12538117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:27.577 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: 1.b scrub starts 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: 1.b scrub ok 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[215775]: osdmap e560: 8 total, 6 up, 7 in 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: pgmap v1456: 119 pgs: 5 peering, 10 unknown, 22 active+clean+remapped, 14 undersized+peered, 21 active+undersized, 11 active+undersized+remapped, 2 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.7 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 57/234 objects degraded (24.359%); 13/234 objects misplaced (5.556%) 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: 1.12 scrub starts 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: 1.12 scrub ok 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.578 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 18 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:27.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/12538117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:27.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: 1.b scrub starts 2023-09-23T22:41:27.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: 1.b scrub ok 2023-09-23T22:41:27.579 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:27 smithi026 ceph-mon[223014]: osdmap e560: 8 total, 6 up, 7 in 2023-09-23T22:41:27.631 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:27.658 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:27.853 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:27.854 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: pgmap v1456: 119 pgs: 5 peering, 10 unknown, 22 active+clean+remapped, 14 undersized+peered, 21 active+undersized, 11 active+undersized+remapped, 2 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.7 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 57/234 objects degraded (24.359%); 13/234 objects misplaced (5.556%) 2023-09-23T22:41:27.854 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: 1.12 scrub starts 2023-09-23T22:41:27.854 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: 1.12 scrub ok 2023-09-23T22:41:27.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: failed to encode map e559 with expected crc 2023-09-23T22:41:27.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 18 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:27.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/12538117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:27.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: 1.b scrub starts 2023-09-23T22:41:27.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: 1.b scrub ok 2023-09-23T22:41:27.855 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:27 smithi070 ceph-mon[192652]: osdmap e560: 8 total, 6 up, 7 in 2023-09-23T22:41:27.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:27.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:27.929 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:41:27.956 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:27.997 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:28.040 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:28.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:28.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:28.148 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:28.177 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:28.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 17 16 67 51 0.187475 0 - 0.00470231 2023-09-23T22:41:28.277 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:28.331 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:28.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:28.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:28.434 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:28.461 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:28.505 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:28.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:28.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:28.609 INFO:tasks.thrashosds.thrasher:in_osds: [5, 0, 1, 6, 2, 7, 3] out_osds: [4] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:41:28.609 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:41:28.609 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [5, 0, 1, 6, 2, 7, 3] 2023-09-23T22:41:28.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T22:41:28.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:28.642 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[215775]: 1.5 scrub starts 2023-09-23T22:41:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[215775]: 1.5 scrub ok 2023-09-23T22:41:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[215775]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[215775]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[215775]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[215775]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 57/234 objects degraded (24.359%), 16 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[223014]: 1.5 scrub starts 2023-09-23T22:41:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[223014]: 1.5 scrub ok 2023-09-23T22:41:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[223014]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[223014]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[223014]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[223014]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 57/234 objects degraded (24.359%), 16 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:28.746 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:28.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:28.832 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:28 smithi070 ceph-mon[192652]: 1.5 scrub starts 2023-09-23T22:41:28.832 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:28 smithi070 ceph-mon[192652]: 1.5 scrub ok 2023-09-23T22:41:28.832 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:28 smithi070 ceph-mon[192652]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.833 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:28 smithi070 ceph-mon[192652]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.833 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:28 smithi070 ceph-mon[192652]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.833 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:28 smithi070 ceph-mon[192652]: failed to encode map e560 with expected crc 2023-09-23T22:41:28.833 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 57/234 objects degraded (24.359%), 16 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:28.849 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:28.875 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:28.919 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:28.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:28.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:28.984 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:29.023 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:29.053 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:29.116 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:29.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:29.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:29.219 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:29.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 18 16 67 51 0.17706 0 - 0.00470231 2023-09-23T22:41:29.247 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:29.293 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:29.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:29.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:29.398 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:29.404 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:41:29.428 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:29.455 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:41:29.526 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:29 smithi070 ceph-mon[192652]: pgmap v1459: 119 pgs: 5 peering, 6 unknown, 24 active+clean+remapped, 14 undersized+peered, 22 active+undersized, 12 active+undersized+remapped, 2 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.7 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 57/234 objects degraded (24.359%); 13/234 objects misplaced (5.556%) 2023-09-23T22:41:29.526 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:29 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2510656015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.527 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:29 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:29 smithi026 ceph-mon[215775]: pgmap v1459: 119 pgs: 5 peering, 6 unknown, 24 active+clean+remapped, 14 undersized+peered, 22 active+undersized, 12 active+undersized+remapped, 2 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.7 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 57/234 objects degraded (24.359%); 13/234 objects misplaced (5.556%) 2023-09-23T22:41:29.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:29 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2510656015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:29 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:29 smithi026 ceph-mon[223014]: pgmap v1459: 119 pgs: 5 peering, 6 unknown, 24 active+clean+remapped, 14 undersized+peered, 22 active+undersized, 12 active+undersized+remapped, 2 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.7 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 57/234 objects degraded (24.359%); 13/234 objects misplaced (5.556%) 2023-09-23T22:41:29.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:29 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2510656015' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:29 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:29.743 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:41:29.743 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:41:29.743 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:41:29.743 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:41:29.743 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:41:29.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:41:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:41:29] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-09-23T22:41:30.042 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:30.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:30.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:30.146 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:30.176 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:30.229 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 19 16 67 51 0.167741 0 - 0.00470231 2023-09-23T22:41:30.377 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:30.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:30.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:30.446 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:41:30.482 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:30.512 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:30.624 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:30.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:30.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:30.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:30.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[223014]: failed to encode map e561 with expected crc 2023-09-23T22:41:30.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[223014]: failed to encode map e561 with expected crc 2023-09-23T22:41:30.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[223014]: failed to encode map e561 with expected crc 2023-09-23T22:41:30.695 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[223014]: osdmap e561: 8 total, 6 up, 7 in 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[223014]: 1.1b scrub starts 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[223014]: 1.1b scrub ok 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[215775]: failed to encode map e561 with expected crc 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[215775]: failed to encode map e561 with expected crc 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[215775]: failed to encode map e561 with expected crc 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[215775]: osdmap e561: 8 total, 6 up, 7 in 2023-09-23T22:41:30.696 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[215775]: 1.1b scrub starts 2023-09-23T22:41:30.697 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:30 smithi026 ceph-mon[215775]: 1.1b scrub ok 2023-09-23T22:41:30.728 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:30.753 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:30 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:41:30.753 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:30 smithi070 ceph-mon[192652]: failed to encode map e561 with expected crc 2023-09-23T22:41:30.753 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:30 smithi070 ceph-mon[192652]: failed to encode map e561 with expected crc 2023-09-23T22:41:30.753 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:30 smithi070 ceph-mon[192652]: failed to encode map e561 with expected crc 2023-09-23T22:41:30.753 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:30 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:30.753 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:30 smithi070 ceph-mon[192652]: osdmap e561: 8 total, 6 up, 7 in 2023-09-23T22:41:30.754 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:30 smithi070 ceph-mon[192652]: 1.1b scrub starts 2023-09-23T22:41:30.754 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:30 smithi070 ceph-mon[192652]: 1.1b scrub ok 2023-09-23T22:41:30.756 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:30.790 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:30.803 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:30.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:30.808 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:30.872 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:30.908 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:30.935 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.028 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:31.131 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:31.159 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:41:31.230228+0000 min lat: 0.00118217 max lat: 0.00897703 avg lat: 0.00470231 2023-09-23T22:41:31.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:41:31.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 20 16 67 51 0.159354 0 - 0.00470231 2023-09-23T22:41:31.352 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:31.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:31.392 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:31 smithi026 conmon[215751]: 2023-09-23T22:41:31.198+0000 7f4580f04700 -1 mon.a@0(leader).osd e561 definitely_dead 0 2023-09-23T22:41:31.457 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:31.470 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.7. 2023-09-23T22:41:31.501 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.547 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:31.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:31 smithi070 ceph-mon[192652]: pgmap v1461: 119 pgs: 5 peering, 6 unknown, 24 active+clean+remapped, 14 undersized+peered, 22 active+undersized, 12 active+undersized+remapped, 2 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.7 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 57/234 objects degraded (24.359%); 13/234 objects misplaced (5.556%) 2023-09-23T22:41:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/936678342' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:41:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:31 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:41:31.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:31 smithi070 ceph-mon[192652]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:41:31.650 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:31.678 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:31 smithi026 ceph-mon[223014]: pgmap v1461: 119 pgs: 5 peering, 6 unknown, 24 active+clean+remapped, 14 undersized+peered, 22 active+undersized, 12 active+undersized+remapped, 2 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.7 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 57/234 objects degraded (24.359%); 13/234 objects misplaced (5.556%) 2023-09-23T22:41:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/936678342' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:41:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:31 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:41:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:31 smithi026 ceph-mon[223014]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:41:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:31 smithi026 ceph-mon[215775]: pgmap v1461: 119 pgs: 5 peering, 6 unknown, 24 active+clean+remapped, 14 undersized+peered, 22 active+undersized, 12 active+undersized+remapped, 2 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.7 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 57/234 objects degraded (24.359%); 13/234 objects misplaced (5.556%) 2023-09-23T22:41:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/936678342' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:41:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:31 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T22:41:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:31 smithi026 ceph-mon[215775]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T22:41:31.756 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:31.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:31.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:31.888 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:31.987 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:31.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:31.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:32.091 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:32.119 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:32.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 21 16 67 51 0.151766 0 - 0.00470231 2023-09-23T22:41:32.272 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:41:32.273 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:41:32.273 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:41:32.273 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:41:32.273 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.273 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48942 9.c 9.a863b59c (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:41:32.273 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.682106+0000", 2023-09-23T22:41:32.273 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 340.58778029000001, 2023-09-23T22:41:32.273 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99119241000000002, 2023-09-23T22:41:32.273 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48942 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682106+0000", 2023-09-23T22:41:32.274 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682106+0000", 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682105+0000", 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:32.275 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682183+0000", 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8666999999999997e-05 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682195+0000", 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2023e-05 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.276 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682203+0000", 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9549999999999992e-06 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691884+0000", 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096805209999999992 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691901+0000", 2023-09-23T22:41:32.277 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7312e-05 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673292+0000", 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98139073399999999 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673299+0000", 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7959999999999998e-06 2023-09-23T22:41:32.278 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48944 9.c 9.f6505a7c (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.682215+0000", 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 340.58767170900001, 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99109994300000004, 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.279 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48944 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682215+0000", 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.280 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682215+0000", 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682214+0000", 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682280+0000", 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5757000000000001e-05 2023-09-23T22:41:32.281 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682288+0000", 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2889999999999998e-06 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682296+0000", 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2710000000000003e-06 2023-09-23T22:41:32.282 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691924+0000", 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096284600000000001 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691930+0000", 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.122e-06 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.283 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673310+0000", 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.981380107 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673315+0000", 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5009999999999998e-06 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.284 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48956 9.e 9.222599de (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.682302+0000", 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 340.58758505999998, 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98747526900000004, 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48956 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.285 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682302+0000", 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682302+0000", 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.286 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682301+0000", 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682355+0000", 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3656999999999998e-05 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682362+0000", 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2960000000000001e-06 2023-09-23T22:41:32.287 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682367+0000", 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7790000000000003e-06 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.693185+0000", 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010818533 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.288 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.693190+0000", 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9110000000000001e-06 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.669765+0000", 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97657507700000001 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.289 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.669777+0000", 2023-09-23T22:41:32.290 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1501999999999999e-05 2023-09-23T22:41:32.290 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.290 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.290 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.290 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.290 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.290 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75839 9.e 9:747fb0cf:::benchmark_data_smithi026_555954_object75838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.254321+0000", 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 309.01556552900001, 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.009841719, 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75839 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.291 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254321+0000", 2023-09-23T22:41:32.292 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.292 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.292 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.292 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.292 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254321+0000", 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254319+0000", 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.293 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254433+0000", 2023-09-23T22:41:32.294 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113632 2023-09-23T22:41:32.294 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.294 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.294 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.294 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254450+0000", 2023-09-23T22:41:32.295 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6667000000000002e-05 2023-09-23T22:41:32.295 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.295 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.295 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.295 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254456+0000", 2023-09-23T22:41:32.295 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7610000000000002e-06 2023-09-23T22:41:32.295 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.296 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.296 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254520+0000", 2023-09-23T22:41:32.296 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4104999999999997e-05 2023-09-23T22:41:32.296 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.296 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.297 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.297 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254529+0000", 2023-09-23T22:41:32.297 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8799999999999997e-06 2023-09-23T22:41:32.297 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.297 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.298 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.298 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.239678+0000", 2023-09-23T22:41:32.298 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98514882599999998 2023-09-23T22:41:32.298 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.298 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.298 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.298 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.239842+0000", 2023-09-23T22:41:32.299 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016353099999999999 2023-09-23T22:41:32.299 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.299 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.299 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:41:32.299 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.240035+0000", 2023-09-23T22:41:32.299 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019364800000000001 2023-09-23T22:41:32.299 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.300 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.300 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.249930+0000", 2023-09-23T22:41:32.300 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.009894524 2023-09-23T22:41:32.300 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.300 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.301 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.301 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264057+0000", 2023-09-23T22:41:32.301 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014127293000000001 2023-09-23T22:41:32.301 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.301 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.301 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.302 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264105+0000", 2023-09-23T22:41:32.302 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7617999999999997e-05 2023-09-23T22:41:32.302 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.302 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.302 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264138+0000", 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3302000000000001e-05 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264163+0000", 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4776000000000001e-05 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.303 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75851 9.e 9:72e98536:::benchmark_data_smithi026_555954_object75850:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.266486+0000", 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 309.00340046600002, 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.997817069, 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.304 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75851 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266486+0000", 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266486+0000", 2023-09-23T22:41:32.305 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266485+0000", 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266560+0000", 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4641999999999994e-05 2023-09-23T22:41:32.306 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266591+0000", 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1872999999999998e-05 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266600+0000", 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0960000000000007e-06 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.307 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266665+0000", 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4324000000000003e-05 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266675+0000", 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0036e-05 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.241436+0000", 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97476125800000002 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.308 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.241566+0000", 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012998400000000001 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.241731+0000", 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000164981 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.309 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.249937+0000", 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082056939999999995 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264197+0000", 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014260324 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264251+0000", 2023-09-23T22:41:32.310 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3449000000000003e-05 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264281+0000", 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0531000000000003e-05 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264303+0000", 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2221999999999998e-05 2023-09-23T22:41:32.311 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75870 9.e 9:7ffb9527:::benchmark_data_smithi026_555954_object75869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.291109+0000", 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 308.97877728899999, 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97326064000000001, 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.312 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75870 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291109+0000", 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.313 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291109+0000", 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291108+0000", 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291155+0000", 2023-09-23T22:41:32.314 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6782e-05 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291172+0000", 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7436e-05 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291180+0000", 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.994e-06 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.315 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291269+0000", 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8403e-05 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291288+0000", 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9711000000000002e-05 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.316 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.243282+0000", 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.951993172 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.243406+0000", 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000124828 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.243551+0000", 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014428499999999999 2023-09-23T22:41:32.317 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.253042+0000", 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094910329999999994 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264225+0000", 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011183383 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.318 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264322+0000", 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6733999999999999e-05 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264349+0000", 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6846999999999999e-05 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.319 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264370+0000", 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1325999999999999e-05 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104674 9.e 9:7c43a090:::benchmark_data_smithi026_555954_object9945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 258.01696840900001, 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0412800680000001, 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.320 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104674 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:41:32.321 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:41:32.322 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252921+0000", 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6509999999999999e-06 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252938+0000", 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6416999999999999e-05 2023-09-23T22:41:32.323 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252943+0000", 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8679999999999998e-06 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.254941+0000", 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001998395 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.324 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.255082+0000", 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014090000000000001 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.255225+0000", 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014290599999999999 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.325 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.260880+0000", 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056549929999999997 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.261192+0000", 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031205499999999999 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.280742+0000", 2023-09-23T22:41:32.326 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019549997 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.280749+0000", 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2590000000000003e-06 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.294129+0000", 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0133797609999999 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.327 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.294198+0000", 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9536999999999995e-05 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104670 9.4 9:28c1ceab:::benchmark_data_smithi026_555954_object9941:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.293142+0000", 2023-09-23T22:41:32.328 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 257.976744467, 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0530680600000002, 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104670 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.329 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293142+0000", 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293142+0000", 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293141+0000", 2023-09-23T22:41:32.330 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293146+0000", 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6229999999999996e-06 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293155+0000", 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4839999999999993e-06 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.331 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293163+0000", 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8669999999999994e-06 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293213+0000", 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9536999999999997e-05 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.332 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293222+0000", 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1740000000000006e-06 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.261387+0000", 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9681656429999999 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.265588+0000", 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042009059999999999 2023-09-23T22:41:32.333 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.334 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.342211+0000", 2023-09-23T22:41:32.334 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.076622814999999997 2023-09-23T22:41:32.334 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.334 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.334 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.334 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.346210+0000", 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039992270000000002 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104676 9.4 9:2b72389e:::benchmark_data_smithi026_555954_object9947:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 257.97670964700001, 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0644994749999999, 2023-09-23T22:41:32.335 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104676 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.336 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.337 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293180+0000", 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.898e-06 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293186+0000", 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5080000000000002e-06 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.338 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293191+0000", 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7829999999999998e-06 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293244+0000", 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2927999999999999e-05 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293249+0000", 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1440000000000002e-06 2023-09-23T22:41:32.339 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.265969+0000", 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9727195919999998 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.270138+0000", 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041699170000000004 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.340 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.277809+0000", 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0076701460000000001 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.310996+0000", 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033187649999999999 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.341 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.347606+0000", 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036609292000000002 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.349036+0000", 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014300789999999999 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.351072+0000", 2023-09-23T22:41:32.342 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020369070000000001 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.357677+0000", 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066040279999999996 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.343 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104680 9.4 9:208db86f:::benchmark_data_smithi026_555954_object9951:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:41:32.344 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.293206+0000", 2023-09-23T22:41:32.344 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 257.97668115099998, 2023-09-23T22:41:32.344 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0756980610000002, 2023-09-23T22:41:32.344 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.344 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.344 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.344 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.344 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104680 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293206+0000", 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293206+0000", 2023-09-23T22:41:32.345 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293205+0000", 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293208+0000", 2023-09-23T22:41:32.346 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5409999999999999e-06 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293214+0000", 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2210000000000002e-06 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293218+0000", 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0640000000000004e-06 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.347 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293268+0000", 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9852999999999999e-05 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293273+0000", 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8940000000000001e-06 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.348 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.287851+0000", 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9945780060000002 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.293072+0000", 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052214150000000001 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.300397+0000", 2023-09-23T22:41:32.349 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073247130000000001 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.317253+0000", 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016856401 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.358723+0000", 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.041469682000000001 2023-09-23T22:41:32.350 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.359839+0000", 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011156460000000001 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.361542+0000", 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001703148 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.351 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.368904+0000", 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073618570000000003 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(9.c 489/478 [PushOp(9:31531dd3:::benchmark_data_smithi026_555954_object9982:head, version: 460'627, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:31531dd3:::benchmark_data_smithi026_555954_object9982:head@460'627, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.334517+0000", 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 256.93536980300001, 2023-09-23T22:41:32.352 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6523379540000001, 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334517+0000", 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334517+0000", 2023-09-23T22:41:32.353 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.354 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334516+0000", 2023-09-23T22:41:32.354 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:41:32.354 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.354 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.354 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334620+0000", 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010427799999999999 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334637+0000", 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6889999999999999e-05 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334646+0000", 2023-09-23T22:41:32.355 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9209999999999998e-06 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.984277+0000", 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6496310489999999 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.984284+0000", 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8950000000000001e-06 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.356 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.986855+0000", 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002571041 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104715 9.4 9:22773e84:::benchmark_data_smithi026_555954_object9986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.341862+0000", 2023-09-23T22:41:32.357 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 256.928024924, 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2123820830000001, 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104715 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.358 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341862+0000", 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341862+0000", 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341859+0000", 2023-09-23T22:41:32.359 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341870+0000", 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0808999999999999e-05 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341889+0000", 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8604e-05 2023-09-23T22:41:32.360 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341900+0000", 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1754e-05 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.306973+0000", 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.965072514 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.361 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.310651+0000", 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036783609999999998 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.383378+0000", 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.072726329000000006 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.362 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.387721+0000", 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043429649999999998 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.502428+0000", 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114707641 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.363 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.506100+0000", 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036718440000000001 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.512882+0000", 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067814260000000001 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.543660+0000", 2023-09-23T22:41:32.364 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030778920000000001 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.544694+0000", 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001033574 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.545296+0000", 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00060233699999999999 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.365 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.547534+0000", 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022377349999999998 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.554244+0000", 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067097160000000001 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.366 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(9.c 489/478 [PushOp(9:3a91d401:::benchmark_data_smithi026_555954_object9993:head, version: 460'628, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:3a91d401:::benchmark_data_smithi026_555954_object9993:head@460'628, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.362751+0000", 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 256.90713613499997, 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6258491149999998, 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.367 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362751+0000", 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362751+0000", 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362750+0000", 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.368 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362865+0000", 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115596 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362882+0000", 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7062e-05 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362897+0000", 2023-09-23T22:41:32.369 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4198e-05 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.986936+0000", 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6240388779999999 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.986943+0000", 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7989999999999994e-06 2023-09-23T22:41:32.370 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.988600+0000", 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001656364 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104723 9.4 9:24a0630c:::benchmark_data_smithi026_555954_object9994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.365986+0000", 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 256.90390020500001, 2023-09-23T22:41:32.371 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2005458440000001, 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104723 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365986+0000", 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.372 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365986+0000", 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365985+0000", 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.373 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365992+0000", 2023-09-23T22:41:32.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6150000000000002e-06 2023-09-23T22:41:32.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.374 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.374 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.374 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.366002+0000", 2023-09-23T22:41:32.374 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0249e-05 2023-09-23T22:41:32.374 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.366007+0000", 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1039999999999998e-06 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.393428+0000", 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.027421414 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:32.375 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.398310+0000", 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048813049999999998 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.425313+0000", 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027003367 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.429163+0000", 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038503410000000002 2023-09-23T22:41:32.376 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.527354+0000", 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098190762000000001 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.530473+0000", 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031186880000000001 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.377 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.537672+0000", 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0071995219999999999 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.555295+0000", 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017622819000000001 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.378 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.556659+0000", 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013637510000000001 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.557146+0000", 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048692499999999998 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.559282+0000", 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021364320000000002 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.379 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.566532+0000", 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00725007 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104756 9.4 9:258d171c:::benchmark_data_smithi026_555954_object10027:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:16.626866+0000", 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 255.64302078200001, 2023-09-23T22:41:32.380 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.026819157, 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104756 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626866+0000", 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.381 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626866+0000", 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626865+0000", 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:32.382 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626872+0000", 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2779999999999996e-06 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626886+0000", 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4470999999999999e-05 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.383 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626893+0000", 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5139999999999998e-06 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.398840+0000", 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.77194742100000002 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:32.384 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.402776+0000", 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039360929999999999 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.442574+0000", 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039797499 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.446766+0000", 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041918859999999997 2023-09-23T22:41:32.385 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.575609+0000", 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12884293299999999 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.578674+0000", 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00306552 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.386 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.586425+0000", 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077511760000000002 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.616679+0000", 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030253802 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.387 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.617743+0000", 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010636319999999999 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.644498+0000", 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026755226999999999 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.646717+0000", 2023-09-23T22:41:32.388 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022190299999999999 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.653685+0000", 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069680569999999997 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112987 9.c 9:3a416566:::benchmark_data_smithi026_555954_object18258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:32.389 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:05.354955+0000", 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 146.91493202699999, 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4743666209999999, 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112987 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.390 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354955+0000", 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354955+0000", 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354953+0000", 2023-09-23T22:41:32.391 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354958+0000", 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.69e-06 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354968+0000", 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4560000000000006e-06 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.392 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354973+0000", 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2769999999999996e-06 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.355025+0000", 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2425e-05 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.355032+0000", 2023-09-23T22:41:32.393 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1999999999999999e-06 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.714595+0000", 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3595632559999999 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.717359+0000", 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027637450000000002 2023-09-23T22:41:32.394 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.738228+0000", 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020869011 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.741256+0000", 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030285960000000002 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.395 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.777617+0000", 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036360874000000001 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.781007+0000", 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003390452 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.786955+0000", 2023-09-23T22:41:32.396 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059470399999999998 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.819024+0000", 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.032069822999999997 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.820025+0000", 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010011379999999999 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.397 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.820409+0000", 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038331999999999998 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.822395+0000", 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019860360000000001 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.398 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.829321+0000", 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069264519999999996 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:113617 9.c 9:3bc4b131:::benchmark_data_smithi026_555954_object18888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:08.817731+0000", 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 143.45215542099999, 2023-09-23T22:41:32.399 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98456004699999999, 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 113617 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817731+0000", 2023-09-23T22:41:32.400 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817731+0000", 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817730+0000", 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.401 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817740+0000", 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0345e-05 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817754+0000", 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3906e-05 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.402 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817762+0000", 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.072e-06 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.878893+0000", 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.061130914000000001 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:32.403 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.882737+0000", 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038442099999999998 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.933676+0000", 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050939367999999999 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.936972+0000", 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003295769 2023-09-23T22:41:32.404 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.740736+0000", 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.80376360499999999 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.743337+0000", 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026008390000000002 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.405 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.748613+0000", 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052764719999999999 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.775493+0000", 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026879872999999999 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.406 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.792909+0000", 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017415588999999999 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.794028+0000", 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00111911 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.795686+0000", 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001658723 2023-09-23T22:41:32.407 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.802291+0000", 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066049189999999999 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:113663 9.c 9:352b5b63:::benchmark_data_smithi026_555954_object18934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:41:32.408 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:09.125940+0000", 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 143.14394621599999, 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0012365830000001, 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 113663 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.409 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125940+0000", 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125940+0000", 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.410 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125939+0000", 2023-09-23T22:41:32.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:32.411 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.411 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.411 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.411 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125956+0000", 2023-09-23T22:41:32.411 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.73e-05 2023-09-23T22:41:32.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.412 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125970+0000", 2023-09-23T22:41:32.412 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3583e-05 2023-09-23T22:41:32.412 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.412 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.412 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125976+0000", 2023-09-23T22:41:32.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2389999999999999e-06 2023-09-23T22:41:32.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.413 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.413 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.413 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.224933+0000", 2023-09-23T22:41:32.413 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098956855999999996 2023-09-23T22:41:32.413 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.414 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:32.414 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.229210+0000", 2023-09-23T22:41:32.414 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042773409999999996 2023-09-23T22:41:32.414 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.414 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.288240+0000", 2023-09-23T22:41:32.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.059029617 2023-09-23T22:41:32.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.415 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.415 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:41:32.415 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.291651+0000", 2023-09-23T22:41:32.415 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034113630000000002 2023-09-23T22:41:32.415 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.416 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.085535+0000", 2023-09-23T22:41:32.416 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79388345299999996 2023-09-23T22:41:32.416 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.416 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.416 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.088218+0000", 2023-09-23T22:41:32.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026835880000000002 2023-09-23T22:41:32.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.417 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:32.417 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.093377+0000", 2023-09-23T22:41:32.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051590380000000003 2023-09-23T22:41:32.417 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.418 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.118971+0000", 2023-09-23T22:41:32.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025593576999999999 2023-09-23T22:41:32.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.418 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.418 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.119343+0000", 2023-09-23T22:41:32.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00037188599999999998 2023-09-23T22:41:32.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.120218+0000", 2023-09-23T22:41:32.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00087530799999999999 2023-09-23T22:41:32.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.121636+0000", 2023-09-23T22:41:32.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014175069999999999 2023-09-23T22:41:32.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.127177+0000", 2023-09-23T22:41:32.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055414499999999998 2023-09-23T22:41:32.421 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.421 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.421 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.422 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:117481 9.c 9:3122b071:::benchmark_data_smithi026_555954_object22752:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T22:41:32.422 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:22.273591+0000", 2023-09-23T22:41:32.422 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 129.996295285, 2023-09-23T22:41:32.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.062280133, 2023-09-23T22:41:32.422 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.422 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.422 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.422 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.423 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.423 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 117481 2023-09-23T22:41:32.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.423 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273591+0000", 2023-09-23T22:41:32.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273591+0000", 2023-09-23T22:41:32.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273590+0000", 2023-09-23T22:41:32.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:32.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273596+0000", 2023-09-23T22:41:32.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0970000000000002e-06 2023-09-23T22:41:32.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.426 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273609+0000", 2023-09-23T22:41:32.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3054000000000001e-05 2023-09-23T22:41:32.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.426 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273615+0000", 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2859999999999996e-06 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.293079+0000", 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019464043 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.294561+0000", 2023-09-23T22:41:32.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001482209 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.821444+0000", 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.52688237299999996 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.824388+0000", 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029445890000000001 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.300187+0000", 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47579873099999997 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.302383+0000", 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021957130000000002 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.306849+0000", 2023-09-23T22:41:32.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044659249999999999 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.328972+0000", 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022122882999999999 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.329730+0000", 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00075875699999999999 2023-09-23T22:41:32.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.329972+0000", 2023-09-23T22:41:32.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024183599999999999 2023-09-23T22:41:32.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.431 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.431 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.331151+0000", 2023-09-23T22:41:32.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001178988 2023-09-23T22:41:32.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.335872+0000", 2023-09-23T22:41:32.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047203760000000001 2023-09-23T22:41:32.432 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.432 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.432 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:118575 9.c 9:36238a6d:::benchmark_data_smithi026_555954_object23846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:24.992467+0000", 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 127.27741966000001, 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0767739080000001, 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:32.433 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 118575 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992467+0000", 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992467+0000", 2023-09-23T22:41:32.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992466+0000", 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992470+0000", 2023-09-23T22:41:32.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.783e-06 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992478+0000", 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5189999999999992e-06 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992483+0000", 2023-09-23T22:41:32.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1320000000000004e-06 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:25.004684+0000", 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012201900999999999 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:25.006194+0000", 2023-09-23T22:41:32.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015095289999999999 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:25.486627+0000", 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.48043316600000002 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:25.489033+0000", 2023-09-23T22:41:32.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024054979999999998 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.034240+0000", 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.54520759500000004 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.036416+0000", 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021752960000000002 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.040515+0000", 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040993469999999997 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.061630+0000", 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021115373 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.062311+0000", 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068060899999999999 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.063050+0000", 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00073889099999999998 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.064177+0000", 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011268160000000001 2023-09-23T22:41:32.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:41:32.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:41:32.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:41:32.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.069241+0000", 2023-09-23T22:41:32.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050643290000000002 2023-09-23T22:41:32.442 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.442 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.442 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.442 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:41:32.443 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:41:32.443 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:41:32.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:32.444 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:32.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:32.544 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:32.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:32 smithi070 ceph-mon[192652]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.570 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:32 smithi070 ceph-mon[192652]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:32 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:41:32.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:32 smithi070 ceph-mon[192652]: osdmap e562: 8 total, 6 up, 6 in 2023-09-23T22:41:32.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:32 smithi070 ceph-mon[192652]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.571 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:32 smithi070 ceph-mon[192652]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.574 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:32.623 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:32.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:32.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:32.640 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:41:32.689 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[223014]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[223014]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:41:32.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[223014]: osdmap e562: 8 total, 6 up, 6 in 2023-09-23T22:41:32.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[223014]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[223014]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.690 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[215775]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.690 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[215775]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T22:41:32.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[215775]: osdmap e562: 8 total, 6 up, 6 in 2023-09-23T22:41:32.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[215775]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:32 smithi026 ceph-mon[215775]: failed to encode map e562 with expected crc 2023-09-23T22:41:32.726 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:32.755 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:32.808 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:32.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:32.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:32.882 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:41:32.903 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:41:32.913 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:32.925 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:41:32.946 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:32.950 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:41:32.975 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:41:33.000 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:41:33.000 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:41:33.000 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:41:33.000 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:41:33.001 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:41:33.001 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:41:33.001 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:41:33.001 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:41:33.001 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:41:33.002 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:33.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:33.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:33.013 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:41:33.033 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:33.109 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:33.136 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:33.177 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:33.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:33.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:33.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 22 16 67 51 0.144868 0 - 0.00470231 2023-09-23T22:41:33.282 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:33.308 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:33.349 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:33.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:33.354 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:33.455 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:33.487 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:33.533 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:33.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:33.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: pgmap v1463: 119 pgs: 25 active+clean+remapped, 20 undersized+peered, 24 active+undersized, 13 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 19 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: osdmap e563: 8 total, 6 up, 6 in 2023-09-23T22:41:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1381372544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 59/237 objects degraded (24.895%), 17 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:33.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:33 smithi070 ceph-mon[192652]: osdmap e564: 8 total, 6 up, 6 in 2023-09-23T22:41:33.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:33.671 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:33.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: pgmap v1463: 119 pgs: 25 active+clean+remapped, 20 undersized+peered, 24 active+undersized, 13 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:33.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 19 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:33.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: osdmap e563: 8 total, 6 up, 6 in 2023-09-23T22:41:33.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1381372544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:33.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 59/237 objects degraded (24.895%), 17 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:33.730 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[223014]: osdmap e564: 8 total, 6 up, 6 in 2023-09-23T22:41:33.730 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: pgmap v1463: 119 pgs: 25 active+clean+remapped, 20 undersized+peered, 24 active+undersized, 13 active+undersized+remapped, 3 undersized+degraded+peered, 14 active+undersized+degraded, 20 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 59/237 objects degraded (24.895%); 13/237 objects misplaced (5.485%) 2023-09-23T22:41:33.731 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 19 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:33.731 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.731 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.731 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.731 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: osdmap e563: 8 total, 6 up, 6 in 2023-09-23T22:41:33.731 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: failed to encode map e563 with expected crc 2023-09-23T22:41:33.731 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1381372544' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:33.731 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 59/237 objects degraded (24.895%), 17 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:33.731 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:33 smithi026 ceph-mon[215775]: osdmap e564: 8 total, 6 up, 6 in 2023-09-23T22:41:33.732 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:33.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:33.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:33.749 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:41:33.750 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:41:33.837 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:33.865 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:33.910 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:33.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:33.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:33.977 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:34.014 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:34.041 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:34.085 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:34.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:34.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:34.193 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:34.221 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:34.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 23 16 67 51 0.138569 0 - 0.00470231 2023-09-23T22:41:34.267 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:34.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:34.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:34.370 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:34.398 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:34.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:34 smithi026 ceph-mon[223014]: failed to encode map e564 with expected crc 2023-09-23T22:41:34.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:34 smithi026 ceph-mon[223014]: failed to encode map e564 with expected crc 2023-09-23T22:41:34.687 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:34 smithi026 ceph-mon[223014]: failed to encode map e564 with expected crc 2023-09-23T22:41:34.687 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:34 smithi026 ceph-mon[215775]: failed to encode map e564 with expected crc 2023-09-23T22:41:34.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:34 smithi026 ceph-mon[215775]: failed to encode map e564 with expected crc 2023-09-23T22:41:34.688 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:34 smithi026 ceph-mon[215775]: failed to encode map e564 with expected crc 2023-09-23T22:41:34.694 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:34.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:34.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:34.797 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:34.823 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:34 smithi070 ceph-mon[192652]: failed to encode map e564 with expected crc 2023-09-23T22:41:34.824 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:34 smithi070 ceph-mon[192652]: failed to encode map e564 with expected crc 2023-09-23T22:41:34.824 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:34 smithi070 ceph-mon[192652]: failed to encode map e564 with expected crc 2023-09-23T22:41:34.826 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:34.871 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:34.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:34.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:34.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:35.005 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.047 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:35.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:35.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:35.154 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:35.184 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 24 16 67 51 0.132796 0 - 0.00470231 2023-09-23T22:41:35.230 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:35.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:35.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:35.334 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:35.360 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.402 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:35.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:35.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:35.507 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:35.536 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.585 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:35.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:35.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:35 smithi070 ceph-mon[192652]: pgmap v1466: 119 pgs: 1 peering, 19 remapped+peering, 22 active+clean+remapped, 18 undersized+peered, 18 active+undersized, 12 active+undersized+remapped, 3 undersized+degraded+peered, 12 active+undersized+degraded, 14 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 51/237 objects degraded (21.519%); 7/237 objects misplaced (2.954%) 2023-09-23T22:41:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:35 smithi070 ceph-mon[192652]: 1.d scrub starts 2023-09-23T22:41:35.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:35 smithi070 ceph-mon[192652]: 1.d scrub ok 2023-09-23T22:41:35.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:35 smithi070 ceph-mon[192652]: failed to encode map e565 with expected crc 2023-09-23T22:41:35.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:35 smithi070 ceph-mon[192652]: failed to encode map e565 with expected crc 2023-09-23T22:41:35.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:35 smithi070 ceph-mon[192652]: osdmap e565: 8 total, 6 up, 6 in 2023-09-23T22:41:35.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:35 smithi070 ceph-mon[192652]: failed to encode map e565 with expected crc 2023-09-23T22:41:35.690 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:35.715 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[215775]: pgmap v1466: 119 pgs: 1 peering, 19 remapped+peering, 22 active+clean+remapped, 18 undersized+peered, 18 active+undersized, 12 active+undersized+remapped, 3 undersized+degraded+peered, 12 active+undersized+degraded, 14 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 51/237 objects degraded (21.519%); 7/237 objects misplaced (2.954%) 2023-09-23T22:41:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[215775]: 1.d scrub starts 2023-09-23T22:41:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[215775]: 1.d scrub ok 2023-09-23T22:41:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[215775]: failed to encode map e565 with expected crc 2023-09-23T22:41:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[215775]: failed to encode map e565 with expected crc 2023-09-23T22:41:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[215775]: osdmap e565: 8 total, 6 up, 6 in 2023-09-23T22:41:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[215775]: failed to encode map e565 with expected crc 2023-09-23T22:41:35.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[223014]: pgmap v1466: 119 pgs: 1 peering, 19 remapped+peering, 22 active+clean+remapped, 18 undersized+peered, 18 active+undersized, 12 active+undersized+remapped, 3 undersized+degraded+peered, 12 active+undersized+degraded, 14 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 51/237 objects degraded (21.519%); 7/237 objects misplaced (2.954%) 2023-09-23T22:41:35.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[223014]: 1.d scrub starts 2023-09-23T22:41:35.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[223014]: 1.d scrub ok 2023-09-23T22:41:35.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[223014]: failed to encode map e565 with expected crc 2023-09-23T22:41:35.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[223014]: failed to encode map e565 with expected crc 2023-09-23T22:41:35.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[223014]: osdmap e565: 8 total, 6 up, 6 in 2023-09-23T22:41:35.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:35 smithi026 ceph-mon[223014]: failed to encode map e565 with expected crc 2023-09-23T22:41:35.750 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:41:35.756 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:35.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:35.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:35.862 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:35.888 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:35.930 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:35.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:35.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:36.036 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:36.061 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:36.103 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:36.104 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:36.104 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:36.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:36.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:36.210 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:36.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 25 16 67 51 0.127484 0 - 0.00470231 2023-09-23T22:41:36.236 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:36.277 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:36.282 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:36.383 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:36.408 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:36.450 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:36.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:36.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:36.497 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:41:36.557 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:36.587 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:36.829 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:36.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[215775]: failed to encode map e566 with expected crc 2023-09-23T22:41:36.831 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[215775]: failed to encode map e566 with expected crc 2023-09-23T22:41:36.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[215775]: failed to encode map e566 with expected crc 2023-09-23T22:41:36.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[215775]: osdmap e566: 8 total, 6 up, 6 in 2023-09-23T22:41:36.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[215775]: 1.10 scrub starts 2023-09-23T22:41:36.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[215775]: 1.10 scrub ok 2023-09-23T22:41:36.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3937940413' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:36.832 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:36.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[223014]: failed to encode map e566 with expected crc 2023-09-23T22:41:36.832 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[223014]: failed to encode map e566 with expected crc 2023-09-23T22:41:36.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[223014]: failed to encode map e566 with expected crc 2023-09-23T22:41:36.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[223014]: osdmap e566: 8 total, 6 up, 6 in 2023-09-23T22:41:36.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[223014]: 1.10 scrub starts 2023-09-23T22:41:36.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[223014]: 1.10 scrub ok 2023-09-23T22:41:36.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3937940413' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:36.833 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:36.842 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:36.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:36.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:36.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:41:36.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:36 smithi070 ceph-mon[192652]: failed to encode map e566 with expected crc 2023-09-23T22:41:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:36 smithi070 ceph-mon[192652]: failed to encode map e566 with expected crc 2023-09-23T22:41:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:36 smithi070 ceph-mon[192652]: failed to encode map e566 with expected crc 2023-09-23T22:41:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:36 smithi070 ceph-mon[192652]: osdmap e566: 8 total, 6 up, 6 in 2023-09-23T22:41:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:36 smithi070 ceph-mon[192652]: 1.10 scrub starts 2023-09-23T22:41:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:36 smithi070 ceph-mon[192652]: 1.10 scrub ok 2023-09-23T22:41:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:36 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3937940413' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:41:36.947 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:36.975 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:37.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 26 16 67 51 0.122581 0 - 0.00470231 2023-09-23T22:41:37.300 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:37.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:37.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:37.404 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:37.437 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:37.486 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:37.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:37.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:37.522 INFO:tasks.thrashosds.thrasher:in_osds: [5, 0, 1, 6, 2, 3] out_osds: [4, 7] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:41:37.522 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:41:37.523 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T22:41:37.523 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T22:41:37.523 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T22:41:37.523 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T22:41:37.555 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[215775]: pgmap v1469: 119 pgs: 1 peering, 19 remapped+peering, 25 active+clean+remapped, 16 undersized+peered, 16 active+undersized, 16 active+undersized+remapped, 2 undersized+degraded+peered, 10 active+undersized+degraded, 14 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 43/237 objects degraded (18.143%); 22/237 objects misplaced (9.283%) 2023-09-23T22:41:37.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:37.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[215775]: failed to encode map e567 with expected crc 2023-09-23T22:41:37.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[215775]: failed to encode map e567 with expected crc 2023-09-23T22:41:37.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:41:37.556 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[215775]: osdmap e567: 8 total, 6 up, 6 in 2023-09-23T22:41:37.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[223014]: pgmap v1469: 119 pgs: 1 peering, 19 remapped+peering, 25 active+clean+remapped, 16 undersized+peered, 16 active+undersized, 16 active+undersized+remapped, 2 undersized+degraded+peered, 10 active+undersized+degraded, 14 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 43/237 objects degraded (18.143%); 22/237 objects misplaced (9.283%) 2023-09-23T22:41:37.556 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:37.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[223014]: failed to encode map e567 with expected crc 2023-09-23T22:41:37.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[223014]: failed to encode map e567 with expected crc 2023-09-23T22:41:37.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:41:37.557 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:37 smithi026 ceph-mon[223014]: osdmap e567: 8 total, 6 up, 6 in 2023-09-23T22:41:37.591 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:37.622 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:37.772 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:37.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:37.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:37.870 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:41:37.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:37 smithi070 ceph-mon[192652]: pgmap v1469: 119 pgs: 1 peering, 19 remapped+peering, 25 active+clean+remapped, 16 undersized+peered, 16 active+undersized, 16 active+undersized+remapped, 2 undersized+degraded+peered, 10 active+undersized+degraded, 14 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 43/237 objects degraded (18.143%); 22/237 objects misplaced (9.283%) 2023-09-23T22:41:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:37 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:37 smithi070 ceph-mon[192652]: failed to encode map e567 with expected crc 2023-09-23T22:41:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:37 smithi070 ceph-mon[192652]: failed to encode map e567 with expected crc 2023-09-23T22:41:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:37 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:41:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:37 smithi070 ceph-mon[192652]: osdmap e567: 8 total, 6 up, 6 in 2023-09-23T22:41:37.875 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:37.903 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:37.948 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:37.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:38.051 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:38.078 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:38.125 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:38.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:38.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:38.228 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:38.229 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:38.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 27 16 67 51 0.118041 0 - 0.00470231 2023-09-23T22:41:38.256 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:38.502 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:38.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:38.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:38.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:38 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 18 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:41:38.597 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 43/237 objects degraded (18.143%), 12 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:38.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:38 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 18 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:41:38.597 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 43/237 objects degraded (18.143%), 12 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:38.606 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:38.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:38.740 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:38.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:38.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:38.845 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:38.870 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:38 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 18 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:41:38.871 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 43/237 objects degraded (18.143%), 12 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:38.873 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:38.918 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:38.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:38.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:39.025 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:39.055 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:39.230 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 28 16 67 51 0.113826 0 - 0.00470231 2023-09-23T22:41:39.353 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:39.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:39.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:39.392 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:41:39.399 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:41:39.399 INFO:teuthology.orchestra.run.smithi070.stdout: "success": "" 2023-09-23T22:41:39.400 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:41:39.420 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:41:39.434 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:41:39.439 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:41:39.457 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:39.461 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:41:39.482 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:41:39.486 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:39.508 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:41:39.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:41:39.508 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:41:39.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:41:39.509 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:41:39.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:41:39.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:41:39.509 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:41:39.509 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:41:39.547 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:39.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:39.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:39.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:39 smithi026 ceph-mon[223014]: pgmap v1471: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 25 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 198 KiB/s, 2 keys/s, 4 objects/s recovering 2023-09-23T22:41:39.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:39.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:39 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3312596910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:39.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:39 smithi026 ceph-mon[215775]: pgmap v1471: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 25 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 198 KiB/s, 2 keys/s, 4 objects/s recovering 2023-09-23T22:41:39.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:39.570 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:39 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3312596910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:39.651 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:39.681 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:39.724 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:39.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:39.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:39.745 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:41:39.745 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:41:39.745 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:41:39.745 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:41:39.745 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:41:39.829 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:39.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:39 smithi070 ceph-mon[192652]: pgmap v1471: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 25 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 198 KiB/s, 2 keys/s, 4 objects/s recovering 2023-09-23T22:41:39.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:39.852 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:39 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3312596910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:39.855 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:39.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:41:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:41:39] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.43.0" 2023-09-23T22:41:40.010 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:40.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:40.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:40.114 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:40.140 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:40.181 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:40.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:40.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:40.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 29 16 67 51 0.109901 0 - 0.00470231 2023-09-23T22:41:40.287 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:40.319 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:40.500 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:40 smithi026 ceph-mon[215775]: pgmap v1472: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 25 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 139 KiB/s, 1 keys/s, 2 objects/s recovering 2023-09-23T22:41:40.500 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:40 smithi026 ceph-mon[223014]: pgmap v1472: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 25 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 139 KiB/s, 1 keys/s, 2 objects/s recovering 2023-09-23T22:41:40.563 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:41:40.564 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:41:40.632 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:40.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:40.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:40.688 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:41:40.736 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:40.763 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:40 smithi070 ceph-mon[192652]: pgmap v1472: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 25 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 139 KiB/s, 1 keys/s, 2 objects/s recovering 2023-09-23T22:41:40.765 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:40.786 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:40.810 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:40.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:40.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:40.917 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:40.943 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:40.984 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:40.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:40.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:41.088 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:41.117 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:41.169 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:41.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:41.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:41.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 30 16 67 51 0.106237 0 - 0.00470231 2023-09-23T22:41:41.281 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:41.308 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:41.351 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:41.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:41.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:41.457 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:41.483 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:41.524 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:41.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:41.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:41.629 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:41.659 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:41.725 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:41.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:41.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:41.760 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:41.829 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:41.858 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:41.960 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:41.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:41.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:42.064 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:42.091 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:42.134 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:42.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:42.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:42.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 31 16 67 51 0.10281 0 - 0.00470231 2023-09-23T22:41:42.239 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:42.270 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:42.565 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:41:42.856 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:42 smithi026 ceph-mon[223014]: pgmap v1473: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 116 KiB/s, 1 keys/s, 2 objects/s recovering 2023-09-23T22:41:42.856 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:42 smithi026 ceph-mon[215775]: pgmap v1473: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 116 KiB/s, 1 keys/s, 2 objects/s recovering 2023-09-23T22:41:42.882 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:42.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:42.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:42.974 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:42.988 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:43.021 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:43.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:42 smithi070 ceph-mon[192652]: pgmap v1473: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 116 KiB/s, 1 keys/s, 2 objects/s recovering 2023-09-23T22:41:43.124 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:43.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:43.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:43.229 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:43.232 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 32 16 67 51 0.0995977 0 - 0.00470231 2023-09-23T22:41:43.263 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:43.384 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:43.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:43.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:43.401 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:41:43.401 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:41:43.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:41:43.401 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:41:43.401 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.401 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112962 9.a 9:52973282:::benchmark_data_smithi026_555954_object18233:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.401 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328512+0000", 2023-09-23T22:41:43.401 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.069973127, 2023-09-23T22:41:43.401 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98606011100000002, 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112962 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.402 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328512+0000", 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328512+0000", 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328511+0000", 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:43.403 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328518+0000", 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5150000000000003e-06 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328529+0000", 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1212e-05 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.404 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328542+0000", 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2842e-05 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328594+0000", 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1993999999999997e-05 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328604+0000", 2023-09-23T22:41:43.405 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0642000000000001e-05 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.308385+0000", 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97978035900000005 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.308529+0000", 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014458099999999999 2023-09-23T22:41:43.406 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.308664+0000", 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134287 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314379+0000", 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057154340000000001 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.407 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314423+0000", 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4088000000000002e-05 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314474+0000", 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1251000000000001e-05 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.408 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314573+0000", 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8189000000000005e-05 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112963 9.a 9:52aabdda:::benchmark_data_smithi026_555954_object18234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328560+0000", 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.069925084, 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99365022599999997, 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.409 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112963 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328560+0000", 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.410 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328560+0000", 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328560+0000", 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.411 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328563+0000", 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4599999999999999e-06 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328571+0000", 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7770000000000002e-06 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328578+0000", 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4919999999999998e-06 2023-09-23T22:41:43.412 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328682+0000", 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010416000000000001 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328691+0000", 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6969999999999999e-06 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.413 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.312186+0000", 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98349548499999995 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.312326+0000", 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014028200000000001 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.414 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.312467+0000", 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140822 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322021+0000", 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095533800000000002 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322035+0000", 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4491000000000001e-05 2023-09-23T22:41:43.415 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322095+0000", 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9570000000000001e-05 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322211+0000", 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011613 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.416 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112964 9.a 9:5eeb1e94:::benchmark_data_smithi026_555954_object18235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328584+0000", 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.06990205700001, 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.004056794, 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.417 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112964 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328584+0000", 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328584+0000", 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.418 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328583+0000", 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328586+0000", 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8080000000000001e-06 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.419 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328593+0000", 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8360000000000003e-06 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328598+0000", 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2989999999999996e-06 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328724+0000", 2023-09-23T22:41:43.420 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012606499999999999 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328729+0000", 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9400000000000001e-06 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314618+0000", 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98588852199999999 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.421 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314723+0000", 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000105509 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.314816+0000", 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3148999999999999e-05 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.327349+0000", 2023-09-23T22:41:43.422 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012532857999999999 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332500+0000", 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051508040000000001 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332541+0000", 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1054999999999999e-05 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.423 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332640+0000", 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9347999999999998e-05 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112967 9.a 9:571abd47:::benchmark_data_smithi026_555954_object18238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328729+0000", 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.069756654, 2023-09-23T22:41:43.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0040623639999999, 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112967 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328729+0000", 2023-09-23T22:41:43.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328729+0000", 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328728+0000", 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:41:43.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328734+0000", 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7930000000000003e-06 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328744+0000", 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8709999999999992e-06 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328749+0000", 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2889999999999997e-06 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328789+0000", 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0247000000000003e-05 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328797+0000", 2023-09-23T22:41:43.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0679999999999997e-06 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.316738+0000", 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98794093299999997 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.316885+0000", 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014646900000000001 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.317035+0000", 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015076900000000001 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.327390+0000", 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010354531 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332650+0000", 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052601890000000002 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332697+0000", 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7135000000000002e-05 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332791+0000", 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3982999999999998e-05 2023-09-23T22:41:43.431 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112968 9.2 9:4d21db9d:::benchmark_data_smithi026_555954_object18239:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328773+0000", 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.06971232800001, 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.008910618, 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.432 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112968 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328773+0000", 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328773+0000", 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328773+0000", 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328776+0000", 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8070000000000001e-06 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328784+0000", 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2840000000000001e-06 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328789+0000", 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7030000000000003e-06 2023-09-23T22:41:43.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328815+0000", 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5333999999999998e-05 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328819+0000", 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2710000000000003e-06 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322249+0000", 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99342982999999996 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322398+0000", 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014946299999999999 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,7", 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.322843+0000", 2023-09-23T22:41:43.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044482300000000001 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.333772+0000", 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010928927999999999 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.334363+0000", 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059104700000000004 2023-09-23T22:41:43.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.337515+0000", 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031519579999999998 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.337571+0000", 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5683000000000002e-05 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.337684+0000", 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000113097 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112969 9.3 9:caad47cb:::benchmark_data_smithi026_555954_object18240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.069505967, 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6657709080000001, 2023-09-23T22:41:43.440 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112969 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328979+0000", 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328983+0000", 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.693e-06 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328991+0000", 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2220000000000002e-06 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328999+0000", 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4870000000000002e-06 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.329055+0000", 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6026999999999997e-05 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.329062+0000", 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.161e-06 2023-09-23T22:41:43.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.976223+0000", 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6471613999999999 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.977503+0000", 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012796629999999999 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:41:43.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.979185+0000", 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001682149 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.984402+0000", 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052169599999999997 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.991082+0000", 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066800590000000003 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.992088+0000", 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001005964 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.992639+0000", 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055137800000000005 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.994750+0000", 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021113310000000001 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112986 9.a 9:56c0f53c:::benchmark_data_smithi026_555954_object18257:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.354912+0000", 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.04357398499999, 2023-09-23T22:41:43.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97904756500000001, 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112986 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354912+0000", 2023-09-23T22:41:43.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354912+0000", 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354911+0000", 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354916+0000", 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3979999999999998e-06 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354926+0000", 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0438000000000001e-05 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354932+0000", 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.694e-06 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354986+0000", 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.427e-05 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354997+0000", 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0295999999999999e-05 2023-09-23T22:41:43.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.319413+0000", 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96441663899999996 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.319544+0000", 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000130939 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.319686+0000", 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014142899999999999 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.332804+0000", 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013118094 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.454 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.333812+0000", 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001008461 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.333865+0000", 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2939999999999998e-05 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.333959+0000", 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3908999999999998e-05 2023-09-23T22:41:43.455 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112990 9.3 9:c43d78c1:::benchmark_data_smithi026_555954_object18261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.03941932199999, 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.640044252, 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.456 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112990 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:41:43.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359065+0000", 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359070+0000", 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9409999999999998e-06 2023-09-23T22:41:43.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359083+0000", 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2697999999999999e-05 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359089+0000", 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7200000000000003e-06 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.459 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359144+0000", 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5954e-05 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359157+0000", 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2853e-05 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.984626+0000", 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.625468431 2023-09-23T22:41:43.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.986258+0000", 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016318319999999999 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.988082+0000", 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018244540000000001 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.995233+0000", 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071506429999999999 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.995368+0000", 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135067 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.996086+0000", 2023-09-23T22:41:43.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00071818499999999998 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.996659+0000", 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057308399999999996 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.999110+0000", 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024514530000000001 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.463 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112992 9.3 9:cb6055b3:::benchmark_data_smithi026_555954_object18263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.03796121400001, 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6580400470000001, 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.464 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112992 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:41:43.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360528+0000", 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4259999999999996e-06 2023-09-23T22:41:43.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360540+0000", 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1824999999999999e-05 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360545+0000", 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5289999999999999e-06 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360597+0000", 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1746000000000002e-05 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360609+0000", 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1953000000000001e-05 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.999299+0000", 2023-09-23T22:41:43.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.638690135 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.000875+0000", 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015752990000000001 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.002719+0000", 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018446700000000001 2023-09-23T22:41:43.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015384+0000", 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012664969999999999 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015497+0000", 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011238 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015923+0000", 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042655200000000002 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.016447+0000", 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052434900000000002 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.471 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.018564+0000", 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021169520000000001 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112993 9.3 9:cd412e89:::benchmark_data_smithi026_555954_object18264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 158.03451387999999, 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.658274966, 2023-09-23T22:41:43.472 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112993 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363971+0000", 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363975+0000", 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0169999999999998e-06 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363986+0000", 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0569e-05 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363991+0000", 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0599999999999998e-06 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364039+0000", 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8767999999999998e-05 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364049+0000", 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8279999999999998e-06 2023-09-23T22:41:43.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.008138+0000", 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6440885409999999 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.009772+0000", 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016337540000000001 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.477 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.011820+0000", 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020481570000000001 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.018990+0000", 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071701450000000002 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.019117+0000", 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012743300000000001 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.019436+0000", 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000318239 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.020027+0000", 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059157899999999995 2023-09-23T22:41:43.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.022247+0000", 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022195629999999999 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188104 9.e 9:7689f13c:::benchmark_data_smithi026_555954_object93375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:41:43.480 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.984819553000001, 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.09525067, 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188104 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:41:43.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413668+0000", 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6299999999999998e-06 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413674+0000", 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.241e-06 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413680+0000", 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1270000000000002e-06 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419176+0000", 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054967100000000001 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419186+0000", 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2289999999999992e-06 2023-09-23T22:41:43.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.502956+0000", 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0837702039999999 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.503056+0000", 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100003 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.503139+0000", 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2886999999999994e-05 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505036+0000", 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001897756 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506288+0000", 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001251816 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508853+0000", 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025648149999999998 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508885+0000", 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2224999999999997e-05 2023-09-23T22:41:43.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508917+0000", 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1417000000000001e-05 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188105 9.e 9:72ff4f7f:::benchmark_data_smithi026_555954_object93376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.415823+0000", 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.982662529999999, 2023-09-23T22:41:43.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094443748, 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188105 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415823+0000", 2023-09-23T22:41:43.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415823+0000", 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415822+0000", 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:41:43.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415828+0000", 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9649999999999996e-06 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415843+0000", 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4371e-05 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415851+0000", 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4130000000000006e-06 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419209+0000", 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003357942 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419219+0000", 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0190999999999999e-05 2023-09-23T22:41:43.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505077+0000", 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0858583820000001 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505177+0000", 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9227000000000004e-05 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505256+0000", 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8937000000000007e-05 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508765+0000", 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035097739999999998 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508778+0000", 2023-09-23T22:41:43.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2464e-05 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510218+0000", 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001440059 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510245+0000", 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6982e-05 2023-09-23T22:41:43.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510267+0000", 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1889e-05 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188109 9.e 9:78387bd9:::benchmark_data_smithi026_555954_object93380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.421149+0000", 2023-09-23T22:41:43.496 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.977336780999998, 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094876424, 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188109 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421149+0000", 2023-09-23T22:41:43.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421149+0000", 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421148+0000", 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:43.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421154+0000", 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7089999999999996e-06 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421165+0000", 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0801e-05 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421173+0000", 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9389999999999997e-06 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.422829+0000", 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001656299 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.422839+0000", 2023-09-23T22:41:43.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7890000000000006e-06 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506315+0000", 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0834760299999999 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506388+0000", 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3413999999999999e-05 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.501 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506451+0000", 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2315000000000002e-05 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508814+0000", 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023629089999999998 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.502 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508933+0000", 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119478 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.515963+0000", 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070293259999999998 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.515994+0000", 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1168999999999999e-05 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.503 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516025+0000", 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1522000000000001e-05 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188140 9.e 9:78d2bdae:::benchmark_data_smithi026_555954_object93411:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.458307+0000", 2023-09-23T22:41:43.504 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.940178838000001, 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.057861621, 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188140 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458307+0000", 2023-09-23T22:41:43.505 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458307+0000", 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458306+0000", 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.506 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458310+0000", 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.208e-06 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458315+0000", 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9190000000000004e-06 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.507 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458320+0000", 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.989e-06 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471337+0000", 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013016099 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471347+0000", 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0324e-05 2023-09-23T22:41:43.508 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.507497+0000", 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0361505470000001 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.507566+0000", 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8753999999999999e-05 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.509 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.507632+0000", 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5599999999999995e-05 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508942+0000", 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001310077 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510149+0000", 2023-09-23T22:41:43.510 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012073489999999999 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516108+0000", 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059589150000000004 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516143+0000", 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5042000000000001e-05 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.511 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516168+0000", 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5201000000000001e-05 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188143 9.e 9:7424b661:::benchmark_data_smithi026_555954_object93414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.463773+0000", 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.934712161, 2023-09-23T22:41:43.512 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0524969799999999, 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188143 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463773+0000", 2023-09-23T22:41:43.513 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463773+0000", 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463772+0000", 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:43.514 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463779+0000", 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5320000000000003e-06 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463791+0000", 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2319e-05 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.515 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463803+0000", 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2082000000000001e-05 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471367+0000", 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075638820000000001 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471373+0000", 2023-09-23T22:41:43.516 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0279999999999997e-06 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508966+0000", 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0375934039999999 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.509038+0000", 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1544999999999993e-05 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.517 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.509111+0000", 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3699e-05 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.511865+0000", 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002753148 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.518 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.513751+0000", 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001886307 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516216+0000", 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024647459999999999 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516246+0000", 2023-09-23T22:41:43.519 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0256999999999998e-05 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516270+0000", 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4433000000000001e-05 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188144 9.e 9:73d3d7cc:::benchmark_data_smithi026_555954_object93415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:41:43.520 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.463813+0000", 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.934672591000002, 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.052639544, 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188144 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.521 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463813+0000", 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463813+0000", 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463812+0000", 2023-09-23T22:41:43.522 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463816+0000", 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2349999999999999e-06 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463823+0000", 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.294e-06 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.523 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463828+0000", 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5160000000000002e-06 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471392+0000", 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075633810000000001 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.524 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471398+0000", 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7570000000000001e-06 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510314+0000", 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.038916022 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510382+0000", 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8048999999999998e-05 2023-09-23T22:41:43.525 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510445+0000", 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3137000000000002e-05 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.511918+0000", 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001472714 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516042+0000", 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041248190000000001 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516374+0000", 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000331746 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516419+0000", 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4904999999999998e-05 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516453+0000", 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3528000000000002e-05 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188148 9.e 9:7e5e885c:::benchmark_data_smithi026_555954_object93419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:41:43.528 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.469012+0000", 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.929473891000001, 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.050155932, 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188148 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.469012+0000", 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.469012+0000", 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.469010+0000", 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.469018+0000", 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.354e-06 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.469034+0000", 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5333e-05 2023-09-23T22:41:43.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.469043+0000", 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0350000000000007e-06 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471416+0000", 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002373146 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471422+0000", 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6099999999999997e-06 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.511955+0000", 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.040533208 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.512077+0000", 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122633 2023-09-23T22:41:43.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.512194+0000", 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116255 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516281+0000", 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040870139999999999 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516331+0000", 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0640999999999998e-05 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519077+0000", 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027459429999999998 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519126+0000", 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8538999999999997e-05 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519168+0000", 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1829999999999998e-05 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.536 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188151 9.e 9:7d27ae6e:::benchmark_data_smithi026_555954_object93422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.477000+0000", 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.921485273000002, 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0422953129999999, 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188151 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.477000+0000", 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.477000+0000", 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.476998+0000", 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.477007+0000", 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3259999999999996e-06 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.477023+0000", 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6171000000000002e-05 2023-09-23T22:41:43.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.477032+0000", 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3819999999999993e-06 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.478020+0000", 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098777700000000001 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.478031+0000", 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.081e-05 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.513804+0000", 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0357731729999999 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.513924+0000", 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011969299999999999 2023-09-23T22:41:43.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.514017+0000", 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3337000000000004e-05 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516488+0000", 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024711239999999999 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519031+0000", 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002542636 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519211+0000", 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018065799999999999 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519258+0000", 2023-09-23T22:41:43.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6754e-05 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519296+0000", 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7509000000000003e-05 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188166 9.e 9:7a39430d:::benchmark_data_smithi026_555954_object93437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.492851+0000", 2023-09-23T22:41:43.544 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.905634239000001, 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0335386740000001, 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188166 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.492851+0000", 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.492851+0000", 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.492851+0000", 2023-09-23T22:41:43.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.492854+0000", 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4479999999999999e-06 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.492862+0000", 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5340000000000002e-06 2023-09-23T22:41:43.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.492868+0000", 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9009999999999999e-06 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.492923+0000", 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4988999999999998e-05 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.492932+0000", 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9849999999999995e-06 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516508+0000", 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0235767680000001 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516618+0000", 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109178 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516737+0000", 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119667 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.521455+0000", 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0047182099999999996 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.521464+0000", 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3809999999999999e-06 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.526298+0000", 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0048344740000000001 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.526352+0000", 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3539999999999999e-05 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.526390+0000", 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8158999999999998e-05 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188214 9.e 9:7901f1c5:::benchmark_data_smithi026_555954_object93485:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.555807+0000", 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 52.842678343000003, 2023-09-23T22:41:43.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.97563505299999997, 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188214 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.555807+0000", 2023-09-23T22:41:43.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.555807+0000", 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.555806+0000", 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.555816+0000", 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0131e-05 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.555828+0000", 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1505000000000001e-05 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.555835+0000", 2023-09-23T22:41:43.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3250000000000002e-06 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.555882+0000", 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7030999999999997e-05 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.555888+0000", 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3570000000000003e-06 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519325+0000", 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96343637199999999 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519452+0000", 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012739500000000001 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:41:43.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.519587+0000", 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013479400000000001 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.526240+0000", 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066528139999999999 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.526455+0000", 2023-09-23T22:41:43.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000215825 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.531357+0000", 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049014100000000001 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.531400+0000", 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3504999999999998e-05 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:41:43.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:41:43.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.531442+0000", 2023-09-23T22:41:43.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1860999999999998e-05 2023-09-23T22:41:43.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.560 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.560 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:41:43.560 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:41:43.560 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:41:43.560 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:43.587 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:43.632 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:43.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:43.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:43.737 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:43.745 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:41:43.763 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:43.816 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:43.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:43.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/601406234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:43 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:41:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:43 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:43 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 10 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 39/237 objects degraded (16.456%), 10 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:43.919 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:43.946 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/601406234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:43.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 10 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 39/237 objects degraded (16.456%), 10 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/601406234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:41:43.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:41:43.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:43.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 10 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:43.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 39/237 objects degraded (16.456%), 10 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:44.040 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:44.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:44.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:44.147 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:44.174 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:44.220 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:44.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:44.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:44.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 33 16 67 51 0.0965797 0 - 0.00470231 2023-09-23T22:41:44.326 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:44.370 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:44.411 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:44.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:44.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:44.515 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:44.547 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:44.585 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:41:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[215775]: pgmap v1474: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 102 KiB/s, 1 keys/s, 4 objects/s recovering 2023-09-23T22:41:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[215775]: failed to encode map e568 with expected crc 2023-09-23T22:41:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[215775]: failed to encode map e568 with expected crc 2023-09-23T22:41:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[215775]: failed to encode map e568 with expected crc 2023-09-23T22:41:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[215775]: osdmap e568: 8 total, 6 up, 6 in 2023-09-23T22:41:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[223014]: pgmap v1474: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 102 KiB/s, 1 keys/s, 4 objects/s recovering 2023-09-23T22:41:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[223014]: failed to encode map e568 with expected crc 2023-09-23T22:41:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[223014]: failed to encode map e568 with expected crc 2023-09-23T22:41:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[223014]: failed to encode map e568 with expected crc 2023-09-23T22:41:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:44.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:44 smithi026 ceph-mon[223014]: osdmap e568: 8 total, 6 up, 6 in 2023-09-23T22:41:45.087 INFO:tasks.thrashosds.thrasher:in_osds: [5, 0, 1, 6, 2, 3] out_osds: [4, 7] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:41:45.087 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:41:45.087 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [5, 0, 1, 6, 2, 3] 2023-09-23T22:41:45.087 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T22:41:45.100 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:45.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:45.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:45.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:44 smithi070 ceph-mon[192652]: pgmap v1474: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 102 KiB/s, 1 keys/s, 4 objects/s recovering 2023-09-23T22:41:45.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:44 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:45.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:44 smithi070 ceph-mon[192652]: failed to encode map e568 with expected crc 2023-09-23T22:41:45.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:44 smithi070 ceph-mon[192652]: failed to encode map e568 with expected crc 2023-09-23T22:41:45.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:44 smithi070 ceph-mon[192652]: failed to encode map e568 with expected crc 2023-09-23T22:41:45.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:44 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:41:45.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:44 smithi070 ceph-mon[192652]: osdmap e568: 8 total, 6 up, 6 in 2023-09-23T22:41:45.206 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:45.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 34 16 67 51 0.0937392 0 - 0.00470231 2023-09-23T22:41:45.244 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:45.245 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:45.313 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:45.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:45.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:45.417 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:45.452 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:45.679 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:41:45.698 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:41:45.718 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:41:45.744 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:41:45.753 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:45.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:45.777 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:41:45.800 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:41:45.800 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:41:45.800 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:41:45.800 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:41:45.800 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:41:45.801 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:41:45.801 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:41:45.801 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:41:45.801 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:41:45.862 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:45.899 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:46.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:45 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/147458162' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:46.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 35 16 67 51 0.0910609 0 - 0.00470231 2023-09-23T22:41:46.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:45 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/147458162' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:46.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:45 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/147458162' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:46.648 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:46.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:46.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:46.751 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:46.780 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:47.014 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:41:47.014 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:41:47.015 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:47.033 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:47.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:47.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:47.092 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:41:47.143 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:47.171 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:47.213 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:47.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:47.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:47.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 36 16 67 51 0.0885315 0 - 0.00470231 2023-09-23T22:41:47.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:47.347 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:47.388 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:47.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:47.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:47.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:47 smithi026 ceph-mon[223014]: pgmap v1476: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 49 KiB/s, 1 keys/s, 3 objects/s recovering 2023-09-23T22:41:47.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:47 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:47.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:47 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:47.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:47 smithi026 ceph-mon[215775]: pgmap v1476: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 49 KiB/s, 1 keys/s, 3 objects/s recovering 2023-09-23T22:41:47.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:47 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:47.466 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:47 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:47.487 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:47.487 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:47.495 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:47.521 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:47.563 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:47.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:47.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:47.669 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:47.695 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:47.736 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:47.742 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:47 smithi070 ceph-mon[192652]: pgmap v1476: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%); 49 KiB/s, 1 keys/s, 3 objects/s recovering 2023-09-23T22:41:47.742 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:47 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:47.742 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:47 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:47.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:47.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:47.843 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:47.869 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:47.910 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:47.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:47.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:48.016 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:48.042 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:48.084 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:48.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:48.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:48.189 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:48.224 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:48.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 37 16 67 51 0.0861388 0 - 0.00470231 2023-09-23T22:41:48.318 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:41:48.374 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:48.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:48.479 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:48.508 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:48.550 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:48.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:48.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:48.657 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:48.683 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:48.725 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:48.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:48.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:48.804 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:48.830 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:48.870 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:49.000 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:49.015 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:41:49.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:49.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:49.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:49.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:49.217 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:49.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 38 16 67 51 0.0838721 0 - 0.00470231 2023-09-23T22:41:49.601 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:49.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:49 smithi026 ceph-mon[223014]: pgmap v1477: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%) 2023-09-23T22:41:49.603 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:49 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 39/237 objects degraded (16.456%), 10 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:49.603 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:49 smithi026 ceph-mon[215775]: pgmap v1477: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%) 2023-09-23T22:41:49.604 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:49 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 39/237 objects degraded (16.456%), 10 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:49.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:49.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:49 smithi070 ceph-mon[192652]: pgmap v1477: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%) 2023-09-23T22:41:49.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:49 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 39/237 objects degraded (16.456%), 10 pgs degraded, 25 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:49.705 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:49.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:49.782 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:49.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:49.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:49.889 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:49.917 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:49.960 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:49.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:49.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:41:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:41:49] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.43.0" 2023-09-23T22:41:49.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:49 smithi026 conmon[215751]: 2023-09-23T22:41:49.804+0000 7f4580f04700 -1 mon.a@0(leader).osd e568 definitely_dead 0 2023-09-23T22:41:50.068 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:50.093 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:50.135 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:50.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:50.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:50.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 39 16 67 51 0.0817215 0 - 0.00470231 2023-09-23T22:41:50.242 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:50.274 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:50.349 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3083932726' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:41:50.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:50 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:41:50.350 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:50 smithi026 ceph-mon[223014]: Client client.admin marked osd.3 out, after it was down for 165 seconds 2023-09-23T22:41:50.390 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.3. 2023-09-23T22:41:50.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3083932726' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:41:50.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:50 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:41:50.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:50 smithi070 ceph-mon[192652]: Client client.admin marked osd.3 out, after it was down for 165 seconds 2023-09-23T22:41:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3083932726' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:41:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:50 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T22:41:50.665 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:50 smithi026 ceph-mon[215775]: Client client.admin marked osd.3 out, after it was down for 165 seconds 2023-09-23T22:41:50.708 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:50.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:50.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:50.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:50.841 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:51.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:41:51.232140+0000 min lat: 0.00118217 max lat: 0.00897703 avg lat: 0.00470231 2023-09-23T22:41:51.232 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:41:51.232 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 40 16 67 51 0.0796784 0 - 0.00470231 2023-09-23T22:41:51.292 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:51.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:51.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:51.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:51.461 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:51.542 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[215775]: pgmap v1478: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%) 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[215775]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[215775]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[215775]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[215775]: osdmap e569: 8 total, 6 up, 5 in 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[215775]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[215775]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[223014]: pgmap v1478: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%) 2023-09-23T22:41:51.574 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[223014]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T22:41:51.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[223014]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[223014]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T22:41:51.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[223014]: osdmap e569: 8 total, 6 up, 5 in 2023-09-23T22:41:51.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[223014]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:51 smithi026 ceph-mon[223014]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:51 smithi070 ceph-mon[192652]: pgmap v1478: 119 pgs: 49 active+clean+remapped, 13 undersized+peered, 10 active+undersized, 1 undersized+degraded+peered, 25 active+undersized+remapped, 9 active+undersized+degraded, 12 active+clean; 7.8 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 39/237 objects degraded (16.456%); 50/237 objects misplaced (21.097%) 2023-09-23T22:41:51.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:51 smithi070 ceph-mon[192652]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T22:41:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:51 smithi070 ceph-mon[192652]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:51 smithi070 ceph-mon[192652]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:51 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T22:41:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:51 smithi070 ceph-mon[192652]: osdmap e569: 8 total, 6 up, 5 in 2023-09-23T22:41:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:51 smithi070 ceph-mon[192652]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:51 smithi070 ceph-mon[192652]: failed to encode map e569 with expected crc 2023-09-23T22:41:51.658 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:51.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:51.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:51.765 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:51.798 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:51.957 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:51.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:51.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:52.061 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:52.094 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:52.231 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 41 16 211 195 0.297222 0.225 0.00274731 3.30196 2023-09-23T22:41:52.379 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:41:52.612 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:52.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[223014]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[223014]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[223014]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[223014]: osdmap e570: 8 total, 6 up, 5 in 2023-09-23T22:41:52.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[223014]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[223014]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[223014]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1304588939' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:52.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[215775]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[215775]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[215775]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[215775]: osdmap e570: 8 total, 6 up, 5 in 2023-09-23T22:41:52.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[215775]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[215775]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[215775]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1304588939' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:52.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:52.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:52.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:52 smithi070 ceph-mon[192652]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:52 smithi070 ceph-mon[192652]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:52 smithi070 ceph-mon[192652]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:52 smithi070 ceph-mon[192652]: osdmap e570: 8 total, 6 up, 5 in 2023-09-23T22:41:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:52 smithi070 ceph-mon[192652]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:52 smithi070 ceph-mon[192652]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:52 smithi070 ceph-mon[192652]: failed to encode map e570 with expected crc 2023-09-23T22:41:52.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1304588939' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:41:52.717 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:52.744 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:52.785 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:52.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:52.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:52.839 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:41:52.839 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:41:52.891 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:52.918 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:52.963 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:52.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:52.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:53.069 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:53.137 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:53.232 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 42 16 1815 1799 2.67678 100.25 0.0049084 0.373562 2023-09-23T22:41:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[215775]: pgmap v1481: 119 pgs: 3 peering, 28 remapped+peering, 39 active+clean+remapped, 11 undersized+peered, 9 active+undersized, 1 undersized+degraded+peered, 8 active+undersized+remapped, 9 active+undersized+degraded, 11 active+clean; 7.8 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 39/237 objects degraded (16.456%); 29/237 objects misplaced (12.236%) 2023-09-23T22:41:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[215775]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[215775]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[215775]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1304588939' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[215775]: osdmap e571: 8 total, 6 up, 5 in 2023-09-23T22:41:53.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[215775]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[223014]: pgmap v1481: 119 pgs: 3 peering, 28 remapped+peering, 39 active+clean+remapped, 11 undersized+peered, 9 active+undersized, 1 undersized+degraded+peered, 8 active+undersized+remapped, 9 active+undersized+degraded, 11 active+clean; 7.8 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 39/237 objects degraded (16.456%); 29/237 objects misplaced (12.236%) 2023-09-23T22:41:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[223014]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[223014]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[223014]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1304588939' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[223014]: osdmap e571: 8 total, 6 up, 5 in 2023-09-23T22:41:53.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:53 smithi026 ceph-mon[223014]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:53 smithi070 ceph-mon[192652]: pgmap v1481: 119 pgs: 3 peering, 28 remapped+peering, 39 active+clean+remapped, 11 undersized+peered, 9 active+undersized, 1 undersized+degraded+peered, 8 active+undersized+remapped, 9 active+undersized+degraded, 11 active+clean; 7.8 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 39/237 objects degraded (16.456%); 29/237 objects misplaced (12.236%) 2023-09-23T22:41:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:53 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:41:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:53 smithi070 ceph-mon[192652]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:53 smithi070 ceph-mon[192652]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:53 smithi070 ceph-mon[192652]: failed to encode map e571 with expected crc 2023-09-23T22:41:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:53 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1304588939' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:41:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:53 smithi070 ceph-mon[192652]: osdmap e571: 8 total, 6 up, 5 in 2023-09-23T22:41:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:53 smithi070 ceph-mon[192652]: failed to encode map e571 with expected crc 2023-09-23T22:41:54.232 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 43 16 4743 4727 6.86984 183 0.00378039 0.145551 2023-09-23T22:41:54.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[215775]: 1.1d scrub starts 2023-09-23T22:41:54.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[215775]: 1.1d scrub ok 2023-09-23T22:41:54.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[215775]: failed to encode map e572 with expected crc 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[215775]: failed to encode map e572 with expected crc 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[215775]: osdmap e572: 8 total, 6 up, 5 in 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[215775]: failed to encode map e572 with expected crc 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[215775]: osdmap e573: 8 total, 6 up, 5 in 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[223014]: 1.1d scrub starts 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[223014]: 1.1d scrub ok 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[223014]: failed to encode map e572 with expected crc 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[223014]: failed to encode map e572 with expected crc 2023-09-23T22:41:54.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[223014]: osdmap e572: 8 total, 6 up, 5 in 2023-09-23T22:41:54.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[223014]: failed to encode map e572 with expected crc 2023-09-23T22:41:54.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:54.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:54 smithi026 ceph-mon[223014]: osdmap e573: 8 total, 6 up, 5 in 2023-09-23T22:41:54.760 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:41:54.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:54.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:41:54.866 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:54 smithi070 ceph-mon[192652]: 1.1d scrub starts 2023-09-23T22:41:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:54 smithi070 ceph-mon[192652]: 1.1d scrub ok 2023-09-23T22:41:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:54 smithi070 ceph-mon[192652]: failed to encode map e572 with expected crc 2023-09-23T22:41:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:54 smithi070 ceph-mon[192652]: failed to encode map e572 with expected crc 2023-09-23T22:41:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:54 smithi070 ceph-mon[192652]: osdmap e572: 8 total, 6 up, 5 in 2023-09-23T22:41:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:54 smithi070 ceph-mon[192652]: failed to encode map e572 with expected crc 2023-09-23T22:41:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:41:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:54 smithi070 ceph-mon[192652]: osdmap e573: 8 total, 6 up, 5 in 2023-09-23T22:41:54.893 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:54.938 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:54.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:54.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:55.043 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:41:55.070 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:55.115 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:41:55.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:55.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:41:55.221 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:55.233 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 44 16 7362 7346 10.4332 163.688 0.00713626 0.0958349 2023-09-23T22:41:55.286 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[223014]: pgmap v1484: 119 pgs: 3 peering, 29 remapped+peering, 38 active+clean+remapped, 5 undersized+peered, 8 active+undersized, 1 active+undersized+remapped, 11 active+undersized+degraded, 24 active+clean; 24 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 4.1 MiB/s wr, 65 op/s; 155/1023 objects degraded (15.152%); 37/1023 objects misplaced (3.617%); 320 KiB/s, 12 objects/s recovering 2023-09-23T22:41:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[223014]: failed to encode map e573 with expected crc 2023-09-23T22:41:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[223014]: failed to encode map e573 with expected crc 2023-09-23T22:41:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[223014]: failed to encode map e573 with expected crc 2023-09-23T22:41:55.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 7 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:55.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 155/1023 objects degraded (15.152%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:55.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[215775]: pgmap v1484: 119 pgs: 3 peering, 29 remapped+peering, 38 active+clean+remapped, 5 undersized+peered, 8 active+undersized, 1 active+undersized+remapped, 11 active+undersized+degraded, 24 active+clean; 24 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 4.1 MiB/s wr, 65 op/s; 155/1023 objects degraded (15.152%); 37/1023 objects misplaced (3.617%); 320 KiB/s, 12 objects/s recovering 2023-09-23T22:41:55.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[215775]: failed to encode map e573 with expected crc 2023-09-23T22:41:55.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[215775]: failed to encode map e573 with expected crc 2023-09-23T22:41:55.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[215775]: failed to encode map e573 with expected crc 2023-09-23T22:41:55.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 7 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:55.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:55 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 155/1023 objects degraded (15.152%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:55.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:55 smithi070 ceph-mon[192652]: pgmap v1484: 119 pgs: 3 peering, 29 remapped+peering, 38 active+clean+remapped, 5 undersized+peered, 8 active+undersized, 1 active+undersized+remapped, 11 active+undersized+degraded, 24 active+clean; 24 MiB data, 3.9 GiB used, 443 GiB / 447 GiB avail; 4.1 MiB/s wr, 65 op/s; 155/1023 objects degraded (15.152%); 37/1023 objects misplaced (3.617%); 320 KiB/s, 12 objects/s recovering 2023-09-23T22:41:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:55 smithi070 ceph-mon[192652]: failed to encode map e573 with expected crc 2023-09-23T22:41:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:55 smithi070 ceph-mon[192652]: failed to encode map e573 with expected crc 2023-09-23T22:41:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:55 smithi070 ceph-mon[192652]: failed to encode map e573 with expected crc 2023-09-23T22:41:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:55 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 7 pgs inactive (PG_AVAILABILITY) 2023-09-23T22:41:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:55 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 155/1023 objects degraded (15.152%), 11 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:41:56.233 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 45 16 9826 9810 13.6232 154 0.00353828 0.0733959 2023-09-23T22:41:56.477 INFO:tasks.thrashosds.thrasher:in_osds: [5, 0, 1, 6, 2] out_osds: [4, 7, 3] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:41:56.478 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:41:56.478 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [5, 0, 1, 6, 2] 2023-09-23T22:41:56.478 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd out 0 2023-09-23T22:41:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:56 smithi026 ceph-mon[223014]: 1.0 scrub starts 2023-09-23T22:41:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:56 smithi026 ceph-mon[223014]: 1.0 scrub ok 2023-09-23T22:41:56.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:56 smithi026 ceph-mon[215775]: 1.0 scrub starts 2023-09-23T22:41:56.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:56 smithi026 ceph-mon[215775]: 1.0 scrub ok 2023-09-23T22:41:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:56 smithi070 ceph-mon[192652]: 1.0 scrub starts 2023-09-23T22:41:56.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:56 smithi070 ceph-mon[192652]: 1.0 scrub ok 2023-09-23T22:41:57.237 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 46 16 12043 12027 16.3388 138.562 0.00424766 0.061191 2023-09-23T22:41:57.278 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:57.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:57.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:57.363 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:41:57.406 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:57.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[223014]: pgmap v1486: 119 pgs: 3 peering, 28 remapped+peering, 37 active+clean+remapped, 4 undersized+peered, 9 active+undersized, 11 active+undersized+degraded, 27 active+clean; 55 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 11 MiB/s wr, 173 op/s; 646/2496 objects degraded (25.881%); 36/2496 objects misplaced (1.442%); 296 KiB/s, 12 objects/s recovering 2023-09-23T22:41:57.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[223014]: 1.1c scrub starts 2023-09-23T22:41:57.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[223014]: 1.1c scrub ok 2023-09-23T22:41:57.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:57.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:57.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[215775]: pgmap v1486: 119 pgs: 3 peering, 28 remapped+peering, 37 active+clean+remapped, 4 undersized+peered, 9 active+undersized, 11 active+undersized+degraded, 27 active+clean; 55 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 11 MiB/s wr, 173 op/s; 646/2496 objects degraded (25.881%); 36/2496 objects misplaced (1.442%); 296 KiB/s, 12 objects/s recovering 2023-09-23T22:41:57.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[215775]: 1.1c scrub starts 2023-09-23T22:41:57.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[215775]: 1.1c scrub ok 2023-09-23T22:41:57.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:57.428 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:57 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:57.513 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:57.748 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:57.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:57.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:57.815 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:57.853 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:41:57.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:57 smithi070 ceph-mon[192652]: pgmap v1486: 119 pgs: 3 peering, 28 remapped+peering, 37 active+clean+remapped, 4 undersized+peered, 9 active+undersized, 11 active+undersized+degraded, 27 active+clean; 55 MiB data, 4.0 GiB used, 443 GiB / 447 GiB avail; 11 MiB/s wr, 173 op/s; 646/2496 objects degraded (25.881%); 36/2496 objects misplaced (1.442%); 296 KiB/s, 12 objects/s recovering 2023-09-23T22:41:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:57 smithi070 ceph-mon[192652]: 1.1c scrub starts 2023-09-23T22:41:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:57 smithi070 ceph-mon[192652]: 1.1c scrub ok 2023-09-23T22:41:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:57 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:57 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:57.961 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:58.002 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:41:58.045 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:41:58.108 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:41:58.141 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:41:58.149 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:41:58.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:58.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:41:58.186 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:41:58.227 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:41:58.227 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:41:58.228 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:41:58.228 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:41:58.228 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:41:58.228 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:41:58.228 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:41:58.228 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:41:58.228 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:41:58.237 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 47 16 13371 13355 17.7555 83 0.0128788 0.0563133 2023-09-23T22:41:58.258 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:58.283 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:58.327 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:58.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:58.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:58.435 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:41:58.495 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:58.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:58 smithi070 ceph-mon[192652]: 1.15 scrub starts 2023-09-23T22:41:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:58 smithi070 ceph-mon[192652]: 1.15 scrub ok 2023-09-23T22:41:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3323327938' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:41:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:41:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[223014]: 1.15 scrub starts 2023-09-23T22:41:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[223014]: 1.15 scrub ok 2023-09-23T22:41:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3323327938' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:41:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:41:58.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:58.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[215775]: 1.15 scrub starts 2023-09-23T22:41:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[215775]: 1.15 scrub ok 2023-09-23T22:41:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3323327938' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:41:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:41:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:41:58.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:41:58.974 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:41:59.026 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:41:59.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:59.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:41:59.130 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:59.157 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:59.201 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:59.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:59.237 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 48 16 15410 15394 20.04 127.438 0.00863077 0.0498908 2023-09-23T22:41:59.306 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:41:59.323 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:41:59.333 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:59.377 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:41:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:59.382 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:41:59.482 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:41:59.509 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:59.552 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:41:59.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:41:59.560 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:41:59.660 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:41:59.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:59 smithi026 ceph-mon[215775]: pgmap v1487: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 681 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 112 MiB/s wr, 1.79k op/s; 5181/32535 objects degraded (15.924%); 1325/32535 objects misplaced (4.073%); 320 KiB/s, 11 objects/s recovering 2023-09-23T22:41:59.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:59 smithi026 ceph-mon[215775]: 1.9 scrub starts 2023-09-23T22:41:59.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:59 smithi026 ceph-mon[215775]: 1.9 scrub ok 2023-09-23T22:41:59.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:41:59 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive) 2023-09-23T22:41:59.739 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:41:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:41:59] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-09-23T22:41:59.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:59 smithi026 ceph-mon[223014]: pgmap v1487: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 681 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 112 MiB/s wr, 1.79k op/s; 5181/32535 objects degraded (15.924%); 1325/32535 objects misplaced (4.073%); 320 KiB/s, 11 objects/s recovering 2023-09-23T22:41:59.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:59 smithi026 ceph-mon[223014]: 1.9 scrub starts 2023-09-23T22:41:59.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:59 smithi026 ceph-mon[223014]: 1.9 scrub ok 2023-09-23T22:41:59.739 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:41:59 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive) 2023-09-23T22:41:59.749 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:41:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:59 smithi070 ceph-mon[192652]: pgmap v1487: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 681 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 112 MiB/s wr, 1.79k op/s; 5181/32535 objects degraded (15.924%); 1325/32535 objects misplaced (4.073%); 320 KiB/s, 11 objects/s recovering 2023-09-23T22:41:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:59 smithi070 ceph-mon[192652]: 1.9 scrub starts 2023-09-23T22:41:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:59 smithi070 ceph-mon[192652]: 1.9 scrub ok 2023-09-23T22:41:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:41:59 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 6 pgs inactive) 2023-09-23T22:42:00.237 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 49 16 17247 17231 21.9737 114.812 0.00124599 0.0454645 2023-09-23T22:42:01.239 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 50 16 18942 18926 23.652 105.938 0.00645053 0.0422736 2023-09-23T22:42:01.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:01 smithi026 ceph-mon[215775]: pgmap v1488: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 681 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 92 MiB/s wr, 1.47k op/s; 5181/32535 objects degraded (15.924%); 1325/32535 objects misplaced (4.073%); 262 KiB/s, 9 objects/s recovering 2023-09-23T22:42:01.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:01 smithi026 ceph-mon[223014]: pgmap v1488: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 681 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 92 MiB/s wr, 1.47k op/s; 5181/32535 objects degraded (15.924%); 1325/32535 objects misplaced (4.073%); 262 KiB/s, 9 objects/s recovering 2023-09-23T22:42:01.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:01 smithi070 ceph-mon[192652]: pgmap v1488: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 681 MiB data, 5.1 GiB used, 442 GiB / 447 GiB avail; 92 MiB/s wr, 1.47k op/s; 5181/32535 objects degraded (15.924%); 1325/32535 objects misplaced (4.073%); 262 KiB/s, 9 objects/s recovering 2023-09-23T22:42:02.213 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:02.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:02.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:02.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 51 16 20325 20309 24.8827 86.4375 0.0124274 0.040183 2023-09-23T22:42:02.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:02.346 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:02.382 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:42:02.382 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:42:02.390 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:02.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:02.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:02 smithi026 ceph-mon[215775]: 1.14 scrub starts 2023-09-23T22:42:02.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:02 smithi026 ceph-mon[215775]: 1.14 scrub ok 2023-09-23T22:42:02.499 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:02.585 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:02.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:02 smithi070 ceph-mon[192652]: 1.14 scrub starts 2023-09-23T22:42:02.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:02 smithi070 ceph-mon[192652]: 1.14 scrub ok 2023-09-23T22:42:02.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:02 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7591/44862 objects degraded (16.921%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:02.698 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:02.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:02.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:02.728 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:02 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7591/44862 objects degraded (16.921%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:02.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:02 smithi026 ceph-mon[223014]: 1.14 scrub starts 2023-09-23T22:42:02.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:02 smithi026 ceph-mon[223014]: 1.14 scrub ok 2023-09-23T22:42:02.729 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:02 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7591/44862 objects degraded (16.921%), 11 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:02.808 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:02.875 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:02.920 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:03.239 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 52 16 21895 21879 26.2908 98.125 0.00710426 0.0380284 2023-09-23T22:42:03.333 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:03.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:03.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:03.447 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:03.473 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:03.518 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:03.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:03.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:03.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:03 smithi026 ceph-mon[223014]: pgmap v1489: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 937 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 110 MiB/s wr, 1.75k op/s; 7591/44862 objects degraded (16.921%); 1909/44862 objects misplaced (4.255%); 77 KiB/s, 2 objects/s recovering 2023-09-23T22:42:03.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:03 smithi026 ceph-mon[223014]: 1.4 scrub starts 2023-09-23T22:42:03.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:03 smithi026 ceph-mon[223014]: 1.4 scrub ok 2023-09-23T22:42:03.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:03 smithi026 ceph-mon[215775]: pgmap v1489: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 937 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 110 MiB/s wr, 1.75k op/s; 7591/44862 objects degraded (16.921%); 1909/44862 objects misplaced (4.255%); 77 KiB/s, 2 objects/s recovering 2023-09-23T22:42:03.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:03 smithi026 ceph-mon[215775]: 1.4 scrub starts 2023-09-23T22:42:03.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:03 smithi026 ceph-mon[215775]: 1.4 scrub ok 2023-09-23T22:42:03.623 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:03.685 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:03.819 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:03.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:03.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:03.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:03 smithi070 ceph-mon[192652]: pgmap v1489: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 937 MiB data, 6.2 GiB used, 441 GiB / 447 GiB avail; 110 MiB/s wr, 1.75k op/s; 7591/44862 objects degraded (16.921%); 1909/44862 objects misplaced (4.255%); 77 KiB/s, 2 objects/s recovering 2023-09-23T22:42:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:03 smithi070 ceph-mon[192652]: 1.4 scrub starts 2023-09-23T22:42:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:03 smithi070 ceph-mon[192652]: 1.4 scrub ok 2023-09-23T22:42:03.928 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:03.955 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:03.999 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:04.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:04.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:04.105 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:04.174 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:04.226 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:04.227 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:03 smithi026 conmon[215751]: 2023-09-23T22:42:03.924+0000 7f4580f04700 -1 mon.a@0(leader).osd e573 definitely_dead 0 2023-09-23T22:42:04.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:04.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:04.239 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 53 16 23431 23415 27.6058 96 0.0073225 0.0361789 2023-09-23T22:42:04.330 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:04.356 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:04.383 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:42:04.402 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:04.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:04.500 INFO:teuthology.orchestra.run.smithi026.stderr:marked out osd.0. 2023-09-23T22:42:04.506 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:04.532 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:04.576 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:04.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:04.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:04.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1167269988' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T22:42:04.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:04 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T22:42:04.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:04 smithi070 ceph-mon[192652]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T22:42:04.683 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:04.710 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:04.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1167269988' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T22:42:04.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:04 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T22:42:04.779 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:04 smithi026 ceph-mon[223014]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T22:42:04.779 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1167269988' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T22:42:04.779 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:04 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T22:42:04.779 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:04 smithi026 ceph-mon[215775]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T22:42:05.211 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:05.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:05.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:05.239 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 54 16 23711 23695 27.4186 17.5 0.0180085 0.0359674 2023-09-23T22:42:05.315 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:05.345 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:05.672 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:05.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:05.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:05.776 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:05.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[215775]: pgmap v1490: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 1.2 GiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 121 MiB/s wr, 1.93k op/s; 9737/58959 objects degraded (16.515%); 2411/58959 objects misplaced (4.089%); 64 KiB/s, 1 objects/s recovering 2023-09-23T22:42:05.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[215775]: 1.c scrub starts 2023-09-23T22:42:05.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[215775]: 1.c scrub ok 2023-09-23T22:42:05.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[215775]: failed to encode map e574 with expected crc 2023-09-23T22:42:05.791 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[215775]: failed to encode map e574 with expected crc 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[215775]: failed to encode map e574 with expected crc 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[215775]: osdmap e574: 8 total, 6 up, 4 in 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[223014]: pgmap v1490: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 1.2 GiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 121 MiB/s wr, 1.93k op/s; 9737/58959 objects degraded (16.515%); 2411/58959 objects misplaced (4.089%); 64 KiB/s, 1 objects/s recovering 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[223014]: 1.c scrub starts 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[223014]: 1.c scrub ok 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[223014]: failed to encode map e574 with expected crc 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[223014]: failed to encode map e574 with expected crc 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[223014]: failed to encode map e574 with expected crc 2023-09-23T22:42:05.792 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T22:42:05.793 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:05 smithi026 ceph-mon[223014]: osdmap e574: 8 total, 6 up, 4 in 2023-09-23T22:42:05.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:05 smithi070 ceph-mon[192652]: pgmap v1490: 119 pgs: 58 active+clean+remapped, 15 active+undersized, 11 active+undersized+degraded, 35 active+clean; 1.2 GiB data, 6.5 GiB used, 440 GiB / 447 GiB avail; 121 MiB/s wr, 1.93k op/s; 9737/58959 objects degraded (16.515%); 2411/58959 objects misplaced (4.089%); 64 KiB/s, 1 objects/s recovering 2023-09-23T22:42:05.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:05 smithi070 ceph-mon[192652]: 1.c scrub starts 2023-09-23T22:42:05.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:05 smithi070 ceph-mon[192652]: 1.c scrub ok 2023-09-23T22:42:05.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:05 smithi070 ceph-mon[192652]: failed to encode map e574 with expected crc 2023-09-23T22:42:05.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:05 smithi070 ceph-mon[192652]: failed to encode map e574 with expected crc 2023-09-23T22:42:05.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:05 smithi070 ceph-mon[192652]: failed to encode map e574 with expected crc 2023-09-23T22:42:05.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:05 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T22:42:05.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:05 smithi070 ceph-mon[192652]: osdmap e574: 8 total, 6 up, 4 in 2023-09-23T22:42:05.808 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:05.857 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:05.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:05.962 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:05.997 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:06.239 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 55 16 23711 23695 26.9201 0 - 0.0359674 2023-09-23T22:42:06.545 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:06.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:06.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:06.599 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:06.654 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:06.681 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:06.727 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:06.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:06.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:06.832 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:06.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:06 smithi070 ceph-mon[192652]: failed to encode map e575 with expected crc 2023-09-23T22:42:06.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:06 smithi070 ceph-mon[192652]: failed to encode map e575 with expected crc 2023-09-23T22:42:06.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:06 smithi070 ceph-mon[192652]: failed to encode map e575 with expected crc 2023-09-23T22:42:06.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:06 smithi070 ceph-mon[192652]: osdmap e575: 8 total, 6 up, 4 in 2023-09-23T22:42:06.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:06 smithi070 ceph-mon[192652]: failed to encode map e575 with expected crc 2023-09-23T22:42:06.856 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:06 smithi070 ceph-mon[192652]: pgmap v1493: 119 pgs: 2 remapped+peering, 3 peering, 57 active+clean+remapped, 15 active+undersized, 10 active+undersized+degraded, 32 active+clean; 1.2 GiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s; 8607/60237 objects degraded (14.289%); 2411/60237 objects misplaced (4.003%) 2023-09-23T22:42:06.858 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:06.901 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:06.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:06.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151873 8.6 8:675a56d2:::benchmark_data_smithi026_497658_object52981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 501.85418120000003, 2023-09-23T22:42:06.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6652286869999999, 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151873 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:42:06.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048417+0000", 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048419+0000", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7810000000000001e-06 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048428+0000", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2649999999999996e-06 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048434+0000", 2023-09-23T22:42:06.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2419999999999997e-06 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048492+0000", 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7899999999999998e-05 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048608+0000", 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116111 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 4,6", 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.048734+0000", 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012566199999999999 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.057680+0000", 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.008946173 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202205+0000", 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.144524878 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202215+0000", 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0217000000000001e-05 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198539+0000", 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99632416099999999 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198553+0000", 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4236999999999999e-05 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713565+0000", 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5150114879999999 2023-09-23T22:42:06.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713646+0000", 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1019000000000005e-05 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151874 8.e 8:70ae8a39:::benchmark_data_smithi026_497658_object52982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:42:06.917 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 501.84782580400002, 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6486478249999998, 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151874 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054772+0000", 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054771+0000", 2023-09-23T22:42:06.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054778+0000", 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7599999999999997e-06 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054794+0000", 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6138999999999999e-05 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054803+0000", 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8610000000000004e-06 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.054857+0000", 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3900000000000002e-05 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.055039+0000", 2023-09-23T22:42:06.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018265100000000001 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.055175+0000", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013610799999999999 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.057826+0000", 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00265037 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.060056+0000", 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022297520000000002 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202153+0000", 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.142097219 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.202169+0000", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5707000000000001e-05 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198586+0000", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99641781500000004 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198596+0000", 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.662e-06 2023-09-23T22:42:06.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703316+0000", 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.504720442 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703420+0000", 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010388800000000001 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.925 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151883 8.e 8:73a3339c:::benchmark_data_smithi026_497658_object52991:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 501.60403109200001, 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4185350429999999, 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:06.926 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151883 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298567+0000", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298566+0000", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298574+0000", 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6419999999999996e-06 2023-09-23T22:42:06.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298592+0000", 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.749e-05 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298603+0000", 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1551e-05 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298672+0000", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8804000000000001e-05 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.298688+0000", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6004000000000001e-05 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198619+0000", 2023-09-23T22:42:06.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89993140699999996 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198625+0000", 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0419999999999999e-06 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703671+0000", 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.505045454 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703821+0000", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014994300000000001 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.703937+0000", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116397 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713345+0000", 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0094075259999999994 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713414+0000", 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9202999999999995e-05 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713603+0000", 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000189373 2023-09-23T22:42:06.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716935+0000", 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033319270000000002 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716988+0000", 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3121000000000002e-05 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.717102+0000", 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011384599999999999 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151885 8.e 8:7d6f345f:::benchmark_data_smithi026_497658_object52993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 501.60304033, 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4256615969999999, 2023-09-23T22:42:06.935 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151885 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.936 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299558+0000", 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299557+0000", 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299563+0000", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1279999999999996e-06 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299579+0000", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5736e-05 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299589+0000", 2023-09-23T22:42:06.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9720000000000004e-06 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299662+0000", 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3312000000000006e-05 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.299675+0000", 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3414999999999999e-05 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198647+0000", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89897187599999995 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198655+0000", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1219999999999995e-06 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706694+0000", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5080385459999999 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706833+0000", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013966099999999999 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.706985+0000", 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151915 2023-09-23T22:42:06.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713514+0000", 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065285760000000003 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713609+0000", 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5025000000000006e-05 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716876+0000", 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032670960000000001 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725068+0000", 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0081923059999999999 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725134+0000", 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5544999999999997e-05 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725220+0000", 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5604000000000002e-05 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151886 8.e 8:74052c51:::benchmark_data_smithi026_497658_object52994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 501.60075877000003, 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4236035519999999, 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151886 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301840+0000", 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301838+0000", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301844+0000", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2099999999999998e-06 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301857+0000", 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2943e-05 2023-09-23T22:42:06.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301867+0000", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3810000000000005e-06 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301924+0000", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7370000000000001e-05 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.301936+0000", 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2551999999999999e-05 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198678+0000", 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.89674120599999996 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.198686+0000", 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0330000000000008e-06 2023-09-23T22:42:06.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.709941+0000", 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5112555969999999 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.710081+0000", 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013977400000000001 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,2,4", 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.710232+0000", 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015061799999999999 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.713594+0000", 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033623500000000001 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716839+0000", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032453930000000001 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.716906+0000", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7041999999999999e-05 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725274+0000", 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083672850000000003 2023-09-23T22:42:06.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725329+0000", 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5031000000000002e-05 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:47.725443+0000", 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000114305 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.953 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104711 9.c 9:31531dd3:::benchmark_data_smithi026_555954_object9982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.333837+0000", 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 291.56876124899998, 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9901661960000001, 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:06.954 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104711 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333837+0000", 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333837+0000", 2023-09-23T22:42:06.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333835+0000", 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333844+0000", 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9309999999999997e-06 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333865+0000", 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0749000000000001e-05 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333875+0000", 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9089999999999995e-06 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.333951+0000", 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5932000000000004e-05 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.334444+0000", 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049377199999999996 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.997213+0000", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6627689000000001 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.997330+0000", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011688699999999999 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:42:06.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:16.997482+0000", 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015183899999999999 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.014559+0000", 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017076560000000001 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045008+0000", 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.030448994 2023-09-23T22:42:06.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045090+0000", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2562e-05 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.320023+0000", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.27493322999999997 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:06.961 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.320097+0000", 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3244000000000005e-05 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.324003+0000", 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039067069999999997 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104722 9.c 9:3a91d401:::benchmark_data_smithi026_555954_object9993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:15.362114+0000", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 291.54048475000002, 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9830620269999999, 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104722 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.963 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362114+0000", 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362114+0000", 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362112+0000", 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362119+0000", 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5490000000000003e-06 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362132+0000", 2023-09-23T22:42:06.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3563e-05 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362140+0000", 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5209999999999997e-06 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362187+0000", 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7746e-05 2023-09-23T22:42:06.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.362696+0000", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00050887800000000004 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.009860+0000", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6471635419999999 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.009977+0000", 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011697300000000001 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,4,7", 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.010139+0000", 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000161762 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.022087+0000", 2023-09-23T22:42:06.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.011948652000000001 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045050+0000", 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022963063999999998 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.045126+0000", 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5426000000000006e-05 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.342067+0000", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.29694130099999999 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.342108+0000", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1276999999999998e-05 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.970 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:17.345176+0000", 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030672630000000002 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119013 9.9 9:97fb371b:::benchmark_data_smithi026_555954_object24284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.616004+0000", 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.28659398299999, 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.772435714, 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.971 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119013 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616004+0000", 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616004+0000", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616002+0000", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616016+0000", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4297999999999999e-05 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616050+0000", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3912000000000003e-05 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.616066+0000", 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5749000000000001e-05 2023-09-23T22:42:06.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.636982+0000", 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020916489999999999 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.636996+0000", 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3326000000000001e-05 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.372328+0000", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.73533267700000005 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.372459+0000", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013030600000000001 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.372900+0000", 2023-09-23T22:42:06.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044181100000000002 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388194+0000", 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015293958999999999 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388300+0000", 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106106 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388352+0000", 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1922999999999997e-05 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388440+0000", 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7700000000000004e-05 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119014 9.5 9:a7fed247:::benchmark_data_smithi026_555954_object24285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.622481+0000", 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.28011694599999, 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.78627524299999996, 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119014 2023-09-23T22:42:06.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622481+0000", 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622481+0000", 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622479+0000", 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622492+0000", 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3371e-05 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622526+0000", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3466000000000001e-05 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622542+0000", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6580999999999999e-05 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622626+0000", 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3819999999999994e-05 2023-09-23T22:42:06.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.622642+0000", 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6061000000000001e-05 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.377283+0000", 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.75464055699999999 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.983 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.377429+0000", 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014568900000000001 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.377864+0000", 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043552899999999998 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.379798+0000", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001934285 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.387859+0000", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080606210000000005 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.408640+0000", 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.020780664000000001 2023-09-23T22:42:06.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.408698+0000", 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8724000000000002e-05 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.408757+0000", 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8183000000000002e-05 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.986 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119027 9.9 9:95b0ebe6:::benchmark_data_smithi026_555954_object24298:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.792678+0000", 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.10991993799999, 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60675180200000001, 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.987 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119027 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792678+0000", 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.988 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792678+0000", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792677+0000", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792683+0000", 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9030000000000001e-06 2023-09-23T22:42:06.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792693+0000", 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8879999999999992e-06 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792699+0000", 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7769999999999999e-06 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792745+0000", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6686e-05 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.792752+0000", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3999999999999997e-06 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.386094+0000", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.59334233000000003 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.386227+0000", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013239799999999999 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 7", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.386635+0000", 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040815300000000002 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.388625+0000", 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001989925 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.399291+0000", 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010666375000000001 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:06.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.399345+0000", 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4129000000000003e-05 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.399430+0000", 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5032999999999997e-05 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.994 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:119032 9.15 9:abacdbfd:::benchmark_data_smithi026_555954_object24303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:26.811691+0000", 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 160.09090758299999, 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.61184655600000004, 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 119032 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:06.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811691+0000", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811691+0000", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811689+0000", 2023-09-23T22:42:06.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811700+0000", 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1749999999999999e-05 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811732+0000", 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1522000000000001e-05 2023-09-23T22:42:06.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811738+0000", 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2349999999999996e-06 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811780+0000", 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1488e-05 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:26.811786+0000", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1310000000000003e-06 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.417633+0000", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.60584729699999995 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.417795+0000", 2023-09-23T22:42:06.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016233099999999999 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.418138+0000", 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034242300000000001 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.421355+0000", 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032175089999999999 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.423411+0000", 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020557570000000001 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.423454+0000", 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2836000000000001e-05 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.423501+0000", 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6959000000000002e-05 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:27.423537+0000", 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6437999999999999e-05 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.002 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:169105 9.15 9:ae0dab03:::benchmark_data_smithi026_555954_object74376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:27.661466+0000", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 99.241131917000004, 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20122447700000001, 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 169105 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:07.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661466+0000", 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661466+0000", 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:07.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661466+0000", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661469+0000", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9699999999999999e-06 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661475+0000", 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3400000000000003e-06 2023-09-23T22:42:07.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661481+0000", 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6459999999999998e-06 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.664517+0000", 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030358999999999998 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.664639+0000", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012202299999999999 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.664976+0000", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033671900000000003 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.668612+0000", 2023-09-23T22:42:07.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036362550000000001 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.669990+0000", 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013781310000000001 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.862600+0000", 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.192610429 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.862651+0000", 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0909999999999999e-05 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.862691+0000", 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9598999999999998e-05 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:169106 9.15 9:a81cf775:::benchmark_data_smithi026_555954_object74377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:27.661513+0000", 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 99.241085757999997, 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20626381099999999, 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:07.010 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 169106 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661513+0000", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661513+0000", 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661512+0000", 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661515+0000", 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2179999999999999e-06 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661523+0000", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.238e-06 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.661529+0000", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0929999999999999e-06 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.666957+0000", 2023-09-23T22:42:07.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00542824 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.667040+0000", 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2954999999999994e-05 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.667225+0000", 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018559199999999999 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.668701+0000", 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0014756389999999999 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.670095+0000", 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0013935480000000001 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.867671+0000", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.19757656800000001 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.867736+0000", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4695000000000003e-05 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:27.867776+0000", 2023-09-23T22:42:07.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0519999999999998e-05 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:170922 9.5 9:a405daf9:::benchmark_data_smithi026_555954_object76193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e536)", 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:29.540540+0000", 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 97.362057848999996, 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.18062175499999999, 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.017 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 170922 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540540+0000", 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540540+0000", 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540540+0000", 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540543+0000", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5919999999999999e-06 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540549+0000", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8249999999999999e-06 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.540555+0000", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7830000000000004e-06 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.546762+0000", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0062069550000000001 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.546881+0000", 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011924 2023-09-23T22:42:07.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.547331+0000", 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044946599999999998 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.550352+0000", 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030210749999999998 2023-09-23T22:42:07.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.552181+0000", 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001829027 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.721082+0000", 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.168900772 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.721126+0000", 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4659000000000001e-05 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:29.721162+0000", 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5778999999999997e-05 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:186574 9.5 9:a2b7e1ed:::benchmark_data_smithi026_555954_object91845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:48.576335+0000", 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 78.326263737999994, 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26174019599999998, 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 186574 2023-09-23T22:42:07.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576335+0000", 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:07.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576335+0000", 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576333+0000", 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576340+0000", 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6750000000000002e-06 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576359+0000", 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8476999999999999e-05 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576368+0000", 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7099999999999996e-06 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576428+0000", 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0409999999999999e-05 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576560+0000", 2023-09-23T22:42:07.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013158700000000001 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.577032+0000", 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000471881 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.582645+0000", 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056133629999999997 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.582675+0000", 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9960000000000001e-05 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.837976+0000", 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.25530079500000002 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838033+0000", 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7073000000000001e-05 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838075+0000", 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2055999999999998e-05 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:186575 9.5 9:a1f249a0:::benchmark_data_smithi026_555954_object91846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:48.576557+0000", 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 78.326041187000001, 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.26162006599999998, 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 186575 2023-09-23T22:42:07.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576557+0000", 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576557+0000", 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576556+0000", 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576562+0000", 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6659999999999996e-06 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576574+0000", 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2204e-05 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.576582+0000", 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0770000000000008e-06 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:07.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.579664+0000", 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030818740000000001 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.579795+0000", 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013058899999999999 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.580252+0000", 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045709100000000002 2023-09-23T22:42:07.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:07.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.583584+0000", 2023-09-23T22:42:07.040 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:07.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003332645 2023-09-23T22:42:07.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.584967+0000", 2023-09-23T22:42:07.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001382538 2023-09-23T22:42:07.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838112+0000", 2023-09-23T22:42:07.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.253144753 2023-09-23T22:42:07.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838147+0000", 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5052000000000003e-05 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:48.838177+0000", 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0528000000000003e-05 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.043 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:187640 9.15 9:af3445ae:::benchmark_data_smithi026_555954_object92911:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:49.804222+0000", 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 77.098375844000003, 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.227025541, 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:07.044 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 187640 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804222+0000", 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804222+0000", 2023-09-23T22:42:07.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804221+0000", 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804227+0000", 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5969999999999999e-06 2023-09-23T22:42:07.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804236+0000", 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4879999999999996e-06 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804247+0000", 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0560999999999999e-05 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804296+0000", 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9296999999999999e-05 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804421+0000", 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000124919 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:07.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.804874+0000", 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045281999999999999 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.806885+0000", 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020113520000000001 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:49.807344+0000", 2023-09-23T22:42:07.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045847100000000001 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.031156+0000", 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.22381212 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.031210+0000", 2023-09-23T22:42:07.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3794e-05 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.031248+0000", 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8448999999999997e-05 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46634.0:19566 10.5 e573/569 10:a5d059b7:::benchmark_data_smithi026_640738_object19565:head v 573'1236, mlcod=573'1231)", 2023-09-23T22:42:07.051 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:01.553839+0000", 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.3487595470000002, 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.232879372, 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.553839+0000", 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.553839+0000", 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.553838+0000", 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.553855+0000", 2023-09-23T22:42:07.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7252999999999999e-05 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.553862+0000", 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6649999999999997e-06 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.553866+0000", 2023-09-23T22:42:07.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9299999999999996e-06 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.553912+0000", 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5874000000000002e-05 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.553931+0000", 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9505e-05 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.554877+0000", 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00094575499999999995 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.786718+0000", 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.231840983 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46634.0:19553 10.6 e573/549 10:6f96123f:::benchmark_data_smithi026_640738_object19552:head v 573'1196, mlcod=573'1194)", 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:01.580875+0000", 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.321722802, 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20765726200000001, 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.580875+0000", 2023-09-23T22:42:07.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.580875+0000", 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.580874+0000", 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:07.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.580897+0000", 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3023999999999998e-05 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.786650+0000", 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20575267699999999 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.786663+0000", 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2833000000000001e-05 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.786722+0000", 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9202e-05 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:07.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.786755+0000", 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2759000000000003e-05 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.788496+0000", 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017415180000000001 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.788533+0000", 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6511999999999999e-05 2023-09-23T22:42:07.061 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46634.0:19569 10.4 e573/569 10:25d56f9d:::benchmark_data_smithi026_640738_object19568:head v 573'1169, mlcod=573'1167)", 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:01.583218+0000", 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 5.3193804370000004, 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.205765956, 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:07.062 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.583218+0000", 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.583218+0000", 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.583216+0000", 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.583246+0000", 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9555000000000001e-05 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.786673+0000", 2023-09-23T22:42:07.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.20342718800000001 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.786686+0000", 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3025e-05 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.786742+0000", 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5800000000000001e-05 2023-09-23T22:42:07.065 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.786757+0000", 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5129e-05 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.788948+0000", 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021906080000000001 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:07.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:07.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:07.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:01.788984+0000", 2023-09-23T22:42:07.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6177000000000003e-05 2023-09-23T22:42:07.067 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.067 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.067 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.067 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:07.067 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:07.067 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[215775]: failed to encode map e575 with expected crc 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[215775]: failed to encode map e575 with expected crc 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[215775]: failed to encode map e575 with expected crc 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[215775]: osdmap e575: 8 total, 6 up, 4 in 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[215775]: failed to encode map e575 with expected crc 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[215775]: pgmap v1493: 119 pgs: 2 remapped+peering, 3 peering, 57 active+clean+remapped, 15 active+undersized, 10 active+undersized+degraded, 32 active+clean; 1.2 GiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s; 8607/60237 objects degraded (14.289%); 2411/60237 objects misplaced (4.003%) 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[223014]: failed to encode map e575 with expected crc 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[223014]: failed to encode map e575 with expected crc 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[223014]: failed to encode map e575 with expected crc 2023-09-23T22:42:07.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[223014]: osdmap e575: 8 total, 6 up, 4 in 2023-09-23T22:42:07.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[223014]: failed to encode map e575 with expected crc 2023-09-23T22:42:07.069 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:06 smithi026 ceph-mon[223014]: pgmap v1493: 119 pgs: 2 remapped+peering, 3 peering, 57 active+clean+remapped, 15 active+undersized, 10 active+undersized+degraded, 32 active+clean; 1.2 GiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 72 MiB/s wr, 1.15k op/s; 8607/60237 objects degraded (14.289%); 2411/60237 objects misplaced (4.003%) 2023-09-23T22:42:07.069 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:07.107 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:07.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:07.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:07.212 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:07.239 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:07.239 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 56 15 24153 24138 26.9337 13.8438 0.0181206 0.0371209 2023-09-23T22:42:07.282 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:07.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:07.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:07.387 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:07.462 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:07.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[223014]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[223014]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[223014]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[223014]: osdmap e576: 8 total, 6 up, 4 in 2023-09-23T22:42:07.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[223014]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[215775]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[215775]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[215775]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[215775]: osdmap e576: 8 total, 6 up, 4 in 2023-09-23T22:42:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:07 smithi026 ceph-mon[215775]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:07 smithi070 ceph-mon[192652]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:07 smithi070 ceph-mon[192652]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:07 smithi070 ceph-mon[192652]: failed to encode map e576 with expected crc 2023-09-23T22:42:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:07 smithi070 ceph-mon[192652]: osdmap e576: 8 total, 6 up, 4 in 2023-09-23T22:42:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:07 smithi070 ceph-mon[192652]: failed to encode map e576 with expected crc 2023-09-23T22:42:08.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 57 16 24930 24914 27.312 48.5 0.0111799 0.0366071 2023-09-23T22:42:08.478 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:42:08.518 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:08.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:08.623 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:08.650 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:08.689 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[223014]: failed to encode map e577 with expected crc 2023-09-23T22:42:08.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[223014]: failed to encode map e577 with expected crc 2023-09-23T22:42:08.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[223014]: failed to encode map e577 with expected crc 2023-09-23T22:42:08.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[223014]: osdmap e577: 8 total, 6 up, 4 in 2023-09-23T22:42:08.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[223014]: pgmap v1496: 119 pgs: 2 active+recovery_wait+remapped, 43 remapped+peering, 3 peering, 43 active+clean+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 4 active+clean; 1.5 GiB data, 5.1 GiB used, 352 GiB / 358 GiB avail; 70 MiB/s wr, 1.12k op/s; 7492/72453 objects degraded (10.340%); 12160/72453 objects misplaced (16.783%); 736 KiB/s, 12 objects/s recovering 2023-09-23T22:42:08.690 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 8607/60237 objects degraded (14.289%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:08.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[215775]: failed to encode map e577 with expected crc 2023-09-23T22:42:08.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[215775]: failed to encode map e577 with expected crc 2023-09-23T22:42:08.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[215775]: failed to encode map e577 with expected crc 2023-09-23T22:42:08.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[215775]: osdmap e577: 8 total, 6 up, 4 in 2023-09-23T22:42:08.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[215775]: pgmap v1496: 119 pgs: 2 active+recovery_wait+remapped, 43 remapped+peering, 3 peering, 43 active+clean+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 4 active+clean; 1.5 GiB data, 5.1 GiB used, 352 GiB / 358 GiB avail; 70 MiB/s wr, 1.12k op/s; 7492/72453 objects degraded (10.340%); 12160/72453 objects misplaced (16.783%); 736 KiB/s, 12 objects/s recovering 2023-09-23T22:42:08.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:08 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 8607/60237 objects degraded (14.289%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:08.698 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:08.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:08.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:08.804 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:08.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:08 smithi070 ceph-mon[192652]: failed to encode map e577 with expected crc 2023-09-23T22:42:08.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:08 smithi070 ceph-mon[192652]: failed to encode map e577 with expected crc 2023-09-23T22:42:08.829 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:08 smithi070 ceph-mon[192652]: failed to encode map e577 with expected crc 2023-09-23T22:42:08.830 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:08 smithi070 ceph-mon[192652]: osdmap e577: 8 total, 6 up, 4 in 2023-09-23T22:42:08.830 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:08 smithi070 ceph-mon[192652]: pgmap v1496: 119 pgs: 2 active+recovery_wait+remapped, 43 remapped+peering, 3 peering, 43 active+clean+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 4 active+clean; 1.5 GiB data, 5.1 GiB used, 352 GiB / 358 GiB avail; 70 MiB/s wr, 1.12k op/s; 7492/72453 objects degraded (10.340%); 12160/72453 objects misplaced (16.783%); 736 KiB/s, 12 objects/s recovering 2023-09-23T22:42:08.830 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:08 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 8607/60237 objects degraded (14.289%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:08.831 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:08.837 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:08.877 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:08.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:08.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:08.982 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:09.058 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:09.181 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:09.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:09.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:09.239 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 58 16 25738 25722 27.7116 50.5 0.0184582 0.0360353 2023-09-23T22:42:09.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:09.315 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:09.375 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:09.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:09.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:09.485 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:09.514 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:09.556 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:09.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:09.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:09.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[215775]: failed to encode map e578 with expected crc 2023-09-23T22:42:09.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[215775]: failed to encode map e578 with expected crc 2023-09-23T22:42:09.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[215775]: osdmap e578: 8 total, 6 up, 4 in 2023-09-23T22:42:09.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[215775]: failed to encode map e578 with expected crc 2023-09-23T22:42:09.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:42:09.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:09.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3113105146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:09.617 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:09.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[223014]: failed to encode map e578 with expected crc 2023-09-23T22:42:09.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[223014]: failed to encode map e578 with expected crc 2023-09-23T22:42:09.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[223014]: osdmap e578: 8 total, 6 up, 4 in 2023-09-23T22:42:09.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[223014]: failed to encode map e578 with expected crc 2023-09-23T22:42:09.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:42:09.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:09.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3113105146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:09.618 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:09 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:09.625 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:42:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:42:09] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-09-23T22:42:09.659 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:09.728 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:09.853 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:09.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:09.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:09 smithi070 ceph-mon[192652]: failed to encode map e578 with expected crc 2023-09-23T22:42:09.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:09 smithi070 ceph-mon[192652]: failed to encode map e578 with expected crc 2023-09-23T22:42:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:09 smithi070 ceph-mon[192652]: osdmap e578: 8 total, 6 up, 4 in 2023-09-23T22:42:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:09 smithi070 ceph-mon[192652]: failed to encode map e578 with expected crc 2023-09-23T22:42:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:42:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:09 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3113105146' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:09 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:09.960 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:10.030 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:10.167 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:10.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:10.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:10.173 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:42:10.246 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 59 16 26559 26543 28.1113 51.3125 0.015363 0.0355649 2023-09-23T22:42:10.274 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:10.388 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:10.462 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:42:10.462 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:42:10.462 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:42:10.462 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:42:10.490 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:10.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:10.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:10.611 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:10.641 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:10.672 INFO:tasks.thrashosds.thrasher:in_osds: [5, 1, 6, 2] out_osds: [4, 7, 3, 0] dead_osds: [6, 3] live_osds: [0, 5, 7, 2, 4, 1] 2023-09-23T22:42:10.672 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2023-09-23T22:42:10.672 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2023-09-23T22:42:10.673 DEBUG:teuthology.orchestra.run.smithi026:osd.3> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T22:42:10.681 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl start ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T22:42:10.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:10 smithi026 ceph-mon[215775]: pgmap v1498: 119 pgs: 2 active+recovery_wait+remapped, 43 remapped+peering, 3 peering, 43 active+clean+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 4 active+clean; 1.5 GiB data, 5.1 GiB used, 352 GiB / 358 GiB avail; 61 MiB/s wr, 971 op/s; 7492/72453 objects degraded (10.340%); 12160/72453 objects misplaced (16.783%); 702 KiB/s, 11 objects/s recovering 2023-09-23T22:42:10.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:10 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:42:10.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:10.745 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:10 smithi026 ceph-mon[215775]: osdmap e579: 8 total, 6 up, 4 in 2023-09-23T22:42:10.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:10 smithi026 ceph-mon[223014]: pgmap v1498: 119 pgs: 2 active+recovery_wait+remapped, 43 remapped+peering, 3 peering, 43 active+clean+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 4 active+clean; 1.5 GiB data, 5.1 GiB used, 352 GiB / 358 GiB avail; 61 MiB/s wr, 971 op/s; 7492/72453 objects degraded (10.340%); 12160/72453 objects misplaced (16.783%); 702 KiB/s, 11 objects/s recovering 2023-09-23T22:42:10.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:10 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:42:10.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:10.746 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:10 smithi026 ceph-mon[223014]: osdmap e579: 8 total, 6 up, 4 in 2023-09-23T22:42:10.753 INFO:journalctl@ceph.osd.3.smithi026.stdout:-- Logs begin at Sat 2023-09-23 21:41:51 UTC. -- 2023-09-23T22:42:10.785 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:10.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:10.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:10 smithi070 ceph-mon[192652]: pgmap v1498: 119 pgs: 2 active+recovery_wait+remapped, 43 remapped+peering, 3 peering, 43 active+clean+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 4 active+clean; 1.5 GiB data, 5.1 GiB used, 352 GiB / 358 GiB avail; 61 MiB/s wr, 971 op/s; 7492/72453 objects degraded (10.340%); 12160/72453 objects misplaced (16.783%); 702 KiB/s, 11 objects/s recovering 2023-09-23T22:42:10.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:10 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:42:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:10 smithi070 ceph-mon[192652]: osdmap e579: 8 total, 6 up, 4 in 2023-09-23T22:42:10.889 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:10.960 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:11.070 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:42:11.087 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:11.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:11.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:11.094 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:10 smithi026 systemd[1]: Starting Ceph osd.3 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:42:11.120 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:42:11.171 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:42:11.195 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:11.216 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:42:11.237 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:42:11.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:42:11.240880+0000 min lat: 0.000971618 max lat: 40.1955 avg lat: 0.0350413 2023-09-23T22:42:11.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:42:11.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 60 16 27411 27395 28.5301 53.25 0.0263405 0.0350413 2023-09-23T22:42:11.260 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:42:11.268 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:11.298 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:42:11.298 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:42:11.299 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:42:11.299 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:42:11.299 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:42:11.299 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:42:11.299 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:42:11.299 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:42:11.299 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:42:11.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:11 smithi070 ceph-mon[192652]: failed to encode map e579 with expected crc 2023-09-23T22:42:11.565 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:11 smithi070 ceph-mon[192652]: failed to encode map e579 with expected crc 2023-09-23T22:42:11.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:11 smithi070 ceph-mon[192652]: failed to encode map e579 with expected crc 2023-09-23T22:42:11.566 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/755656553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:11.579 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:11.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:11 smithi026 ceph-mon[215775]: failed to encode map e579 with expected crc 2023-09-23T22:42:11.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:11 smithi026 ceph-mon[215775]: failed to encode map e579 with expected crc 2023-09-23T22:42:11.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:11 smithi026 ceph-mon[215775]: failed to encode map e579 with expected crc 2023-09-23T22:42:11.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/755656553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:11.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:11 smithi026 ceph-mon[223014]: failed to encode map e579 with expected crc 2023-09-23T22:42:11.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:11 smithi026 ceph-mon[223014]: failed to encode map e579 with expected crc 2023-09-23T22:42:11.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:11 smithi026 ceph-mon[223014]: failed to encode map e579 with expected crc 2023-09-23T22:42:11.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/755656553' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:12.139 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:12.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:12.240 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 61 16 28183 28167 28.8532 48.25 0.00742489 0.0346503 2023-09-23T22:42:12.247 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:12.310 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:12 smithi070 ceph-mon[192652]: pgmap v1500: 119 pgs: 3 active+recovery_wait+remapped, 15 remapped+peering, 65 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 1.6 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 42 MiB/s wr, 669 op/s; 16361/76878 objects degraded (21.282%); 24012/76878 objects misplaced (31.234%); 629 KiB/s, 10 objects/s recovering 2023-09-23T22:42:12.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:12 smithi026 ceph-mon[223014]: pgmap v1500: 119 pgs: 3 active+recovery_wait+remapped, 15 remapped+peering, 65 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 1.6 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 42 MiB/s wr, 669 op/s; 16361/76878 objects degraded (21.282%); 24012/76878 objects misplaced (31.234%); 629 KiB/s, 10 objects/s recovering 2023-09-23T22:42:12.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:12 smithi026 ceph-mon[215775]: pgmap v1500: 119 pgs: 3 active+recovery_wait+remapped, 15 remapped+peering, 65 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 1.6 GiB data, 5.3 GiB used, 352 GiB / 358 GiB avail; 42 MiB/s wr, 669 op/s; 16361/76878 objects degraded (21.282%); 24012/76878 objects misplaced (31.234%); 629 KiB/s, 10 objects/s recovering 2023-09-23T22:42:13.084 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:13.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:13.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:13.107 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:13.108 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:13.167 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:42:13.167 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:42:13.196 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:13.246 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 62 16 29022 29006 29.2327 52.4375 0.0217352 0.0341996 2023-09-23T22:42:13.285 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:13.420 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:13.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:13.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:13.530 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:13.561 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:13.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:13 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 16361/76878 objects degraded (21.282%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:13.733 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:13 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 16361/76878 objects degraded (21.282%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:13.744 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:13.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:13.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:13.796 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:42:13.820 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:13 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 16361/76878 objects degraded (21.282%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:13.849 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:13.950 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:14.089 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:14.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:14.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:14.141 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:14.193 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:14.220 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:14.242 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 63 16 29913 29897 29.6524 55.6875 0.016392 0.0337181 2023-09-23T22:42:14.262 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:14.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:14.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:14.366 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:14.442 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:14.537 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:14.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:14.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:14.645 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:14.717 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:14 smithi026 ceph-mon[223014]: pgmap v1501: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 1.7 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 13 KiB/s rd, 59 MiB/s wr, 946 op/s; 16878/83787 objects degraded (20.144%); 25339/83787 objects misplaced (30.242%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T22:42:14.718 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:14 smithi026 ceph-mon[215775]: pgmap v1501: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 1.7 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 13 KiB/s rd, 59 MiB/s wr, 946 op/s; 16878/83787 objects degraded (20.144%); 25339/83787 objects misplaced (30.242%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T22:42:14.723 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:14.840 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:14.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:14.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:14.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:14 smithi070 ceph-mon[192652]: pgmap v1501: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 1.7 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 13 KiB/s rd, 59 MiB/s wr, 946 op/s; 16878/83787 objects degraded (20.144%); 25339/83787 objects misplaced (30.242%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T22:42:14.945 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:14.985 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:14 smithi026 bash[658804]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:42:14.985 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:14 smithi026 bash[658804]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T22:42:14.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:14 smithi026 bash[658804]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T22:42:14.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:14 smithi026 bash[658804]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T22:42:14.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:14 smithi026 bash[658804]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T22:42:14.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:14 smithi026 bash[658804]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T22:42:14.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:14 smithi026 bash[658804]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T22:42:14.986 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:14 smithi026 bash[658804]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T22:42:15.016 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:15.138 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:42:15.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:42:15.235 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:15.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:15.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:15.248 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 64 16 30743 30727 29.9995 51.875 0.0188232 0.0333237 2023-09-23T22:42:15.342 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:15.371 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:15.441 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:15.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:15.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:15.544 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:15.595 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:15.769 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:15.769 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:16.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 65 16 31633 31617 30.3936 55.625 0.00880173 0.0328945 2023-09-23T22:42:16.444 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:42:16.794 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:17.035 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:16 smithi070 ceph-mon[192652]: pgmap v1502: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 1.7 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 10 KiB/s rd, 31 MiB/s wr, 510 op/s; 16878/84513 objects degraded (19.971%); 25581/84513 objects misplaced (30.269%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:42:17.057 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:17.068 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:16 smithi026 ceph-mon[223014]: pgmap v1502: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 1.7 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 10 KiB/s rd, 31 MiB/s wr, 510 op/s; 16878/84513 objects degraded (19.971%); 25581/84513 objects misplaced (30.269%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:42:17.068 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:16 smithi026 ceph-mon[215775]: pgmap v1502: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 1.7 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 10 KiB/s rd, 31 MiB/s wr, 510 op/s; 16878/84513 objects degraded (19.971%); 25581/84513 objects misplaced (30.269%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T22:42:17.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:17.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:17.174 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:17.203 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:17.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 66 16 32597 32581 30.8458 60.25 0.0164081 0.032409 2023-09-23T22:42:17.400 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:17.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:17.470 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:17.503 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:17.523 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:17.584 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:17.782 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:17.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:17.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:17.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:17.996 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:18.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 67 16 33323 33307 31.0626 45.375 0.019531 0.0321853 2023-09-23T22:42:18.336 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:18.336 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:18.979 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:42:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:18 smithi070 ceph-mon[192652]: pgmap v1503: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.0 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 9.1 KiB/s rd, 55 MiB/s wr, 882 op/s; 17444/96516 objects degraded (18.074%); 28693/96516 objects misplaced (29.729%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:42:19.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17444/96516 objects degraded (18.074%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:19.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:18 smithi026 ceph-mon[223014]: pgmap v1503: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.0 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 9.1 KiB/s rd, 55 MiB/s wr, 882 op/s; 17444/96516 objects degraded (18.074%); 28693/96516 objects misplaced (29.729%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:42:19.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17444/96516 objects degraded (18.074%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:18 smithi026 ceph-mon[215775]: pgmap v1503: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.0 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 9.1 KiB/s rd, 55 MiB/s wr, 882 op/s; 17444/96516 objects degraded (18.074%); 28693/96516 objects misplaced (29.729%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T22:42:19.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17444/96516 objects degraded (18.074%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:19.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 68 16 34267 34251 31.4733 59 0.0142833 0.0317641 2023-09-23T22:42:19.274 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:19.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:19.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:19.337 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:19.385 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:19.413 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:19.455 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:19.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:19.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:19.559 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:19.622 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:42:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:42:19] "GET /metrics HTTP/1.1" 200 36525 "" "Prometheus/2.43.0" 2023-09-23T22:42:19.639 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:20.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 69 16 35153 35137 31.8196 55.375 0.0137193 0.0314197 2023-09-23T22:42:20.676 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:20.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:20.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:20.781 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:20.812 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:20.860 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:20.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:20.879 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:20.879 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:20.901 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:20 smithi070 ceph-mon[192652]: pgmap v1504: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.0 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 8.3 KiB/s rd, 50 MiB/s wr, 807 op/s; 17444/96516 objects degraded (18.074%); 28693/96516 objects misplaced (29.729%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:42:20.959 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:20 smithi026 ceph-mon[215775]: pgmap v1504: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.0 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 8.3 KiB/s rd, 50 MiB/s wr, 807 op/s; 17444/96516 objects degraded (18.074%); 28693/96516 objects misplaced (29.729%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:42:20.960 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:20 smithi026 ceph-mon[223014]: pgmap v1504: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.0 GiB data, 6.2 GiB used, 351 GiB / 358 GiB avail; 8.3 KiB/s rd, 50 MiB/s wr, 807 op/s; 17444/96516 objects degraded (18.074%); 28693/96516 objects misplaced (29.729%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T22:42:20.964 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:20.991 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:21.086 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:21.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:21.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:21.190 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:21.216 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:21.243 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 70 16 35855 35839 31.9917 43.875 0.02617 0.0312492 2023-09-23T22:42:21.258 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:21.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:21.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:21.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:42:21.362 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:21.387 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:21.485 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:21 smithi026 bash[659986]: 4ea5b94d55508b1fa5eccefdfc5080a42c5577bf2a7553119010f66b80218f87 2023-09-23T22:42:21.485 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:21 smithi026 systemd[1]: Started Ceph osd.3 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:42:21.584 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:42:21.593 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:21.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:21.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:21.698 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:21.724 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:21.767 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:21.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:21.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:21.872 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:21.899 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:21.928 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:21 smithi026 conmon[660192]: 2023-09-23T22:42:21.566+0000 7f84e50aa640 -1 Falling back to public interface 2023-09-23T22:42:21.948 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:21.997 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:22.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:22.070 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:42:22.103 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:22.127 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:42:22.128 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:22.174 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:22.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:22.184 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:42:22.238 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:42:22.244 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 71 16 36563 36547 32.1636 44.25 0.0236111 0.0310821 2023-09-23T22:42:22.277 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:42:22.279 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:22.280 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:22.307 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:22.317 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:42:22.317 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:42:22.317 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:42:22.317 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:42:22.317 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T22:42:22.317 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:42:22.318 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:42:22.318 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:42:22.318 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:42:22.501 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:22.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:22.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:22.605 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:22.697 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:22.843 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:22.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:22 smithi026 ceph-mon[223014]: pgmap v1505: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.1 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 7.2 KiB/s rd, 53 MiB/s wr, 859 op/s; 17444/102105 objects degraded (17.084%); 30556/102105 objects misplaced (29.926%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T22:42:22.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1296038311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:22.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/409897617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:22.851 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:22 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:22.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:22 smithi026 ceph-mon[215775]: pgmap v1505: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.1 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 7.2 KiB/s rd, 53 MiB/s wr, 859 op/s; 17444/102105 objects degraded (17.084%); 30556/102105 objects misplaced (29.926%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T22:42:22.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1296038311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:22.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/409897617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:22.852 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:22 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:22.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:22.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:22.855 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:42:22.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:23.028 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:23.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:22 smithi070 ceph-mon[192652]: pgmap v1505: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.1 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 7.2 KiB/s rd, 53 MiB/s wr, 859 op/s; 17444/102105 objects degraded (17.084%); 30556/102105 objects misplaced (29.926%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T22:42:23.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1296038311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:23.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/409897617' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:23.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:22 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:42:23.161 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:23.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:23.165 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:23.248 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 72 16 37322 37306 32.3742 47.4375 0.0160714 0.0308804 2023-09-23T22:42:23.265 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:23.346 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:23.417 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:42:23.417 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:42:23.417 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:42:23.417 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:42:23.417 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.417 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9495 9.d 9:b6f72c49:::benchmark_data_smithi026_555954_object9494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.350545+0000", 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 416.06392433600001, 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.022504189, 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9495 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.418 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350545+0000", 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350545+0000", 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.419 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350543+0000", 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350613+0000", 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0388000000000004e-05 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.420 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350628+0000", 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4489e-05 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350635+0000", 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6150000000000002e-06 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350683+0000", 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8446000000000003e-05 2023-09-23T22:42:23.421 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350811+0000", 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128374 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350977+0000", 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016547600000000001 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.422 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.352085+0000", 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00110797 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.352551+0000", 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00046640899999999999 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.423 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370196+0000", 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017644389999999999 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370205+0000", 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4939999999999992e-06 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372976+0000", 2023-09-23T22:42:23.424 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0027705790000001 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373049+0000", 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3143000000000001e-05 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9496 9.c 9:3bf3d114:::benchmark_data_smithi026_555954_object9495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:42:23.425 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.370618+0000", 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 416.043851098, 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.001084182, 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9496 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.426 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370618+0000", 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370618+0000", 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.427 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370617+0000", 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370666+0000", 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9101999999999998e-05 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370677+0000", 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1586999999999999e-05 2023-09-23T22:42:23.428 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370684+0000", 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2400000000000004e-06 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370725+0000", 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1288000000000002e-05 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.429 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370731+0000", 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8649999999999998e-06 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371493+0000", 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.000761907 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.430 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371702+0000", 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020950600000000001 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9509 9.c 9:31b495ae:::benchmark_data_smithi026_555954_object9508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.442632+0000", 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 415.97183714499999, 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93168581900000003, 2023-09-23T22:42:23.431 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9509 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442632+0000", 2023-09-23T22:42:23.432 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442632+0000", 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442630+0000", 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.433 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442697+0000", 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6773000000000006e-05 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442707+0000", 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5789999999999993e-06 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.434 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442714+0000", 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0979999999999996e-06 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442752+0000", 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8105000000000002e-05 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.435 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442757+0000", 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5249999999999996e-06 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371728+0000", 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92897071200000003 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.436 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371868+0000", 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000139462 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372052+0000", 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000184446 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373673+0000", 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001620511 2023-09-23T22:42:23.437 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374175+0000", 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000502203 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374228+0000", 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2825999999999999e-05 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.438 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374318+0000", 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0082999999999998e-05 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9535 9.d 9:b4eb0ec8:::benchmark_data_smithi026_555954_object9534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.455847+0000", 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 415.95862207499999, 2023-09-23T22:42:23.439 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92259973200000001, 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9535 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.440 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455847+0000", 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455847+0000", 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455846+0000", 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.441 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455906+0000", 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0608e-05 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455923+0000", 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6758000000000001e-05 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.442 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455930+0000", 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7780000000000002e-06 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455990+0000", 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0136000000000002e-05 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.443 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456000+0000", 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8220000000000002e-06 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373303+0000", 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91730366500000005 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373384+0000", 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0984999999999998e-05 2023-09-23T22:42:23.444 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373486+0000", 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000101313 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.375253+0000", 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001767565 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.445 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378309+0000", 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030558080000000001 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378360+0000", 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0806000000000001e-05 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.446 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378447+0000", 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6707999999999997e-05 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9536 9.c 9:382e37e1:::benchmark_data_smithi026_555954_object9535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.456300+0000", 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 415.95816873799998, 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92362326400000005, 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.447 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9536 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456300+0000", 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.448 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456300+0000", 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456299+0000", 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.449 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456394+0000", 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4780999999999999e-05 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456405+0000", 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0876999999999999e-05 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456411+0000", 2023-09-23T22:42:23.450 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7259999999999999e-06 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456453+0000", 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2178000000000002e-05 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456466+0000", 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3247e-05 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.451 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372550+0000", 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91608398700000004 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372617+0000", 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7168000000000005e-05 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:42:23.452 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372714+0000", 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6972000000000006e-05 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373682+0000", 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00096728799999999996 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379826+0000", 2023-09-23T22:42:23.453 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061445110000000001 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379868+0000", 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1446999999999998e-05 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379924+0000", 2023-09-23T22:42:23.454 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5908000000000001e-05 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9540 9.d 9:bd7479be:::benchmark_data_smithi026_555954_object9539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 415.956560685, 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92071582799999996, 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.455 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9540 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:42:23.456 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.457 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457972+0000", 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3987999999999998e-05 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457986+0000", 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3869999999999999e-05 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.458 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457992+0000", 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6499999999999999e-06 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.458039+0000", 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6647000000000002e-05 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.459 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.458044+0000", 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0610000000000003e-06 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373970+0000", 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91592621799999996 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374074+0000", 2023-09-23T22:42:23.460 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103479 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374200+0000", 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000126317 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.375260+0000", 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010597790000000001 2023-09-23T22:42:23.461 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378498+0000", 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032380619999999999 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378543+0000", 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5624999999999998e-05 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.462 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378624+0000", 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0729999999999994e-05 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9544 9.d 9:b6a337e4:::benchmark_data_smithi026_555954_object9543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.459597+0000", 2023-09-23T22:42:23.463 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 415.95487194899999, 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91919865700000003, 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9544 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.464 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459597+0000", 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459597+0000", 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459596+0000", 2023-09-23T22:42:23.465 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459657+0000", 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0798999999999998e-05 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459668+0000", 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0868e-05 2023-09-23T22:42:23.466 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459673+0000", 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4889999999999997e-06 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459722+0000", 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8953999999999999e-05 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.467 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459728+0000", 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7449999999999998e-06 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374739+0000", 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91501085900000001 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.468 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374843+0000", 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104354 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374936+0000", 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2578999999999996e-05 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.375383+0000", 2023-09-23T22:42:23.469 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044722000000000002 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378674+0000", 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032907280000000001 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378716+0000", 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2168e-05 2023-09-23T22:42:23.470 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378796+0000", 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9506999999999996e-05 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9551 9.c 9:30ade42e:::benchmark_data_smithi026_555954_object9550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.465719+0000", 2023-09-23T22:42:23.471 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 415.94875043799999, 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.914369392, 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9551 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.472 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465719+0000", 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465719+0000", 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465718+0000", 2023-09-23T22:42:23.473 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465753+0000", 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5376999999999997e-05 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465780+0000", 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7169e-05 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.474 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465785+0000", 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5059999999999997e-06 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465841+0000", 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6444999999999999e-05 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.475 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465846+0000", 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9490000000000004e-06 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373010+0000", 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90716359499999999 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373077+0000", 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7021999999999996e-05 2023-09-23T22:42:23.476 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373205+0000", 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012822300000000001 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374744+0000", 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015386899999999999 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.477 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379977+0000", 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052328569999999996 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.380030+0000", 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3511000000000003e-05 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.478 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.380088+0000", 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7877000000000002e-05 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104683 9.d 9:b37e8cbe:::benchmark_data_smithi026_555954_object9954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.265880+0000", 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 309.14858889300001, 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.041256663, 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.479 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104683 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265880+0000", 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.480 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265880+0000", 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265879+0000", 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265884+0000", 2023-09-23T22:42:23.481 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3159999999999996e-06 2023-09-23T22:42:23.482 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.482 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.482 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.482 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265896+0000", 2023-09-23T22:42:23.482 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2296000000000001e-05 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265901+0000", 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2499999999999997e-06 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265962+0000", 2023-09-23T22:42:23.483 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0973000000000001e-05 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266094+0000", 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131686 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266227+0000", 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013251100000000001 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.484 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.269476+0000", 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032489149999999998 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.269598+0000", 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012262800000000001 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.485 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.297704+0000", 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028105502000000001 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.297712+0000", 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3180000000000007e-06 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307040+0000", 2023-09-23T22:42:23.486 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0093283340000001 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307137+0000", 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6446999999999993e-05 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104689 9.d 9:b5dd9fdc:::benchmark_data_smithi026_555954_object9960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:42:23.487 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.299991+0000", 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 309.11447777400002, 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.01128229, 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104689 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.488 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.299991+0000", 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.299991+0000", 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.489 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.299989+0000", 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.299999+0000", 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0040999999999999e-05 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.300014+0000", 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4395e-05 2023-09-23T22:42:23.490 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.300022+0000", 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3820000000000004e-06 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.300898+0000", 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000875894 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.491 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.300908+0000", 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0465000000000001e-05 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307652+0000", 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.006743991 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.492 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307757+0000", 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010459900000000001 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307872+0000", 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115252 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.311035+0000", 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031630389999999999 2023-09-23T22:42:23.493 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.311127+0000", 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1836000000000006e-05 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.311182+0000", 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4568000000000003e-05 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.494 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.311273+0000", 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1748000000000003e-05 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104691 9.3 9.d6892893 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.306137+0000", 2023-09-23T22:42:23.495 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 309.10833239599998, 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97170118400000005, 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104691 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.496 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306137+0000", 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306137+0000", 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306135+0000", 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.497 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306142+0000", 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1369999999999996e-06 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306156+0000", 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4102e-05 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.498 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306163+0000", 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8770000000000004e-06 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306207+0000", 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3531e-05 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.499 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306214+0000", 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5469999999999999e-06 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.277827+0000", 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97161278600000001 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.277838+0000", 2023-09-23T22:42:23.500 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0689e-05 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104697 9.3 9.c29b12d3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.352031+0000", 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 309.06243759900002, 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92583439899999997, 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:42:23.501 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104697 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352031+0000", 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.502 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352031+0000", 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352030+0000", 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.503 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352037+0000", 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.472e-06 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352050+0000", 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3534e-05 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352056+0000", 2023-09-23T22:42:23.504 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4689999999999996e-06 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352104+0000", 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8214999999999998e-05 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352112+0000", 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3950000000000001e-06 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.505 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.277858+0000", 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92574585099999995 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.277866+0000", 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7500000000000003e-06 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.506 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112809 9.1 9.c9f688c1 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:38:54.412199+0000", 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 209.00227002, 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97345260199999994, 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.507 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112809 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412199+0000", 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412199+0000", 2023-09-23T22:42:23.508 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412197+0000", 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412204+0000", 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3919999999999997e-06 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.509 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412220+0000", 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.594e-05 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412227+0000", 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4089999999999997e-06 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.510 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412275+0000", 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7166999999999998e-05 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412282+0000", 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2490000000000004e-06 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.385645+0000", 2023-09-23T22:42:23.511 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97336309399999998 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.385652+0000", 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6599999999999998e-06 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112816 9.1 9.8fe24c21 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:42:23.512 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:38:54.592383+0000", 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 208.82208580700001, 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79328532900000004, 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112816 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.513 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592383+0000", 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592383+0000", 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.514 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592382+0000", 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592387+0000", 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3170000000000001e-06 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592400+0000", 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3156000000000001e-05 2023-09-23T22:42:23.515 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592422+0000", 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1756999999999999e-05 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592463+0000", 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1282000000000002e-05 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.516 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592474+0000", 2023-09-23T22:42:23.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0373e-05 2023-09-23T22:42:23.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.517 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.385665+0000", 2023-09-23T22:42:23.517 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79319085899999997 2023-09-23T22:42:23.517 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.517 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.517 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.385669+0000", 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9090000000000002e-06 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112831 9.a 9:5995d63d:::benchmark_data_smithi026_555954_object18102:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:38:54.851121+0000", 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 208.563347996, 2023-09-23T22:42:23.518 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.58216017499999995, 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112831 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.519 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851121+0000", 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851121+0000", 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851120+0000", 2023-09-23T22:42:23.520 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851127+0000", 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0439999999999998e-06 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851139+0000", 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2616000000000001e-05 2023-09-23T22:42:23.521 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851145+0000", 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1850000000000003e-06 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851191+0000", 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6767999999999997e-05 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.522 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851199+0000", 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2350000000000002e-06 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.416893+0000", 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.56569482400000004 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.523 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.417034+0000", 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000140464 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.417161+0000", 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000126771 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.424475+0000", 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073143879999999998 2023-09-23T22:42:23.524 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.425894+0000", 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014191729999999999 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433157+0000", 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072629180000000002 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.525 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433195+0000", 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8127000000000003e-05 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433281+0000", 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5987000000000002e-05 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.526 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112839 9.a 9:5f6027c3:::benchmark_data_smithi026_555954_object18110:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:38:55.082571+0000", 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 208.33189766000001, 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.35082294800000002, 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112839 2023-09-23T22:42:23.527 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082571+0000", 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082571+0000", 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.528 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082570+0000", 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082577+0000", 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6730000000000003e-06 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082591+0000", 2023-09-23T22:42:23.529 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4158999999999999e-05 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082596+0000", 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.288e-06 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082644+0000", 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7542e-05 2023-09-23T22:42:23.530 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082652+0000", 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4190000000000002e-06 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.419978+0000", 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33732560499999997 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.531 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.420067+0000", 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9258999999999994e-05 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.420139+0000", 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2515000000000006e-05 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.532 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.425926+0000", 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057866640000000004 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.427909+0000", 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019834150000000001 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433305+0000", 2023-09-23T22:42:23.533 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0053956860000000002 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433336+0000", 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0824999999999997e-05 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433394+0000", 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8346e-05 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.534 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:119011 9.3 9:c70114b4:::benchmark_data_smithi026_555954_object24282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:26.604881+0000", 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 176.809587932, 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.77412598399999999, 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.535 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119011 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604881+0000", 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.536 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604881+0000", 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604880+0000", 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604886+0000", 2023-09-23T22:42:23.537 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2570000000000004e-06 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604900+0000", 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4287999999999999e-05 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604906+0000", 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4009999999999996e-06 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.538 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604968+0000", 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2269999999999998e-05 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604978+0000", 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7210000000000006e-06 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.539 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.364445+0000", 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75946789699999995 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.364580+0000", 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000134905 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.364883+0000", 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000302928 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.540 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.367536+0000", 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002652298 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.368262+0000", 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00072598599999999997 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.541 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.378827+0000", 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010565827 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.378888+0000", 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0173999999999997e-05 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.542 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.379007+0000", 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119449 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:119015 9.1 9:83bc4cab:::benchmark_data_smithi026_555954_object24286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:26.628581+0000", 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 176.78588832700001, 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73202734300000005, 2023-09-23T22:42:23.543 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119015 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628581+0000", 2023-09-23T22:42:23.544 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628581+0000", 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628579+0000", 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:23.545 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628585+0000", 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.524e-06 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628596+0000", 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1279e-05 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.546 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628601+0000", 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.87e-06 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628649+0000", 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7799000000000001e-05 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.547 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628656+0000", 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6000000000000003e-06 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.356235+0000", 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.72757985199999997 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.356371+0000", 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013551000000000001 2023-09-23T22:42:23.548 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.356908+0000", 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00053668599999999998 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360435+0000", 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035278319999999998 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.549 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360477+0000", 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2073999999999997e-05 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360526+0000", 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8738e-05 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.550 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360608+0000", 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1797000000000003e-05 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:119016 9.3 9:c7a55f5e:::benchmark_data_smithi026_555954_object24287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:26.635619+0000", 2023-09-23T22:42:23.551 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 176.778850358, 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.743613733, 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119016 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.552 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635619+0000", 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635619+0000", 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635618+0000", 2023-09-23T22:42:23.553 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635623+0000", 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1569999999999999e-06 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635636+0000", 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2728000000000001e-05 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.554 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635641+0000", 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0939999999999998e-06 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635685+0000", 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4427e-05 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.555 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635692+0000", 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4860000000000002e-06 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.366319+0000", 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.730627684 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.366456+0000", 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013700499999999999 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.556 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.366620+0000", 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016379 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.367810+0000", 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011903129999999999 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.557 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.368409+0000", 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059821199999999998 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.379061+0000", 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010652017999999999 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.379118+0000", 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7512000000000001e-05 2023-09-23T22:42:23.558 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.379232+0000", 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011414299999999999 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:119017 9.1 9:86eab180:::benchmark_data_smithi026_555954_object24288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:26.642933+0000", 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 176.77153614100001, 2023-09-23T22:42:23.559 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.71978275999999997, 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119017 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642933+0000", 2023-09-23T22:42:23.560 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642933+0000", 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642932+0000", 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.561 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642937+0000", 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8400000000000002e-06 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642956+0000", 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9249000000000002e-05 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642962+0000", 2023-09-23T22:42:23.562 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4979999999999997e-06 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642997+0000", 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5170999999999999e-05 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.643003+0000", 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6939999999999998e-06 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.563 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.358600+0000", 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.715596071 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.358742+0000", 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014219300000000001 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T22:42:23.564 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.359002+0000", 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026045499999999999 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360621+0000", 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016183829999999999 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.362562+0000", 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001941702 2023-09-23T22:42:23.565 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.362623+0000", 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1169999999999999e-05 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.362716+0000", 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2162000000000003e-05 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.566 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.567 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:42:23.567 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:42:23.567 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:42:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:23 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:23 smithi070 ceph-mon[192652]: failed to encode map e580 with expected crc 2023-09-23T22:42:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:23 smithi070 ceph-mon[192652]: failed to encode map e580 with expected crc 2023-09-23T22:42:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:23 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:42:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:23 smithi070 ceph-mon[192652]: osdmap e580: 8 total, 6 up, 4 in 2023-09-23T22:42:23.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:23 smithi070 ceph-mon[192652]: failed to encode map e580 with expected crc 2023-09-23T22:42:23.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17444/102105 objects degraded (17.084%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:24.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:42:24.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:24.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[215775]: failed to encode map e580 with expected crc 2023-09-23T22:42:24.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[215775]: failed to encode map e580 with expected crc 2023-09-23T22:42:24.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:42:24.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[215775]: osdmap e580: 8 total, 6 up, 4 in 2023-09-23T22:42:24.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[215775]: failed to encode map e580 with expected crc 2023-09-23T22:42:24.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17444/102105 objects degraded (17.084%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:24.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:24.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[223014]: failed to encode map e580 with expected crc 2023-09-23T22:42:24.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[223014]: failed to encode map e580 with expected crc 2023-09-23T22:42:24.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:42:24.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[223014]: osdmap e580: 8 total, 6 up, 4 in 2023-09-23T22:42:24.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[223014]: failed to encode map e580 with expected crc 2023-09-23T22:42:24.098 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17444/102105 objects degraded (17.084%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:24.250 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 73 16 37876 37860 32.4041 34.625 0.0299512 0.0308526 2023-09-23T22:42:24.995 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:25.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:25.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:25.027 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:42:25.027 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:42:25.079 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:24 smithi026 ceph-mon[215775]: pgmap v1507: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.2 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 52 MiB/s wr, 831 op/s; 17949/108750 objects degraded (16.505%); 31819/108750 objects misplaced (29.259%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:42:25.079 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:25.079 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:24 smithi026 ceph-mon[223014]: pgmap v1507: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.2 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 52 MiB/s wr, 831 op/s; 17949/108750 objects degraded (16.505%); 31819/108750 objects misplaced (29.259%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:42:25.080 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:25.104 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:25.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:24 smithi070 ceph-mon[192652]: pgmap v1507: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.2 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 52 MiB/s wr, 831 op/s; 17949/108750 objects degraded (16.505%); 31819/108750 objects misplaced (29.259%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T22:42:25.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:25.209 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:25.250 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 74 16 38490 38474 32.4847 38.375 0.0154714 0.0307759 2023-09-23T22:42:25.389 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:25.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:25.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:25.505 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:25.533 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:25.577 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:25.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:25.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:25.682 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:25.711 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:25.755 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:25.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:25.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:25.863 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:25.890 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:25.934 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:25.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:25.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:26.040 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:26.066 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:26.111 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:26.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:26.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:26.217 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:26.244 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:26.250 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 75 16 38848 38832 32.3499 22.375 0.00335838 0.0306481 2023-09-23T22:42:26.290 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:26.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:26.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:26.395 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:26.421 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:26.464 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:26.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:26.498 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:26.569 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:26 smithi070 ceph-mon[192652]: failed to encode map e581 with expected crc 2023-09-23T22:42:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:26 smithi070 ceph-mon[192652]: osdmap e581: 8 total, 6 up, 4 in 2023-09-23T22:42:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:26 smithi070 ceph-mon[192652]: failed to encode map e581 with expected crc 2023-09-23T22:42:26.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:26 smithi070 ceph-mon[192652]: failed to encode map e581 with expected crc 2023-09-23T22:42:26.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:26 smithi026 ceph-mon[215775]: failed to encode map e581 with expected crc 2023-09-23T22:42:26.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:26 smithi026 ceph-mon[215775]: osdmap e581: 8 total, 6 up, 4 in 2023-09-23T22:42:26.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:26 smithi026 ceph-mon[215775]: failed to encode map e581 with expected crc 2023-09-23T22:42:26.675 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:26 smithi026 ceph-mon[215775]: failed to encode map e581 with expected crc 2023-09-23T22:42:26.678 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:26 smithi026 ceph-mon[223014]: failed to encode map e581 with expected crc 2023-09-23T22:42:26.678 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:26 smithi026 ceph-mon[223014]: osdmap e581: 8 total, 6 up, 4 in 2023-09-23T22:42:26.678 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:26 smithi026 ceph-mon[223014]: failed to encode map e581 with expected crc 2023-09-23T22:42:26.678 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:26 smithi026 ceph-mon[223014]: failed to encode map e581 with expected crc 2023-09-23T22:42:26.680 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:26.841 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:26.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:26.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:26.955 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:27.028 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:42:27.073 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:27.080 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:42:27.253 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 76 16 39303 39287 32.2973 28.4375 0.128591 0.0309505 2023-09-23T22:42:27.386 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[223014]: pgmap v1509: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.2 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 34 MiB/s wr, 536 op/s; 18852/109386 objects degraded (17.234%); 31075/109386 objects misplaced (28.409%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:42:27.685 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:27.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:27.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:27.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[215775]: pgmap v1509: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.2 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 34 MiB/s wr, 536 op/s; 18852/109386 objects degraded (17.234%); 31075/109386 objects misplaced (28.409%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:42:27.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[215775]: failed to encode map e582 with expected crc 2023-09-23T22:42:27.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[215775]: failed to encode map e582 with expected crc 2023-09-23T22:42:27.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[215775]: failed to encode map e582 with expected crc 2023-09-23T22:42:27.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[215775]: osdmap e582: 8 total, 6 up, 4 in 2023-09-23T22:42:27.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[223014]: failed to encode map e582 with expected crc 2023-09-23T22:42:27.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[223014]: failed to encode map e582 with expected crc 2023-09-23T22:42:27.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[223014]: failed to encode map e582 with expected crc 2023-09-23T22:42:27.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:27 smithi026 ceph-mon[223014]: osdmap e582: 8 total, 6 up, 4 in 2023-09-23T22:42:27.798 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:27 smithi070 ceph-mon[192652]: pgmap v1509: 119 pgs: 3 active+recovery_wait+remapped, 80 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 6 active+recovery_wait+degraded+remapped, 5 active+clean; 2.2 GiB data, 6.7 GiB used, 351 GiB / 358 GiB avail; 34 MiB/s wr, 536 op/s; 18852/109386 objects degraded (17.234%); 31075/109386 objects misplaced (28.409%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T22:42:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:27 smithi070 ceph-mon[192652]: failed to encode map e582 with expected crc 2023-09-23T22:42:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:27 smithi070 ceph-mon[192652]: failed to encode map e582 with expected crc 2023-09-23T22:42:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:27 smithi070 ceph-mon[192652]: failed to encode map e582 with expected crc 2023-09-23T22:42:27.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:27 smithi070 ceph-mon[192652]: osdmap e582: 8 total, 6 up, 4 in 2023-09-23T22:42:27.929 INFO:teuthology.orchestra.run.smithi026.stderr:no valid command found; 10 closest matches: 2023-09-23T22:42:27.929 INFO:teuthology.orchestra.run.smithi026.stderr:0 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:1 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:2 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:abort 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:assert 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs debug_inject_read_zeros 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs files list 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:bluefs stats 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:bluestore allocator dump block 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:bluestore allocator fragmentation block 2023-09-23T22:42:27.930 INFO:teuthology.orchestra.run.smithi026.stderr:admin_socket: invalid command 2023-09-23T22:42:27.934 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:28.069 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:28.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:28.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:28.177 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:28.204 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:28.247 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:28.252 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:28.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:28.255 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 77 16 39843 39827 32.3153 33.75 0.00950037 0.0309375 2023-09-23T22:42:28.353 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:28.379 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:28.421 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:28.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:28.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:28.526 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:28.555 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:28.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 18852/109386 objects degraded (17.234%), 20 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:28.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 18852/109386 objects degraded (17.234%), 20 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:28.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 18852/109386 objects degraded (17.234%), 20 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:28.737 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:28 smithi026 conmon[660192]: 2023-09-23T22:42:28.590+0000 7f84e50aa640 -1 osd.3 516 log_to_monitors true 2023-09-23T22:42:29.209 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:29.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:29.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:29.255 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 78 16 40358 40342 32.3135 32.1875 0.00893695 0.030933 2023-09-23T22:42:29.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:29.346 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:29.391 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:29.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:29.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:29.409 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:29 smithi026 ceph-mon[215775]: pgmap v1511: 119 pgs: 3 active+recovery_wait+remapped, 82 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.1 GiB used, 351 GiB / 358 GiB avail; 54 MiB/s wr, 868 op/s; 16709/117735 objects degraded (14.192%); 35782/117735 objects misplaced (30.392%); 7.4 MiB/s, 119 objects/s recovering 2023-09-23T22:42:29.410 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:29 smithi026 ceph-mon[215775]: from='osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:42:29.497 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:29.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:29 smithi070 ceph-mon[192652]: pgmap v1511: 119 pgs: 3 active+recovery_wait+remapped, 82 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.1 GiB used, 351 GiB / 358 GiB avail; 54 MiB/s wr, 868 op/s; 16709/117735 objects degraded (14.192%); 35782/117735 objects misplaced (30.392%); 7.4 MiB/s, 119 objects/s recovering 2023-09-23T22:42:29.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:29 smithi070 ceph-mon[192652]: from='osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:42:29.649 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:29.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:42:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:42:29] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2023-09-23T22:42:29.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:29 smithi026 ceph-mon[223014]: pgmap v1511: 119 pgs: 3 active+recovery_wait+remapped, 82 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.1 GiB used, 351 GiB / 358 GiB avail; 54 MiB/s wr, 868 op/s; 16709/117735 objects degraded (14.192%); 35782/117735 objects misplaced (30.392%); 7.4 MiB/s, 119 objects/s recovering 2023-09-23T22:42:29.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:29 smithi026 ceph-mon[223014]: from='osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T22:42:29.833 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T22:42:29.833 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2023-09-23T22:42:29.852 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:29.951 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:29.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:29.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:30.057 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:30.126 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:42:29 smithi026 conmon[660192]: 2023-09-23T22:42:29.878+0000 7f84d4d14700 -1 osd.3 583 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:42:30.133 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:30.255 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 79 16 40947 40931 32.3704 36.8125 0.0377477 0.0308842 2023-09-23T22:42:30.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[215775]: failed to encode map e583 with expected crc 2023-09-23T22:42:30.440 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[215775]: from='osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:42:30.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[215775]: osdmap e583: 8 total, 6 up, 4 in 2023-09-23T22:42:30.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[215775]: failed to encode map e583 with expected crc 2023-09-23T22:42:30.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[215775]: failed to encode map e583 with expected crc 2023-09-23T22:42:30.441 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[215775]: from='osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:42:30.467 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:30.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:30.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:30.576 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:30.597 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:42:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:42:30.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46634.0:41165 10.0 e584/575 10:08329270:::benchmark_data_smithi026_640738_object41164:head v 584'2597, mlcod=0'0)", 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.587614+0000", 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0050612809999999999, 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005107599, 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587614+0000", 2023-09-23T22:42:30.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587614+0000", 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587613+0000", 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587637+0000", 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3836999999999999e-05 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587644+0000", 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7139999999999996e-06 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587649+0000", 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7910000000000004e-06 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591328+0000", 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0036787970000000001 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591353+0000", 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5270999999999999e-05 2023-09-23T22:42:30.601 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:41167 10.7 10:edc08d4b:::benchmark_data_smithi026_640738_object41166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591146+0000", 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0015292190000000001, 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016477729999999999, 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:30.602 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 41167 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591146+0000", 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591146+0000", 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591144+0000", 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591167+0000", 2023-09-23T22:42:30.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3071999999999999e-05 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591177+0000", 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5719999999999992e-06 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591183+0000", 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3439999999999997e-06 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591231+0000", 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8174999999999999e-05 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591378+0000", 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000146821 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:30.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591530+0000", 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015140999999999999 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:41168 10.4 10:2a4695e6:::benchmark_data_smithi026_640738_object41167:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591195+0000", 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0014794579999999999, 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001675988, 2023-09-23T22:42:30.607 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 41168 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591195+0000", 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591195+0000", 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591195+0000", 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591218+0000", 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3127000000000001e-05 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591225+0000", 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3869999999999997e-06 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591232+0000", 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8419999999999999e-06 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591284+0000", 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2083999999999998e-05 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591424+0000", 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013983800000000001 2023-09-23T22:42:30.611 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591572+0000", 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014828199999999999 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46634.0:41159 10.7 e584/581)", 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591298+0000", 2023-09-23T22:42:30.612 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001376366, 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016382219999999999, 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591298+0000", 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.613 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591298+0000", 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591298+0000", 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591302+0000", 2023-09-23T22:42:30.614 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2320000000000003e-06 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591311+0000", 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7909999999999993e-06 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591317+0000", 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.268e-06 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.615 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46634.0:41160 10.7 e584/581)", 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591326+0000", 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0013484219999999999, 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016571489999999999, 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.616 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591326+0000", 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591326+0000", 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.617 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591326+0000", 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591329+0000", 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5610000000000001e-06 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591335+0000", 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8329999999999999e-06 2023-09-23T22:42:30.618 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591340+0000", 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.977e-06 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46634.0:41156 10.4 e584/575)", 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591344+0000", 2023-09-23T22:42:30.619 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0013310049999999999, 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016877389999999999, 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591344+0000", 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.620 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591344+0000", 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591344+0000", 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.621 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591346+0000", 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0990000000000001e-06 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591351+0000", 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4029999999999998e-06 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591356+0000", 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5639999999999998e-06 2023-09-23T22:42:30.622 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46634.0:41166 10.2 e584/575)", 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591382+0000", 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001292844, 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017004069999999999, 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.623 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591382+0000", 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591382+0000", 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.624 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591382+0000", 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591406+0000", 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4006999999999999e-05 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591413+0000", 2023-09-23T22:42:30.625 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0670000000000003e-06 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591419+0000", 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.981e-06 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593082+0000", 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001662742 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.626 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46634.0:41159 10.7 e584/581)", 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591521+0000", 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0011541240000000001, 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016213600000000001, 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.627 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591521+0000", 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591521+0000", 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591520+0000", 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:30.628 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591523+0000", 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5799999999999999e-06 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591529+0000", 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0399999999999998e-06 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.629 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591534+0000", 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6550000000000003e-06 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593130+0000", 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015965440000000001 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593137+0000", 2023-09-23T22:42:30.630 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2450000000000003e-06 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46634.0:41160 10.7 e584/581)", 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591537+0000", 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0011374040000000001, 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016683539999999999, 2023-09-23T22:42:30.631 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591537+0000", 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591537+0000", 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.632 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591537+0000", 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591539+0000", 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9829999999999999e-06 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.633 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591544+0000", 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.972e-06 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591549+0000", 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4410000000000003e-06 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.634 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46634.0:41156 10.4 e584/575 ondisk, result = 0)", 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591552+0000", 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0011225530000000001, 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016995269999999999, 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.635 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591552+0000", 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591552+0000", 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591552+0000", 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.636 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591554+0000", 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8199999999999999e-06 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591560+0000", 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.066e-06 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.637 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591565+0000", 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7539999999999997e-06 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593161+0000", 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015957429999999999 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593168+0000", 2023-09-23T22:42:30.638 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6079999999999998e-06 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46634.0:41157 10.4 e584/575)", 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591569+0000", 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0011058050000000001, 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017383450000000001, 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:42:30.639 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591569+0000", 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591569+0000", 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.640 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591569+0000", 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591571+0000", 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8870000000000001e-06 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591576+0000", 2023-09-23T22:42:30.641 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0919999999999997e-06 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591580+0000", 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6199999999999998e-06 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.642 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46634.0:41161 10.4 e584/575)", 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591584+0000", 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.001090555, 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017680930000000001, 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591584+0000", 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.643 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591584+0000", 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591584+0000", 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.644 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591586+0000", 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.928e-06 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591591+0000", 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0220000000000004e-06 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591595+0000", 2023-09-23T22:42:30.645 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2180000000000001e-06 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop_reply(client.46634.0:41163 10.e e584/575)", 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.591604+0000", 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0010707360000000001, 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001793982, 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "queued for pg", 2023-09-23T22:42:30.646 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591604+0000", 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591604+0000", 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.647 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591604+0000", 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591606+0000", 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4540000000000001e-06 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591612+0000", 2023-09-23T22:42:30.648 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8810000000000001e-06 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591617+0000", 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5499999999999996e-06 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.649 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:41157 10.4 10:2bc3284f:::benchmark_data_smithi026_640738_object41156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.576054+0000", 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.016620721000000001, 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017398611000000001, 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 41157 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.650 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576054+0000", 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576054+0000", 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.651 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576054+0000", 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576078+0000", 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4369e-05 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.652 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576085+0000", 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9809999999999997e-06 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576090+0000", 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0969999999999996e-06 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.578546+0000", 2023-09-23T22:42:30.653 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024556930000000001 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.578668+0000", 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012209300000000001 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.578811+0000", 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000143378 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.654 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593332+0000", 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01452064 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593373+0000", 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1003e-05 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.655 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:41158 10.e 10:7d2a6062:::benchmark_data_smithi026_640738_object41157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.576095+0000", 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.016579732, 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017445816999999999, 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 41158 2023-09-23T22:42:30.656 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576095+0000", 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576095+0000", 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.657 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576095+0000", 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576121+0000", 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6143999999999999e-05 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.658 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576128+0000", 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1339999999999998e-06 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576133+0000", 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8640000000000004e-06 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.659 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576180+0000", 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6690999999999997e-05 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576322+0000", 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014235300000000001 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576482+0000", 2023-09-23T22:42:30.660 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159779 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.578746+0000", 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022648 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.591595+0000", 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01284858 2023-09-23T22:42:30.661 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:41161 10.4 10:2e879f03:::benchmark_data_smithi026_640738_object41160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.576241+0000", 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.016434107, 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017386064999999999, 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:30.662 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 41161 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576241+0000", 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.663 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576241+0000", 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576240+0000", 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576262+0000", 2023-09-23T22:42:30.664 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1543999999999999e-05 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576268+0000", 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7100000000000001e-06 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.576274+0000", 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4979999999999997e-06 2023-09-23T22:42:30.665 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580162+0000", 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0038875060000000002 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580312+0000", 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000150882 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:30.666 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580477+0000", 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000164993 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593338+0000", 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012860373 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593412+0000", 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4281000000000003e-05 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.667 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:41163 10.e 10:7f12dbc6:::benchmark_data_smithi026_640738_object41162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.580011+0000", 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.012663409, 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013738481, 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:30.668 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 41163 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580011+0000", 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580011+0000", 2023-09-23T22:42:30.669 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580010+0000", 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580031+0000", 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0588999999999999e-05 2023-09-23T22:42:30.670 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580044+0000", 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2801e-05 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580053+0000", 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1670000000000005e-06 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.671 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580103+0000", 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9808999999999997e-05 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580248+0000", 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145468 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.580406+0000", 2023-09-23T22:42:30.672 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015830900000000001 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.586079+0000", 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0056723349999999997 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.673 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(10.a 584/575 [PushOp(10:59eb7574:::benchmark_data_smithi026_640738_object8916:head, version: 573'601, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:59eb7574:::benchmark_data_smithi026_640738_object8916:head@573'601, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.587223+0000", 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0054519779999999997, 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066254790000000001, 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587223+0000", 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.674 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587223+0000", 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587221+0000", 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.675 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587279+0000", 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7624000000000003e-05 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587308+0000", 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9244999999999999e-05 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587327+0000", 2023-09-23T22:42:30.676 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9082000000000001e-05 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593079+0000", 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0057519829999999996 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.593095+0000", 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5158e-05 2023-09-23T22:42:30.677 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_repop(client.46634.0:41164 10.2 e584/575 10:4ec704a7:::benchmark_data_smithi026_640738_object41163:head v 584'2552, mlcod=0'0)", 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:30.587533+0000", 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0051416659999999996, 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063790460000000002, 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:30.678 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587533+0000", 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587533+0000", 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:30.679 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587533+0000", 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587559+0000", 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6044999999999999e-05 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587565+0000", 2023-09-23T22:42:30.680 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3849999999999999e-06 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.587570+0000", 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8239999999999999e-06 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.590457+0000", 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028871180000000001 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:30.681 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:30.590488+0000", 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0351000000000001e-05 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 19 2023-09-23T22:42:30.682 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:42:30.683 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:30.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[223014]: failed to encode map e583 with expected crc 2023-09-23T22:42:30.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[223014]: from='osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:42:30.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[223014]: osdmap e583: 8 total, 6 up, 4 in 2023-09-23T22:42:30.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[223014]: failed to encode map e583 with expected crc 2023-09-23T22:42:30.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[223014]: failed to encode map e583 with expected crc 2023-09-23T22:42:30.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:30 smithi026 ceph-mon[223014]: from='osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:42:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:30 smithi070 ceph-mon[192652]: failed to encode map e583 with expected crc 2023-09-23T22:42:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:30 smithi070 ceph-mon[192652]: from='osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T22:42:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:30 smithi070 ceph-mon[192652]: osdmap e583: 8 total, 6 up, 4 in 2023-09-23T22:42:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:30 smithi070 ceph-mon[192652]: failed to encode map e583 with expected crc 2023-09-23T22:42:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:30 smithi070 ceph-mon[192652]: failed to encode map e583 with expected crc 2023-09-23T22:42:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:30 smithi070 ceph-mon[192652]: from='osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi026", "root=default"]}]: dispatch 2023-09-23T22:42:31.256 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:2023-09-23T22:42:31.256407+0000 min lat: 0.000971618 max lat: 40.1955 avg lat: 0.0307139 2023-09-23T22:42:31.256 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T22:42:31.256 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 80 16 41666 41650 32.5273 44.9375 0.0293058 0.0307139 2023-09-23T22:42:31.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[215775]: pgmap v1513: 119 pgs: 3 active+recovery_wait+remapped, 82 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.1 GiB used, 351 GiB / 358 GiB avail; 31 MiB/s wr, 498 op/s; 16709/117735 objects degraded (14.192%); 35782/117735 objects misplaced (30.392%); 2.7 MiB/s, 45 objects/s recovering 2023-09-23T22:42:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[215775]: failed to encode map e584 with expected crc 2023-09-23T22:42:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[215775]: failed to encode map e584 with expected crc 2023-09-23T22:42:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[215775]: osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290] boot 2023-09-23T22:42:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[215775]: osdmap e584: 8 total, 7 up, 4 in 2023-09-23T22:42:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[215775]: failed to encode map e584 with expected crc 2023-09-23T22:42:31.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:42:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[223014]: pgmap v1513: 119 pgs: 3 active+recovery_wait+remapped, 82 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.1 GiB used, 351 GiB / 358 GiB avail; 31 MiB/s wr, 498 op/s; 16709/117735 objects degraded (14.192%); 35782/117735 objects misplaced (30.392%); 2.7 MiB/s, 45 objects/s recovering 2023-09-23T22:42:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[223014]: failed to encode map e584 with expected crc 2023-09-23T22:42:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[223014]: failed to encode map e584 with expected crc 2023-09-23T22:42:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[223014]: osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290] boot 2023-09-23T22:42:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[223014]: osdmap e584: 8 total, 7 up, 4 in 2023-09-23T22:42:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[223014]: failed to encode map e584 with expected crc 2023-09-23T22:42:31.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:31 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:42:31.843 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:31.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:31.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:31 smithi070 ceph-mon[192652]: pgmap v1513: 119 pgs: 3 active+recovery_wait+remapped, 82 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.1 GiB used, 351 GiB / 358 GiB avail; 31 MiB/s wr, 498 op/s; 16709/117735 objects degraded (14.192%); 35782/117735 objects misplaced (30.392%); 2.7 MiB/s, 45 objects/s recovering 2023-09-23T22:42:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:31 smithi070 ceph-mon[192652]: failed to encode map e584 with expected crc 2023-09-23T22:42:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:31 smithi070 ceph-mon[192652]: failed to encode map e584 with expected crc 2023-09-23T22:42:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:31 smithi070 ceph-mon[192652]: osd.3 [v2:172.21.15.26:6826/2763378290,v1:172.21.15.26:6827/2763378290] boot 2023-09-23T22:42:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:31 smithi070 ceph-mon[192652]: osdmap e584: 8 total, 7 up, 4 in 2023-09-23T22:42:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:31 smithi070 ceph-mon[192652]: failed to encode map e584 with expected crc 2023-09-23T22:42:31.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:31 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T22:42:31.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:32.047 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:32.256 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 81 16 42206 42190 32.5424 33.75 0.0179262 0.0307212 2023-09-23T22:42:32.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:32 smithi026 ceph-mon[215775]: failed to encode map e585 with expected crc 2023-09-23T22:42:32.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:32 smithi026 ceph-mon[215775]: failed to encode map e585 with expected crc 2023-09-23T22:42:32.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:32 smithi026 ceph-mon[215775]: osdmap e585: 8 total, 7 up, 4 in 2023-09-23T22:42:32.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:32 smithi026 ceph-mon[215775]: failed to encode map e585 with expected crc 2023-09-23T22:42:32.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:32 smithi026 ceph-mon[223014]: failed to encode map e585 with expected crc 2023-09-23T22:42:32.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:32 smithi026 ceph-mon[223014]: failed to encode map e585 with expected crc 2023-09-23T22:42:32.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:32 smithi026 ceph-mon[223014]: osdmap e585: 8 total, 7 up, 4 in 2023-09-23T22:42:32.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:32 smithi026 ceph-mon[223014]: failed to encode map e585 with expected crc 2023-09-23T22:42:32.748 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:32.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:32.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:32.858 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:32 smithi070 ceph-mon[192652]: failed to encode map e585 with expected crc 2023-09-23T22:42:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:32 smithi070 ceph-mon[192652]: failed to encode map e585 with expected crc 2023-09-23T22:42:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:32 smithi070 ceph-mon[192652]: osdmap e585: 8 total, 7 up, 4 in 2023-09-23T22:42:32.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:32 smithi070 ceph-mon[192652]: failed to encode map e585 with expected crc 2023-09-23T22:42:32.884 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:32.928 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:32.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:32.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:32.965 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:42:33.034 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:33.060 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:33.104 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:33.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:33.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:33.209 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:33.236 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:33.256 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 82 16 42876 42860 32.6561 41.875 0.0167909 0.0306154 2023-09-23T22:42:33.277 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:33.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:33.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:33.385 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:33.413 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:33 smithi070 ceph-mon[192652]: pgmap v1516: 119 pgs: 3 active+recovery_wait+remapped, 82 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+degraded+remapped, 6 active+clean; 2.5 GiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 31 MiB/s wr, 490 op/s; 16844/120987 objects degraded (13.922%); 36213/120987 objects misplaced (29.931%); 8.5 MiB/s, 138 objects/s recovering 2023-09-23T22:42:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:33 smithi070 ceph-mon[192652]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:33 smithi070 ceph-mon[192652]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:33 smithi070 ceph-mon[192652]: osdmap e586: 8 total, 7 up, 4 in 2023-09-23T22:42:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:33 smithi070 ceph-mon[192652]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:33 smithi070 ceph-mon[192652]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:33 smithi070 ceph-mon[192652]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 16844/120987 objects degraded (13.922%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:33.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[223014]: pgmap v1516: 119 pgs: 3 active+recovery_wait+remapped, 82 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+degraded+remapped, 6 active+clean; 2.5 GiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 31 MiB/s wr, 490 op/s; 16844/120987 objects degraded (13.922%); 36213/120987 objects misplaced (29.931%); 8.5 MiB/s, 138 objects/s recovering 2023-09-23T22:42:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[223014]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[223014]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[223014]: osdmap e586: 8 total, 7 up, 4 in 2023-09-23T22:42:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[223014]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[223014]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[223014]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 16844/120987 objects degraded (13.922%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[215775]: pgmap v1516: 119 pgs: 3 active+recovery_wait+remapped, 82 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 9 active+undersized, 1 active+recovering+undersized+degraded+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 5 active+recovery_wait+degraded+remapped, 6 active+clean; 2.5 GiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 31 MiB/s wr, 490 op/s; 16844/120987 objects degraded (13.922%); 36213/120987 objects misplaced (29.931%); 8.5 MiB/s, 138 objects/s recovering 2023-09-23T22:42:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[215775]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[215775]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[215775]: osdmap e586: 8 total, 7 up, 4 in 2023-09-23T22:42:33.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[215775]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[215775]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[215775]: failed to encode map e586 with expected crc 2023-09-23T22:42:33.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 16844/120987 objects degraded (13.922%), 18 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:33.879 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:33.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:33.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:33.984 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:34.104 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:34.217 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:34.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:34.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:34.256 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 83 16 43508 43492 32.7385 39.5 0.0231261 0.0305356 2023-09-23T22:42:34.260 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:34.268 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:42:34.304 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:42:34.325 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:34.350 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:34.352 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:42:34.392 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:34.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:34.398 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:42:34.437 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:42:34.481 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:42:34.498 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:34.524 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:34.550 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:42:34.550 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:42:34.550 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:42:34.550 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:42:34.550 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:42:34.550 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:42:34.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:42:34.551 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:42:34.551 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:42:34.565 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:34.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:34.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:34.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:34 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1481690410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:34.670 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:34.759 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:34 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1481690410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:34.760 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:34 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1481690410' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:34.780 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:34.834 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:42:35.232 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:35.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:35.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:35.256 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 84 16 44160 44144 32.8336 40.75 0.0361795 0.0304501 2023-09-23T22:42:35.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:35.369 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:35.540 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:42:35.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:35 smithi070 ceph-mon[192652]: pgmap v1518: 119 pgs: 3 active+recovery_wait+remapped, 85 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 6 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.6 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 40 MiB/s wr, 633 op/s; 17017/125886 objects degraded (13.518%); 37673/125886 objects misplaced (29.926%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T22:42:35.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3285161168' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:35.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:35 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:35.898 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:35 smithi026 ceph-mon[223014]: pgmap v1518: 119 pgs: 3 active+recovery_wait+remapped, 85 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 6 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.6 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 40 MiB/s wr, 633 op/s; 17017/125886 objects degraded (13.518%); 37673/125886 objects misplaced (29.926%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T22:42:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3285161168' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:35.902 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:35 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:35.903 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:35 smithi026 ceph-mon[215775]: pgmap v1518: 119 pgs: 3 active+recovery_wait+remapped, 85 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 6 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.6 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 40 MiB/s wr, 633 op/s; 17017/125886 objects degraded (13.518%); 37673/125886 objects misplaced (29.926%); 7.6 MiB/s, 120 objects/s recovering 2023-09-23T22:42:35.903 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3285161168' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:35.903 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:35 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:35.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:35.949 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:42:35.949 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:42:36.005 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:36.031 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:36.076 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:36.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:36.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:36.183 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:36.251 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:36.257 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 85 16 44728 44712 32.8649 35.5 0.0104966 0.0304137 2023-09-23T22:42:36.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:36.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[223014]: failed to encode map e587 with expected crc 2023-09-23T22:42:36.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:36.538 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[223014]: osdmap e587: 8 total, 7 up, 4 in 2023-09-23T22:42:36.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:36.538 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[215775]: failed to encode map e587 with expected crc 2023-09-23T22:42:36.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:36.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[215775]: osdmap e587: 8 total, 7 up, 4 in 2023-09-23T22:42:36.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[215775]: failed to encode map e587 with expected crc 2023-09-23T22:42:36.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[215775]: failed to encode map e587 with expected crc 2023-09-23T22:42:36.539 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[215775]: pgmap v1520: 119 pgs: 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.6 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 33 MiB/s wr, 527 op/s; 17209/126462 objects degraded (13.608%); 37087/126462 objects misplaced (29.327%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:42:36.801 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[223014]: failed to encode map e587 with expected crc 2023-09-23T22:42:36.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[223014]: failed to encode map e587 with expected crc 2023-09-23T22:42:36.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:36 smithi026 ceph-mon[223014]: pgmap v1520: 119 pgs: 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.6 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 33 MiB/s wr, 527 op/s; 17209/126462 objects degraded (13.608%); 37087/126462 objects misplaced (29.327%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:42:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:36 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:36 smithi070 ceph-mon[192652]: failed to encode map e587 with expected crc 2023-09-23T22:42:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:36 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:36 smithi070 ceph-mon[192652]: osdmap e587: 8 total, 7 up, 4 in 2023-09-23T22:42:36.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:36 smithi070 ceph-mon[192652]: failed to encode map e587 with expected crc 2023-09-23T22:42:36.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:36 smithi070 ceph-mon[192652]: failed to encode map e587 with expected crc 2023-09-23T22:42:36.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:36 smithi070 ceph-mon[192652]: pgmap v1520: 119 pgs: 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.6 GiB data, 7.5 GiB used, 350 GiB / 358 GiB avail; 33 MiB/s wr, 527 op/s; 17209/126462 objects degraded (13.608%); 37087/126462 objects misplaced (29.327%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T22:42:37.030 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:37.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:37.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:37.136 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:37.164 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:37.261 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 86 16 45256 45240 32.8652 33 0.0102299 0.0304223 2023-09-23T22:42:37.482 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:37.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:37.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:37.589 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:37.614 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:37.656 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:37.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:37.731 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:37.763 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:37.868 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:37.950 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:42:38.091 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:38.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:38.099 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:38.200 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:38.262 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 87 16 45930 45914 32.971 42.125 0.0467729 0.0303192 2023-09-23T22:42:38.337 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:38.367 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:38.367 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:38.518 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:38.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:38.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:38.629 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:38.656 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:38.698 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:38.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:38.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:38.804 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:38.829 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:38.872 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:38.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:38.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:38.981 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:39.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:38 smithi026 ceph-mon[223014]: pgmap v1521: 119 pgs: 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 48 MiB/s wr, 764 op/s; 17425/135351 objects degraded (12.874%); 39834/135351 objects misplaced (29.430%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T22:42:39.067 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 17425/135351 objects degraded (12.874%), 18 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:39.073 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:38 smithi026 ceph-mon[215775]: pgmap v1521: 119 pgs: 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 48 MiB/s wr, 764 op/s; 17425/135351 objects degraded (12.874%); 39834/135351 objects misplaced (29.430%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T22:42:39.073 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 17425/135351 objects degraded (12.874%), 18 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:39.073 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:39.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:38 smithi070 ceph-mon[192652]: pgmap v1521: 119 pgs: 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 48 MiB/s wr, 764 op/s; 17425/135351 objects degraded (12.874%); 39834/135351 objects misplaced (29.430%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T22:42:39.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 17425/135351 objects degraded (12.874%), 18 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:39.220 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:39.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:39.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:39.262 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 88 16 46618 46602 33.0847 43 0.0240377 0.0302183 2023-09-23T22:42:39.325 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:42:39.334 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:39.360 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:39.403 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:39.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:39.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:39.508 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:39.536 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:39.585 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:39.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:39.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:39.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:39.715 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:39.756 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:39.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:39.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:39.863 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:39.887 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:42:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:42:39] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-09-23T22:42:39.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:39.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:39 smithi026 ceph-mon[215775]: failed to encode map e588 with expected crc 2023-09-23T22:42:39.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:39 smithi026 ceph-mon[215775]: osdmap e588: 8 total, 7 up, 4 in 2023-09-23T22:42:39.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:39 smithi026 ceph-mon[215775]: failed to encode map e588 with expected crc 2023-09-23T22:42:39.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:39.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:39 smithi026 ceph-mon[223014]: failed to encode map e588 with expected crc 2023-09-23T22:42:39.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:39 smithi026 ceph-mon[223014]: osdmap e588: 8 total, 7 up, 4 in 2023-09-23T22:42:39.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:39 smithi026 ceph-mon[223014]: failed to encode map e588 with expected crc 2023-09-23T22:42:39.890 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:40.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:39 smithi070 ceph-mon[192652]: failed to encode map e588 with expected crc 2023-09-23T22:42:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:39 smithi070 ceph-mon[192652]: osdmap e588: 8 total, 7 up, 4 in 2023-09-23T22:42:40.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:39 smithi070 ceph-mon[192652]: failed to encode map e588 with expected crc 2023-09-23T22:42:40.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout: 89 16 46903 46887 32.9132 17.8125 0.00224255 0.0302702 2023-09-23T22:42:40.568 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:42:40.575 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:40.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:40.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:40.687 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:40.780 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[223014]: failed to encode map e588 with expected crc 2023-09-23T22:42:40.781 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[223014]: pgmap v1523: 119 pgs: 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 41 MiB/s wr, 661 op/s; 17425/135351 objects degraded (12.874%); 39834/135351 objects misplaced (29.430%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:42:40.781 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[223014]: failed to encode map e589 with expected crc 2023-09-23T22:42:40.781 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[223014]: osdmap e589: 8 total, 7 up, 4 in 2023-09-23T22:42:40.781 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[223014]: failed to encode map e589 with expected crc 2023-09-23T22:42:40.806 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:40.808 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:40.869 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:41.119 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:41.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:40 smithi070 ceph-mon[192652]: failed to encode map e588 with expected crc 2023-09-23T22:42:41.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:40 smithi070 ceph-mon[192652]: pgmap v1523: 119 pgs: 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 41 MiB/s wr, 661 op/s; 17425/135351 objects degraded (12.874%); 39834/135351 objects misplaced (29.430%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:42:41.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:40 smithi070 ceph-mon[192652]: failed to encode map e589 with expected crc 2023-09-23T22:42:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:40 smithi070 ceph-mon[192652]: osdmap e589: 8 total, 7 up, 4 in 2023-09-23T22:42:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:40 smithi070 ceph-mon[192652]: failed to encode map e589 with expected crc 2023-09-23T22:42:41.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[215775]: failed to encode map e588 with expected crc 2023-09-23T22:42:41.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[215775]: pgmap v1523: 119 pgs: 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 7.9 GiB used, 350 GiB / 358 GiB avail; 41 MiB/s wr, 661 op/s; 17425/135351 objects degraded (12.874%); 39834/135351 objects misplaced (29.430%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T22:42:41.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[215775]: failed to encode map e589 with expected crc 2023-09-23T22:42:41.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[215775]: osdmap e589: 8 total, 7 up, 4 in 2023-09-23T22:42:41.125 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:40 smithi026 ceph-mon[215775]: failed to encode map e589 with expected crc 2023-09-23T22:42:41.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:41.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:41.227 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:41.258 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:41.262 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total time run: 90.0098 2023-09-23T22:42:41.262 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Total writes made: 47393 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Write size: 65536 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Object size: 65536 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Bandwidth (MB/sec): 32.9082 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Bandwidth: 42.2708 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max bandwidth (MB/sec): 183 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min bandwidth (MB/sec): 0 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average IOPS: 526 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev IOPS: 676.341 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max IOPS: 2928 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min IOPS: 0 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Average Latency(s): 0.0303833 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Stddev Latency(s): 0.739594 2023-09-23T22:42:41.263 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Max latency(s): 40.1955 2023-09-23T22:42:41.264 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Min latency(s): 0.000971618 2023-09-23T22:42:41.267 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T22:42:41.306 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:41.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:41.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:41.412 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:41.439 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:41.483 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:41.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:41.537 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:41.537 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:41.588 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:41.676 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:42.059 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:41 smithi026 ceph-mon[215775]: failed to encode map e589 with expected crc 2023-09-23T22:42:42.059 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:41 smithi026 ceph-mon[223014]: failed to encode map e589 with expected crc 2023-09-23T22:42:42.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:41 smithi070 ceph-mon[192652]: failed to encode map e589 with expected crc 2023-09-23T22:42:42.228 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:42.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:42.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:42.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:42.403 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:43.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:42 smithi070 ceph-mon[192652]: pgmap v1525: 119 pgs: 1 remapped+peering, 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 41 MiB/s wr, 660 op/s; 16074/138762 objects degraded (11.584%); 38829/138762 objects misplaced (27.982%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:42:43.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:42 smithi026 ceph-mon[215775]: pgmap v1525: 119 pgs: 1 remapped+peering, 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 41 MiB/s wr, 660 op/s; 16074/138762 objects degraded (11.584%); 38829/138762 objects misplaced (27.982%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:42:43.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:42 smithi026 ceph-mon[223014]: pgmap v1525: 119 pgs: 1 remapped+peering, 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 8.1 GiB used, 350 GiB / 358 GiB avail; 41 MiB/s wr, 660 op/s; 16074/138762 objects degraded (11.584%); 38829/138762 objects misplaced (27.982%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T22:42:44.101 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:44.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:44.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:44.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 16074/138762 objects degraded (11.584%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:44.111 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 16074/138762 objects degraded (11.584%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:44.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 16074/138762 objects degraded (11.584%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:44.208 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:44.291 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:44.801 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2023-09-23T22:42:44.801 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2023-09-23T22:42:44.801 DEBUG:teuthology.orchestra.run.smithi070:osd.6> sudo journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T22:42:44.804 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl start ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T22:42:44.839 INFO:journalctl@ceph.osd.6.smithi070.stdout:-- Logs begin at Sat 2023-09-23 21:42:03 UTC. -- 2023-09-23T22:42:44.980 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:44.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:44.990 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:45.091 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:45.124 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:44 smithi070 systemd[1]: Starting Ceph osd.6 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:42:45.164 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:45.299 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:45.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:45.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:45.406 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:45.409 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:45.439 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:45.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:45 smithi026 ceph-mon[223014]: pgmap v1526: 119 pgs: 1 remapped+peering, 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.9 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 127 B/s rd, 40 MiB/s wr, 718 op/s; 16115/140013 objects degraded (11.510%); 39205/140013 objects misplaced (28.001%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:42:45.493 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:45 smithi026 ceph-mon[215775]: pgmap v1526: 119 pgs: 1 remapped+peering, 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.9 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 127 B/s rd, 40 MiB/s wr, 718 op/s; 16115/140013 objects degraded (11.510%); 39205/140013 objects misplaced (28.001%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:42:45.536 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:45.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:45.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:45.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:45 smithi070 ceph-mon[192652]: pgmap v1526: 119 pgs: 1 remapped+peering, 3 active+recovery_wait+remapped, 86 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.9 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 127 B/s rd, 40 MiB/s wr, 718 op/s; 16115/140013 objects degraded (11.510%); 39205/140013 objects misplaced (28.001%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:42:45.641 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:45.727 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:45.911 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:45.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:45.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:46.028 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:42:46.043 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:46.061 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:42:46.095 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:42:46.111 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:46.139 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:42:46.173 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:42:46.211 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:42:46.249 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:42:46.249 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:42:46.250 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:42:46.250 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:42:46.250 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:42:46.250 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:42:46.250 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:42:46.250 INFO:teuthology.orchestra.run.smithi026.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T22:42:46.250 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:42:46.374 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:46.374 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:46 smithi070 bash[409630]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T22:42:46.374 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:46 smithi070 bash[409630]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T22:42:46.375 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:46 smithi070 bash[409630]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2023-09-23T22:42:46.375 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:46 smithi070 bash[409630]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2023-09-23T22:42:46.375 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:46 smithi070 bash[409630]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2023-09-23T22:42:46.375 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:46 smithi070 bash[409630]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T22:42:46.375 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:46 smithi070 bash[409630]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T22:42:46.375 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:46 smithi070 bash[409630]: --> ceph-volume lvm activate successful for osd ID: 6 2023-09-23T22:42:46.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:46.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:46.443 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:42:46.467 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/823283910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:46.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/42775584' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:46.468 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:46 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:46.468 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/823283910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:46.468 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/42775584' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:46.469 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:46 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:46.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:46.510 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:46.557 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:46.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:46.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:46.661 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:46.728 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/823283910' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/42775584' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:46 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:42:47.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[215775]: pgmap v1527: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 127 B/s rd, 17 MiB/s wr, 377 op/s; 16115/139593 objects degraded (11.544%); 42010/139593 objects misplaced (30.095%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:42:47.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:47.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[215775]: failed to encode map e590 with expected crc 2023-09-23T22:42:47.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[215775]: failed to encode map e590 with expected crc 2023-09-23T22:42:47.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[215775]: failed to encode map e590 with expected crc 2023-09-23T22:42:47.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:42:47.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[215775]: osdmap e590: 8 total, 7 up, 4 in 2023-09-23T22:42:47.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[223014]: pgmap v1527: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 127 B/s rd, 17 MiB/s wr, 377 op/s; 16115/139593 objects degraded (11.544%); 42010/139593 objects misplaced (30.095%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:42:47.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:47.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[223014]: failed to encode map e590 with expected crc 2023-09-23T22:42:47.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[223014]: failed to encode map e590 with expected crc 2023-09-23T22:42:47.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[223014]: failed to encode map e590 with expected crc 2023-09-23T22:42:47.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:42:47.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:47 smithi026 ceph-mon[223014]: osdmap e590: 8 total, 7 up, 4 in 2023-09-23T22:42:47.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:47 smithi070 ceph-mon[192652]: pgmap v1527: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 8.2 GiB used, 349 GiB / 358 GiB avail; 127 B/s rd, 17 MiB/s wr, 377 op/s; 16115/139593 objects degraded (11.544%); 42010/139593 objects misplaced (30.095%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T22:42:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:47 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:42:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:47 smithi070 ceph-mon[192652]: failed to encode map e590 with expected crc 2023-09-23T22:42:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:47 smithi070 ceph-mon[192652]: failed to encode map e590 with expected crc 2023-09-23T22:42:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:47 smithi070 ceph-mon[192652]: failed to encode map e590 with expected crc 2023-09-23T22:42:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:47 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:42:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:47 smithi070 ceph-mon[192652]: osdmap e590: 8 total, 7 up, 4 in 2023-09-23T22:42:48.069 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:42:48.374 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:48 smithi070 ceph-osd[410206]: -- 172.21.15.70:0/3856862661 <== mon.1 v2:172.21.15.70:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d6cf1d6b60 con 0x55d6cf19cc00 2023-09-23T22:42:48.374 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:48 smithi070 bash[410141]: 8e7c9aa0f23e8eda39b15167307ee7af79ef52bb843450474a377ea585bb5188 2023-09-23T22:42:48.374 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:48 smithi070 systemd[1]: Started Ceph osd.6 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:42:48.472 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:48.672 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:48 smithi070 conmon[410182]: 2023-09-23T22:42:48.582+0000 7fe392e7b540 -1 Falling back to public interface 2023-09-23T22:42:48.673 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 16115/139593 objects degraded (11.544%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 16115/139593 objects degraded (11.544%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:48.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 16115/139593 objects degraded (11.544%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:49.317 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:49.328 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:49.428 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:49.460 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:49.510 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:49.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:49.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:49.592 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:49 smithi026 ceph-mon[215775]: pgmap v1529: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 8.1 GiB used, 349 GiB / 358 GiB avail; 127 B/s rd, 18 MiB/s wr, 536 op/s; 15949/136290 objects degraded (11.702%); 40822/136290 objects misplaced (29.952%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T22:42:49.600 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:49 smithi026 ceph-mon[223014]: pgmap v1529: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 8.1 GiB used, 349 GiB / 358 GiB avail; 127 B/s rd, 18 MiB/s wr, 536 op/s; 15949/136290 objects degraded (11.702%); 40822/136290 objects misplaced (29.952%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T22:42:49.615 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:49.628 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T22:42:49.629 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T22:42:49.701 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:49.749 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:49 smithi070 ceph-mon[192652]: pgmap v1529: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.8 GiB data, 8.1 GiB used, 349 GiB / 358 GiB avail; 127 B/s rd, 18 MiB/s wr, 536 op/s; 15949/136290 objects degraded (11.702%); 40822/136290 objects misplaced (29.952%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T22:42:49.858 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:49.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:49.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:49.880 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:42:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:42:49] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:no valid command found; 10 closest matches: 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:0 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:1 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:2 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:abort 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:assert 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:config diff 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:config diff get 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:config get 2023-09-23T22:42:49.966 INFO:teuthology.orchestra.run.smithi070.stderr:config help [] 2023-09-23T22:42:49.967 INFO:teuthology.orchestra.run.smithi070.stderr:config set ... 2023-09-23T22:42:49.967 INFO:teuthology.orchestra.run.smithi070.stderr:admin_socket: invalid command 2023-09-23T22:42:49.975 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:50.057 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:50.177 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:42:50.177 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:151868 8.f 8.3587fcdf (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e415)", 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:33:45.193100+0000", 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 544.98046167799998, 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0010652879999999, 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:50.178 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 151868 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193100+0000", 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193100+0000", 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193098+0000", 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.180 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193104+0000", 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9959999999999999e-06 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193112+0000", 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8630000000000008e-06 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.193120+0000", 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8820000000000001e-06 2023-09-23T22:42:50.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.205704+0000", 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012583321999999999 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:45.205719+0000", 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5328999999999998e-05 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.194140+0000", 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98842133099999996 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:33:46.194165+0000", 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4830000000000001e-05 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188384 8.e 8:7bd73a78:::benchmark_data_smithi026_497658_object89492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.107077+0000", 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 487.06648417700001, 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0056152540000001, 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188384 2023-09-23T22:42:50.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107077+0000", 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107077+0000", 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107076+0000", 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107082+0000", 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9000000000000003e-06 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107094+0000", 2023-09-23T22:42:50.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2679e-05 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107102+0000", 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3640000000000001e-06 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107151+0000", 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9509999999999999e-05 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.107162+0000", 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0845999999999999e-05 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.104169+0000", 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99700736599999995 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.104250+0000", 2023-09-23T22:42:50.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0839000000000003e-05 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.104359+0000", 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109159 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112412+0000", 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080528330000000006 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112558+0000", 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014534199999999999 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112597+0000", 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9882000000000002e-05 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112651+0000", 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.363e-05 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112693+0000", 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1485e-05 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.191 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188418 8.e 8:72a1b165:::benchmark_data_smithi026_497658_object89526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.147012+0000", 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 487.02654907599998, 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98431729099999998, 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188418 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.192 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147012+0000", 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147012+0000", 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147011+0000", 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147016+0000", 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2360000000000004e-06 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147028+0000", 2023-09-23T22:42:50.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1131999999999999e-05 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147035+0000", 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0099999999999998e-06 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147095+0000", 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0294000000000002e-05 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.147104+0000", 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.4949999999999997e-06 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.110529+0000", 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96342510100000001 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.110634+0000", 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000104362 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.110768+0000", 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000133886 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112828+0000", 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002059988 2023-09-23T22:42:50.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.112973+0000", 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000145816 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131241+0000", 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.018267965000000001 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131290+0000", 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8979e-05 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.131330+0000", 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9229e-05 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188425 8.e 8:7aac6c1a:::benchmark_data_smithi026_497658_object89533:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.155148+0000", 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 487.01841357900003, 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98108856700000002, 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188425 2023-09-23T22:42:50.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155148+0000", 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155148+0000", 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155147+0000", 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155150+0000", 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0929999999999999e-06 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155158+0000", 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7659999999999998e-06 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155164+0000", 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8130000000000001e-06 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155232+0000", 2023-09-23T22:42:50.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8145999999999994e-05 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155238+0000", 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9680000000000003e-06 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.115278+0000", 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96004005299999995 2023-09-23T22:42:50.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.115389+0000", 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000110332 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.115491+0000", 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010268999999999999 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120676+0000", 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051844309999999998 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.125970+0000", 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052943809999999999 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136166+0000", 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010195533 2023-09-23T22:42:50.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136200+0000", 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4057999999999998e-05 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136236+0000", 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6789999999999998e-05 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.207 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.45842.0:188426 8.e 8:7e42bbf5:::benchmark_data_smithi026_497658_object89534:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 487.01839239999998, 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98118840799999996, 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.45842", 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/4109252893", 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188426 2023-09-23T22:42:50.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155169+0000", 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155171+0000", 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6089999999999999e-06 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155179+0000", 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9230000000000002e-06 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155185+0000", 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.468e-06 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155258+0000", 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3825999999999994e-05 2023-09-23T22:42:50.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:43.155277+0000", 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8732999999999998e-05 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.117481+0000", 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96220415800000003 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.117631+0000", 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149794 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,4", 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.117792+0000", 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016125699999999999 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.120845+0000", 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030528819999999998 2023-09-23T22:42:50.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.126058+0000", 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052129309999999996 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136274+0000", 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010215504 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136323+0000", 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9487000000000002e-05 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:34:44.136357+0000", 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4270999999999998e-05 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.215 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104685 9.2 9:46fec5fe:::benchmark_data_smithi026_555954_object9956:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.292511+0000", 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.88105019199998, 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0057272559999999, 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104685 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292511+0000", 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292511+0000", 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292509+0000", 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292518+0000", 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4230000000000005e-06 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292537+0000", 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9038999999999998e-05 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292550+0000", 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3446e-05 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292613+0000", 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2655000000000002e-05 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292625+0000", 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1747e-05 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298147+0000", 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0055220330000001 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298238+0000", 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.1748000000000003e-05 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.220 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104686 9.2 9:4bc296ca:::benchmark_data_smithi026_555954_object9957:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.88099156300001, 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.032939702, 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.221 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104686 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:42:50.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292569+0000", 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292573+0000", 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6500000000000002e-06 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292581+0000", 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8080000000000005e-06 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292589+0000", 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4209999999999995e-06 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292656+0000", 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7243999999999995e-05 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292666+0000", 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3850000000000008e-06 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298265+0000", 2023-09-23T22:42:50.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.005598754 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298394+0000", 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012897599999999999 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298532+0000", 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001381 2023-09-23T22:42:50.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.308571+0000", 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010039792000000001 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.308872+0000", 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030035000000000001 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325328+0000", 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016456529000000001 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325391+0000", 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2385999999999995e-05 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325510+0000", 2023-09-23T22:42:50.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118841 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104695 9.6 9:6983d852:::benchmark_data_smithi026_555954_object9966:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.320260+0000", 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 335.85330192499998, 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.000856502, 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.229 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104695 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320260+0000", 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320260+0000", 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320258+0000", 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320267+0000", 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6309999999999992e-06 2023-09-23T22:42:50.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320286+0000", 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.893e-05 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320294+0000", 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1999999999999994e-06 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320349+0000", 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5631999999999999e-05 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.320362+0000", 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3037000000000001e-05 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.309531+0000", 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98916838299999998 2023-09-23T22:42:50.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.309666+0000", 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135649 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 6", 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.309801+0000", 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134764 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.317002+0000", 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0072011929999999998 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.320998+0000", 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0039957480000000004 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.321034+0000", 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5407999999999997e-05 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.321116+0000", 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2433000000000002e-05 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.236 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112814 9.6 9:6d8bc28c:::benchmark_data_smithi026_555954_object18085:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:38:54.442835+0000", 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 235.730726433, 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.98434545699999998, 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112814 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442835+0000", 2023-09-23T22:42:50.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442835+0000", 2023-09-23T22:42:50.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442834+0000", 2023-09-23T22:42:50.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.239 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442840+0000", 2023-09-23T22:42:50.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0310000000000004e-06 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442852+0000", 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2702e-05 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442860+0000", 2023-09-23T22:42:50.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2420000000000003e-06 2023-09-23T22:42:50.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442901+0000", 2023-09-23T22:42:50.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1551000000000002e-05 2023-09-23T22:42:50.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:54.442909+0000", 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3880000000000002e-06 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.408301+0000", 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.965391954 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.408440+0000", 2023-09-23T22:42:50.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013930600000000001 2023-09-23T22:42:50.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1", 2023-09-23T22:42:50.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.408565+0000", 2023-09-23T22:42:50.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012514199999999999 2023-09-23T22:42:50.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.416305+0000", 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0077403649999999999 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.427051+0000", 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010745265 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.427084+0000", 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3386999999999997e-05 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:38:55.427180+0000", 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6440999999999994e-05 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112966 9.7 9:e8a30d92:::benchmark_data_smithi026_555954_object18237:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328682+0000", 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 224.844878978, 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.030748977, 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112966 2023-09-23T22:42:50.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328682+0000", 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328682+0000", 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328680+0000", 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999976 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328690+0000", 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7580000000000004e-06 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328733+0000", 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3245999999999999e-05 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328745+0000", 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1802e-05 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328817+0000", 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2485999999999997e-05 2023-09-23T22:42:50.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328829+0000", 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1195999999999999e-05 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.340011+0000", 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.011182614 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.340147+0000", 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013621 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.340232+0000", 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4806000000000001e-05 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346112+0000", 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.005879843 2023-09-23T22:42:50.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346121+0000", 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9819999999999997e-06 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359213+0000", 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013091635000000001 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359279+0000", 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5773999999999997e-05 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359431+0000", 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001529 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112980 9.7 9:e8e4b400:::benchmark_data_smithi026_555954_object18251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.348335+0000", 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 224.825226829, 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.011279043, 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112980 2023-09-23T22:42:50.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348335+0000", 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348335+0000", 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348333+0000", 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348339+0000", 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4689999999999996e-06 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348351+0000", 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2146e-05 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348356+0000", 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4840000000000003e-06 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348409+0000", 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2997000000000001e-05 2023-09-23T22:42:50.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.348419+0000", 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3209999999999994e-06 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346148+0000", 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99772916899999997 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346286+0000", 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013786899999999999 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.346413+0000", 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012667900000000001 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.354459+0000", 2023-09-23T22:42:50.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080463510000000002 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.357736+0000", 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003277355 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359472+0000", 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001736118 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359503+0000", 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0645000000000002e-05 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359614+0000", 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011066899999999999 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112988 9.7 9:e725ecc4:::benchmark_data_smithi026_555954_object18259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.354932+0000", 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 224.81862940799999, 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0114405769999999, 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:42:50.262 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112988 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354932+0000", 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354932+0000", 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354931+0000", 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354937+0000", 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5400000000000001e-06 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354955+0000", 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7791999999999998e-05 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.354964+0000", 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6969999999999999e-06 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.355023+0000", 2023-09-23T22:42:50.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9036999999999997e-05 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.355035+0000", 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2679e-05 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.350800+0000", 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.99576413200000002 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.350941+0000", 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014160400000000001 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.351070+0000", 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012856799999999999 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.357793+0000", 2023-09-23T22:42:50.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0067230629999999996 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.359698+0000", 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00190493 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.363020+0000", 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0033221280000000001 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.363075+0000", 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5172e-05 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.366373+0000", 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032975359999999998 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23700 10.4 10:2b626a3e:::benchmark_data_smithi026_640738_object23699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.405861+0000", 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.767700847, 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3690022009999998, 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23700 2023-09-23T22:42:50.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405861+0000", 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405861+0000", 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405859+0000", 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405882+0000", 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2935999999999999e-05 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405893+0000", 2023-09-23T22:42:50.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1070000000000001e-05 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405899+0000", 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8880000000000002e-06 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405942+0000", 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2895000000000002e-05 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.406020+0000", 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7571999999999995e-05 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.541149+0000", 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.135129573 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.547471+0000", 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063218190000000002 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.570649+0000", 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023177474999999999 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.570663+0000", 2023-09-23T22:42:50.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4408e-05 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.539860+0000", 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96919705199999995 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.539868+0000", 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0099999999999995e-06 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.747326+0000", 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.207457716 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.749304+0000", 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001978133 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774797+0000", 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025492642999999999 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774863+0000", 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6105000000000005e-05 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23703 10.9 10:9dc130d6:::benchmark_data_smithi026_640738_object23702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:42:50.278 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.544380+0000", 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.629181619999997, 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2154338739999999, 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23703 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544380+0000", 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544380+0000", 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544379+0000", 2023-09-23T22:42:50.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544402+0000", 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3332999999999998e-05 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544413+0000", 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0912e-05 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544420+0000", 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6690000000000006e-06 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544864+0000", 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044322699999999998 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544998+0000", 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134339 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545158+0000", 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159946 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.550111+0000", 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049527060000000003 2023-09-23T22:42:50.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.550128+0000", 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7799e-05 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.536473+0000", 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.986344318 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.536488+0000", 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5150999999999999e-05 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746698+0000", 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2102103909999999 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.748810+0000", 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021113019999999998 2023-09-23T22:42:50.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.759720+0000", 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010910840999999999 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.759814+0000", 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3207000000000004e-05 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.286 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23704 10.4 10:2f348807:::benchmark_data_smithi026_640738_object23703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.544429+0000", 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.629131989999998, 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.230336844, 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:50.287 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23704 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544429+0000", 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544429+0000", 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544429+0000", 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544450+0000", 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0857e-05 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544457+0000", 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6619999999999999e-06 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544462+0000", 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.631e-06 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544508+0000", 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6322000000000001e-05 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544667+0000", 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015808599999999999 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544859+0000", 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000192194 2023-09-23T22:42:50.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.547477+0000", 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026186289999999999 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.574150+0000", 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026672927999999999 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.574163+0000", 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.296e-05 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.539887+0000", 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96572371099999998 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.539892+0000", 2023-09-23T22:42:50.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9710000000000003e-06 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.751454+0000", 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.211562539 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.752273+0000", 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081840400000000001 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774687+0000", 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022414204 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774766+0000", 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9196999999999994e-05 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23705 10.3 10:c03ba639:::benchmark_data_smithi026_640738_object23704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.555379+0000", 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.618182304999998, 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2153192160000001, 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23705 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555379+0000", 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555379+0000", 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555378+0000", 2023-09-23T22:42:50.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555404+0000", 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5799000000000002e-05 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555415+0000", 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1182999999999999e-05 2023-09-23T22:42:50.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555426+0000", 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0934000000000001e-05 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555779+0000", 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035356400000000001 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555933+0000", 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015408899999999999 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.556090+0000", 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015706300000000001 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.562662+0000", 2023-09-23T22:42:50.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065715909999999999 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579088+0000", 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016426211999999999 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579100+0000", 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1781e-05 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.540506+0000", 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96140631600000004 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.540519+0000", 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3169999999999999e-05 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.750405+0000", 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.209885898 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.751359+0000", 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00095339299999999997 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770625+0000", 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019265971 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770698+0000", 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3579999999999997e-05 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23706 10.5 10:ae9e38a0:::benchmark_data_smithi026_640738_object23705:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.555441+0000", 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.618120654000002, 2023-09-23T22:42:50.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2154646439999999, 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23706 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555441+0000", 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555441+0000", 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555440+0000", 2023-09-23T22:42:50.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555462+0000", 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1421e-05 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555470+0000", 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9990000000000008e-06 2023-09-23T22:42:50.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555484+0000", 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3971000000000001e-05 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.580064+0000", 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024580547000000001 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.580073+0000", 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6330000000000002e-06 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.543265+0000", 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96319191999999998 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.543280+0000", 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5228000000000001e-05 2023-09-23T22:42:50.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.752233+0000", 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2089533160000001 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.752376+0000", 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000142898 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.752561+0000", 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018452200000000001 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.768542+0000", 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015980836000000002 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770097+0000", 2023-09-23T22:42:50.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015551639999999999 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770775+0000", 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00067841400000000001 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770816+0000", 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0444000000000001e-05 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770905+0000", 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9864000000000006e-05 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23708 10.7 10:e7b56d34:::benchmark_data_smithi026_640738_object23707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.555490+0000", 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.618071137000001, 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2026997179999999, 2023-09-23T22:42:50.313 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23708 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555490+0000", 2023-09-23T22:42:50.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555490+0000", 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555490+0000", 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555513+0000", 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3119e-05 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555520+0000", 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1709999999999996e-06 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.316 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555526+0000", 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0580000000000003e-06 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.561054+0000", 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055277349999999998 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.561223+0000", 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016923200000000001 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0", 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.561390+0000", 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016651599999999999 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.562883+0000", 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001493692 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579044+0000", 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016160858 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579058+0000", 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3764999999999999e-05 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.516181+0000", 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93712273700000004 2023-09-23T22:42:50.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.516196+0000", 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5617e-05 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.745713+0000", 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.229516936 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.748581+0000", 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028682149999999999 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.758101+0000", 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095193750000000001 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.758190+0000", 2023-09-23T22:42:50.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9208000000000004e-05 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23709 10.d 10:b530215a:::benchmark_data_smithi026_640738_object23708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.571919+0000", 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 45.601642863999999, 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1996814210000002, 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.322 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23709 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.571919+0000", 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.571919+0000", 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.571917+0000", 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.571946+0000", 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9686e-05 2023-09-23T22:42:50.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.572715+0000", 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00076831999999999996 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.572735+0000", 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0168000000000001e-05 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579332+0000", 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065975699999999997 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579343+0000", 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0319999999999999e-05 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.543315+0000", 2023-09-23T22:42:50.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96397255000000004 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.543328+0000", 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2292e-05 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.750428+0000", 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2071007579999999 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.750600+0000", 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000171657 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.750815+0000", 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021523700000000001 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:42:50.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.753636+0000", 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028208420000000001 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770038+0000", 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016402067999999999 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770362+0000", 2023-09-23T22:42:50.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000323476 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770429+0000", 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7350999999999999e-05 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.771600+0000", 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011710710000000001 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.330 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.331 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.331 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:38586 10.7 10:e7840033:::benchmark_data_smithi026_640738_object38585:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e580)", 2023-09-23T22:42:50.331 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:25.388027+0000", 2023-09-23T22:42:50.331 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.785534163000001, 2023-09-23T22:42:50.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.015240863, 2023-09-23T22:42:50.331 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:42:50.331 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 38586 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.388027+0000", 2023-09-23T22:42:50.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.388027+0000", 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.388026+0000", 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.388053+0000", 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.7368999999999998e-05 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.388067+0000", 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3489000000000001e-05 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.388075+0000", 2023-09-23T22:42:50.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.875e-06 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.388123+0000", 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8661e-05 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.388271+0000", 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000147455 2023-09-23T22:42:50.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.388430+0000", 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159796 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.394970+0000", 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065392059999999997 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:42:50.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:25.394982+0000", 2023-09-23T22:42:50.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2818e-05 2023-09-23T22:42:50.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:42:50.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:26.403214+0000", 2023-09-23T22:42:50.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0082310670000001 2023-09-23T22:42:50.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:26.403268+0000", 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4605000000000003e-05 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:42:50.339 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:42:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:50.340 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:50.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:50.441 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:50.473 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:50.630 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:50.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:50.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:50.694 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T22:42:50.695 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-09-23T22:42:50.734 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:50.805 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:50.851 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:50.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:50.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:50.954 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:50.981 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:51.025 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:51.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:51.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:51.128 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:51.157 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:51.200 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:51.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:51.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:51.306 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:51.332 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:51.376 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:51.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:51.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:51.482 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:51.521 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:42:51.561 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:51.630 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:42:51.674 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:51.674 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:42:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:51.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:51.691 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:51 smithi026 ceph-mon[223014]: pgmap v1530: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.7 GiB data, 8.0 GiB used, 350 GiB / 358 GiB avail; 109 B/s rd, 14 MiB/s wr, 653 op/s; 15738/130434 objects degraded (12.066%); 39081/130434 objects misplaced (29.962%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:42:51.692 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:51 smithi026 ceph-mon[215775]: pgmap v1530: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.7 GiB data, 8.0 GiB used, 350 GiB / 358 GiB avail; 109 B/s rd, 14 MiB/s wr, 653 op/s; 15738/130434 objects degraded (12.066%); 39081/130434 objects misplaced (29.962%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:42:51.792 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:51.820 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:51 smithi070 ceph-mon[192652]: pgmap v1530: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.7 GiB data, 8.0 GiB used, 350 GiB / 358 GiB avail; 109 B/s rd, 14 MiB/s wr, 653 op/s; 15738/130434 objects degraded (12.066%); 39081/130434 objects misplaced (29.962%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T22:42:51.823 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:51.871 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:51.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:51.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:51.931 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:51.977 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:52.062 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:52.173 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:52.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:52.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:52.286 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:42:52.359 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:52.427 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:52.439 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:42:52.490 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:42:52.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:52.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:42:52.597 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:52.656 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:52.798 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:52.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:52.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:52.912 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:53.004 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:53.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:53 smithi026 ceph-mon[215775]: pgmap v1531: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.6 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 102 B/s rd, 7.3 MiB/s wr, 669 op/s; 15516/125685 objects degraded (12.345%); 37415/125685 objects misplaced (29.769%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:42:53.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15516/125685 objects degraded (12.345%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:53.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:53 smithi026 ceph-mon[223014]: pgmap v1531: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.6 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 102 B/s rd, 7.3 MiB/s wr, 669 op/s; 15516/125685 objects degraded (12.345%); 37415/125685 objects misplaced (29.769%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:42:53.491 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15516/125685 objects degraded (12.345%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:53.679 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:42:53.679 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:42:53.679 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:42:53.679 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:42:53.703 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:53 smithi070 ceph-mon[192652]: pgmap v1531: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.6 GiB data, 7.8 GiB used, 350 GiB / 358 GiB avail; 102 B/s rd, 7.3 MiB/s wr, 669 op/s; 15516/125685 objects degraded (12.345%); 37415/125685 objects misplaced (29.769%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:42:53.703 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15516/125685 objects degraded (12.345%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:54.427 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:54 smithi070 conmon[410182]: 2023-09-23T22:42:54.167+0000 7fe392e7b540 -1 osd.6 512 log_to_monitors true 2023-09-23T22:42:54.446 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:42:54.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:54.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:54 smithi026 ceph-mon[215775]: from='osd.6 [v2:172.21.15.70:6816/1249954970,v1:172.21.15.70:6817/1249954970]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:42:54.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:54 smithi026 ceph-mon[215775]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:42:54.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:54.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:54 smithi026 ceph-mon[223014]: from='osd.6 [v2:172.21.15.70:6816/1249954970,v1:172.21.15.70:6817/1249954970]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:42:54.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:54 smithi026 ceph-mon[223014]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:42:54.868 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:54.873 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:42:54 smithi070 conmon[410182]: 2023-09-23T22:42:54.625+0000 7fe385c5f700 -1 osd.6 591 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T22:42:54.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:42:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:54 smithi070 ceph-mon[192652]: from='osd.6 [v2:172.21.15.70:6816/1249954970,v1:172.21.15.70:6817/1249954970]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:42:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:54 smithi070 ceph-mon[192652]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T22:42:55.696 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:42:55.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[215775]: pgmap v1532: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 857 KiB/s wr, 704 op/s; 15290/119655 objects degraded (12.778%); 35631/119655 objects misplaced (29.778%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:42:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[215775]: failed to encode map e591 with expected crc 2023-09-23T22:42:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[215775]: failed to encode map e591 with expected crc 2023-09-23T22:42:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[215775]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:42:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[215775]: osdmap e591: 8 total, 7 up, 4 in 2023-09-23T22:42:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[215775]: from='osd.6 [v2:172.21.15.70:6816/1249954970,v1:172.21.15.70:6817/1249954970]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:42:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[215775]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:42:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[215775]: failed to encode map e591 with expected crc 2023-09-23T22:42:55.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[223014]: pgmap v1532: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 857 KiB/s wr, 704 op/s; 15290/119655 objects degraded (12.778%); 35631/119655 objects misplaced (29.778%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:42:55.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[223014]: failed to encode map e591 with expected crc 2023-09-23T22:42:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[223014]: failed to encode map e591 with expected crc 2023-09-23T22:42:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[223014]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:42:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[223014]: osdmap e591: 8 total, 7 up, 4 in 2023-09-23T22:42:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[223014]: from='osd.6 [v2:172.21.15.70:6816/1249954970,v1:172.21.15.70:6817/1249954970]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:42:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[223014]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:42:55.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:55 smithi026 ceph-mon[223014]: failed to encode map e591 with expected crc 2023-09-23T22:42:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:55 smithi070 ceph-mon[192652]: pgmap v1532: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.7 GiB used, 350 GiB / 358 GiB avail; 857 KiB/s wr, 704 op/s; 15290/119655 objects degraded (12.778%); 35631/119655 objects misplaced (29.778%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T22:42:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:55 smithi070 ceph-mon[192652]: failed to encode map e591 with expected crc 2023-09-23T22:42:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:55 smithi070 ceph-mon[192652]: failed to encode map e591 with expected crc 2023-09-23T22:42:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:55 smithi070 ceph-mon[192652]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T22:42:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:55 smithi070 ceph-mon[192652]: osdmap e591: 8 total, 7 up, 4 in 2023-09-23T22:42:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:55 smithi070 ceph-mon[192652]: from='osd.6 [v2:172.21.15.70:6816/1249954970,v1:172.21.15.70:6817/1249954970]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:42:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:55 smithi070 ceph-mon[192652]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi070", "root=default"]}]: dispatch 2023-09-23T22:42:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:55 smithi070 ceph-mon[192652]: failed to encode map e591 with expected crc 2023-09-23T22:42:55.920 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:42:55.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:55.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:42:56.027 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:56.057 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:56.106 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:56.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:56.111 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:56.212 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:42:56.217 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:56.243 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:56.318 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:42:56.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:56.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:42:56.422 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:56.450 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:56.675 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:42:56.699 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:42:56.720 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:42:56.736 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:56 smithi070 ceph-mon[192652]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:42:56.736 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:56 smithi070 ceph-mon[192652]: failed to encode map e592 with expected crc 2023-09-23T22:42:56.736 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:56 smithi070 ceph-mon[192652]: osd.6 [v2:172.21.15.70:6816/1249954970,v1:172.21.15.70:6817/1249954970] boot 2023-09-23T22:42:56.736 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:56 smithi070 ceph-mon[192652]: osdmap e592: 8 total, 8 up, 4 in 2023-09-23T22:42:56.736 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:56 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:42:56.736 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:56 smithi070 ceph-mon[192652]: failed to encode map e592 with expected crc 2023-09-23T22:42:56.736 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:56 smithi070 ceph-mon[192652]: failed to encode map e592 with expected crc 2023-09-23T22:42:56.736 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:56 smithi070 ceph-mon[192652]: osdmap e593: 8 total, 8 up, 4 in 2023-09-23T22:42:56.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[215775]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:42:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[215775]: failed to encode map e592 with expected crc 2023-09-23T22:42:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[215775]: osd.6 [v2:172.21.15.70:6816/1249954970,v1:172.21.15.70:6817/1249954970] boot 2023-09-23T22:42:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[215775]: osdmap e592: 8 total, 8 up, 4 in 2023-09-23T22:42:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:42:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[215775]: failed to encode map e592 with expected crc 2023-09-23T22:42:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[215775]: failed to encode map e592 with expected crc 2023-09-23T22:42:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[215775]: osdmap e593: 8 total, 8 up, 4 in 2023-09-23T22:42:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[223014]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T22:42:56.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[223014]: failed to encode map e592 with expected crc 2023-09-23T22:42:56.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[223014]: osd.6 [v2:172.21.15.70:6816/1249954970,v1:172.21.15.70:6817/1249954970] boot 2023-09-23T22:42:56.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[223014]: osdmap e592: 8 total, 8 up, 4 in 2023-09-23T22:42:56.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T22:42:56.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[223014]: failed to encode map e592 with expected crc 2023-09-23T22:42:56.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[223014]: failed to encode map e592 with expected crc 2023-09-23T22:42:56.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:56 smithi026 ceph-mon[223014]: osdmap e593: 8 total, 8 up, 4 in 2023-09-23T22:42:56.742 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:42:56.764 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:42:56.785 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:42:56.805 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:42:56.825 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:42:56.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:42:56.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:42:56.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:42:56.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:42:56.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:42:56.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:42:56.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:42:56.827 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:42:56.879 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:42:56.879 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:42:56.879 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:42:56.879 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:42:56.879 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:42:57.417 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:57.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:57.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:57.475 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:42:57.521 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:57.560 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:57.567 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:42:57.716 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:57.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:57.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:57.731 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: failed to encode map e513 with expected crc 2023-09-23T22:42:57.732 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: failed to encode map e553 with expected crc 2023-09-23T22:42:57.732 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: failed to encode map e592 with expected crc 2023-09-23T22:42:57.732 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: pgmap v1535: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 724 op/s; 15290/118902 objects degraded (12.859%); 35380/118902 objects misplaced (29.756%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:42:57.732 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.732 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.732 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.732 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: 10.8 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3676 2023-09-23T22:42:57.732 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.732 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: 10.4 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3517 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: 10.2 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3491 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: 10.3 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3714 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: 10.7 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3530 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: 10.1 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3650 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: 10.5 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3579 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3891240510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3102076117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:57.733 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:57 smithi070 ceph-mon[192652]: osdmap e594: 8 total, 8 up, 4 in 2023-09-23T22:42:57.821 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:57.872 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: failed to encode map e513 with expected crc 2023-09-23T22:42:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: failed to encode map e553 with expected crc 2023-09-23T22:42:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: failed to encode map e592 with expected crc 2023-09-23T22:42:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: pgmap v1535: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 724 op/s; 15290/118902 objects degraded (12.859%); 35380/118902 objects misplaced (29.756%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:42:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: 10.8 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3676 2023-09-23T22:42:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: 10.4 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3517 2023-09-23T22:42:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: 10.2 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3491 2023-09-23T22:42:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: 10.3 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3714 2023-09-23T22:42:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: 10.7 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3530 2023-09-23T22:42:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: 10.1 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3650 2023-09-23T22:42:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: 10.5 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3579 2023-09-23T22:42:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3891240510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3102076117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:42:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:57.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[223014]: osdmap e594: 8 total, 8 up, 4 in 2023-09-23T22:42:57.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: failed to encode map e513 with expected crc 2023-09-23T22:42:57.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: failed to encode map e553 with expected crc 2023-09-23T22:42:57.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: failed to encode map e592 with expected crc 2023-09-23T22:42:57.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: pgmap v1535: 119 pgs: 3 active+recovery_wait+remapped, 87 active+clean+remapped, 1 active+recovery_wait+undersized+remapped, 5 active+undersized, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.4 GiB data, 7.4 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 724 op/s; 15290/118902 objects degraded (12.859%); 35380/118902 objects misplaced (29.756%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T22:42:57.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: 10.8 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3676 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: failed to encode map e593 with expected crc 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: 10.4 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3517 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: 10.2 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3491 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: 10.3 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3714 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: 10.7 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3530 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: 10.1 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3650 2023-09-23T22:42:57.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: 10.5 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3579 2023-09-23T22:42:57.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3891240510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:57.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:42:57.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3102076117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:42:57.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:42:57.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:42:57.990 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:57 smithi026 ceph-mon[215775]: osdmap e594: 8 total, 8 up, 4 in 2023-09-23T22:42:58.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.547 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: 10.0 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3579 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: 10.b starting backfill to osd.6 from (0'0,0'0] MAX to 591'3582 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: 10.6 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3562 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: 10.f starting backfill to osd.6 from (0'0,0'0] MAX to 591'3635 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: 10.c starting backfill to osd.6 from (0'0,0'0] MAX to 591'3594 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: 10.9 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3509 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: 10.d starting backfill to osd.6 from (0'0,0'0] MAX to 591'3587 2023-09-23T22:42:58.548 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: 10.e starting backfill to osd.6 from (0'0,0'0] MAX to 591'3616 2023-09-23T22:42:58.549 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 15290/118902 objects degraded (12.859%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:58.549 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: osdmap e595: 8 total, 8 up, 4 in 2023-09-23T22:42:58.549 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:58 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:42:58.635 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:58.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:58.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:58.739 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:42:58.790 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:58.866 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:42:58.949 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:42:58.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:58.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: 10.0 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3579 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: 10.b starting backfill to osd.6 from (0'0,0'0] MAX to 591'3582 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: 10.6 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3562 2023-09-23T22:42:58.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: 10.f starting backfill to osd.6 from (0'0,0'0] MAX to 591'3635 2023-09-23T22:42:58.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: 10.c starting backfill to osd.6 from (0'0,0'0] MAX to 591'3594 2023-09-23T22:42:58.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: 10.9 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3509 2023-09-23T22:42:58.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: 10.d starting backfill to osd.6 from (0'0,0'0] MAX to 591'3587 2023-09-23T22:42:58.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: 10.e starting backfill to osd.6 from (0'0,0'0] MAX to 591'3616 2023-09-23T22:42:58.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 15290/118902 objects degraded (12.859%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:58.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: osdmap e595: 8 total, 8 up, 4 in 2023-09-23T22:42:58.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:42:58.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: failed to encode map e594 with expected crc 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: 10.0 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3579 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: 10.b starting backfill to osd.6 from (0'0,0'0] MAX to 591'3582 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: 10.6 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3562 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: 10.f starting backfill to osd.6 from (0'0,0'0] MAX to 591'3635 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: 10.c starting backfill to osd.6 from (0'0,0'0] MAX to 591'3594 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: 10.9 starting backfill to osd.6 from (0'0,0'0] MAX to 591'3509 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: 10.d starting backfill to osd.6 from (0'0,0'0] MAX to 591'3587 2023-09-23T22:42:58.989 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: 10.e starting backfill to osd.6 from (0'0,0'0] MAX to 591'3616 2023-09-23T22:42:58.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 15290/118902 objects degraded (12.859%), 17 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T22:42:58.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: osdmap e595: 8 total, 8 up, 4 in 2023-09-23T22:42:58.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:58 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:42:59.061 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:42:59.169 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:59.297 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:42:59.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:59.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:42:59.408 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:42:59.436 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:42:59.436 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:42:59.455 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:59.508 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:42:59.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:59.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:42:59.612 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:42:59.696 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:42:59.831 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:42:59.845 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:42:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:42:59] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2023-09-23T22:42:59.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[223014]: pgmap v1538: 119 pgs: 1 peering, 40 remapped+peering, 2 active+recovery_wait+remapped, 54 active+clean+remapped, 4 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.3 GiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 364 op/s; 6930/115284 objects degraded (6.011%); 21936/115284 objects misplaced (19.028%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:42:59.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[223014]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[223014]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[223014]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[223014]: 10.2 continuing backfill to osd.6 from (573'800,591'3491] 10:40e084bb:::benchmark_data_smithi026_640738_object41542:head to 591'3491 2023-09-23T22:42:59.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[223014]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[223014]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[223014]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.846 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[223014]: osdmap e596: 8 total, 8 up, 4 in 2023-09-23T22:42:59.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[215775]: pgmap v1538: 119 pgs: 1 peering, 40 remapped+peering, 2 active+recovery_wait+remapped, 54 active+clean+remapped, 4 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.3 GiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 364 op/s; 6930/115284 objects degraded (6.011%); 21936/115284 objects misplaced (19.028%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:42:59.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[215775]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[215775]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[215775]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[215775]: 10.2 continuing backfill to osd.6 from (573'800,591'3491] 10:40e084bb:::benchmark_data_smithi026_640738_object41542:head to 591'3491 2023-09-23T22:42:59.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[215775]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.847 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[215775]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[215775]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.848 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:42:59 smithi026 ceph-mon[215775]: osdmap e596: 8 total, 8 up, 4 in 2023-09-23T22:42:59.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:42:59.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:42:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:59 smithi070 ceph-mon[192652]: pgmap v1538: 119 pgs: 1 peering, 40 remapped+peering, 2 active+recovery_wait+remapped, 54 active+clean+remapped, 4 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 7 active+recovery_wait+degraded+remapped, 6 active+clean; 2.3 GiB data, 7.3 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 364 op/s; 6930/115284 objects degraded (6.011%); 21936/115284 objects misplaced (19.028%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T22:42:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:59 smithi070 ceph-mon[192652]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:59 smithi070 ceph-mon[192652]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:59 smithi070 ceph-mon[192652]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:59 smithi070 ceph-mon[192652]: 10.2 continuing backfill to osd.6 from (573'800,591'3491] 10:40e084bb:::benchmark_data_smithi026_640738_object41542:head to 591'3491 2023-09-23T22:42:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:59 smithi070 ceph-mon[192652]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:59 smithi070 ceph-mon[192652]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:59 smithi070 ceph-mon[192652]: failed to encode map e595 with expected crc 2023-09-23T22:42:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:42:59 smithi070 ceph-mon[192652]: osdmap e596: 8 total, 8 up, 4 in 2023-09-23T22:42:59.949 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:42:59.995 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:00.077 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:00.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:00.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:00.170 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:43:00.189 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:00.226 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:00.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[215775]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[215775]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[215775]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[215775]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.560 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[215775]: pgmap v1541: 119 pgs: 1 activating+degraded+remapped, 1 peering, 25 remapped+peering, 2 active+recovery_wait+degraded, 4 active+clean+remapped, 2 active+undersized, 5 active+remapped+backfill_wait, 6 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+recovery_wait+degraded+remapped, 68 active+clean; 2.3 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 550 op/s; 6949/112296 objects degraded (6.188%); 20907/112296 objects misplaced (18.618%); 6.1 MiB/s, 2 keys/s, 96 objects/s recovering 2023-09-23T22:43:00.560 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[223014]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[223014]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[223014]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[223014]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.561 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:00 smithi026 ceph-mon[223014]: pgmap v1541: 119 pgs: 1 activating+degraded+remapped, 1 peering, 25 remapped+peering, 2 active+recovery_wait+degraded, 4 active+clean+remapped, 2 active+undersized, 5 active+remapped+backfill_wait, 6 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+recovery_wait+degraded+remapped, 68 active+clean; 2.3 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 550 op/s; 6949/112296 objects degraded (6.188%); 20907/112296 objects misplaced (18.618%); 6.1 MiB/s, 2 keys/s, 96 objects/s recovering 2023-09-23T22:43:00.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:00 smithi070 ceph-mon[192652]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:00 smithi070 ceph-mon[192652]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:00 smithi070 ceph-mon[192652]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:00 smithi070 ceph-mon[192652]: failed to encode map e596 with expected crc 2023-09-23T22:43:00.862 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:00 smithi070 ceph-mon[192652]: pgmap v1541: 119 pgs: 1 activating+degraded+remapped, 1 peering, 25 remapped+peering, 2 active+recovery_wait+degraded, 4 active+clean+remapped, 2 active+undersized, 5 active+remapped+backfill_wait, 6 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded, 2 active+recovery_wait+degraded+remapped, 68 active+clean; 2.3 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 550 op/s; 6949/112296 objects degraded (6.188%); 20907/112296 objects misplaced (18.618%); 6.1 MiB/s, 2 keys/s, 96 objects/s recovering 2023-09-23T22:43:00.939 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:01.263 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:43:02.293 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:02.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:02.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:02.399 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:02.436 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:43:02.466 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:03.109 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:02 smithi070 ceph-mon[192652]: pgmap v1542: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 4 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 6 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 92 active+clean; 2.2 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 346 op/s; 12242/109815 objects degraded (11.148%); 33917/109815 objects misplaced (30.886%); 14 MiB/s, 1 keys/s, 221 objects/s recovering 2023-09-23T22:43:03.218 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:03.218 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:03.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:02 smithi026 ceph-mon[215775]: pgmap v1542: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 4 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 6 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 92 active+clean; 2.2 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 346 op/s; 12242/109815 objects degraded (11.148%); 33917/109815 objects misplaced (30.886%); 14 MiB/s, 1 keys/s, 221 objects/s recovering 2023-09-23T22:43:03.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:02 smithi026 ceph-mon[223014]: pgmap v1542: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 activating+degraded+remapped, 4 active+recovery_wait+degraded, 7 active+remapped+backfill_wait, 6 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 92 active+clean; 2.2 GiB data, 7.2 GiB used, 350 GiB / 358 GiB avail; 0 B/s wr, 346 op/s; 12242/109815 objects degraded (11.148%); 33917/109815 objects misplaced (30.886%); 14 MiB/s, 1 keys/s, 221 objects/s recovering 2023-09-23T22:43:03.722 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:03.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:03.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:03.830 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:03.862 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:03.989 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:03 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 12242/109815 objects degraded (11.148%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:03.990 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:03 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 12242/109815 objects degraded (11.148%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:04.022 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:04.087 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:04.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:03 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 12242/109815 objects degraded (11.148%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:04.427 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:04.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:04.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:04.523 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:43:04.533 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:04.565 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:04.579 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:43:04.620 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:04.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:04.627 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:04.631 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:43:04.677 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:43:04.728 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:04.741 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:43:04.792 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:43:04.835 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:43:04.883 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:04.893 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:43:04.936 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:43:04.951 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:43:04.951 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:43:04.951 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:43:04.951 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:43:04.951 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:43:04.951 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:43:04.952 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:43:04.952 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:43:05.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:05 smithi070 ceph-mon[192652]: pgmap v1543: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.2 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 494 op/s; 11771/106005 objects degraded (11.104%); 32648/106005 objects misplaced (30.799%); 16 MiB/s, 1 keys/s, 261 objects/s recovering 2023-09-23T22:43:05.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:05 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2334471290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:05.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:05 smithi026 ceph-mon[223014]: pgmap v1543: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.2 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 494 op/s; 11771/106005 objects degraded (11.104%); 32648/106005 objects misplaced (30.799%); 16 MiB/s, 1 keys/s, 261 objects/s recovering 2023-09-23T22:43:05.794 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:05 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2334471290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:05.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:05 smithi026 ceph-mon[215775]: pgmap v1543: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.2 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 494 op/s; 11771/106005 objects degraded (11.104%); 32648/106005 objects misplaced (30.799%); 16 MiB/s, 1 keys/s, 261 objects/s recovering 2023-09-23T22:43:05.795 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:05 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2334471290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:06.410 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:06.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:06.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:06.516 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:06.633 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:07.226 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:07 smithi026 ceph-mon[215775]: pgmap v1544: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.1 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 408 op/s; 11771/105471 objects degraded (11.160%); 32025/105471 objects misplaced (30.364%); 18 MiB/s, 1 keys/s, 282 objects/s recovering 2023-09-23T22:43:07.226 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:07 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:07.226 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:07 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:07.422 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:07.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:07.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:07 smithi026 ceph-mon[223014]: pgmap v1544: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.1 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 408 op/s; 11771/105471 objects degraded (11.160%); 32025/105471 objects misplaced (30.364%); 18 MiB/s, 1 keys/s, 282 objects/s recovering 2023-09-23T22:43:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:07 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:07.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:07 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:07.526 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:07.551 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:07 smithi070 ceph-mon[192652]: pgmap v1544: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.1 GiB data, 7.0 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 408 op/s; 11771/105471 objects degraded (11.160%); 32025/105471 objects misplaced (30.364%); 18 MiB/s, 1 keys/s, 282 objects/s recovering 2023-09-23T22:43:07.551 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:07 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:07.552 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:07 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:07.555 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:07.603 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:07.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:07.710 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:07.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:07.783 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:07.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:07.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:07.891 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:07.953 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:08.497 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:08.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:08.605 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:08.635 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:08.686 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:08.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:08.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:08.790 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:08.819 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:08.870 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:08.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:08.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:08.976 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:09.006 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:09.054 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:09.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:09.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:09.155 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:09.161 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:09.245 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:09.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[215775]: pgmap v1545: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.1 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 386 op/s; 10813/101463 objects degraded (10.657%); 30897/101463 objects misplaced (30.451%); 20 MiB/s, 320 objects/s recovering 2023-09-23T22:43:09.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 10813/101463 objects degraded (10.657%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:09.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:09.247 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:09.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:09.248 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:09.373 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:09 smithi070 ceph-mon[192652]: pgmap v1545: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.1 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 386 op/s; 10813/101463 objects degraded (10.657%); 30897/101463 objects misplaced (30.451%); 20 MiB/s, 320 objects/s recovering 2023-09-23T22:43:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:09 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 10813/101463 objects degraded (10.657%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:09.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:09.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:09.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:09.486 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:09.568 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[223014]: pgmap v1545: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.1 GiB data, 6.8 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 386 op/s; 10813/101463 objects degraded (10.657%); 30897/101463 objects misplaced (30.451%); 20 MiB/s, 320 objects/s recovering 2023-09-23T22:43:09.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 10813/101463 objects degraded (10.657%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:09.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:09.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:09.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:09.569 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:09.570 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:09.745 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:09.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:09.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:09.847 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:43:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:43:09] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-09-23T22:43:09.851 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:09.880 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:09.933 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:09.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:09.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:10.039 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:10.125 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:10.241 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:10.243 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:43:10.243 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:43:10.243 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:10.244 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2811789867' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:43:10.244 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:43:10.244 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3497589899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:10.244 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:10 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:10.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:10.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:10.352 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:10.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:43:10.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:43:10.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:10.426 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2811789867' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:43:10.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:43:10.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3497589899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:10.427 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:10.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:43:10.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:43:10.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:10.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2811789867' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:43:10.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:43:10.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3497589899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:10.428 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:10 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:43:10.432 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:10.445 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:43:10.569 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:10.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:10.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:10.678 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:10.710 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:43:10.712 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:10.718 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:43:10.759 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:10.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:10.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:10.865 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:10.893 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:10.940 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:10.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:10.945 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:11.046 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:11.072 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:11.120 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:11.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:11.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:11.225 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:11.259 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:11 smithi070 ceph-mon[192652]: pgmap v1546: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.0 GiB data, 6.6 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 501 op/s; 10214/96744 objects degraded (10.558%); 29325/96744 objects misplaced (30.312%); 22 MiB/s, 347 objects/s recovering 2023-09-23T22:43:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:11 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:11 smithi070 ceph-mon[192652]: failed to encode map e597 with expected crc 2023-09-23T22:43:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:43:11.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:11 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:43:11.375 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:11 smithi070 ceph-mon[192652]: osdmap e597: 8 total, 8 up, 4 in 2023-09-23T22:43:11.456 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:11.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:11.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:11.560 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:11.587 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:11.632 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:11.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:11.640 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[215775]: pgmap v1546: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.0 GiB data, 6.6 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 501 op/s; 10214/96744 objects degraded (10.558%); 29325/96744 objects misplaced (30.312%); 22 MiB/s, 347 objects/s recovering 2023-09-23T22:43:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[215775]: failed to encode map e597 with expected crc 2023-09-23T22:43:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:43:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:43:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[215775]: osdmap e597: 8 total, 8 up, 4 in 2023-09-23T22:43:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[223014]: pgmap v1546: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 4 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 7 active+recovery_wait+degraded+remapped, 98 active+clean; 2.0 GiB data, 6.6 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 501 op/s; 10214/96744 objects degraded (10.558%); 29325/96744 objects misplaced (30.312%); 22 MiB/s, 347 objects/s recovering 2023-09-23T22:43:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[223014]: failed to encode map e597 with expected crc 2023-09-23T22:43:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:43:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:43:11.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:11 smithi026 ceph-mon[223014]: osdmap e597: 8 total, 8 up, 4 in 2023-09-23T22:43:11.741 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:11.826 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:12.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: failed to encode map e597 with expected crc 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: osdmap e598: 8 total, 8 up, 4 in 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4016 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.1 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4067 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.6 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4002 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 596'3946 2023-09-23T22:43:12.487 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4120 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.d starting backfill to osd.5 from (0'0,0'0] MAX to 596'4058 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 596'3984 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.0 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4028 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 596'4074 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.3 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4219 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.9 continuing backfill to osd.6 from (573'1100,596'3946] MIN to 596'3946 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.7 continuing backfill to osd.6 from (573'1200,596'3984] MIN to 596'3984 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.2 starting backfill to osd.5 from (0'0,0'0] MAX to 596'3939 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.6 continuing backfill to osd.6 from (573'1300,596'4002] MIN to 596'4002 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.0 continuing backfill to osd.6 from (573'1300,596'4028] MIN to 596'4028 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.e continuing backfill to osd.6 from (573'1200,596'4074] MIN to 596'4074 2023-09-23T22:43:12.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[223014]: 10.d continuing backfill to osd.6 from (573'1200,596'4058] MIN to 596'4058 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: failed to encode map e597 with expected crc 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: osdmap e598: 8 total, 8 up, 4 in 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4016 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.1 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4067 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.6 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4002 2023-09-23T22:43:12.489 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 596'3946 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4120 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.d starting backfill to osd.5 from (0'0,0'0] MAX to 596'4058 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 596'3984 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.0 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4028 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 596'4074 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.3 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4219 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.9 continuing backfill to osd.6 from (573'1100,596'3946] MIN to 596'3946 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.7 continuing backfill to osd.6 from (573'1200,596'3984] MIN to 596'3984 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.2 starting backfill to osd.5 from (0'0,0'0] MAX to 596'3939 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.6 continuing backfill to osd.6 from (573'1300,596'4002] MIN to 596'4002 2023-09-23T22:43:12.490 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.0 continuing backfill to osd.6 from (573'1300,596'4028] MIN to 596'4028 2023-09-23T22:43:12.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.e continuing backfill to osd.6 from (573'1200,596'4074] MIN to 596'4074 2023-09-23T22:43:12.491 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:12 smithi026 ceph-mon[215775]: 10.d continuing backfill to osd.6 from (573'1200,596'4058] MIN to 596'4058 2023-09-23T22:43:12.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: failed to encode map e597 with expected crc 2023-09-23T22:43:12.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: osdmap e598: 8 total, 8 up, 4 in 2023-09-23T22:43:12.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.625 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.5 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4016 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.1 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4067 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: failed to encode map e598 with expected crc 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.6 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4002 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.9 starting backfill to osd.5 from (0'0,0'0] MAX to 596'3946 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4120 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.d starting backfill to osd.5 from (0'0,0'0] MAX to 596'4058 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.7 starting backfill to osd.5 from (0'0,0'0] MAX to 596'3984 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.0 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4028 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 596'4074 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.3 starting backfill to osd.5 from (0'0,0'0] MAX to 596'4219 2023-09-23T22:43:12.626 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.9 continuing backfill to osd.6 from (573'1100,596'3946] MIN to 596'3946 2023-09-23T22:43:12.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.7 continuing backfill to osd.6 from (573'1200,596'3984] MIN to 596'3984 2023-09-23T22:43:12.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.2 starting backfill to osd.5 from (0'0,0'0] MAX to 596'3939 2023-09-23T22:43:12.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.6 continuing backfill to osd.6 from (573'1300,596'4002] MIN to 596'4002 2023-09-23T22:43:12.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.0 continuing backfill to osd.6 from (573'1300,596'4028] MIN to 596'4028 2023-09-23T22:43:12.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.e continuing backfill to osd.6 from (573'1200,596'4074] MIN to 596'4074 2023-09-23T22:43:12.627 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:12 smithi070 ceph-mon[192652]: 10.d continuing backfill to osd.6 from (573'1200,596'4058] MIN to 596'4058 2023-09-23T22:43:12.783 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:12.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:12.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:12.893 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:12.955 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:13.033 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:13.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:13.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:13.140 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:13.169 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:13.221 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:13.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:13.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:13.328 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:13.355 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:13.406 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:13.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:13.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:13.514 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:13.540 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:13 smithi070 ceph-mon[192652]: failed to encode map e598 with expected crc 2023-09-23T22:43:13.540 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:13 smithi070 ceph-mon[192652]: pgmap v1549: 119 pgs: 4 remapped+peering, 12 peering, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 6.5 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 549 op/s; 7389/92817 objects degraded (7.961%); 20000/92817 objects misplaced (21.548%); 28 MiB/s, 444 objects/s recovering 2023-09-23T22:43:13.540 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:13 smithi070 ceph-mon[192652]: failed to encode map e599 with expected crc 2023-09-23T22:43:13.540 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:13 smithi070 ceph-mon[192652]: osdmap e599: 8 total, 8 up, 4 in 2023-09-23T22:43:13.540 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:13 smithi070 ceph-mon[192652]: failed to encode map e599 with expected crc 2023-09-23T22:43:13.543 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:13.581 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[215775]: failed to encode map e598 with expected crc 2023-09-23T22:43:13.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[215775]: pgmap v1549: 119 pgs: 4 remapped+peering, 12 peering, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 6.5 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 549 op/s; 7389/92817 objects degraded (7.961%); 20000/92817 objects misplaced (21.548%); 28 MiB/s, 444 objects/s recovering 2023-09-23T22:43:13.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[215775]: failed to encode map e599 with expected crc 2023-09-23T22:43:13.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[215775]: osdmap e599: 8 total, 8 up, 4 in 2023-09-23T22:43:13.582 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[215775]: failed to encode map e599 with expected crc 2023-09-23T22:43:13.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[223014]: failed to encode map e598 with expected crc 2023-09-23T22:43:13.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[223014]: pgmap v1549: 119 pgs: 4 remapped+peering, 12 peering, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 87 active+clean; 1.9 GiB data, 6.5 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 549 op/s; 7389/92817 objects degraded (7.961%); 20000/92817 objects misplaced (21.548%); 28 MiB/s, 444 objects/s recovering 2023-09-23T22:43:13.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[223014]: failed to encode map e599 with expected crc 2023-09-23T22:43:13.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[223014]: osdmap e599: 8 total, 8 up, 4 in 2023-09-23T22:43:13.582 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:13 smithi026 ceph-mon[223014]: failed to encode map e599 with expected crc 2023-09-23T22:43:13.590 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:13.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:13.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:13.695 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:13.722 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:13.727 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:43:13.768 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:13.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:13.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:13.874 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:13.901 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:13.990 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:14.117 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:14.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:14.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:14.156 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:43:14.236 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:14.299 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:14.423 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:43:14.423 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66182 10.0 10:07d0f08c:::benchmark_data_smithi026_640738_object18786:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.401480+0000", 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017454713, 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017494151999999999, 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "delayed", 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.424 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66182 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401480+0000", 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401480+0000", 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.425 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401478+0000", 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401487+0000", 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6230000000000003e-06 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.426 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401502+0000", 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5435999999999999e-05 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401513+0000", 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0603e-05 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.427 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.404743+0000", 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00322975 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.409903+0000", 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051602389999999996 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.428 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66183 10.b 10:da6594ca:::benchmark_data_smithi026_640738_object18787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.401530+0000", 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017404622000000002, 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017526759999999999, 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66183 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.429 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401530+0000", 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401530+0000", 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.430 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401529+0000", 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401533+0000", 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7910000000000002e-06 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401542+0000", 2023-09-23T22:43:14.431 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3139999999999993e-06 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401549+0000", 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9800000000000001e-06 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.405152+0000", 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003603036 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.432 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.405300+0000", 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000148369 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.405477+0000", 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017672 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.433 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.412461+0000", 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0069842760000000002 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413298+0000", 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00083705599999999998 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.415742+0000", 2023-09-23T22:43:14.434 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002443092 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66184 10.b 10:dd430a1d:::benchmark_data_smithi026_640738_object18788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.401561+0000", 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017373355, 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017591431000000001, 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.435 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66184 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401561+0000", 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.436 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401561+0000", 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401560+0000", 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401564+0000", 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0809999999999998e-06 2023-09-23T22:43:14.437 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401572+0000", 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.011e-06 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401579+0000", 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1289999999999998e-06 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.438 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.407107+0000", 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055282279999999996 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.407244+0000", 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000136649 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.407386+0000", 2023-09-23T22:43:14.439 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014254 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.412513+0000", 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0051268959999999997 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413306+0000", 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00079281100000000004 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.440 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.415782+0000", 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024767169999999998 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66178 10.6 10:66141315:::benchmark_data_smithi026_640738_object18782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.373868+0000", 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.045066924000000001, 2023-09-23T22:43:14.441 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045384339000000003, 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66178 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.373868+0000", 2023-09-23T22:43:14.442 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.373868+0000", 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.373866+0000", 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.443 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.373873+0000", 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5100000000000001e-06 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.373888+0000", 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4542000000000001e-05 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.444 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.373897+0000", 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2580000000000001e-06 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.376666+0000", 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0027686490000000002 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.377568+0000", 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00090272499999999995 2023-09-23T22:43:14.445 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.399691+0000", 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022122829 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.399859+0000", 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016814999999999999 2023-09-23T22:43:14.446 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2,5,6", 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.399958+0000", 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8519999999999999e-05 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417457+0000", 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.01749887 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.447 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417467+0000", 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9960000000000005e-06 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417515+0000", 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8205999999999999e-05 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417553+0000", 2023-09-23T22:43:14.448 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7682999999999998e-05 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66185 10.c 10:361028d6:::benchmark_data_smithi026_640738_object18789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.401584+0000", 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017350287999999998, 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017778439, 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.449 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "delayed", 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66185 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401584+0000", 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.450 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401584+0000", 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401584+0000", 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.451 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401587+0000", 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9399999999999998e-06 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401599+0000", 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2295e-05 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401606+0000", 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.311e-06 2023-09-23T22:43:14.452 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401776+0000", 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016963699999999999 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.402469+0000", 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00069279499999999996 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.453 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66186 10.0 10:0e0ac428:::benchmark_data_smithi026_640738_object18790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.401620+0000", 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017314236, 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017812643999999999, 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "delayed", 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.454 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66186 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401620+0000", 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401620+0000", 2023-09-23T22:43:14.455 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401620+0000", 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401623+0000", 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9569999999999998e-06 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.456 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401631+0000", 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3119999999999994e-06 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401638+0000", 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8410000000000002e-06 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.457 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.410002+0000", 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0083644449999999999 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413512+0000", 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035096089999999999 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.458 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66187 10.f 10:fbae7f8c:::benchmark_data_smithi026_640738_object18791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.401643+0000", 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017291793, 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017865753000000002, 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66187 2023-09-23T22:43:14.459 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401643+0000", 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401643+0000", 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.460 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401642+0000", 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401645+0000", 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4770000000000002e-06 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401652+0000", 2023-09-23T22:43:14.461 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3610000000000003e-06 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401659+0000", 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8979999999999999e-06 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.402614+0000", 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00095493399999999995 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.462 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.402746+0000", 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013243599999999999 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.402881+0000", 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013504599999999999 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:14.463 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413272+0000", 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010390263 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.415554+0000", 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022827580000000002 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.464 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66189 10.f 10:f54a4d11:::benchmark_data_smithi026_640738_object18793:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.401678+0000", 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017256865, 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017924056000000001, 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66189 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.465 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401678+0000", 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401678+0000", 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.466 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401677+0000", 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401680+0000", 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8559999999999998e-06 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401688+0000", 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1929999999999993e-06 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.467 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401695+0000", 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4559999999999997e-06 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.405421+0000", 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037264279999999999 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.468 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.405562+0000", 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000140457 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.405700+0000", 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013859000000000001 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413280+0000", 2023-09-23T22:43:14.469 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.007580313 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.415734+0000", 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024535590000000001 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.470 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66190 10.b 10:d9327291:::benchmark_data_smithi026_640738_object18794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.401701+0000", 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.017233496000000001, 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.017989453999999998, 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66190 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.471 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401701+0000", 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401701+0000", 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.472 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401700+0000", 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401703+0000", 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.74e-06 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401716+0000", 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2689e-05 2023-09-23T22:43:14.473 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.401723+0000", 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8499999999999996e-06 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.410378+0000", 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0086558350000000006 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.474 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.410528+0000", 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149791 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.410684+0000", 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015612199999999999 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.475 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413550+0000", 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002865907 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.415821+0000", 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002270345 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417472+0000", 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016511119999999999 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.476 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66180 10.1 10:8799db43:::benchmark_data_smithi026_640738_object18784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.381925+0000", 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.037009309999999997, 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037868187999999997, 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.477 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66180 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.381925+0000", 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.478 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.381925+0000", 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.381923+0000", 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.381931+0000", 2023-09-23T22:43:14.479 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.396e-06 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.381947+0000", 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6492000000000001e-05 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.381955+0000", 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9330000000000001e-06 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.480 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.383844+0000", 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001889309 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.383991+0000", 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000146592 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T22:43:14.481 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.384127+0000", 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013615799999999999 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.388649+0000", 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0045222209999999999 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.391239+0000", 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002589652 2023-09-23T22:43:14.482 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.400740+0000", 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095007249999999998 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66191 10.f 10:fc5b3824:::benchmark_data_smithi026_640738_object18795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.483 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.413514+0000", 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0054201199999999996, 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063795880000000003, 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66191 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.484 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413514+0000", 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413514+0000", 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413513+0000", 2023-09-23T22:43:14.485 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413518+0000", 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3580000000000002e-06 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413529+0000", 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0165e-05 2023-09-23T22:43:14.486 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413536+0000", 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0369999999999997e-06 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413581+0000", 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5311e-05 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.487 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413692+0000", 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000111087 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413819+0000", 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012727600000000001 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.415811+0000", 2023-09-23T22:43:14.488 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001991549 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417796+0000", 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001985391 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.489 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66193 10.1 10:8dc4e041:::benchmark_data_smithi026_640738_object18797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.413552+0000", 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0053826619999999999, 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064359550000000001, 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66193 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.490 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413552+0000", 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413552+0000", 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.491 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413551+0000", 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413555+0000", 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6289999999999999e-06 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413562+0000", 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9820000000000002e-06 2023-09-23T22:43:14.492 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413568+0000", 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6540000000000004e-06 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413617+0000", 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9718000000000001e-05 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.493 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413779+0000", 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000161824 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413925+0000", 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014594099999999999 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:14.494 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417151+0000", 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0032264559999999999 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:66194 10.1 10:868c1f1e:::benchmark_data_smithi026_640738_object18798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:14.413579+0000", 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 0.0053555790000000001, 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0064904689999999996, 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:14.495 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "waiting for sub ops", 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 66194 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413579+0000", 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.496 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413579+0000", 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413578+0000", 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:14.497 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413582+0000", 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3119999999999998e-06 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413589+0000", 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9909999999999996e-06 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.413594+0000", 2023-09-23T22:43:14.498 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3199999999999999e-06 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417381+0000", 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037873730000000001 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417564+0000", 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000182857 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:14.499 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:14.417762+0000", 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019747300000000001 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout: ], 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 13 2023-09-23T22:43:14.500 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:43:14.511 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:14.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:14.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:14.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[223014]: failed to encode map e599 with expected crc 2023-09-23T22:43:14.516 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[223014]: failed to encode map e599 with expected crc 2023-09-23T22:43:14.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7389/92817 objects degraded (7.961%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:14.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[223014]: failed to encode map e600 with expected crc 2023-09-23T22:43:14.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[223014]: failed to encode map e600 with expected crc 2023-09-23T22:43:14.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[223014]: osdmap e600: 8 total, 8 up, 4 in 2023-09-23T22:43:14.517 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[223014]: failed to encode map e600 with expected crc 2023-09-23T22:43:14.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[215775]: failed to encode map e599 with expected crc 2023-09-23T22:43:14.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[215775]: failed to encode map e599 with expected crc 2023-09-23T22:43:14.517 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7389/92817 objects degraded (7.961%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:14.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[215775]: failed to encode map e600 with expected crc 2023-09-23T22:43:14.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[215775]: failed to encode map e600 with expected crc 2023-09-23T22:43:14.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[215775]: osdmap e600: 8 total, 8 up, 4 in 2023-09-23T22:43:14.518 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:14 smithi026 ceph-mon[215775]: failed to encode map e600 with expected crc 2023-09-23T22:43:14.617 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:14.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:14 smithi070 ceph-mon[192652]: failed to encode map e599 with expected crc 2023-09-23T22:43:14.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:14 smithi070 ceph-mon[192652]: failed to encode map e599 with expected crc 2023-09-23T22:43:14.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:14 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7389/92817 objects degraded (7.961%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:14 smithi070 ceph-mon[192652]: failed to encode map e600 with expected crc 2023-09-23T22:43:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:14 smithi070 ceph-mon[192652]: failed to encode map e600 with expected crc 2023-09-23T22:43:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:14 smithi070 ceph-mon[192652]: osdmap e600: 8 total, 8 up, 4 in 2023-09-23T22:43:14.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:14 smithi070 ceph-mon[192652]: failed to encode map e600 with expected crc 2023-09-23T22:43:14.645 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:14.693 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:14.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:14.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:14.798 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:14.825 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:14.873 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:14.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:14.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:14.979 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:15.050 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:15.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[223014]: pgmap v1552: 119 pgs: 4 remapped+peering, 12 peering, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 81 active+clean; 1.8 GiB data, 6.5 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 614 op/s; 6937/89361 objects degraded (7.763%); 23226/89361 objects misplaced (25.991%); 32 MiB/s, 507 objects/s recovering 2023-09-23T22:43:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[223014]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[223014]: osdmap e601: 8 total, 8 up, 4 in 2023-09-23T22:43:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[223014]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[223014]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[223014]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[215775]: pgmap v1552: 119 pgs: 4 remapped+peering, 12 peering, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 81 active+clean; 1.8 GiB data, 6.5 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 614 op/s; 6937/89361 objects degraded (7.763%); 23226/89361 objects misplaced (25.991%); 32 MiB/s, 507 objects/s recovering 2023-09-23T22:43:15.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[215775]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[215775]: osdmap e601: 8 total, 8 up, 4 in 2023-09-23T22:43:15.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[215775]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[215775]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:15 smithi026 ceph-mon[215775]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:15 smithi070 ceph-mon[192652]: pgmap v1552: 119 pgs: 4 remapped+peering, 12 peering, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 81 active+clean; 1.8 GiB data, 6.5 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 614 op/s; 6937/89361 objects degraded (7.763%); 23226/89361 objects misplaced (25.991%); 32 MiB/s, 507 objects/s recovering 2023-09-23T22:43:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:15 smithi070 ceph-mon[192652]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:15 smithi070 ceph-mon[192652]: osdmap e601: 8 total, 8 up, 4 in 2023-09-23T22:43:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:15 smithi070 ceph-mon[192652]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:15 smithi070 ceph-mon[192652]: failed to encode map e601 with expected crc 2023-09-23T22:43:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:15 smithi070 ceph-mon[192652]: failed to encode map e601 with expected crc 2023-09-23T22:43:16.852 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:16.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:16.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:16.959 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:16.986 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:17.082 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:17.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:17.088 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:17.189 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:17.264 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:17.498 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:43:17.512 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:17 smithi026 ceph-mon[215775]: pgmap v1554: 119 pgs: 4 remapped+peering, 12 peering, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 81 active+clean; 1.8 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 323 op/s; 7621/88914 objects degraded (8.571%); 22747/88914 objects misplaced (25.583%); 14 MiB/s, 231 objects/s recovering 2023-09-23T22:43:17.513 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:17 smithi026 ceph-mon[223014]: pgmap v1554: 119 pgs: 4 remapped+peering, 12 peering, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 81 active+clean; 1.8 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 323 op/s; 7621/88914 objects degraded (8.571%); 22747/88914 objects misplaced (25.583%); 14 MiB/s, 231 objects/s recovering 2023-09-23T22:43:17.527 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:17.545 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:17.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:17.560 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:17.661 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:17.674 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:17.685 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:17 smithi070 ceph-mon[192652]: pgmap v1554: 119 pgs: 4 remapped+peering, 12 peering, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 5 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 81 active+clean; 1.8 GiB data, 6.4 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 323 op/s; 7621/88914 objects degraded (8.571%); 22747/88914 objects misplaced (25.583%); 14 MiB/s, 231 objects/s recovering 2023-09-23T22:43:17.687 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:17.735 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:17.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:17.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:17.841 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:17.868 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:17.913 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:17.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:17.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:18.020 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:18.048 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:18.097 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:18.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:18.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:18.206 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:18.328 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:18.522 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:18.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:18.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:18.639 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:18.658 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:43:18.703 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:43:18.721 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:18 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 7621/88914 objects degraded (8.571%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:18.722 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:18 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 7621/88914 objects degraded (8.571%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:18.725 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:18.743 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:43:18.782 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:43:18.824 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:43:18.860 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:43:18.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:18 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 7621/88914 objects degraded (8.571%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:18.904 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:43:18.945 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:43:18.953 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:43:18.953 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:43:18.953 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:43:18.953 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:43:18.953 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:43:18.954 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:43:18.954 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:43:18.954 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:43:19.242 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:19.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:19.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:19.353 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:19.438 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:19 smithi026 ceph-mon[223014]: pgmap v1555: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 7 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 89 active+clean; 1.7 GiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 424 op/s; 8029/85173 objects degraded (9.427%); 31028/85173 objects misplaced (36.429%); 18 MiB/s, 288 objects/s recovering 2023-09-23T22:43:19.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1675056466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:19 smithi026 ceph-mon[215775]: pgmap v1555: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 7 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 89 active+clean; 1.7 GiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 424 op/s; 8029/85173 objects degraded (9.427%); 31028/85173 objects misplaced (36.429%); 18 MiB/s, 288 objects/s recovering 2023-09-23T22:43:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1675056466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:19.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:19 smithi070 ceph-mon[192652]: pgmap v1555: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 7 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 89 active+clean; 1.7 GiB data, 6.3 GiB used, 351 GiB / 358 GiB avail; 0 B/s wr, 424 op/s; 8029/85173 objects degraded (9.427%); 31028/85173 objects misplaced (36.429%); 18 MiB/s, 288 objects/s recovering 2023-09-23T22:43:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1675056466' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:43:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:43:19] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-09-23T22:43:20.682 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:20.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:20.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:20.795 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:20.826 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:20.872 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:20.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:20.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:20.979 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:21.010 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:21.427 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:21 smithi026 ceph-mon[223014]: pgmap v1556: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.6 GiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 574 op/s; 7503/80748 objects degraded (9.292%); 29357/80748 objects misplaced (36.356%); 20 MiB/s, 322 objects/s recovering 2023-09-23T22:43:21.709 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:21 smithi026 ceph-mon[215775]: pgmap v1556: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.6 GiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 574 op/s; 7503/80748 objects degraded (9.292%); 29357/80748 objects misplaced (36.356%); 20 MiB/s, 322 objects/s recovering 2023-09-23T22:43:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:21 smithi070 ceph-mon[192652]: pgmap v1556: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.6 GiB data, 6.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 574 op/s; 7503/80748 objects degraded (9.292%); 29357/80748 objects misplaced (36.356%); 20 MiB/s, 322 objects/s recovering 2023-09-23T22:43:22.345 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:22.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:22.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:22.450 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:22.481 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:22.531 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:22.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:22.636 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:22.739 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:22.871 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:22.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:22.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:22.981 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:23.045 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:23.085 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:23.227 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:23.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:23.340 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:23.368 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:23.414 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:23.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:23.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:23.520 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:23.591 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:23 smithi026 ceph-mon[215775]: pgmap v1557: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.5 GiB data, 5.9 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 543 op/s; 6543/75753 objects degraded (8.637%); 28052/75753 objects misplaced (37.031%); 19 MiB/s, 335 objects/s recovering 2023-09-23T22:43:23.592 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:23 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6543/75753 objects degraded (8.637%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:23.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:23 smithi026 ceph-mon[223014]: pgmap v1557: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.5 GiB data, 5.9 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 543 op/s; 6543/75753 objects degraded (8.637%); 28052/75753 objects misplaced (37.031%); 19 MiB/s, 335 objects/s recovering 2023-09-23T22:43:23.593 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:23 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6543/75753 objects degraded (8.637%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:23.610 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:23 smithi070 ceph-mon[192652]: pgmap v1557: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.5 GiB data, 5.9 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 543 op/s; 6543/75753 objects degraded (8.637%); 28052/75753 objects misplaced (37.031%); 19 MiB/s, 335 objects/s recovering 2023-09-23T22:43:23.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:23 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6543/75753 objects degraded (8.637%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:23.785 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:23.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:23.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:23.915 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:23.945 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:23.997 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:24.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:24.104 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:24.133 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:24.183 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:24.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:24.288 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:24.351 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:24.469 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:43:24.475 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:43:24.486 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:24.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:24.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:24.590 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:24.616 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:24.658 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:24.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:24.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:24.708 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:43:24.739 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3552718595' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:43:24.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:43:24.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:24.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:24.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1230411112' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:24.740 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:24.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3552718595' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:43:24.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:43:24.740 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:24.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:24.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1230411112' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:24.741 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:24 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:24.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:24.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3552718595' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:43:24.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:24 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:43:24.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:43:24.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:24.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1230411112' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:24.789 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:24 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:24.790 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:24.840 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:24.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:24.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:24.945 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:25.046 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:25.685 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:25.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:25.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[215775]: pgmap v1558: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.4 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 627 op/s; 6064/70533 objects degraded (8.597%); 25933/70533 objects misplaced (36.767%); 19 MiB/s, 327 objects/s recovering 2023-09-23T22:43:25.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:25.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[215775]: failed to encode map e602 with expected crc 2023-09-23T22:43:25.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[215775]: failed to encode map e602 with expected crc 2023-09-23T22:43:25.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[215775]: failed to encode map e602 with expected crc 2023-09-23T22:43:25.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:43:25.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:25.691 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[215775]: osdmap e602: 8 total, 8 up, 4 in 2023-09-23T22:43:25.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[223014]: pgmap v1558: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.4 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 627 op/s; 6064/70533 objects degraded (8.597%); 25933/70533 objects misplaced (36.767%); 19 MiB/s, 327 objects/s recovering 2023-09-23T22:43:25.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:25.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[223014]: failed to encode map e602 with expected crc 2023-09-23T22:43:25.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[223014]: failed to encode map e602 with expected crc 2023-09-23T22:43:25.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[223014]: failed to encode map e602 with expected crc 2023-09-23T22:43:25.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:43:25.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:25.692 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:25 smithi026 ceph-mon[223014]: osdmap e602: 8 total, 8 up, 4 in 2023-09-23T22:43:25.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:25.794 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:25.822 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:25 smithi070 ceph-mon[192652]: pgmap v1558: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.4 GiB data, 5.8 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 627 op/s; 6064/70533 objects degraded (8.597%); 25933/70533 objects misplaced (36.767%); 19 MiB/s, 327 objects/s recovering 2023-09-23T22:43:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:25 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:25 smithi070 ceph-mon[192652]: failed to encode map e602 with expected crc 2023-09-23T22:43:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:25 smithi070 ceph-mon[192652]: failed to encode map e602 with expected crc 2023-09-23T22:43:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:25 smithi070 ceph-mon[192652]: failed to encode map e602 with expected crc 2023-09-23T22:43:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:25 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:43:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:25 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:25 smithi070 ceph-mon[192652]: osdmap e602: 8 total, 8 up, 4 in 2023-09-23T22:43:26.821 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:26.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:26.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:26.932 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:26.964 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:27.021 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:27.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:27.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:27.056 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T22:43:27.127 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:27.158 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:27.211 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:27.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:27.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:27.318 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:27.348 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:27.387 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:27.402 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:27.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:27.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:27.470 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:27 smithi026 ceph-mon[215775]: pgmap v1560: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.4 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 636 op/s; 6064/69819 objects degraded (8.685%); 25933/69819 objects misplaced (37.143%); 16 MiB/s, 279 objects/s recovering 2023-09-23T22:43:27.471 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:27 smithi070 ceph-mon[192652]: pgmap v1560: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.4 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 636 op/s; 6064/69819 objects degraded (8.685%); 25933/69819 objects misplaced (37.143%); 16 MiB/s, 279 objects/s recovering 2023-09-23T22:43:27.475 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:27 smithi026 ceph-mon[223014]: pgmap v1560: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 7 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.4 GiB data, 5.6 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 636 op/s; 6064/69819 objects degraded (8.685%); 25933/69819 objects misplaced (37.143%); 16 MiB/s, 279 objects/s recovering 2023-09-23T22:43:27.507 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:27.529 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:27.529 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:27.542 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:27.591 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:27.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:27.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:27.696 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:27.730 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:28.095 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:28.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:28.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:28.203 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:28.281 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:28 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 6064/69819 objects degraded (8.685%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:28.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:28 smithi026 ceph-mon[215775]: osdmap e603: 8 total, 8 up, 4 in 2023-09-23T22:43:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:28 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 6064/69819 objects degraded (8.685%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:28.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:28 smithi026 ceph-mon[223014]: osdmap e603: 8 total, 8 up, 4 in 2023-09-23T22:43:28.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:28 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 6064/69819 objects degraded (8.685%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:28 smithi070 ceph-mon[192652]: osdmap e603: 8 total, 8 up, 4 in 2023-09-23T22:43:28.943 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:43:29.012 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:29.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:29.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:29.106 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:43:29.122 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:29.182 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:29.248 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:29.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:29.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:29.353 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:29.379 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:29.436 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:29.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:29.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:29.541 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:29.614 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:43:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:43:29] "GET /metrics HTTP/1.1" 200 36560 "" "Prometheus/2.43.0" 2023-09-23T22:43:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[215775]: pgmap v1561: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.3 GiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 668 op/s; 5777/65115 objects degraded (8.872%); 25434/65115 objects misplaced (39.060%); 11 MiB/s, 243 objects/s recovering 2023-09-23T22:43:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[215775]: failed to encode map e603 with expected crc 2023-09-23T22:43:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[215775]: failed to encode map e603 with expected crc 2023-09-23T22:43:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[215775]: failed to encode map e603 with expected crc 2023-09-23T22:43:29.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[215775]: osdmap e604: 8 total, 8 up, 4 in 2023-09-23T22:43:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[223014]: pgmap v1561: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.3 GiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 668 op/s; 5777/65115 objects degraded (8.872%); 25434/65115 objects misplaced (39.060%); 11 MiB/s, 243 objects/s recovering 2023-09-23T22:43:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[223014]: failed to encode map e603 with expected crc 2023-09-23T22:43:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[223014]: failed to encode map e603 with expected crc 2023-09-23T22:43:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[223014]: failed to encode map e603 with expected crc 2023-09-23T22:43:29.616 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:29 smithi026 ceph-mon[223014]: osdmap e604: 8 total, 8 up, 4 in 2023-09-23T22:43:29.635 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:29.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:29 smithi070 ceph-mon[192652]: pgmap v1561: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 6 active+recovery_wait+degraded+remapped, 95 active+clean; 1.3 GiB data, 5.4 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 668 op/s; 5777/65115 objects degraded (8.872%); 25434/65115 objects misplaced (39.060%); 11 MiB/s, 243 objects/s recovering 2023-09-23T22:43:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:29 smithi070 ceph-mon[192652]: failed to encode map e603 with expected crc 2023-09-23T22:43:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:29 smithi070 ceph-mon[192652]: failed to encode map e603 with expected crc 2023-09-23T22:43:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:29 smithi070 ceph-mon[192652]: failed to encode map e603 with expected crc 2023-09-23T22:43:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:29 smithi070 ceph-mon[192652]: osdmap e604: 8 total, 8 up, 4 in 2023-09-23T22:43:30.474 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:30.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:30.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:30.526 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:30.588 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:30.614 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:30.659 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:30.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:30.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:30.749 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:30.767 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:30.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:30 smithi070 ceph-mon[192652]: failed to encode map e604 with expected crc 2023-09-23T22:43:30.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:30 smithi070 ceph-mon[192652]: failed to encode map e604 with expected crc 2023-09-23T22:43:30.794 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:30 smithi070 ceph-mon[192652]: failed to encode map e604 with expected crc 2023-09-23T22:43:30.796 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:30.842 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:30.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:30.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:30.865 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:30 smithi026 ceph-mon[223014]: failed to encode map e604 with expected crc 2023-09-23T22:43:30.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:30 smithi026 ceph-mon[223014]: failed to encode map e604 with expected crc 2023-09-23T22:43:30.866 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:30 smithi026 ceph-mon[223014]: failed to encode map e604 with expected crc 2023-09-23T22:43:30.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:30 smithi026 ceph-mon[215775]: failed to encode map e604 with expected crc 2023-09-23T22:43:30.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:30 smithi026 ceph-mon[215775]: failed to encode map e604 with expected crc 2023-09-23T22:43:30.866 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:30 smithi026 ceph-mon[215775]: failed to encode map e604 with expected crc 2023-09-23T22:43:30.952 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:31.036 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:31.161 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:31.225 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:31.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:31.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:31.332 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:31.359 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:31.405 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:31.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:31.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:31.462 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:43:31.506 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:43:31.511 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:31.557 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:43:31.595 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:43:31.600 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:31.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:31 smithi070 ceph-mon[192652]: pgmap v1564: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 97 active+clean; 1.2 GiB data, 5.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 604 op/s; 4426/59649 objects degraded (7.420%); 21147/59649 objects misplaced (35.452%); 757 KiB/s, 197 objects/s recovering 2023-09-23T22:43:31.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3611194029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:31.638 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:43:31.688 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:43:31.729 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:43:31.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:31 smithi026 ceph-mon[215775]: pgmap v1564: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 97 active+clean; 1.2 GiB data, 5.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 604 op/s; 4426/59649 objects degraded (7.420%); 21147/59649 objects misplaced (35.452%); 757 KiB/s, 197 objects/s recovering 2023-09-23T22:43:31.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3611194029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:31.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:31 smithi026 ceph-mon[223014]: pgmap v1564: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 5 active+recovery_wait+degraded+remapped, 97 active+clean; 1.2 GiB data, 5.1 GiB used, 352 GiB / 358 GiB avail; 0 B/s wr, 604 op/s; 4426/59649 objects degraded (7.420%); 21147/59649 objects misplaced (35.452%); 757 KiB/s, 197 objects/s recovering 2023-09-23T22:43:31.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3611194029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:31.778 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:43:31.779 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:43:31.779 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:43:31.779 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:43:31.779 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:43:31.779 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:43:31.779 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:43:31.779 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:43:31.779 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:43:33.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:33 smithi026 ceph-mon[215775]: pgmap v1565: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 1.1 GiB data, 5.0 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 703 op/s; 3597/55173 objects degraded (6.519%); 18100/55173 objects misplaced (32.806%); 624 KiB/s, 172 objects/s recovering 2023-09-23T22:43:33.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:33 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3597/55173 objects degraded (6.519%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:33.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:33 smithi026 ceph-mon[223014]: pgmap v1565: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 1.1 GiB data, 5.0 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 703 op/s; 3597/55173 objects degraded (6.519%); 18100/55173 objects misplaced (32.806%); 624 KiB/s, 172 objects/s recovering 2023-09-23T22:43:33.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:33 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3597/55173 objects degraded (6.519%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:33.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:33 smithi070 ceph-mon[192652]: pgmap v1565: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 1.1 GiB data, 5.0 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 703 op/s; 3597/55173 objects degraded (6.519%); 18100/55173 objects misplaced (32.806%); 624 KiB/s, 172 objects/s recovering 2023-09-23T22:43:33.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:33 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3597/55173 objects degraded (6.519%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:33.975 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:33.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:33.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:34.080 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:34.108 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:34.162 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:34.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:34.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:34.268 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:34.334 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:35.102 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:35.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:35.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:35.211 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:35.238 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:35.286 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:35.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:35.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:35.392 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:35.419 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:35.466 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:35.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:35.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:35 smithi026 ceph-mon[223014]: pgmap v1566: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 1.0 GiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 842 op/s; 3597/49581 objects degraded (7.255%); 17739/49581 objects misplaced (35.778%); 568 KiB/s, 231 objects/s recovering 2023-09-23T22:43:35.573 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:35.652 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:35.850 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:43:35.850 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:43:35.850 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:43:35.850 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112969 9.3 9:caad47cb:::benchmark_data_smithi026_555954_object18240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 270.51692197099999, 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6657709080000001, 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.851 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112969 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:43:35.852 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328979+0000", 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328983+0000", 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.693e-06 2023-09-23T22:43:35.853 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328991+0000", 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2220000000000002e-06 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328999+0000", 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4870000000000002e-06 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.854 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.329055+0000", 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6026999999999997e-05 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.329062+0000", 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.161e-06 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.855 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.976223+0000", 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6471613999999999 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.977503+0000", 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012796629999999999 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:43:35.856 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.979185+0000", 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001682149 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.984402+0000", 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052169599999999997 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.991082+0000", 2023-09-23T22:43:35.857 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066800590000000003 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.992088+0000", 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001005964 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.992639+0000", 2023-09-23T22:43:35.858 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055137800000000005 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.994750+0000", 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021113310000000001 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.859 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112990 9.3 9:c43d78c1:::benchmark_data_smithi026_555954_object18261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 270.486835326, 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.640044252, 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112990 2023-09-23T22:43:35.860 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.861 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359065+0000", 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359070+0000", 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9409999999999998e-06 2023-09-23T22:43:35.862 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359083+0000", 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2697999999999999e-05 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359089+0000", 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7200000000000003e-06 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.863 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359144+0000", 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5954e-05 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359157+0000", 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2853e-05 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.864 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.984626+0000", 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.625468431 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.986258+0000", 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016318319999999999 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.988082+0000", 2023-09-23T22:43:35.865 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018244540000000001 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.995233+0000", 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071506429999999999 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.995368+0000", 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135067 2023-09-23T22:43:35.866 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.996086+0000", 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00071818499999999998 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.996659+0000", 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057308399999999996 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.867 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.999110+0000", 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024514530000000001 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112992 9.3 9:cb6055b3:::benchmark_data_smithi026_555954_object18263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 270.485377218, 2023-09-23T22:43:35.868 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6580400470000001, 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112992 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:43:35.869 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.870 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360528+0000", 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4259999999999996e-06 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360540+0000", 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1824999999999999e-05 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.873 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360545+0000", 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5289999999999999e-06 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360597+0000", 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1746000000000002e-05 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.874 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360609+0000", 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1953000000000001e-05 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.999299+0000", 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.638690135 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.000875+0000", 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015752990000000001 2023-09-23T22:43:35.875 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.002719+0000", 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018446700000000001 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015384+0000", 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012664969999999999 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.876 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015497+0000", 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011238 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015923+0000", 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042655200000000002 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.877 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.016447+0000", 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052434900000000002 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.018564+0000", 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021169520000000001 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.878 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112993 9.3 9:cd412e89:::benchmark_data_smithi026_555954_object18264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 270.48192988400001, 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.658274966, 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112993 2023-09-23T22:43:35.879 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.880 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363971+0000", 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363975+0000", 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0169999999999998e-06 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.881 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363986+0000", 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0569e-05 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363991+0000", 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0599999999999998e-06 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364039+0000", 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8767999999999998e-05 2023-09-23T22:43:35.882 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364049+0000", 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8279999999999998e-06 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.008138+0000", 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6440885409999999 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.883 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.009772+0000", 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016337540000000001 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.011820+0000", 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020481570000000001 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.884 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.018990+0000", 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071701450000000002 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.019117+0000", 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012743300000000001 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.019436+0000", 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000318239 2023-09-23T22:43:35.885 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.020027+0000", 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059157899999999995 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.022247+0000", 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022195629999999999 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.886 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188104 9.e 9:7689f13c:::benchmark_data_smithi026_555954_object93375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 165.43223555700001, 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.09525067, 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.887 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188104 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.888 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.889 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413668+0000", 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6299999999999998e-06 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413674+0000", 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.241e-06 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413680+0000", 2023-09-23T22:43:35.890 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1270000000000002e-06 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419176+0000", 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054967100000000001 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419186+0000", 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2289999999999992e-06 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.891 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.502956+0000", 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0837702039999999 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.503056+0000", 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100003 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.892 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.503139+0000", 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2886999999999994e-05 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505036+0000", 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001897756 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.893 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506288+0000", 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001251816 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508853+0000", 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025648149999999998 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508885+0000", 2023-09-23T22:43:35.894 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2224999999999997e-05 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508917+0000", 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1417000000000001e-05 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188105 9.e 9:72ff4f7f:::benchmark_data_smithi026_555954_object93376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-09-23T22:43:35.895 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.415823+0000", 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 165.43007853399999, 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094443748, 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188105 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.896 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415823+0000", 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415823+0000", 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415822+0000", 2023-09-23T22:43:35.897 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415828+0000", 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9649999999999996e-06 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415843+0000", 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4371e-05 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.898 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415851+0000", 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4130000000000006e-06 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419209+0000", 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003357942 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.899 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419219+0000", 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0190999999999999e-05 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505077+0000", 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0858583820000001 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505177+0000", 2023-09-23T22:43:35.900 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9227000000000004e-05 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505256+0000", 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8937000000000007e-05 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508765+0000", 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035097739999999998 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.901 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508778+0000", 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2464e-05 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510218+0000", 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001440059 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.902 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510245+0000", 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6982e-05 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510267+0000", 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1889e-05 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.903 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188109 9.e 9:78387bd9:::benchmark_data_smithi026_555954_object93380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.421149+0000", 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 165.42475278500001, 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094876424, 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188109 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.904 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421149+0000", 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421149+0000", 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.905 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421148+0000", 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421154+0000", 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7089999999999996e-06 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.906 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421165+0000", 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0801e-05 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421173+0000", 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9389999999999997e-06 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.422829+0000", 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001656299 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.907 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.422839+0000", 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7890000000000006e-06 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506315+0000", 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0834760299999999 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.908 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506388+0000", 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3413999999999999e-05 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506451+0000", 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2315000000000002e-05 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508814+0000", 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023629089999999998 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.909 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508933+0000", 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119478 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.515963+0000", 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070293259999999998 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.910 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.515994+0000", 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1168999999999999e-05 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516025+0000", 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1522000000000001e-05 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188140 9.e 9:78d2bdae:::benchmark_data_smithi026_555954_object93411:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:43:35.911 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.458307+0000", 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 165.387594842, 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.057861621, 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188140 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.912 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458307+0000", 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458307+0000", 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458306+0000", 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:35.913 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458310+0000", 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.208e-06 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458315+0000", 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9190000000000004e-06 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.914 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458320+0000", 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.989e-06 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471337+0000", 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013016099 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.915 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471347+0000", 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0324e-05 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.507497+0000", 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0361505470000001 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.507566+0000", 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8753999999999999e-05 2023-09-23T22:43:35.916 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.507632+0000", 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5599999999999995e-05 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508942+0000", 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001310077 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510149+0000", 2023-09-23T22:43:35.917 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012073489999999999 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516108+0000", 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059589150000000004 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516143+0000", 2023-09-23T22:43:35.918 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5042000000000001e-05 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516168+0000", 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5201000000000001e-05 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188143 9.e 9:7424b661:::benchmark_data_smithi026_555954_object93414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.463773+0000", 2023-09-23T22:43:35.919 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 165.38212816500001, 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0524969799999999, 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188143 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463773+0000", 2023-09-23T22:43:35.920 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463773+0000", 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463772+0000", 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.921 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463779+0000", 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5320000000000003e-06 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463791+0000", 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2319e-05 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.922 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463803+0000", 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2082000000000001e-05 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471367+0000", 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075638820000000001 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.923 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471373+0000", 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0279999999999997e-06 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508966+0000", 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0375934039999999 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.509038+0000", 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1544999999999993e-05 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.924 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.509111+0000", 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3699e-05 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.511865+0000", 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002753148 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.925 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.513751+0000", 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001886307 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516216+0000", 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024647459999999999 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516246+0000", 2023-09-23T22:43:35.926 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0256999999999998e-05 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516270+0000", 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4433000000000001e-05 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.927 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188144 9.e 9:73d3d7cc:::benchmark_data_smithi026_555954_object93415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.463813+0000", 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 165.382088595, 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.052639544, 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188144 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.928 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463813+0000", 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463813+0000", 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.929 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463812+0000", 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463816+0000", 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2349999999999999e-06 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463823+0000", 2023-09-23T22:43:35.930 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.294e-06 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463828+0000", 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5160000000000002e-06 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471392+0000", 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075633810000000001 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.931 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471398+0000", 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7570000000000001e-06 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510314+0000", 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.038916022 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.932 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510382+0000", 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8048999999999998e-05 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510445+0000", 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3137000000000002e-05 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.511918+0000", 2023-09-23T22:43:35.933 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001472714 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516042+0000", 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041248190000000001 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516374+0000", 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000331746 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.934 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516419+0000", 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4904999999999998e-05 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516453+0000", 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3528000000000002e-05 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.935 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23685 10.c 10:3e82cace:::benchmark_data_smithi026_640738_object23684:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.400587+0000", 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.445314635000003, 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3549325689999998, 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:35.936 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23685 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400587+0000", 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400587+0000", 2023-09-23T22:43:35.937 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400585+0000", 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400611+0000", 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6152999999999998e-05 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.938 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400621+0000", 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0336999999999999e-05 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400629+0000", 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6490000000000008e-06 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.939 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400693+0000", 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4178999999999997e-05 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400843+0000", 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149672 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:43:35.940 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.401003+0000", 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159662 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402788+0000", 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017856790000000001 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.543824+0000", 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.14103579799999999 2023-09-23T22:43:35.941 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566302+0000", 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022477809000000001 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566314+0000", 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1917e-05 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.942 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.523693+0000", 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95737967599999996 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.523720+0000", 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6058e-05 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.755413+0000", 2023-09-23T22:43:35.943 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.23169375 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.755519+0000", 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106198 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23692 10.c 10:3970b700:::benchmark_data_smithi026_640738_object23691:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:43:35.944 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.402440+0000", 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.443461287999995, 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3583861430000002, 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23692 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.945 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402440+0000", 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402440+0000", 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402439+0000", 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:35.946 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402469+0000", 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0212000000000001e-05 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402482+0000", 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2894999999999999e-05 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.947 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402490+0000", 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3620000000000006e-06 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402821+0000", 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033087500000000001 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402963+0000", 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014139999999999999 2023-09-23T22:43:35.948 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403105+0000", 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014212299999999999 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.541814+0000", 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13870915 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.949 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.543831+0000", 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020176510000000001 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.569547+0000", 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025715472999999999 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.569559+0000", 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2438e-05 2023-09-23T22:43:35.950 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.523756+0000", 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.954196453 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.523766+0000", 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0662999999999999e-05 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.951 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.760701+0000", 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.236934706 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.760826+0000", 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012522 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23693 10.2 10:4c04af01:::benchmark_data_smithi026_640738_object23692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:43:35.952 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.402504+0000", 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.443397610999995, 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3437135749999998, 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23693 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.953 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402504+0000", 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402504+0000", 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.954 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402503+0000", 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402528+0000", 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4802000000000001e-05 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402536+0000", 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1480000000000006e-06 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.955 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402542+0000", 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8950000000000003e-06 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403527+0000", 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098492299999999996 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403633+0000", 2023-09-23T22:43:35.956 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106319 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0", 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403772+0000", 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013865399999999999 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.542749+0000", 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13897710899999999 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.957 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564906+0000", 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022157204999999999 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564918+0000", 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.146e-05 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.958 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.515244+0000", 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.950325906 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.515259+0000", 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4796e-05 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746118+0000", 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.23085961 2023-09-23T22:43:35.959 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746218+0000", 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9322999999999998e-05 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23696 10.0 10:07385719:::benchmark_data_smithi026_640738_object23695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.402548+0000", 2023-09-23T22:43:35.960 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.443353787000007, 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3550447210000001, 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23696 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.961 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402548+0000", 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402548+0000", 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.962 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402547+0000", 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402573+0000", 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5388000000000002e-05 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.963 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402580+0000", 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4699999999999996e-06 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402586+0000", 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.699e-06 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403782+0000", 2023-09-23T22:43:35.964 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011965070000000001 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403910+0000", 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012775800000000001 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.404059+0000", 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014904799999999999 2023-09-23T22:43:35.965 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.541946+0000", 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.137886549 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.541955+0000", 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7730000000000005e-06 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.966 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564122+0000", 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022166994999999998 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564139+0000", 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7014999999999999e-05 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.967 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529858+0000", 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96571985500000002 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529874+0000", 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5566999999999999e-05 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757503+0000", 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.227628712 2023-09-23T22:43:35.968 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757592+0000", 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9835999999999998e-05 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23698 10.2 10:402b652c:::benchmark_data_smithi026_640738_object23697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.402600+0000", 2023-09-23T22:43:35.969 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.443301947999998, 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3437404329999998, 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23698 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.970 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402600+0000", 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402600+0000", 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.971 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402599+0000", 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402624+0000", 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4955000000000001e-05 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402632+0000", 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3650000000000004e-06 2023-09-23T22:43:35.972 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402638+0000", 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9370000000000001e-06 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.404701+0000", 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020623759999999999 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.973 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.404809+0000", 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108125 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0", 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.404929+0000", 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120272 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.974 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544366+0000", 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.139437006 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.565658+0000", 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021292285000000001 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.565670+0000", 2023-09-23T22:43:35.975 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1100999999999999e-05 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.515294+0000", 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94962474299999999 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.515305+0000", 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0346999999999999e-05 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.976 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746259+0000", 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.230953959 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746340+0000", 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1451000000000003e-05 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.977 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23701 10.0 10:0aff6243:::benchmark_data_smithi026_640738_object23700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.545466+0000", 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.300435238999995, 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2285612509999999, 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23701 2023-09-23T22:43:35.978 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545466+0000", 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545466+0000", 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.979 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545465+0000", 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545488+0000", 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2617999999999999e-05 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.980 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545499+0000", 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0773e-05 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545506+0000", 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6660000000000008e-06 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564930+0000", 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019424004000000002 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.981 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564941+0000", 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0362e-05 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529900+0000", 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96495990399999998 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.982 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.983 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:35 smithi070 ceph-mon[192652]: pgmap v1566: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 1.0 GiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 842 op/s; 3597/49581 objects degraded (7.255%); 17739/49581 objects misplaced (35.778%); 568 KiB/s, 231 objects/s recovering 2023-09-23T22:43:35.983 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529909+0000", 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4880000000000007e-06 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757621+0000", 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2277121580000001 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.984 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757762+0000", 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000141116 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757929+0000", 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000166899 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.768586+0000", 2023-09-23T22:43:35.985 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010657217 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.769695+0000", 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011086620000000001 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.772781+0000", 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030858219999999998 2023-09-23T22:43:35.986 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.772848+0000", 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6799999999999997e-05 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774028+0000", 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001179951 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.987 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23702 10.0 10:081d1a08:::benchmark_data_smithi026_640738_object23701:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.545512+0000", 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.300389230999997, 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2295653080000002, 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:35.988 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23702 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545512+0000", 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545512+0000", 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.989 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545512+0000", 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545536+0000", 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4450000000000001e-05 2023-09-23T22:43:35.990 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545544+0000", 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5020000000000003e-06 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545550+0000", 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1589999999999998e-06 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.991 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.565069+0000", 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019519334999999999 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.565079+0000", 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5519999999999993e-06 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529932+0000", 2023-09-23T22:43:35.992 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96485308000000003 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529940+0000", 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7149999999999998e-06 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.758947+0000", 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2290078449999999 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.993 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.759087+0000", 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139126 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.759266+0000", 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017959900000000001 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.994 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.768594+0000", 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093276999999999995 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.769787+0000", 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001193103 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774081+0000", 2023-09-23T22:43:35.995 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042944230000000003 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774150+0000", 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8270999999999997e-05 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.775078+0000", 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00092792700000000005 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.996 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23707 10.0 10:0b861545:::benchmark_data_smithi026_640738_object23706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.566617+0000", 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.279284855, 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2095116400000001, 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:35.997 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23707 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566617+0000", 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:35.998 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566617+0000", 2023-09-23T22:43:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:35.999 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566615+0000", 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566644+0000", 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8364000000000002e-05 2023-09-23T22:43:36.000 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566654+0000", 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0806e-05 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566664+0000", 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6160000000000008e-06 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.001 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566717+0000", 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3134000000000003e-05 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566728+0000", 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0526000000000001e-05 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529962+0000", 2023-09-23T22:43:36.002 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96323421600000003 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529970+0000", 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5949999999999999e-06 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.760896+0000", 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2309250709999999 2023-09-23T22:43:36.003 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.761014+0000", 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011832600000000001 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.761123+0000", 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010950099999999999 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.004 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.768601+0000", 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074772730000000004 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.769827+0000", 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001226013 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:36.005 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.775193+0000", 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053661619999999998 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.775241+0000", 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8355e-05 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.776128+0000", 2023-09-23T22:43:36.006 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00088716000000000001 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23711 10.1 10:83d91f22:::benchmark_data_smithi026_640738_object23710:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.592432+0000", 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 91.253469640999995, 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1646562760000001, 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:43:36.007 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23711 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592432+0000", 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.008 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592432+0000", 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592430+0000", 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.009 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592455+0000", 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4997000000000001e-05 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592472+0000", 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6687000000000001e-05 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:36.010 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592481+0000", 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5650000000000007e-06 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592542+0000", 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0534e-05 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592557+0000", 2023-09-23T22:43:36.011 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4868999999999999e-05 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.508097+0000", 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91554035700000003 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.508110+0000", 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.322e-05 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.725178+0000", 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2170675339999999 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.725318+0000", 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139777 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:43:36.013 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.725470+0000", 2023-09-23T22:43:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151924 2023-09-23T22:43:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.736347+0000", 2023-09-23T22:43:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010876798999999999 2023-09-23T22:43:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.743719+0000", 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073726110000000003 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.756953+0000", 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013234364 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757003+0000", 2023-09-23T22:43:36.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9212000000000002e-05 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757088+0000", 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5508999999999997e-05 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.016 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:64668 10.2 10:41157639:::benchmark_data_smithi026_640738_object17272:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:10.634899+0000", 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 25.211002678, 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2374666329999999, 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 64668 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.017 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634899+0000", 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634899+0000", 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634897+0000", 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634905+0000", 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8790000000000003e-06 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:43:36.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087141+0000", 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.45223591200000002 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087153+0000", 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2293000000000001e-05 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087215+0000", 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1610000000000001e-05 2023-09-23T22:43:36.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087373+0000", 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000158455 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087521+0000", 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014800100000000001 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.089367+0000", 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00184593 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.131005+0000", 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041637697000000001 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:43:36.022 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.131017+0000", 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.181e-05 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.856437+0000", 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.72542065 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.856569+0000", 2023-09-23T22:43:36.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013176400000000001 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.872254+0000", 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015684757000000001 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.872366+0000", 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011165799999999999 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:36.024 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:36.025 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:43:36.025 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:43:36.025 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:43:36.025 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:35 smithi026 ceph-mon[215775]: pgmap v1566: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 1.0 GiB data, 4.7 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 842 op/s; 3597/49581 objects degraded (7.255%); 17739/49581 objects misplaced (35.778%); 568 KiB/s, 231 objects/s recovering 2023-09-23T22:43:36.261 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:36.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:36.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:36.369 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:36.441 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:37.378 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:43:37.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:37 smithi026 ceph-mon[215775]: pgmap v1567: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 1019 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 679 op/s; 3597/48786 objects degraded (7.373%); 17739/48786 objects misplaced (36.361%); 64 KiB/s, 177 objects/s recovering 2023-09-23T22:43:37.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:37 smithi026 ceph-mon[223014]: pgmap v1567: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 1019 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 679 op/s; 3597/48786 objects degraded (7.373%); 17739/48786 objects misplaced (36.361%); 64 KiB/s, 177 objects/s recovering 2023-09-23T22:43:37.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:37 smithi070 ceph-mon[192652]: pgmap v1567: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 1019 MiB data, 4.6 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 679 op/s; 3597/48786 objects degraded (7.373%); 17739/48786 objects misplaced (36.361%); 64 KiB/s, 177 objects/s recovering 2023-09-23T22:43:37.930 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:37.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:37.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:38.045 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:38.085 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:43:38.122 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:38.303 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:38.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:38.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:38.408 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:38.437 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:38.484 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:38.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:38.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:38.528 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:38.589 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:38.616 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:38.664 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:38.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:38.771 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:38.827 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:38 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 3597/48786 objects degraded (7.373%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:38.828 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:38 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 3597/48786 objects degraded (7.373%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:38.833 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:38.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:38 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 3597/48786 objects degraded (7.373%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:39.161 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:39.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:39.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:39.275 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:39.343 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:39.491 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:39.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:39.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:39.594 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:39.601 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T22:43:39.678 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:39.822 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:39.836 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:43:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:43:39] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T22:43:39.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:39 smithi026 ceph-mon[215775]: pgmap v1568: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 907 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 594 op/s; 3316/43380 objects degraded (7.644%); 15725/43380 objects misplaced (36.249%); 55 KiB/s, 180 objects/s recovering 2023-09-23T22:43:39.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:39 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/888788003' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:43:39.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:39.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:39.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:39.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:39 smithi026 ceph-mon[223014]: pgmap v1568: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 907 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 594 op/s; 3316/43380 objects degraded (7.644%); 15725/43380 objects misplaced (36.249%); 55 KiB/s, 180 objects/s recovering 2023-09-23T22:43:39.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:39 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/888788003' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:43:39.838 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:39.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:39 smithi070 ceph-mon[192652]: pgmap v1568: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 907 MiB data, 4.4 GiB used, 353 GiB / 358 GiB avail; 0 B/s wr, 594 op/s; 3316/43380 objects degraded (7.644%); 15725/43380 objects misplaced (36.249%); 55 KiB/s, 180 objects/s recovering 2023-09-23T22:43:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:39 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/888788003' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:43:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:39.939 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:39.965 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:40.005 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:40.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:40.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:40.110 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:40.137 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:40.597 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:43:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:40 smithi070 ceph-mon[192652]: failed to encode map e605 with expected crc 2023-09-23T22:43:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:40 smithi070 ceph-mon[192652]: failed to encode map e605 with expected crc 2023-09-23T22:43:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/888788003' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:43:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:40 smithi070 ceph-mon[192652]: osdmap e605: 8 total, 8 up, 4 in 2023-09-23T22:43:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:40 smithi070 ceph-mon[192652]: failed to encode map e605 with expected crc 2023-09-23T22:43:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:40 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1635855260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:40 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:40 smithi070 ceph-mon[192652]: pgmap v1570: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 763 MiB data, 4.1 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 772 op/s; 3316/36468 objects degraded (9.093%); 12587/36468 objects misplaced (34.515%); 0 B/s, 99 objects/s recovering 2023-09-23T22:43:40.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[223014]: failed to encode map e605 with expected crc 2023-09-23T22:43:40.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[223014]: failed to encode map e605 with expected crc 2023-09-23T22:43:40.977 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/888788003' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:43:40.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[223014]: osdmap e605: 8 total, 8 up, 4 in 2023-09-23T22:43:40.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[223014]: failed to encode map e605 with expected crc 2023-09-23T22:43:40.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1635855260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:40.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:40.978 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[223014]: pgmap v1570: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 763 MiB data, 4.1 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 772 op/s; 3316/36468 objects degraded (9.093%); 12587/36468 objects misplaced (34.515%); 0 B/s, 99 objects/s recovering 2023-09-23T22:43:40.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[215775]: failed to encode map e605 with expected crc 2023-09-23T22:43:40.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[215775]: failed to encode map e605 with expected crc 2023-09-23T22:43:40.978 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/888788003' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:43:40.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[215775]: osdmap e605: 8 total, 8 up, 4 in 2023-09-23T22:43:40.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[215775]: failed to encode map e605 with expected crc 2023-09-23T22:43:40.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1635855260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:40.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:43:40.979 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:40 smithi026 ceph-mon[215775]: pgmap v1570: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 97 active+clean; 763 MiB data, 4.1 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 772 op/s; 3316/36468 objects degraded (9.093%); 12587/36468 objects misplaced (34.515%); 0 B/s, 99 objects/s recovering 2023-09-23T22:43:41.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:41 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:41.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:41 smithi070 ceph-mon[192652]: failed to encode map e606 with expected crc 2023-09-23T22:43:41.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:41 smithi070 ceph-mon[192652]: failed to encode map e606 with expected crc 2023-09-23T22:43:41.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:41 smithi070 ceph-mon[192652]: failed to encode map e606 with expected crc 2023-09-23T22:43:41.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:41 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:43:41.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:41 smithi070 ceph-mon[192652]: osdmap e606: 8 total, 8 up, 4 in 2023-09-23T22:43:41.931 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:41.931 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[215775]: failed to encode map e606 with expected crc 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[215775]: failed to encode map e606 with expected crc 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[215775]: failed to encode map e606 with expected crc 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[215775]: osdmap e606: 8 total, 8 up, 4 in 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[223014]: failed to encode map e606 with expected crc 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[223014]: failed to encode map e606 with expected crc 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[223014]: failed to encode map e606 with expected crc 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:43:41.932 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:41 smithi026 ceph-mon[223014]: osdmap e606: 8 total, 8 up, 4 in 2023-09-23T22:43:42.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:42 smithi070 ceph-mon[192652]: pgmap v1572: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 98 active+clean; 656 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 758 op/s; 2983/31362 objects degraded (9.512%); 11359/31362 objects misplaced (36.219%); 0 B/s, 70 objects/s recovering 2023-09-23T22:43:42.936 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:42.942 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:42 smithi026 ceph-mon[215775]: pgmap v1572: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 98 active+clean; 656 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 758 op/s; 2983/31362 objects degraded (9.512%); 11359/31362 objects misplaced (36.219%); 0 B/s, 70 objects/s recovering 2023-09-23T22:43:42.943 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:42.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:42.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:42.946 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:42 smithi026 ceph-mon[223014]: pgmap v1572: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 4 active+recovery_wait+degraded+remapped, 98 active+clean; 656 MiB data, 3.8 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 758 op/s; 2983/31362 objects degraded (9.512%); 11359/31362 objects misplaced (36.219%); 0 B/s, 70 objects/s recovering 2023-09-23T22:43:42.973 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T22:43:43.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:43.156 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:43.374 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:43.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:43.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:43.480 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:43.602 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:43.750 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:43:43.802 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:43:43.850 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:43:43.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:43 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 2983/31362 objects degraded (9.512%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:43.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:43:43.952 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:43:43.971 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:43.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:43.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:43.980 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:43 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 2983/31362 objects degraded (9.512%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:43.983 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:43 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 2983/31362 objects degraded (9.512%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:43.989 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:43:44.029 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:43:44.061 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:43:44.066 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:43:44.066 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:43:44.066 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:43:44.066 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:43:44.066 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:43:44.066 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:43:44.066 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:43:44.067 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:43:44.080 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:44.108 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:44.157 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:44.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:44.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:44.262 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:44.292 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:44.349 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:44.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:44.455 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:44.543 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:44 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1612296807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:44.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:44 smithi070 ceph-mon[192652]: pgmap v1573: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 546 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 944 op/s; 2261/26088 objects degraded (8.667%); 8440/26088 objects misplaced (32.352%); 0 B/s, 106 objects/s recovering 2023-09-23T22:43:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:44 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1612296807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:44.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:44 smithi026 ceph-mon[223014]: pgmap v1573: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 546 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 944 op/s; 2261/26088 objects degraded (8.667%); 8440/26088 objects misplaced (32.352%); 0 B/s, 106 objects/s recovering 2023-09-23T22:43:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:44 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1612296807' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:44.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:44 smithi026 ceph-mon[215775]: pgmap v1573: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 546 MiB data, 3.6 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 944 op/s; 2261/26088 objects degraded (8.667%); 8440/26088 objects misplaced (32.352%); 0 B/s, 106 objects/s recovering 2023-09-23T22:43:45.029 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:43:46.123 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:46.339 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:46.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:46.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:46.444 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:46.471 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:46.521 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:46.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:46.629 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:46.709 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:46.968 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:46.974 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:46 smithi026 ceph-mon[223014]: pgmap v1574: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 533 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 745 op/s; 2261/25464 objects degraded (8.879%); 8440/25464 objects misplaced (33.145%); 0 B/s, 65 objects/s recovering 2023-09-23T22:43:46.975 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:46 smithi026 ceph-mon[215775]: pgmap v1574: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 533 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 745 op/s; 2261/25464 objects degraded (8.879%); 8440/25464 objects misplaced (33.145%); 0 B/s, 65 objects/s recovering 2023-09-23T22:43:46.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:46.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:46.987 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:47.088 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:47.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:46 smithi070 ceph-mon[192652]: pgmap v1574: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 533 MiB data, 3.5 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 745 op/s; 2261/25464 objects degraded (8.879%); 8440/25464 objects misplaced (33.145%); 0 B/s, 65 objects/s recovering 2023-09-23T22:43:47.155 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:47.262 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:47.262 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:47.375 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:47.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:47.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:47.480 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:47.511 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:47.560 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:47.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:47.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:47.665 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:47.734 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:48.476 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:43:49.101 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:49.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:49.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:48 smithi026 ceph-mon[215775]: pgmap v1575: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 437 MiB data, 3.3 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 636 op/s; 1589/20850 objects degraded (7.621%); 6897/20850 objects misplaced (33.079%); 0 B/s, 64 objects/s recovering 2023-09-23T22:43:49.109 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:48 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 1589/20850 objects degraded (7.621%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:49.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:48 smithi026 ceph-mon[223014]: pgmap v1575: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 437 MiB data, 3.3 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 636 op/s; 1589/20850 objects degraded (7.621%); 6897/20850 objects misplaced (33.079%); 0 B/s, 64 objects/s recovering 2023-09-23T22:43:49.110 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:48 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 1589/20850 objects degraded (7.621%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:49.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:49.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:48 smithi070 ceph-mon[192652]: pgmap v1575: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 10 active+remapped+backfill_wait, 3 active+recovery_wait+degraded+remapped, 98 active+clean; 437 MiB data, 3.3 GiB used, 354 GiB / 358 GiB avail; 0 B/s wr, 636 op/s; 1589/20850 objects degraded (7.621%); 6897/20850 objects misplaced (33.079%); 0 B/s, 64 objects/s recovering 2023-09-23T22:43:49.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:48 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 1589/20850 objects degraded (7.621%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:49.211 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:49.240 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:49.290 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:49.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:49.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:49.463 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:49.718 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:49.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:49.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:49.728 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:43:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:43:49] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T22:43:49.826 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:43:49.856 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:43:49.914 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:50.038 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:43:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:50.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:43:50.142 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:50.256 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:50.433 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:50.936 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:50.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:50.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:51.047 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:51.104 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:50 smithi026 ceph-mon[215775]: pgmap v1576: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 98 active+clean; 306 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 730 op/s; 1098/14550 objects degraded (7.546%); 5062/14550 objects misplaced (34.790%); 0 B/s, 87 objects/s recovering 2023-09-23T22:43:51.104 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:50 smithi026 ceph-mon[223014]: pgmap v1576: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 98 active+clean; 306 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 730 op/s; 1098/14550 objects degraded (7.546%); 5062/14550 objects misplaced (34.790%); 0 B/s, 87 objects/s recovering 2023-09-23T22:43:51.113 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:51.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:50 smithi070 ceph-mon[192652]: pgmap v1576: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 98 active+clean; 306 MiB data, 3.0 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 730 op/s; 1098/14550 objects degraded (7.546%); 5062/14550 objects misplaced (34.790%); 0 B/s, 87 objects/s recovering 2023-09-23T22:43:51.864 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T22:43:52.066 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:52.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:52.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:52.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:51 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3032019696' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:43:52.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:51 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:43:52.177 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:52.180 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:51 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3032019696' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:43:52.181 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:51 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:43:52.181 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:51 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3032019696' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:43:52.181 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:51 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:43:52.204 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:52.258 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:52.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:52.263 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:52.363 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:52.391 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:52.441 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:52.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:52.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:52.548 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:52.577 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:52.628 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:52.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:52.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:52.736 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:52.764 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:52.797 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Removed 47393 objects 2023-09-23T22:43:52.797 INFO:tasks.radosbench.radosbench.0.smithi026.stdout:Clean up completed and total clean up time :71.5294 2023-09-23T22:43:52.809 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:52.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:52.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:52.913 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:52.939 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:52.980 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:52.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:53.087 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:53.110 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:52 smithi070 ceph-mon[192652]: pgmap v1577: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 190 MiB data, 2.7 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 821 op/s; 280/8970 objects degraded (3.122%); 3262/8970 objects misplaced (36.366%); 0 B/s, 79 objects/s recovering 2023-09-23T22:43:53.110 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:52 smithi070 ceph-mon[192652]: failed to encode map e607 with expected crc 2023-09-23T22:43:53.111 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:52 smithi070 ceph-mon[192652]: failed to encode map e607 with expected crc 2023-09-23T22:43:53.111 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:52 smithi070 ceph-mon[192652]: failed to encode map e607 with expected crc 2023-09-23T22:43:53.111 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:52 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:43:53.111 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:52 smithi070 ceph-mon[192652]: osdmap e607: 8 total, 8 up, 4 in 2023-09-23T22:43:53.112 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:53.153 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:53.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:53.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:53.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[223014]: pgmap v1577: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 190 MiB data, 2.7 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 821 op/s; 280/8970 objects degraded (3.122%); 3262/8970 objects misplaced (36.366%); 0 B/s, 79 objects/s recovering 2023-09-23T22:43:53.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[223014]: failed to encode map e607 with expected crc 2023-09-23T22:43:53.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[223014]: failed to encode map e607 with expected crc 2023-09-23T22:43:53.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[223014]: failed to encode map e607 with expected crc 2023-09-23T22:43:53.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:43:53.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[223014]: osdmap e607: 8 total, 8 up, 4 in 2023-09-23T22:43:53.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[215775]: pgmap v1577: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 190 MiB data, 2.7 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 821 op/s; 280/8970 objects degraded (3.122%); 3262/8970 objects misplaced (36.366%); 0 B/s, 79 objects/s recovering 2023-09-23T22:43:53.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[215775]: failed to encode map e607 with expected crc 2023-09-23T22:43:53.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[215775]: failed to encode map e607 with expected crc 2023-09-23T22:43:53.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[215775]: failed to encode map e607 with expected crc 2023-09-23T22:43:53.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:43:53.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:52 smithi026 ceph-mon[215775]: osdmap e607: 8 total, 8 up, 4 in 2023-09-23T22:43:53.260 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:43:53.293 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:53.633 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:43:53.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:53.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:43:53.649 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:43:53.650 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:43:53.738 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:53.775 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:53.852 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:53.936 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:53.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:53 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 280/8970 objects degraded (3.122%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:53.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:53 smithi026 ceph-mon[215775]: failed to encode map e608 with expected crc 2023-09-23T22:43:53.939 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:53 smithi026 ceph-mon[215775]: osdmap e608: 8 total, 8 up, 4 in 2023-09-23T22:43:53.939 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:53 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 280/8970 objects degraded (3.122%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:53.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:53 smithi026 ceph-mon[223014]: failed to encode map e608 with expected crc 2023-09-23T22:43:53.940 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:53 smithi026 ceph-mon[223014]: osdmap e608: 8 total, 8 up, 4 in 2023-09-23T22:43:53.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:53.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:54.041 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:54.069 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:54.110 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:54.114 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:53 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 280/8970 objects degraded (3.122%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T22:43:54.114 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:53 smithi070 ceph-mon[192652]: failed to encode map e608 with expected crc 2023-09-23T22:43:54.114 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:53 smithi070 ceph-mon[192652]: osdmap e608: 8 total, 8 up, 4 in 2023-09-23T22:43:54.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:54.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:54.215 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:43:54.241 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:54.282 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:43:54.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:54.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:43:54.389 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:54.411 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:43:54.430 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:54.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[223014]: failed to encode map e608 with expected crc 2023-09-23T22:43:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[223014]: failed to encode map e608 with expected crc 2023-09-23T22:43:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[223014]: pgmap v1580: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 56 MiB data, 2.5 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 954 op/s; 280/2547 objects degraded (10.993%); 624/2547 objects misplaced (24.499%); 0 B/s, 97 objects/s recovering 2023-09-23T22:43:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4292107387' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:54.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:43:54.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[223014]: failed to encode map e609 with expected crc 2023-09-23T22:43:54.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4292107387' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:54.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[223014]: osdmap e609: 8 total, 8 up, 4 in 2023-09-23T22:43:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[215775]: failed to encode map e608 with expected crc 2023-09-23T22:43:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[215775]: failed to encode map e608 with expected crc 2023-09-23T22:43:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[215775]: pgmap v1580: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 56 MiB data, 2.5 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 954 op/s; 280/2547 objects degraded (10.993%); 624/2547 objects misplaced (24.499%); 0 B/s, 97 objects/s recovering 2023-09-23T22:43:54.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:54.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4292107387' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:54.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:43:54.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[215775]: failed to encode map e609 with expected crc 2023-09-23T22:43:54.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4292107387' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:54.988 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:54 smithi026 ceph-mon[215775]: osdmap e609: 8 total, 8 up, 4 in 2023-09-23T22:43:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:54 smithi070 ceph-mon[192652]: failed to encode map e608 with expected crc 2023-09-23T22:43:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:54 smithi070 ceph-mon[192652]: failed to encode map e608 with expected crc 2023-09-23T22:43:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:54 smithi070 ceph-mon[192652]: pgmap v1580: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 56 MiB data, 2.5 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 954 op/s; 280/2547 objects degraded (10.993%); 624/2547 objects misplaced (24.499%); 0 B/s, 97 objects/s recovering 2023-09-23T22:43:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:43:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4292107387' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:43:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:54 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:43:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:54 smithi070 ceph-mon[192652]: failed to encode map e609 with expected crc 2023-09-23T22:43:55.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4292107387' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:43:55.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:54 smithi070 ceph-mon[192652]: osdmap e609: 8 total, 8 up, 4 in 2023-09-23T22:43:55.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:55 smithi026 ceph-mon[223014]: failed to encode map e609 with expected crc 2023-09-23T22:43:55.837 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:55 smithi026 ceph-mon[223014]: failed to encode map e609 with expected crc 2023-09-23T22:43:55.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:55 smithi026 ceph-mon[215775]: failed to encode map e609 with expected crc 2023-09-23T22:43:55.837 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:55 smithi026 ceph-mon[215775]: failed to encode map e609 with expected crc 2023-09-23T22:43:55.894 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T22:43:55.894 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T22:43:56.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:55 smithi070 ceph-mon[192652]: failed to encode map e609 with expected crc 2023-09-23T22:43:56.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:55 smithi070 ceph-mon[192652]: failed to encode map e609 with expected crc 2023-09-23T22:43:56.708 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:56.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:56.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:56.812 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:56.842 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:56.889 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:56.936 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:56.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:56.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:57.043 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:57.048 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:43:57.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:56 smithi026 ceph-mon[223014]: pgmap v1582: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 44 MiB data, 2.4 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 698 op/s; 280/1977 objects degraded (14.163%); 624/1977 objects misplaced (31.563%); 0 B/s, 70 objects/s recovering 2023-09-23T22:43:57.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:56 smithi026 ceph-mon[223014]: failed to encode map e610 with expected crc 2023-09-23T22:43:57.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:56 smithi026 ceph-mon[223014]: failed to encode map e610 with expected crc 2023-09-23T22:43:57.096 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:56 smithi026 ceph-mon[223014]: osdmap e610: 8 total, 8 up, 4 in 2023-09-23T22:43:57.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:56 smithi026 ceph-mon[215775]: pgmap v1582: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 44 MiB data, 2.4 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 698 op/s; 280/1977 objects degraded (14.163%); 624/1977 objects misplaced (31.563%); 0 B/s, 70 objects/s recovering 2023-09-23T22:43:57.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:56 smithi026 ceph-mon[215775]: failed to encode map e610 with expected crc 2023-09-23T22:43:57.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:56 smithi026 ceph-mon[215775]: failed to encode map e610 with expected crc 2023-09-23T22:43:57.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:56 smithi026 ceph-mon[215775]: osdmap e610: 8 total, 8 up, 4 in 2023-09-23T22:43:57.099 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:57.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:56 smithi070 ceph-mon[192652]: pgmap v1582: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 44 MiB data, 2.4 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 698 op/s; 280/1977 objects degraded (14.163%); 624/1977 objects misplaced (31.563%); 0 B/s, 70 objects/s recovering 2023-09-23T22:43:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:56 smithi070 ceph-mon[192652]: failed to encode map e610 with expected crc 2023-09-23T22:43:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:56 smithi070 ceph-mon[192652]: failed to encode map e610 with expected crc 2023-09-23T22:43:57.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:56 smithi070 ceph-mon[192652]: osdmap e610: 8 total, 8 up, 4 in 2023-09-23T22:43:57.215 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:43:57.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:57.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:43:57.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:57.351 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:57.399 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:57.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:57.404 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:57.504 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:43:57.537 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:57.653 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:43:57.653 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:43:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[215775]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[215775]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[215775]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[215775]: osdmap e611: 8 total, 8 up, 4 in 2023-09-23T22:43:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[215775]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[215775]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[215775]: 1.a deep-scrub starts 2023-09-23T22:43:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[215775]: 1.a deep-scrub ok 2023-09-23T22:43:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/179756417' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[223014]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[223014]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[223014]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[223014]: osdmap e611: 8 total, 8 up, 4 in 2023-09-23T22:43:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[223014]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[223014]: failed to encode map e611 with expected crc 2023-09-23T22:43:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[223014]: 1.a deep-scrub starts 2023-09-23T22:43:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[223014]: 1.a deep-scrub ok 2023-09-23T22:43:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/179756417' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:57 smithi070 ceph-mon[192652]: failed to encode map e611 with expected crc 2023-09-23T22:43:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:57 smithi070 ceph-mon[192652]: failed to encode map e611 with expected crc 2023-09-23T22:43:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:57 smithi070 ceph-mon[192652]: failed to encode map e611 with expected crc 2023-09-23T22:43:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:57 smithi070 ceph-mon[192652]: osdmap e611: 8 total, 8 up, 4 in 2023-09-23T22:43:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:57 smithi070 ceph-mon[192652]: failed to encode map e611 with expected crc 2023-09-23T22:43:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:57 smithi070 ceph-mon[192652]: failed to encode map e611 with expected crc 2023-09-23T22:43:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:57 smithi070 ceph-mon[192652]: 1.a deep-scrub starts 2023-09-23T22:43:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:57 smithi070 ceph-mon[192652]: 1.a deep-scrub ok 2023-09-23T22:43:58.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/179756417' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:43:58.480 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:43:58.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:58 smithi026 ceph-mon[215775]: pgmap v1585: 119 pgs: 2 peering, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 34 MiB data, 2.3 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 81 op/s; 280/1485 objects degraded (18.855%); 384/1485 objects misplaced (25.859%) 2023-09-23T22:43:58.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:58 smithi026 ceph-mon[215775]: 1.1a scrub starts 2023-09-23T22:43:58.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:58 smithi026 ceph-mon[215775]: 1.1a scrub ok 2023-09-23T22:43:58.851 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:43:58 smithi026 ceph-mon[215775]: Health check update: Degraded data redundancy: 280/1485 objects degraded (18.855%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:43:58.949 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:43:58.949 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:43:58.949 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:43:58.949 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:43:58.949 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:43:58.949 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:43:58.950 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:43:58.950 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:43:58.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:43:58.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:43:58.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:43:58.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:43:58.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:43:58.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:43:58.950 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:43:58.951 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:43:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:58 smithi026 ceph-mon[223014]: pgmap v1585: 119 pgs: 2 peering, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 34 MiB data, 2.3 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 81 op/s; 280/1485 objects degraded (18.855%); 384/1485 objects misplaced (25.859%) 2023-09-23T22:43:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:58 smithi026 ceph-mon[223014]: 1.1a scrub starts 2023-09-23T22:43:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:58 smithi026 ceph-mon[223014]: 1.1a scrub ok 2023-09-23T22:43:59.108 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:43:58 smithi026 ceph-mon[223014]: Health check update: Degraded data redundancy: 280/1485 objects degraded (18.855%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:43:59.119 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:43:59.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:58 smithi070 ceph-mon[192652]: pgmap v1585: 119 pgs: 2 peering, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+remapped+backfill_wait, 2 active+recovery_wait+degraded+remapped, 99 active+clean; 34 MiB data, 2.3 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 81 op/s; 280/1485 objects degraded (18.855%); 384/1485 objects misplaced (25.859%) 2023-09-23T22:43:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:58 smithi070 ceph-mon[192652]: 1.1a scrub starts 2023-09-23T22:43:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:58 smithi070 ceph-mon[192652]: 1.1a scrub ok 2023-09-23T22:43:59.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:43:58 smithi070 ceph-mon[192652]: Health check update: Degraded data redundancy: 280/1485 objects degraded (18.855%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T22:43:59.263 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:43:59.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:59.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:43:59.368 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:43:59.400 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:59.449 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:43:59.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:59.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:43:59.553 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:43:59.582 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:59.627 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:43:59.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:59.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:43:59.734 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:43:59.760 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:43:59.801 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:43:59.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:43:59.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:43:59.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:43:59.933 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:43:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:43:59] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-09-23T22:43:59.936 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:00.755 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:00.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:00.848 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:00.859 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:00.893 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:00.903 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:00.996 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:00.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:00.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:01.029 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:44:01.032 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:44:01.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[223014]: pgmap v1586: 119 pgs: 2 peering, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 11 MiB data, 2.2 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 120 op/s; 97/372 objects degraded (26.075%); 107/372 objects misplaced (28.763%); 0 B/s, 2 objects/s recovering 2023-09-23T22:44:01.057 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[223014]: 1.8 scrub starts 2023-09-23T22:44:01.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[223014]: 1.8 scrub ok 2023-09-23T22:44:01.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[223014]: 1.19 scrub starts 2023-09-23T22:44:01.058 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[223014]: 1.19 scrub ok 2023-09-23T22:44:01.058 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[215775]: pgmap v1586: 119 pgs: 2 peering, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 11 MiB data, 2.2 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 120 op/s; 97/372 objects degraded (26.075%); 107/372 objects misplaced (28.763%); 0 B/s, 2 objects/s recovering 2023-09-23T22:44:01.058 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[215775]: 1.8 scrub starts 2023-09-23T22:44:01.058 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[215775]: 1.8 scrub ok 2023-09-23T22:44:01.058 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[215775]: 1.19 scrub starts 2023-09-23T22:44:01.058 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:00 smithi026 ceph-mon[215775]: 1.19 scrub ok 2023-09-23T22:44:01.100 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:00 smithi070 ceph-mon[192652]: pgmap v1586: 119 pgs: 2 peering, 1 active+recovering+remapped, 2 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 112 active+clean; 11 MiB data, 2.2 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 120 op/s; 97/372 objects degraded (26.075%); 107/372 objects misplaced (28.763%); 0 B/s, 2 objects/s recovering 2023-09-23T22:44:01.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:00 smithi070 ceph-mon[192652]: 1.8 scrub starts 2023-09-23T22:44:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:00 smithi070 ceph-mon[192652]: 1.8 scrub ok 2023-09-23T22:44:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:00 smithi070 ceph-mon[192652]: 1.19 scrub starts 2023-09-23T22:44:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:00 smithi070 ceph-mon[192652]: 1.19 scrub ok 2023-09-23T22:44:01.131 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:01.180 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:01.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:01.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:01.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:01.315 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:01.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[215775]: 1.0 scrub starts 2023-09-23T22:44:01.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[215775]: 1.0 scrub ok 2023-09-23T22:44:01.860 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[215775]: from='client.36939 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:01.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[215775]: 1.18 scrub starts 2023-09-23T22:44:01.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[215775]: 1.18 scrub ok 2023-09-23T22:44:01.861 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T22:44:01.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[223014]: 1.0 scrub starts 2023-09-23T22:44:01.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[223014]: 1.0 scrub ok 2023-09-23T22:44:01.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[223014]: from='client.36939 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:01.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[223014]: 1.18 scrub starts 2023-09-23T22:44:01.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[223014]: 1.18 scrub ok 2023-09-23T22:44:01.861 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:01 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T22:44:01.963 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:01.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:01.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:02.068 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:02.097 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:01 smithi070 ceph-mon[192652]: 1.0 scrub starts 2023-09-23T22:44:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:01 smithi070 ceph-mon[192652]: 1.0 scrub ok 2023-09-23T22:44:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:01 smithi070 ceph-mon[192652]: from='client.36939 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:01 smithi070 ceph-mon[192652]: 1.18 scrub starts 2023-09-23T22:44:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:01 smithi070 ceph-mon[192652]: 1.18 scrub ok 2023-09-23T22:44:02.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:01 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T22:44:02.552 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:44:02.892 INFO:teuthology.orchestra.run.smithi026.stderr:pool 'unique_pool_5' removed 2023-09-23T22:44:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: 1.10 deep-scrub starts 2023-09-23T22:44:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: 1.10 deep-scrub ok 2023-09-23T22:44:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: pgmap v1587: 119 pgs: 119 active+clean; 4.5 MiB data, 2.2 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 111 op/s; 8.6 KiB/s, 107 objects/s recovering 2023-09-23T22:44:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 97/372 objects degraded (26.075%), 2 pgs degraded) 2023-09-23T22:44:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:44:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T22:44:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: osdmap e612: 8 total, 8 up, 4 in 2023-09-23T22:44:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3518600092' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:44:03.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: 1.10 deep-scrub starts 2023-09-23T22:44:03.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: 1.10 deep-scrub ok 2023-09-23T22:44:03.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: pgmap v1587: 119 pgs: 119 active+clean; 4.5 MiB data, 2.2 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 111 op/s; 8.6 KiB/s, 107 objects/s recovering 2023-09-23T22:44:03.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 97/372 objects degraded (26.075%), 2 pgs degraded) 2023-09-23T22:44:03.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:44:03.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T22:44:03.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: osdmap e612: 8 total, 8 up, 4 in 2023-09-23T22:44:03.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3518600092' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:44:03.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: 1.10 deep-scrub starts 2023-09-23T22:44:03.159 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: 1.10 deep-scrub ok 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: pgmap v1587: 119 pgs: 119 active+clean; 4.5 MiB data, 2.2 GiB used, 355 GiB / 358 GiB avail; 0 B/s wr, 111 op/s; 8.6 KiB/s, 107 objects/s recovering 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 97/372 objects degraded (26.075%), 2 pgs degraded) 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: osdmap e612: 8 total, 8 up, 4 in 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: failed to encode map e612 with expected crc 2023-09-23T22:44:03.160 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3518600092' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T22:44:03.178 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1586,"stamp":"2023-09-23T22:43:59.752231+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":11135321,"num_objects":124,"num_object_clones":0,"num_object_copies":372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":97,"num_objects_misplaced":107,"num_objects_unfound":0,"num_objects_dirty":124,"num_whiteouts":0,"num_read":6027,"num_read_kb":12268,"num_write":95621,"num_write_kb":3046098,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17088,"num_bytes_recovered":866072503,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":50380,"ondisk_log_size":50380,"up":357,"acting":356,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":383,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":2277688,"kb_used_data":690776,"kb_used_omap":35,"kb_used_meta":1586844,"kb_avail":372702920,"statfs":{"total":383980142592,"available":381647790080,"internally_reserved":0,"allocated":707354624,"data_stored":697180748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36602,"internal_metadata":1624928518},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-47513600,"num_objects":-725,"num_object_clones":0,"num_object_copies":-2175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-183,"num_objects_misplaced":-517,"num_objects_unfound":0,"num_objects_dirty":-725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":725,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001642"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":1058,"reported_epoch":605,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T22:43:39.581341+0000","last_change":"2023-09-23T22:43:12.765680+0000","last_active":"2023-09-23T22:43:39.581341+0000","last_peered":"2023-09-23T22:43:39.581341+0000","last_clean":"2023-09-23T22:41:54.358917+0000","last_became_active":"2023-09-23T22:43:12.765147+0000","last_became_peered":"2023-09-23T22:43:12.765147+0000","last_unstale":"2023-09-23T22:43:39.581341+0000","last_undegraded":"2023-09-23T22:43:39.581341+0000","last_fullsized":"2023-09-23T22:43:39.581341+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[1,2,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":569,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.848154+0000","last_change":"2023-09-23T22:43:11.791371+0000","last_active":"2023-09-23T22:43:55.848154+0000","last_peered":"2023-09-23T22:43:55.848154+0000","last_clean":"2023-09-23T22:43:55.848154+0000","last_became_active":"2023-09-23T22:43:11.783461+0000","last_became_peered":"2023-09-23T22:43:11.783461+0000","last_unstale":"2023-09-23T22:43:55.848154+0000","last_undegraded":"2023-09-23T22:43:55.848154+0000","last_fullsized":"2023-09-23T22:43:55.848154+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:19.460031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":346,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.406946+0000","last_change":"2023-09-23T22:43:11.733047+0000","last_active":"2023-09-23T22:43:54.406946+0000","last_peered":"2023-09-23T22:43:54.406946+0000","last_clean":"2023-09-23T22:43:54.406946+0000","last_became_active":"2023-09-23T22:43:11.731356+0000","last_became_peered":"2023-09-23T22:43:11.731356+0000","last_unstale":"2023-09-23T22:43:54.406946+0000","last_undegraded":"2023-09-23T22:43:54.406946+0000","last_fullsized":"2023-09-23T22:43:54.406946+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:09.953038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":821,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.843302+0000","last_change":"2023-09-23T22:43:14.437017+0000","last_active":"2023-09-23T22:43:55.843302+0000","last_peered":"2023-09-23T22:43:55.843302+0000","last_clean":"2023-09-23T22:43:55.843302+0000","last_became_active":"2023-09-23T22:43:14.436749+0000","last_became_peered":"2023-09-23T22:43:14.436749+0000","last_unstale":"2023-09-23T22:43:55.843302+0000","last_undegraded":"2023-09-23T22:43:55.843302+0000","last_fullsized":"2023-09-23T22:43:55.843302+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:29.970215+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:29.970215+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":382,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.408363+0000","last_change":"2023-09-23T22:42:57.702640+0000","last_active":"2023-09-23T22:43:54.408363+0000","last_peered":"2023-09-23T22:43:54.408363+0000","last_clean":"2023-09-23T22:43:54.408363+0000","last_became_active":"2023-09-23T22:42:57.701938+0000","last_became_peered":"2023-09-23T22:42:57.701938+0000","last_unstale":"2023-09-23T22:43:54.408363+0000","last_undegraded":"2023-09-23T22:43:54.408363+0000","last_fullsized":"2023-09-23T22:43:54.408363+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:04.282382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":89,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.407394+0000","last_change":"2023-09-23T22:43:11.805439+0000","last_active":"2023-09-23T22:43:54.407394+0000","last_peered":"2023-09-23T22:43:54.407394+0000","last_clean":"2023-09-23T22:43:54.407394+0000","last_became_active":"2023-09-23T22:43:11.804518+0000","last_became_peered":"2023-09-23T22:43:11.804518+0000","last_unstale":"2023-09-23T22:43:54.407394+0000","last_undegraded":"2023-09-23T22:43:54.407394+0000","last_fullsized":"2023-09-23T22:43:54.407394+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:32.668394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":1006,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579375+0000","last_change":"2023-09-23T22:42:56.531211+0000","last_active":"2023-09-23T22:43:53.579375+0000","last_peered":"2023-09-23T22:43:53.579375+0000","last_clean":"2023-09-23T22:43:53.579375+0000","last_became_active":"2023-09-23T22:42:56.530919+0000","last_became_peered":"2023-09-23T22:42:56.530919+0000","last_unstale":"2023-09-23T22:43:53.579375+0000","last_undegraded":"2023-09-23T22:43:53.579375+0000","last_fullsized":"2023-09-23T22:43:53.579375+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":674,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.841865+0000","last_change":"2023-09-23T22:43:11.861828+0000","last_active":"2023-09-23T22:43:55.841865+0000","last_peered":"2023-09-23T22:43:55.841865+0000","last_clean":"2023-09-23T22:43:55.841865+0000","last_became_active":"2023-09-23T22:43:11.860602+0000","last_became_peered":"2023-09-23T22:43:11.860602+0000","last_unstale":"2023-09-23T22:43:55.841865+0000","last_undegraded":"2023-09-23T22:43:55.841865+0000","last_fullsized":"2023-09-23T22:43:55.841865+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:43.850844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":226,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845068+0000","last_change":"2023-09-23T22:42:56.509390+0000","last_active":"2023-09-23T22:43:55.845068+0000","last_peered":"2023-09-23T22:43:55.845068+0000","last_clean":"2023-09-23T22:43:55.845068+0000","last_became_active":"2023-09-23T22:42:56.509066+0000","last_became_peered":"2023-09-23T22:42:56.509066+0000","last_unstale":"2023-09-23T22:43:55.845068+0000","last_undegraded":"2023-09-23T22:43:55.845068+0000","last_fullsized":"2023-09-23T22:43:55.845068+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:36.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":956,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.848261+0000","last_change":"2023-09-23T22:43:14.457850+0000","last_active":"2023-09-23T22:43:55.848261+0000","last_peered":"2023-09-23T22:43:55.848261+0000","last_clean":"2023-09-23T22:43:55.848261+0000","last_became_active":"2023-09-23T22:43:14.443620+0000","last_became_peered":"2023-09-23T22:43:14.443620+0000","last_unstale":"2023-09-23T22:43:55.848261+0000","last_undegraded":"2023-09-23T22:43:55.848261+0000","last_fullsized":"2023-09-23T22:43:55.848261+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:57.273632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:57.273632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:47.977793+0000","scrub_duration":0.018058204000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.580248+0000","last_change":"2023-09-23T22:43:11.732475+0000","last_active":"2023-09-23T22:43:53.580248+0000","last_peered":"2023-09-23T22:43:53.580248+0000","last_clean":"2023-09-23T22:43:53.580248+0000","last_became_active":"2023-09-23T22:43:11.732134+0000","last_became_peered":"2023-09-23T22:43:11.732134+0000","last_unstale":"2023-09-23T22:43:53.580248+0000","last_undegraded":"2023-09-23T22:43:53.580248+0000","last_fullsized":"2023-09-23T22:43:53.580248+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:22.683794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":204,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.848937+0000","last_change":"2023-09-23T22:42:57.555024+0000","last_active":"2023-09-23T22:43:55.848937+0000","last_peered":"2023-09-23T22:43:55.848937+0000","last_clean":"2023-09-23T22:43:55.848937+0000","last_became_active":"2023-09-23T22:42:57.554656+0000","last_became_peered":"2023-09-23T22:42:57.554656+0000","last_unstale":"2023-09-23T22:43:55.848937+0000","last_undegraded":"2023-09-23T22:43:55.848937+0000","last_fullsized":"2023-09-23T22:43:55.848937+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:56.648868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":768,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579357+0000","last_change":"2023-09-23T22:43:14.436119+0000","last_active":"2023-09-23T22:43:53.579357+0000","last_peered":"2023-09-23T22:43:53.579357+0000","last_clean":"2023-09-23T22:43:53.579357+0000","last_became_active":"2023-09-23T22:43:14.435613+0000","last_became_peered":"2023-09-23T22:43:14.435613+0000","last_unstale":"2023-09-23T22:43:53.579357+0000","last_undegraded":"2023-09-23T22:43:53.579357+0000","last_fullsized":"2023-09-23T22:43:53.579357+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:02.056099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:02.056099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:12.546310+0000","scrub_duration":0.012145022,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.580277+0000","last_change":"2023-09-23T22:43:11.828937+0000","last_active":"2023-09-23T22:43:53.580277+0000","last_peered":"2023-09-23T22:43:53.580277+0000","last_clean":"2023-09-23T22:43:53.580277+0000","last_became_active":"2023-09-23T22:43:11.828512+0000","last_became_peered":"2023-09-23T22:43:11.828512+0000","last_unstale":"2023-09-23T22:43:53.580277+0000","last_undegraded":"2023-09-23T22:43:53.580277+0000","last_fullsized":"2023-09-23T22:43:53.580277+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:56.701943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":207,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.844238+0000","last_change":"2023-09-23T22:42:57.699399+0000","last_active":"2023-09-23T22:43:55.844238+0000","last_peered":"2023-09-23T22:43:55.844238+0000","last_clean":"2023-09-23T22:43:55.844238+0000","last_became_active":"2023-09-23T22:42:57.698650+0000","last_became_peered":"2023-09-23T22:42:57.698650+0000","last_unstale":"2023-09-23T22:43:55.844238+0000","last_undegraded":"2023-09-23T22:43:55.844238+0000","last_fullsized":"2023-09-23T22:43:55.844238+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:17.884820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":896,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.846296+0000","last_change":"2023-09-23T22:43:14.445540+0000","last_active":"2023-09-23T22:43:55.846296+0000","last_peered":"2023-09-23T22:43:55.846296+0000","last_clean":"2023-09-23T22:43:55.846296+0000","last_became_active":"2023-09-23T22:43:14.444963+0000","last_became_peered":"2023-09-23T22:43:14.444963+0000","last_unstale":"2023-09-23T22:43:55.846296+0000","last_undegraded":"2023-09-23T22:43:55.846296+0000","last_fullsized":"2023-09-23T22:43:55.846296+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":49,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845456+0000","last_change":"2023-09-23T22:43:11.882816+0000","last_active":"2023-09-23T22:43:55.845456+0000","last_peered":"2023-09-23T22:43:55.845456+0000","last_clean":"2023-09-23T22:43:55.845456+0000","last_became_active":"2023-09-23T22:43:11.822292+0000","last_became_peered":"2023-09-23T22:43:11.822292+0000","last_unstale":"2023-09-23T22:43:55.845456+0000","last_undegraded":"2023-09-23T22:43:55.845456+0000","last_fullsized":"2023-09-23T22:43:55.845456+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:05.717891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":261,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.406487+0000","last_change":"2023-09-23T22:42:56.567330+0000","last_active":"2023-09-23T22:43:54.406487+0000","last_peered":"2023-09-23T22:43:54.406487+0000","last_clean":"2023-09-23T22:43:54.406487+0000","last_became_active":"2023-09-23T22:42:56.567016+0000","last_became_peered":"2023-09-23T22:42:56.567016+0000","last_unstale":"2023-09-23T22:43:54.406487+0000","last_undegraded":"2023-09-23T22:43:54.406487+0000","last_fullsized":"2023-09-23T22:43:54.406487+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:00.456335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":928,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.847844+0000","last_change":"2023-09-23T22:43:14.443682+0000","last_active":"2023-09-23T22:43:55.847844+0000","last_peered":"2023-09-23T22:43:55.847844+0000","last_clean":"2023-09-23T22:43:55.847844+0000","last_became_active":"2023-09-23T22:43:14.443333+0000","last_became_peered":"2023-09-23T22:43:14.443333+0000","last_unstale":"2023-09-23T22:43:55.847844+0000","last_undegraded":"2023-09-23T22:43:55.847844+0000","last_fullsized":"2023-09-23T22:43:55.847844+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":601,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845610+0000","last_change":"2023-09-23T22:43:11.747332+0000","last_active":"2023-09-23T22:43:55.845610+0000","last_peered":"2023-09-23T22:43:55.845610+0000","last_clean":"2023-09-23T22:43:55.845610+0000","last_became_active":"2023-09-23T22:43:11.733061+0000","last_became_peered":"2023-09-23T22:43:11.733061+0000","last_unstale":"2023-09-23T22:43:55.845610+0000","last_undegraded":"2023-09-23T22:43:55.845610+0000","last_fullsized":"2023-09-23T22:43:55.845610+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:49:43.986136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":42606,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":534,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.843488+0000","last_change":"2023-09-23T22:42:57.485105+0000","last_active":"2023-09-23T22:43:55.843488+0000","last_peered":"2023-09-23T22:43:55.843488+0000","last_clean":"2023-09-23T22:43:55.843488+0000","last_became_active":"2023-09-23T22:42:57.484320+0000","last_became_peered":"2023-09-23T22:42:57.484320+0000","last_unstale":"2023-09-23T22:43:55.843488+0000","last_undegraded":"2023-09-23T22:43:55.843488+0000","last_fullsized":"2023-09-23T22:43:55.843488+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:36.390910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":1020,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579121+0000","last_change":"2023-09-23T22:43:14.434951+0000","last_active":"2023-09-23T22:43:53.579121+0000","last_peered":"2023-09-23T22:43:53.579121+0000","last_clean":"2023-09-23T22:43:53.579121+0000","last_became_active":"2023-09-23T22:43:14.434621+0000","last_became_peered":"2023-09-23T22:43:14.434621+0000","last_unstale":"2023-09-23T22:43:53.579121+0000","last_undegraded":"2023-09-23T22:43:53.579121+0000","last_fullsized":"2023-09-23T22:43:53.579121+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:19.077318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:19.077318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:04.789089+0000","scrub_duration":0.0080980949999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":299,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.842488+0000","last_change":"2023-09-23T22:42:57.736915+0000","last_active":"2023-09-23T22:43:55.842488+0000","last_peered":"2023-09-23T22:43:55.842488+0000","last_clean":"2023-09-23T22:43:55.842488+0000","last_became_active":"2023-09-23T22:42:57.629094+0000","last_became_peered":"2023-09-23T22:42:57.629094+0000","last_unstale":"2023-09-23T22:43:55.842488+0000","last_undegraded":"2023-09-23T22:43:55.842488+0000","last_fullsized":"2023-09-23T22:43:55.842488+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:03.304992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.848575+0000","last_change":"2023-09-23T22:41:52.385773+0000","last_active":"2023-09-23T22:43:55.848575+0000","last_peered":"2023-09-23T22:43:55.848575+0000","last_clean":"2023-09-23T22:43:55.848575+0000","last_became_active":"2023-09-23T22:41:52.385433+0000","last_became_peered":"2023-09-23T22:41:52.385433+0000","last_unstale":"2023-09-23T22:43:55.848575+0000","last_undegraded":"2023-09-23T22:43:55.848575+0000","last_fullsized":"2023-09-23T22:43:55.848575+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:15.934387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":799,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.849251+0000","last_change":"2023-09-23T22:42:57.601397+0000","last_active":"2023-09-23T22:43:55.849251+0000","last_peered":"2023-09-23T22:43:55.849251+0000","last_clean":"2023-09-23T22:43:55.849251+0000","last_became_active":"2023-09-23T22:42:57.599876+0000","last_became_peered":"2023-09-23T22:42:57.599876+0000","last_unstale":"2023-09-23T22:43:55.849251+0000","last_undegraded":"2023-09-23T22:43:55.849251+0000","last_fullsized":"2023-09-23T22:43:55.849251+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:35.896541+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:35.896541+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":80,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.841896+0000","last_change":"2023-09-23T22:43:11.791005+0000","last_active":"2023-09-23T22:43:55.841896+0000","last_peered":"2023-09-23T22:43:55.841896+0000","last_clean":"2023-09-23T22:43:55.841896+0000","last_became_active":"2023-09-23T22:43:11.783005+0000","last_became_peered":"2023-09-23T22:43:11.783005+0000","last_unstale":"2023-09-23T22:43:55.841896+0000","last_undegraded":"2023-09-23T22:43:55.841896+0000","last_fullsized":"2023-09-23T22:43:55.841896+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:23.783502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579677+0000","last_change":"2023-09-23T22:43:11.725543+0000","last_active":"2023-09-23T22:43:53.579677+0000","last_peered":"2023-09-23T22:43:53.579677+0000","last_clean":"2023-09-23T22:43:53.579677+0000","last_became_active":"2023-09-23T22:43:11.725152+0000","last_became_peered":"2023-09-23T22:43:11.725152+0000","last_unstale":"2023-09-23T22:43:53.579677+0000","last_undegraded":"2023-09-23T22:43:53.579677+0000","last_fullsized":"2023-09-23T22:43:53.579677+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:31.160745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":821,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845421+0000","last_change":"2023-09-23T22:43:14.437053+0000","last_active":"2023-09-23T22:43:55.845421+0000","last_peered":"2023-09-23T22:43:55.845421+0000","last_clean":"2023-09-23T22:43:55.845421+0000","last_became_active":"2023-09-23T22:43:14.436624+0000","last_became_peered":"2023-09-23T22:43:14.436624+0000","last_unstale":"2023-09-23T22:43:55.845421+0000","last_undegraded":"2023-09-23T22:43:55.845421+0000","last_fullsized":"2023-09-23T22:43:55.845421+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:57.922489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:57.922489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":454,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.844083+0000","last_change":"2023-09-23T22:43:11.802960+0000","last_active":"2023-09-23T22:43:55.844083+0000","last_peered":"2023-09-23T22:43:55.844083+0000","last_clean":"2023-09-23T22:43:55.844083+0000","last_became_active":"2023-09-23T22:43:11.802684+0000","last_became_peered":"2023-09-23T22:43:11.802684+0000","last_unstale":"2023-09-23T22:43:55.844083+0000","last_undegraded":"2023-09-23T22:43:55.844083+0000","last_fullsized":"2023-09-23T22:43:55.844083+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:21.148096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":90,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.845033+0000","last_change":"2023-09-23T22:43:11.807494+0000","last_active":"2023-09-23T22:43:56.845033+0000","last_peered":"2023-09-23T22:43:56.845033+0000","last_clean":"2023-09-23T22:43:56.845033+0000","last_became_active":"2023-09-23T22:43:11.807103+0000","last_became_peered":"2023-09-23T22:43:11.807103+0000","last_unstale":"2023-09-23T22:43:56.845033+0000","last_undegraded":"2023-09-23T22:43:56.845033+0000","last_fullsized":"2023-09-23T22:43:56.845033+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:50.696866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":955,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.842289+0000","last_change":"2023-09-23T22:43:14.449364+0000","last_active":"2023-09-23T22:43:55.842289+0000","last_peered":"2023-09-23T22:43:55.842289+0000","last_clean":"2023-09-23T22:43:55.842289+0000","last_became_active":"2023-09-23T22:43:14.449003+0000","last_became_peered":"2023-09-23T22:43:14.449003+0000","last_unstale":"2023-09-23T22:43:55.842289+0000","last_undegraded":"2023-09-23T22:43:55.842289+0000","last_fullsized":"2023-09-23T22:43:55.842289+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:26.313042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:26.313042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:00.726138+0000","scrub_duration":0.0017143830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":270,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.849312+0000","last_change":"2023-09-23T22:43:11.844157+0000","last_active":"2023-09-23T22:43:55.849312+0000","last_peered":"2023-09-23T22:43:55.849312+0000","last_clean":"2023-09-23T22:43:55.849312+0000","last_became_active":"2023-09-23T22:43:11.751401+0000","last_became_peered":"2023-09-23T22:43:11.751401+0000","last_unstale":"2023-09-23T22:43:55.849312+0000","last_undegraded":"2023-09-23T22:43:55.849312+0000","last_fullsized":"2023-09-23T22:43:55.849312+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:37.797785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.846825+0000","last_change":"2023-09-23T22:42:57.618059+0000","last_active":"2023-09-23T22:43:56.846825+0000","last_peered":"2023-09-23T22:43:56.846825+0000","last_clean":"2023-09-23T22:43:56.846825+0000","last_became_active":"2023-09-23T22:42:57.617619+0000","last_became_peered":"2023-09-23T22:42:57.617619+0000","last_unstale":"2023-09-23T22:43:56.846825+0000","last_undegraded":"2023-09-23T22:43:56.846825+0000","last_fullsized":"2023-09-23T22:43:56.846825+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:00.845651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.6","version":"606'5808","reported_seq":7592,"reported_epoch":606,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T22:43:51.149050+0000","last_change":"2023-09-23T22:43:11.859697+0000","last_active":"2023-09-23T22:43:51.149050+0000","last_peered":"2023-09-23T22:43:51.149050+0000","last_clean":"2023-09-23T22:41:03.442376+0000","last_became_active":"2023-09-23T22:43:11.830124+0000","last_became_peered":"2023-09-23T22:43:11.830124+0000","last_unstale":"2023-09-23T22:43:51.149050+0000","last_undegraded":"2023-09-23T22:43:11.801762+0000","last_fullsized":"2023-09-23T22:43:51.149050+0000","mapping_epoch":597,"log_start":"589'3100","ondisk_log_start":"589'3100","created":549,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2708,"log_dups_size":3000,"ondisk_log_size":2708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:48:28.018874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6291456,"num_objects":96,"num_object_clones":0,"num_object_copies":288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":96,"num_objects_misplaced":96,"num_objects_unfound":0,"num_objects_dirty":96,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5808,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":846,"num_bytes_recovered":55443456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[1,2,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1,2","objects":96}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":955,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.848351+0000","last_change":"2023-09-23T22:43:14.457577+0000","last_active":"2023-09-23T22:43:55.848351+0000","last_peered":"2023-09-23T22:43:55.848351+0000","last_clean":"2023-09-23T22:43:55.848351+0000","last_became_active":"2023-09-23T22:43:14.443078+0000","last_became_peered":"2023-09-23T22:43:14.443078+0000","last_unstale":"2023-09-23T22:43:55.848351+0000","last_undegraded":"2023-09-23T22:43:55.848351+0000","last_fullsized":"2023-09-23T22:43:55.848351+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:34.147486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:34.147486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:22.398176+0000","scrub_duration":0.0018320190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":150,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.407620+0000","last_change":"2023-09-23T22:43:11.920516+0000","last_active":"2023-09-23T22:43:54.407620+0000","last_peered":"2023-09-23T22:43:54.407620+0000","last_clean":"2023-09-23T22:43:54.407620+0000","last_became_active":"2023-09-23T22:43:11.755782+0000","last_became_peered":"2023-09-23T22:43:11.755782+0000","last_unstale":"2023-09-23T22:43:54.407620+0000","last_undegraded":"2023-09-23T22:43:54.407620+0000","last_fullsized":"2023-09-23T22:43:54.407620+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:16.670619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.578941+0000","last_change":"2023-09-23T22:43:11.831473+0000","last_active":"2023-09-23T22:43:53.578941+0000","last_peered":"2023-09-23T22:43:53.578941+0000","last_clean":"2023-09-23T22:43:53.578941+0000","last_became_active":"2023-09-23T22:43:11.831153+0000","last_became_peered":"2023-09-23T22:43:11.831153+0000","last_unstale":"2023-09-23T22:43:53.578941+0000","last_undegraded":"2023-09-23T22:43:53.578941+0000","last_fullsized":"2023-09-23T22:43:53.578941+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:00:24.678192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"607'5822","reported_seq":8916,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.905645+0000","last_change":"2023-09-23T22:43:56.905645+0000","last_active":"2023-09-23T22:43:56.905645+0000","last_peered":"2023-09-23T22:43:56.905645+0000","last_clean":"2023-09-23T22:43:56.905645+0000","last_became_active":"2023-09-23T22:43:56.881477+0000","last_became_peered":"2023-09-23T22:43:56.881477+0000","last_unstale":"2023-09-23T22:43:56.905645+0000","last_undegraded":"2023-09-23T22:43:56.905645+0000","last_fullsized":"2023-09-23T22:43:56.905645+0000","mapping_epoch":610,"log_start":"589'3000","ondisk_log_start":"589'3000","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2822,"log_dups_size":0,"ondisk_log_size":2822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:29.539818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5822,"num_write_kb":186304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1484,"num_bytes_recovered":97255424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":771,"reported_epoch":606,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T22:43:40.594233+0000","last_change":"2023-09-23T22:43:12.764078+0000","last_active":"2023-09-23T22:43:40.594233+0000","last_peered":"2023-09-23T22:43:40.594233+0000","last_clean":"2023-09-23T22:42:54.458562+0000","last_became_active":"2023-09-23T22:43:12.763523+0000","last_became_peered":"2023-09-23T22:43:12.763523+0000","last_unstale":"2023-09-23T22:43:40.594233+0000","last_undegraded":"2023-09-23T22:43:12.741074+0000","last_fullsized":"2023-09-23T22:43:12.740592+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:04.048606+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:04.048606+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:27.327025+0000","scrub_duration":0.0064015249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[1,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":527,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.408076+0000","last_change":"2023-09-23T22:43:11.899388+0000","last_active":"2023-09-23T22:43:54.408076+0000","last_peered":"2023-09-23T22:43:54.408076+0000","last_clean":"2023-09-23T22:43:54.408076+0000","last_became_active":"2023-09-23T22:43:11.841184+0000","last_became_peered":"2023-09-23T22:43:11.841184+0000","last_unstale":"2023-09-23T22:43:54.408076+0000","last_undegraded":"2023-09-23T22:43:54.408076+0000","last_fullsized":"2023-09-23T22:43:54.408076+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:01.020339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":622,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.406487+0000","last_change":"2023-09-23T22:42:57.702798+0000","last_active":"2023-09-23T22:43:54.406487+0000","last_peered":"2023-09-23T22:43:54.406487+0000","last_clean":"2023-09-23T22:43:54.406487+0000","last_became_active":"2023-09-23T22:42:57.702317+0000","last_became_peered":"2023-09-23T22:42:57.702317+0000","last_unstale":"2023-09-23T22:43:54.406487+0000","last_undegraded":"2023-09-23T22:43:54.406487+0000","last_fullsized":"2023-09-23T22:43:54.406487+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:12.697584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.4","version":"607'5796","reported_seq":5891,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.877253+0000","last_change":"2023-09-23T22:43:56.877014+0000","last_active":"2023-09-23T22:43:56.877253+0000","last_peered":"2023-09-23T22:43:56.877253+0000","last_clean":"2023-09-23T22:43:56.877253+0000","last_became_active":"2023-09-23T22:43:56.855389+0000","last_became_peered":"2023-09-23T22:43:56.855389+0000","last_unstale":"2023-09-23T22:43:56.877253+0000","last_undegraded":"2023-09-23T22:43:56.877253+0000","last_fullsized":"2023-09-23T22:43:56.877253+0000","mapping_epoch":610,"log_start":"589'3000","ondisk_log_start":"589'3000","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2796,"log_dups_size":3000,"ondisk_log_size":2796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:34.761768+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5796,"num_write_kb":185472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":393216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":895,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845242+0000","last_change":"2023-09-23T22:43:14.445750+0000","last_active":"2023-09-23T22:43:55.845242+0000","last_peered":"2023-09-23T22:43:55.845242+0000","last_clean":"2023-09-23T22:43:55.845242+0000","last_became_active":"2023-09-23T22:43:14.445359+0000","last_became_peered":"2023-09-23T22:43:14.445359+0000","last_unstale":"2023-09-23T22:43:55.845242+0000","last_undegraded":"2023-09-23T22:43:55.845242+0000","last_fullsized":"2023-09-23T22:43:55.845242+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":25,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579581+0000","last_change":"2023-09-23T22:43:11.918943+0000","last_active":"2023-09-23T22:43:53.579581+0000","last_peered":"2023-09-23T22:43:53.579581+0000","last_clean":"2023-09-23T22:43:53.579581+0000","last_became_active":"2023-09-23T22:43:11.827406+0000","last_became_peered":"2023-09-23T22:43:11.827406+0000","last_unstale":"2023-09-23T22:43:53.579581+0000","last_undegraded":"2023-09-23T22:43:53.579581+0000","last_fullsized":"2023-09-23T22:43:53.579581+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:57.335665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":244,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.843861+0000","last_change":"2023-09-23T22:42:57.480776+0000","last_active":"2023-09-23T22:43:55.843861+0000","last_peered":"2023-09-23T22:43:55.843861+0000","last_clean":"2023-09-23T22:43:55.843861+0000","last_became_active":"2023-09-23T22:42:57.480378+0000","last_became_peered":"2023-09-23T22:42:57.480378+0000","last_unstale":"2023-09-23T22:43:55.843861+0000","last_undegraded":"2023-09-23T22:43:55.843861+0000","last_fullsized":"2023-09-23T22:43:55.843861+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.689275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"607'5876","reported_seq":6000,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.878600+0000","last_change":"2023-09-23T22:43:56.878600+0000","last_active":"2023-09-23T22:43:56.878600+0000","last_peered":"2023-09-23T22:43:56.878600+0000","last_clean":"2023-09-23T22:43:56.878600+0000","last_became_active":"2023-09-23T22:43:56.857694+0000","last_became_peered":"2023-09-23T22:43:56.857694+0000","last_unstale":"2023-09-23T22:43:56.878600+0000","last_undegraded":"2023-09-23T22:43:56.878600+0000","last_fullsized":"2023-09-23T22:43:56.878600+0000","mapping_epoch":610,"log_start":"589'3000","ondisk_log_start":"589'3000","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2876,"log_dups_size":3000,"ondisk_log_size":2876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:30.609336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5876,"num_write_kb":188032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":927,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.847567+0000","last_change":"2023-09-23T22:43:14.456284+0000","last_active":"2023-09-23T22:43:55.847567+0000","last_peered":"2023-09-23T22:43:55.847567+0000","last_clean":"2023-09-23T22:43:55.847567+0000","last_became_active":"2023-09-23T22:43:14.455903+0000","last_became_peered":"2023-09-23T22:43:14.455903+0000","last_unstale":"2023-09-23T22:43:55.847567+0000","last_undegraded":"2023-09-23T22:43:55.847567+0000","last_fullsized":"2023-09-23T22:43:55.847567+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579443+0000","last_change":"2023-09-23T22:43:11.732060+0000","last_active":"2023-09-23T22:43:53.579443+0000","last_peered":"2023-09-23T22:43:53.579443+0000","last_clean":"2023-09-23T22:43:53.579443+0000","last_became_active":"2023-09-23T22:43:11.731696+0000","last_became_peered":"2023-09-23T22:43:11.731696+0000","last_unstale":"2023-09-23T22:43:53.579443+0000","last_undegraded":"2023-09-23T22:43:53.579443+0000","last_fullsized":"2023-09-23T22:43:53.579443+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:41.169298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":415,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.406992+0000","last_change":"2023-09-23T22:43:11.733183+0000","last_active":"2023-09-23T22:43:54.406992+0000","last_peered":"2023-09-23T22:43:54.406992+0000","last_clean":"2023-09-23T22:43:54.406992+0000","last_became_active":"2023-09-23T22:43:11.731600+0000","last_became_peered":"2023-09-23T22:43:11.731600+0000","last_unstale":"2023-09-23T22:43:54.406992+0000","last_undegraded":"2023-09-23T22:43:54.406992+0000","last_fullsized":"2023-09-23T22:43:54.406992+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:17:17.223104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.2","version":"607'5790","reported_seq":9060,"reported_epoch":610,"state":"peering","last_fresh":"2023-09-23T22:43:55.845813+0000","last_change":"2023-09-23T22:43:55.845990+0000","last_active":"2023-09-23T22:43:55.845813+0000","last_peered":"2023-09-23T22:43:55.434486+0000","last_clean":"2023-09-23T22:43:55.434486+0000","last_became_active":"2023-09-23T22:43:11.844376+0000","last_became_peered":"2023-09-23T22:43:11.844376+0000","last_unstale":"2023-09-23T22:43:55.845813+0000","last_undegraded":"2023-09-23T22:43:55.845813+0000","last_fullsized":"2023-09-23T22:43:55.845813+0000","mapping_epoch":610,"log_start":"590'3100","ondisk_log_start":"590'3100","created":549,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2690,"log_dups_size":3000,"ondisk_log_size":2690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:05.922951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5790,"num_write_kb":185280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1577,"num_bytes_recovered":103284736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,5,6],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":965,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.842880+0000","last_change":"2023-09-23T22:42:59.395878+0000","last_active":"2023-09-23T22:43:55.842880+0000","last_peered":"2023-09-23T22:43:55.842880+0000","last_clean":"2023-09-23T22:43:55.842880+0000","last_became_active":"2023-09-23T22:42:59.395538+0000","last_became_peered":"2023-09-23T22:42:59.395538+0000","last_unstale":"2023-09-23T22:43:55.842880+0000","last_undegraded":"2023-09-23T22:43:55.842880+0000","last_fullsized":"2023-09-23T22:43:55.842880+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:58.211060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:58.211060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:50.018093+0000","scrub_duration":0.0035013240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":92,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845558+0000","last_change":"2023-09-23T22:43:11.840050+0000","last_active":"2023-09-23T22:43:55.845558+0000","last_peered":"2023-09-23T22:43:55.845558+0000","last_clean":"2023-09-23T22:43:55.845558+0000","last_became_active":"2023-09-23T22:43:11.780186+0000","last_became_peered":"2023-09-23T22:43:11.780186+0000","last_unstale":"2023-09-23T22:43:55.845558+0000","last_undegraded":"2023-09-23T22:43:55.845558+0000","last_fullsized":"2023-09-23T22:43:55.845558+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:00.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":452,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845539+0000","last_change":"2023-09-23T22:42:57.606366+0000","last_active":"2023-09-23T22:43:55.845539+0000","last_peered":"2023-09-23T22:43:55.845539+0000","last_clean":"2023-09-23T22:43:55.845539+0000","last_became_active":"2023-09-23T22:42:57.604437+0000","last_became_peered":"2023-09-23T22:42:57.604437+0000","last_unstale":"2023-09-23T22:43:55.845539+0000","last_undegraded":"2023-09-23T22:43:55.845539+0000","last_fullsized":"2023-09-23T22:43:55.845539+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:34.648244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"607'6104","reported_seq":6232,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.881348+0000","last_change":"2023-09-23T22:43:56.881348+0000","last_active":"2023-09-23T22:43:56.881348+0000","last_peered":"2023-09-23T22:43:56.881348+0000","last_clean":"2023-09-23T22:43:56.881348+0000","last_became_active":"2023-09-23T22:43:56.857716+0000","last_became_peered":"2023-09-23T22:43:56.857716+0000","last_unstale":"2023-09-23T22:43:56.881348+0000","last_undegraded":"2023-09-23T22:43:56.881348+0000","last_fullsized":"2023-09-23T22:43:56.881348+0000","mapping_epoch":610,"log_start":"590'3300","ondisk_log_start":"590'3300","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2804,"log_dups_size":3000,"ondisk_log_size":2804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:33.018108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6104,"num_write_kb":195328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":1058,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.578975+0000","last_change":"2023-09-23T22:43:14.434832+0000","last_active":"2023-09-23T22:43:53.578975+0000","last_peered":"2023-09-23T22:43:53.578975+0000","last_clean":"2023-09-23T22:43:53.578975+0000","last_became_active":"2023-09-23T22:43:14.434417+0000","last_became_peered":"2023-09-23T22:43:14.434417+0000","last_unstale":"2023-09-23T22:43:53.578975+0000","last_undegraded":"2023-09-23T22:43:53.578975+0000","last_fullsized":"2023-09-23T22:43:53.578975+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:39.551683+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:39.551683+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0040238799999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":241,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.844514+0000","last_change":"2023-09-23T22:42:57.480722+0000","last_active":"2023-09-23T22:43:55.844514+0000","last_peered":"2023-09-23T22:43:55.844514+0000","last_clean":"2023-09-23T22:43:55.844514+0000","last_became_active":"2023-09-23T22:42:57.480180+0000","last_became_peered":"2023-09-23T22:42:57.480180+0000","last_unstale":"2023-09-23T22:43:55.844514+0000","last_undegraded":"2023-09-23T22:43:55.844514+0000","last_fullsized":"2023-09-23T22:43:55.844514+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:00.944581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":260,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.847999+0000","last_change":"2023-09-23T22:42:57.671849+0000","last_active":"2023-09-23T22:43:55.847999+0000","last_peered":"2023-09-23T22:43:55.847999+0000","last_clean":"2023-09-23T22:43:55.847999+0000","last_became_active":"2023-09-23T22:42:57.667120+0000","last_became_peered":"2023-09-23T22:42:57.667120+0000","last_unstale":"2023-09-23T22:43:55.847999+0000","last_undegraded":"2023-09-23T22:43:55.847999+0000","last_fullsized":"2023-09-23T22:43:55.847999+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:11.794628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.0","version":"607'5920","reported_seq":8955,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.888458+0000","last_change":"2023-09-23T22:43:56.888458+0000","last_active":"2023-09-23T22:43:56.888458+0000","last_peered":"2023-09-23T22:43:56.888458+0000","last_clean":"2023-09-23T22:43:56.888458+0000","last_became_active":"2023-09-23T22:43:56.860544+0000","last_became_peered":"2023-09-23T22:43:56.860544+0000","last_unstale":"2023-09-23T22:43:56.888458+0000","last_undegraded":"2023-09-23T22:43:56.888458+0000","last_fullsized":"2023-09-23T22:43:56.888458+0000","mapping_epoch":610,"log_start":"590'3200","ondisk_log_start":"590'3200","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2720,"log_dups_size":0,"ondisk_log_size":2720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:38.629981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5920,"num_write_kb":189440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54853632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":821,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845112+0000","last_change":"2023-09-23T22:43:14.435092+0000","last_active":"2023-09-23T22:43:55.845112+0000","last_peered":"2023-09-23T22:43:55.845112+0000","last_clean":"2023-09-23T22:43:55.845112+0000","last_became_active":"2023-09-23T22:43:14.434702+0000","last_became_peered":"2023-09-23T22:43:14.434702+0000","last_unstale":"2023-09-23T22:43:55.845112+0000","last_undegraded":"2023-09-23T22:43:55.845112+0000","last_fullsized":"2023-09-23T22:43:55.845112+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:26.963124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:26.963124+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":584,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.843504+0000","last_change":"2023-09-23T22:43:11.854114+0000","last_active":"2023-09-23T22:43:55.843504+0000","last_peered":"2023-09-23T22:43:55.843504+0000","last_clean":"2023-09-23T22:43:55.843504+0000","last_became_active":"2023-09-23T22:43:11.841381+0000","last_became_peered":"2023-09-23T22:43:11.841381+0000","last_unstale":"2023-09-23T22:43:55.843504+0000","last_undegraded":"2023-09-23T22:43:55.843504+0000","last_fullsized":"2023-09-23T22:43:55.843504+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:46.359977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":541,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.847682+0000","last_change":"2023-09-23T22:43:11.817112+0000","last_active":"2023-09-23T22:43:55.847682+0000","last_peered":"2023-09-23T22:43:55.847682+0000","last_clean":"2023-09-23T22:43:55.847682+0000","last_became_active":"2023-09-23T22:43:11.816563+0000","last_became_peered":"2023-09-23T22:43:11.816563+0000","last_unstale":"2023-09-23T22:43:55.847682+0000","last_undegraded":"2023-09-23T22:43:55.847682+0000","last_fullsized":"2023-09-23T22:43:55.847682+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:56.365553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"607'5974","reported_seq":6110,"reported_epoch":610,"state":"peering","last_fresh":"2023-09-23T22:43:55.853373+0000","last_change":"2023-09-23T22:43:55.853537+0000","last_active":"2023-09-23T22:43:55.853373+0000","last_peered":"2023-09-23T22:43:55.420952+0000","last_clean":"2023-09-23T22:43:55.420952+0000","last_became_active":"2023-09-23T22:43:11.853652+0000","last_became_peered":"2023-09-23T22:43:11.853652+0000","last_unstale":"2023-09-23T22:43:55.853373+0000","last_undegraded":"2023-09-23T22:43:55.853373+0000","last_fullsized":"2023-09-23T22:43:55.853373+0000","mapping_epoch":610,"log_start":"589'3200","ondisk_log_start":"589'3200","created":549,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2774,"log_dups_size":3000,"ondisk_log_size":2774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:12.499341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5974,"num_write_kb":191168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":524288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,5],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":1007,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579476+0000","last_change":"2023-09-23T22:42:56.531203+0000","last_active":"2023-09-23T22:43:53.579476+0000","last_peered":"2023-09-23T22:43:53.579476+0000","last_clean":"2023-09-23T22:43:53.579476+0000","last_became_active":"2023-09-23T22:42:56.530915+0000","last_became_peered":"2023-09-23T22:42:56.530915+0000","last_unstale":"2023-09-23T22:43:53.579476+0000","last_undegraded":"2023-09-23T22:43:53.579476+0000","last_fullsized":"2023-09-23T22:43:53.579476+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:36.566715+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:36.566715+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":366,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.407474+0000","last_change":"2023-09-23T22:43:11.862025+0000","last_active":"2023-09-23T22:43:54.407474+0000","last_peered":"2023-09-23T22:43:54.407474+0000","last_clean":"2023-09-23T22:43:54.407474+0000","last_became_active":"2023-09-23T22:43:11.860993+0000","last_became_peered":"2023-09-23T22:43:11.860993+0000","last_unstale":"2023-09-23T22:43:54.407474+0000","last_undegraded":"2023-09-23T22:43:54.407474+0000","last_fullsized":"2023-09-23T22:43:54.407474+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:37.123461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":432,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.407771+0000","last_change":"2023-09-23T22:43:11.756165+0000","last_active":"2023-09-23T22:43:54.407771+0000","last_peered":"2023-09-23T22:43:54.407771+0000","last_clean":"2023-09-23T22:43:54.407771+0000","last_became_active":"2023-09-23T22:43:11.755617+0000","last_became_peered":"2023-09-23T22:43:11.755617+0000","last_unstale":"2023-09-23T22:43:54.407771+0000","last_undegraded":"2023-09-23T22:43:54.407771+0000","last_fullsized":"2023-09-23T22:43:54.407771+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:21.873764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"607'5936","reported_seq":8944,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.891611+0000","last_change":"2023-09-23T22:43:56.891611+0000","last_active":"2023-09-23T22:43:56.891611+0000","last_peered":"2023-09-23T22:43:56.891611+0000","last_clean":"2023-09-23T22:43:56.891611+0000","last_became_active":"2023-09-23T22:43:56.856094+0000","last_became_peered":"2023-09-23T22:43:56.856094+0000","last_unstale":"2023-09-23T22:43:56.891611+0000","last_undegraded":"2023-09-23T22:43:56.891611+0000","last_fullsized":"2023-09-23T22:43:56.891611+0000","mapping_epoch":610,"log_start":"590'3200","ondisk_log_start":"590'3200","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2736,"log_dups_size":0,"ondisk_log_size":2736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:16:07.747632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5936,"num_write_kb":189952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1465,"num_bytes_recovered":54984704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":896,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.844967+0000","last_change":"2023-09-23T22:43:14.445326+0000","last_active":"2023-09-23T22:43:55.844967+0000","last_peered":"2023-09-23T22:43:55.844967+0000","last_clean":"2023-09-23T22:43:55.844967+0000","last_became_active":"2023-09-23T22:43:14.444896+0000","last_became_peered":"2023-09-23T22:43:14.444896+0000","last_unstale":"2023-09-23T22:43:55.844967+0000","last_undegraded":"2023-09-23T22:43:55.844967+0000","last_fullsized":"2023-09-23T22:43:55.844967+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":984,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.849129+0000","last_change":"2023-09-23T22:43:54.413786+0000","last_active":"2023-09-23T22:43:55.849129+0000","last_peered":"2023-09-23T22:43:55.849129+0000","last_clean":"2023-09-23T22:43:55.849129+0000","last_became_active":"2023-09-23T22:43:54.413159+0000","last_became_peered":"2023-09-23T22:43:54.413159+0000","last_unstale":"2023-09-23T22:43:55.849129+0000","last_undegraded":"2023-09-23T22:43:55.849129+0000","last_fullsized":"2023-09-23T22:43:55.849129+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:37.708567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579707+0000","last_change":"2023-09-23T22:43:11.827683+0000","last_active":"2023-09-23T22:43:53.579707+0000","last_peered":"2023-09-23T22:43:53.579707+0000","last_clean":"2023-09-23T22:43:53.579707+0000","last_became_active":"2023-09-23T22:43:11.827004+0000","last_became_peered":"2023-09-23T22:43:11.827004+0000","last_unstale":"2023-09-23T22:43:53.579707+0000","last_undegraded":"2023-09-23T22:43:53.579707+0000","last_fullsized":"2023-09-23T22:43:53.579707+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:21.277304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":367,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.843411+0000","last_change":"2023-09-23T22:43:11.772491+0000","last_active":"2023-09-23T22:43:55.843411+0000","last_peered":"2023-09-23T22:43:55.843411+0000","last_clean":"2023-09-23T22:43:55.843411+0000","last_became_active":"2023-09-23T22:43:11.772145+0000","last_became_peered":"2023-09-23T22:43:11.772145+0000","last_unstale":"2023-09-23T22:43:55.843411+0000","last_undegraded":"2023-09-23T22:43:55.843411+0000","last_fullsized":"2023-09-23T22:43:55.843411+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:22.248227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"607'5986","reported_seq":15201,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.843696+0000","last_change":"2023-09-23T22:43:29.419126+0000","last_active":"2023-09-23T22:43:55.843696+0000","last_peered":"2023-09-23T22:43:55.843696+0000","last_clean":"2023-09-23T22:43:55.843696+0000","last_became_active":"2023-09-23T22:43:29.402722+0000","last_became_peered":"2023-09-23T22:43:29.402722+0000","last_unstale":"2023-09-23T22:43:55.843696+0000","last_undegraded":"2023-09-23T22:43:55.843696+0000","last_fullsized":"2023-09-23T22:43:55.843696+0000","mapping_epoch":603,"log_start":"589'3177","ondisk_log_start":"589'3177","created":549,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2809,"log_dups_size":0,"ondisk_log_size":2809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:15.645601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5986,"num_write_kb":191552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4549,"num_bytes_recovered":253820928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":1021,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579250+0000","last_change":"2023-09-23T22:43:14.435063+0000","last_active":"2023-09-23T22:43:53.579250+0000","last_peered":"2023-09-23T22:43:53.579250+0000","last_clean":"2023-09-23T22:43:53.579250+0000","last_became_active":"2023-09-23T22:43:14.434698+0000","last_became_peered":"2023-09-23T22:43:14.434698+0000","last_unstale":"2023-09-23T22:43:53.579250+0000","last_undegraded":"2023-09-23T22:43:53.579250+0000","last_fullsized":"2023-09-23T22:43:53.579250+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:16.067396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:16.067396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:58.817933+0000","scrub_duration":0.0072300120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1521,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579649+0000","last_change":"2023-09-23T22:42:56.548561+0000","last_active":"2023-09-23T22:43:53.579649+0000","last_peered":"2023-09-23T22:43:53.579649+0000","last_clean":"2023-09-23T22:43:53.579649+0000","last_became_active":"2023-09-23T22:42:56.548134+0000","last_became_peered":"2023-09-23T22:42:56.548134+0000","last_unstale":"2023-09-23T22:43:53.579649+0000","last_undegraded":"2023-09-23T22:43:53.579649+0000","last_fullsized":"2023-09-23T22:43:53.579649+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:36.511009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":504,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845160+0000","last_change":"2023-09-23T22:42:57.574455+0000","last_active":"2023-09-23T22:43:55.845160+0000","last_peered":"2023-09-23T22:43:55.845160+0000","last_clean":"2023-09-23T22:43:55.845160+0000","last_became_active":"2023-09-23T22:42:57.573786+0000","last_became_peered":"2023-09-23T22:42:57.573786+0000","last_unstale":"2023-09-23T22:43:55.845160+0000","last_undegraded":"2023-09-23T22:43:55.845160+0000","last_fullsized":"2023-09-23T22:43:55.845160+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:09.464699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":259,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.849283+0000","last_change":"2023-09-23T22:42:57.591303+0000","last_active":"2023-09-23T22:43:55.849283+0000","last_peered":"2023-09-23T22:43:55.849283+0000","last_clean":"2023-09-23T22:43:55.849283+0000","last_became_active":"2023-09-23T22:42:57.587701+0000","last_became_peered":"2023-09-23T22:42:57.587701+0000","last_unstale":"2023-09-23T22:43:55.849283+0000","last_undegraded":"2023-09-23T22:43:55.849283+0000","last_fullsized":"2023-09-23T22:43:55.849283+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:32.621790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.9","version":"607'5778","reported_seq":8450,"reported_epoch":607,"state":"active+recovering+remapped","last_fresh":"2023-09-23T22:43:52.963258+0000","last_change":"2023-09-23T22:43:47.354967+0000","last_active":"2023-09-23T22:43:52.963258+0000","last_peered":"2023-09-23T22:43:52.963258+0000","last_clean":"2023-09-23T22:42:04.544690+0000","last_became_active":"2023-09-23T22:43:11.840886+0000","last_became_peered":"2023-09-23T22:43:11.840886+0000","last_unstale":"2023-09-23T22:43:52.963258+0000","last_undegraded":"2023-09-23T22:43:52.963258+0000","last_fullsized":"2023-09-23T22:43:52.963258+0000","mapping_epoch":597,"log_start":"589'3000","ondisk_log_start":"589'3000","created":549,"last_epoch_clean":564,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2778,"log_dups_size":3000,"ondisk_log_size":2778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:42.228610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5778,"num_write_kb":184896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1284,"num_bytes_recovered":53673984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[2,1,0],"avail_no_missing":["2","0","5","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":955,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.842200+0000","last_change":"2023-09-23T22:43:14.448779+0000","last_active":"2023-09-23T22:43:55.842200+0000","last_peered":"2023-09-23T22:43:55.842200+0000","last_clean":"2023-09-23T22:43:55.842200+0000","last_became_active":"2023-09-23T22:43:14.448360+0000","last_became_peered":"2023-09-23T22:43:14.448360+0000","last_unstale":"2023-09-23T22:43:55.842200+0000","last_undegraded":"2023-09-23T22:43:55.842200+0000","last_fullsized":"2023-09-23T22:43:55.842200+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:07.291196+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:07.291196+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:38.426551+0000","scrub_duration":0.0016895390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":370,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.843417+0000","last_change":"2023-09-23T22:43:54.199739+0000","last_active":"2023-09-23T22:43:55.843417+0000","last_peered":"2023-09-23T22:43:55.843417+0000","last_clean":"2023-09-23T22:43:55.843417+0000","last_became_active":"2023-09-23T22:42:57.672992+0000","last_became_peered":"2023-09-23T22:42:57.672992+0000","last_unstale":"2023-09-23T22:43:55.843417+0000","last_undegraded":"2023-09-23T22:43:55.843417+0000","last_fullsized":"2023-09-23T22:43:55.843417+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:41.870977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":241,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.850421+0000","last_change":"2023-09-23T22:42:56.548974+0000","last_active":"2023-09-23T22:43:55.850421+0000","last_peered":"2023-09-23T22:43:55.850421+0000","last_clean":"2023-09-23T22:43:55.850421+0000","last_became_active":"2023-09-23T22:42:56.548663+0000","last_became_peered":"2023-09-23T22:42:56.548663+0000","last_unstale":"2023-09-23T22:43:55.850421+0000","last_undegraded":"2023-09-23T22:43:55.850421+0000","last_fullsized":"2023-09-23T22:43:55.850421+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:06.786917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":301,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.408016+0000","last_change":"2023-09-23T22:43:11.756366+0000","last_active":"2023-09-23T22:43:54.408016+0000","last_peered":"2023-09-23T22:43:54.408016+0000","last_clean":"2023-09-23T22:43:54.408016+0000","last_became_active":"2023-09-23T22:43:11.756048+0000","last_became_peered":"2023-09-23T22:43:11.756048+0000","last_unstale":"2023-09-23T22:43:54.408016+0000","last_undegraded":"2023-09-23T22:43:54.408016+0000","last_fullsized":"2023-09-23T22:43:54.408016+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:30.924506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"607'5904","reported_seq":8992,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.883040+0000","last_change":"2023-09-23T22:43:56.883040+0000","last_active":"2023-09-23T22:43:56.883040+0000","last_peered":"2023-09-23T22:43:56.883040+0000","last_clean":"2023-09-23T22:43:56.883040+0000","last_became_active":"2023-09-23T22:43:56.854988+0000","last_became_peered":"2023-09-23T22:43:56.854988+0000","last_unstale":"2023-09-23T22:43:56.883040+0000","last_undegraded":"2023-09-23T22:43:56.883040+0000","last_fullsized":"2023-09-23T22:43:56.883040+0000","mapping_epoch":610,"log_start":"590'3200","ondisk_log_start":"590'3200","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2704,"log_dups_size":0,"ondisk_log_size":2704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:28:39.794411+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5904,"num_write_kb":188928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1494,"num_bytes_recovered":56623104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1520,"reported_epoch":605,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T22:43:39.579476+0000","last_change":"2023-09-23T22:43:12.765614+0000","last_active":"2023-09-23T22:43:39.579476+0000","last_peered":"2023-09-23T22:43:39.579476+0000","last_clean":"2023-09-23T22:41:54.359370+0000","last_became_active":"2023-09-23T22:43:12.765203+0000","last_became_peered":"2023-09-23T22:43:12.765203+0000","last_unstale":"2023-09-23T22:43:39.579476+0000","last_undegraded":"2023-09-23T22:43:39.579476+0000","last_fullsized":"2023-09-23T22:43:39.579476+0000","mapping_epoch":598,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:51.142811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[1,2,6],"avail_no_missing":["1","2","6"],"object_location_counts":[{"shards":"1,2,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":800,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.848847+0000","last_change":"2023-09-23T22:42:57.602628+0000","last_active":"2023-09-23T22:43:55.848847+0000","last_peered":"2023-09-23T22:43:55.848847+0000","last_clean":"2023-09-23T22:43:55.848847+0000","last_became_active":"2023-09-23T22:42:57.600816+0000","last_became_peered":"2023-09-23T22:42:57.600816+0000","last_unstale":"2023-09-23T22:43:55.848847+0000","last_undegraded":"2023-09-23T22:43:55.848847+0000","last_fullsized":"2023-09-23T22:43:55.848847+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:55.059777+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:55.059777+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0043637340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579400+0000","last_change":"2023-09-23T22:43:11.806599+0000","last_active":"2023-09-23T22:43:53.579400+0000","last_peered":"2023-09-23T22:43:53.579400+0000","last_clean":"2023-09-23T22:43:53.579400+0000","last_became_active":"2023-09-23T22:43:11.806297+0000","last_became_peered":"2023-09-23T22:43:11.806297+0000","last_unstale":"2023-09-23T22:43:53.579400+0000","last_undegraded":"2023-09-23T22:43:53.579400+0000","last_fullsized":"2023-09-23T22:43:53.579400+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:11:17.339834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":256,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.846685+0000","last_change":"2023-09-23T22:42:57.493464+0000","last_active":"2023-09-23T22:43:55.846685+0000","last_peered":"2023-09-23T22:43:55.846685+0000","last_clean":"2023-09-23T22:43:55.846685+0000","last_became_active":"2023-09-23T22:42:57.492614+0000","last_became_peered":"2023-09-23T22:42:57.492614+0000","last_unstale":"2023-09-23T22:43:55.846685+0000","last_undegraded":"2023-09-23T22:43:55.846685+0000","last_fullsized":"2023-09-23T22:43:55.846685+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:13.562710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"607'6050","reported_seq":6210,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.873041+0000","last_change":"2023-09-23T22:43:56.873041+0000","last_active":"2023-09-23T22:43:56.873041+0000","last_peered":"2023-09-23T22:43:56.873041+0000","last_clean":"2023-09-23T22:43:56.873041+0000","last_became_active":"2023-09-23T22:43:56.853496+0000","last_became_peered":"2023-09-23T22:43:56.853496+0000","last_unstale":"2023-09-23T22:43:56.873041+0000","last_undegraded":"2023-09-23T22:43:56.873041+0000","last_fullsized":"2023-09-23T22:43:56.873041+0000","mapping_epoch":610,"log_start":"589'3200","ondisk_log_start":"589'3200","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2850,"log_dups_size":3000,"ondisk_log_size":2850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:22.549434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6050,"num_write_kb":193600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":822,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.844811+0000","last_change":"2023-09-23T22:43:14.428011+0000","last_active":"2023-09-23T22:43:55.844811+0000","last_peered":"2023-09-23T22:43:55.844811+0000","last_clean":"2023-09-23T22:43:55.844811+0000","last_became_active":"2023-09-23T22:43:14.427691+0000","last_became_peered":"2023-09-23T22:43:14.427691+0000","last_unstale":"2023-09-23T22:43:55.844811+0000","last_undegraded":"2023-09-23T22:43:55.844811+0000","last_fullsized":"2023-09-23T22:43:55.844811+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:56.928006+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:56.928006+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":1078,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.409097+0000","last_change":"2023-09-23T22:43:47.341779+0000","last_active":"2023-09-23T22:43:54.409097+0000","last_peered":"2023-09-23T22:43:54.409097+0000","last_clean":"2023-09-23T22:43:54.409097+0000","last_became_active":"2023-09-23T22:43:11.808371+0000","last_became_peered":"2023-09-23T22:43:11.808371+0000","last_unstale":"2023-09-23T22:43:54.409097+0000","last_undegraded":"2023-09-23T22:43:54.409097+0000","last_fullsized":"2023-09-23T22:43:54.409097+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:38.132844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":117,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.578443+0000","last_change":"2023-09-23T22:42:57.480877+0000","last_active":"2023-09-23T22:43:53.578443+0000","last_peered":"2023-09-23T22:43:53.578443+0000","last_clean":"2023-09-23T22:43:53.578443+0000","last_became_active":"2023-09-23T22:42:57.480576+0000","last_became_peered":"2023-09-23T22:42:57.480576+0000","last_unstale":"2023-09-23T22:43:53.578443+0000","last_undegraded":"2023-09-23T22:43:53.578443+0000","last_fullsized":"2023-09-23T22:43:53.578443+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:39.895915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":444,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.843941+0000","last_change":"2023-09-23T22:43:11.809271+0000","last_active":"2023-09-23T22:43:55.843941+0000","last_peered":"2023-09-23T22:43:55.843941+0000","last_clean":"2023-09-23T22:43:55.843941+0000","last_became_active":"2023-09-23T22:43:11.809001+0000","last_became_peered":"2023-09-23T22:43:11.809001+0000","last_unstale":"2023-09-23T22:43:55.843941+0000","last_undegraded":"2023-09-23T22:43:55.843941+0000","last_fullsized":"2023-09-23T22:43:55.843941+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:18.092293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.d","version":"607'5980","reported_seq":8968,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.889576+0000","last_change":"2023-09-23T22:43:56.889576+0000","last_active":"2023-09-23T22:43:56.889576+0000","last_peered":"2023-09-23T22:43:56.889576+0000","last_clean":"2023-09-23T22:43:56.889576+0000","last_became_active":"2023-09-23T22:43:56.860286+0000","last_became_peered":"2023-09-23T22:43:56.860286+0000","last_unstale":"2023-09-23T22:43:56.889576+0000","last_undegraded":"2023-09-23T22:43:56.889576+0000","last_fullsized":"2023-09-23T22:43:56.889576+0000","mapping_epoch":610,"log_start":"590'3200","ondisk_log_start":"590'3200","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2780,"log_dups_size":0,"ondisk_log_size":2780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:14:47.736177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5980,"num_write_kb":191297,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1450,"num_bytes_recovered":55902208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":928,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.844506+0000","last_change":"2023-09-23T22:43:14.445554+0000","last_active":"2023-09-23T22:43:55.844506+0000","last_peered":"2023-09-23T22:43:55.844506+0000","last_clean":"2023-09-23T22:43:55.844506+0000","last_became_active":"2023-09-23T22:43:14.445311+0000","last_became_peered":"2023-09-23T22:43:14.445311+0000","last_unstale":"2023-09-23T22:43:55.844506+0000","last_undegraded":"2023-09-23T22:43:55.844506+0000","last_fullsized":"2023-09-23T22:43:55.844506+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":50,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.845347+0000","last_change":"2023-09-23T22:43:27.513607+0000","last_active":"2023-09-23T22:43:55.845347+0000","last_peered":"2023-09-23T22:43:55.845347+0000","last_clean":"2023-09-23T22:43:55.845347+0000","last_became_active":"2023-09-23T22:42:57.674485+0000","last_became_peered":"2023-09-23T22:42:57.674485+0000","last_unstale":"2023-09-23T22:43:55.845347+0000","last_undegraded":"2023-09-23T22:43:55.845347+0000","last_fullsized":"2023-09-23T22:43:55.845347+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:57.916453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":263,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.849358+0000","last_change":"2023-09-23T22:43:11.749267+0000","last_active":"2023-09-23T22:43:55.849358+0000","last_peered":"2023-09-23T22:43:55.849358+0000","last_clean":"2023-09-23T22:43:55.849358+0000","last_became_active":"2023-09-23T22:43:11.748938+0000","last_became_peered":"2023-09-23T22:43:11.748938+0000","last_unstale":"2023-09-23T22:43:55.849358+0000","last_undegraded":"2023-09-23T22:43:55.849358+0000","last_fullsized":"2023-09-23T22:43:55.849358+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:38.249466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579822+0000","last_change":"2023-09-23T22:43:11.794701+0000","last_active":"2023-09-23T22:43:53.579822+0000","last_peered":"2023-09-23T22:43:53.579822+0000","last_clean":"2023-09-23T22:43:53.579822+0000","last_became_active":"2023-09-23T22:43:11.794419+0000","last_became_peered":"2023-09-23T22:43:11.794419+0000","last_unstale":"2023-09-23T22:43:53.579822+0000","last_undegraded":"2023-09-23T22:43:53.579822+0000","last_fullsized":"2023-09-23T22:43:53.579822+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:21.897331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"607'5916","reported_seq":9046,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.910587+0000","last_change":"2023-09-23T22:43:56.910587+0000","last_active":"2023-09-23T22:43:56.910587+0000","last_peered":"2023-09-23T22:43:56.910587+0000","last_clean":"2023-09-23T22:43:56.910587+0000","last_became_active":"2023-09-23T22:43:56.894396+0000","last_became_peered":"2023-09-23T22:43:56.894396+0000","last_unstale":"2023-09-23T22:43:56.910587+0000","last_undegraded":"2023-09-23T22:43:56.910587+0000","last_fullsized":"2023-09-23T22:43:56.910587+0000","mapping_epoch":610,"log_start":"589'3100","ondisk_log_start":"589'3100","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2816,"log_dups_size":0,"ondisk_log_size":2816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:36.096086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5916,"num_write_kb":189312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1506,"num_bytes_recovered":55574528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":955,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.848092+0000","last_change":"2023-09-23T22:43:14.458068+0000","last_active":"2023-09-23T22:43:55.848092+0000","last_peered":"2023-09-23T22:43:55.848092+0000","last_clean":"2023-09-23T22:43:55.848092+0000","last_became_active":"2023-09-23T22:43:14.444011+0000","last_became_peered":"2023-09-23T22:43:14.444011+0000","last_unstale":"2023-09-23T22:43:55.848092+0000","last_undegraded":"2023-09-23T22:43:55.848092+0000","last_fullsized":"2023-09-23T22:43:55.848092+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:27.290379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:27.290379+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:55.282838+0000","scrub_duration":0.0016473779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":92,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.580261+0000","last_change":"2023-09-23T22:42:05.533737+0000","last_active":"2023-09-23T22:43:53.580261+0000","last_peered":"2023-09-23T22:43:53.580261+0000","last_clean":"2023-09-23T22:43:53.580261+0000","last_became_active":"2023-09-23T22:42:05.525900+0000","last_became_peered":"2023-09-23T22:42:05.525900+0000","last_unstale":"2023-09-23T22:43:53.580261+0000","last_undegraded":"2023-09-23T22:43:53.580261+0000","last_fullsized":"2023-09-23T22:43:53.580261+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:31.248241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":1209,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.407925+0000","last_change":"2023-09-23T22:43:39.799350+0000","last_active":"2023-09-23T22:43:54.407925+0000","last_peered":"2023-09-23T22:43:54.407925+0000","last_clean":"2023-09-23T22:43:54.407925+0000","last_became_active":"2023-09-23T22:42:57.585398+0000","last_became_peered":"2023-09-23T22:42:57.585398+0000","last_unstale":"2023-09-23T22:43:54.407925+0000","last_undegraded":"2023-09-23T22:43:54.407925+0000","last_fullsized":"2023-09-23T22:43:54.407925+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:06.542366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579076+0000","last_change":"2023-09-23T22:43:11.831381+0000","last_active":"2023-09-23T22:43:53.579076+0000","last_peered":"2023-09-23T22:43:53.579076+0000","last_clean":"2023-09-23T22:43:53.579076+0000","last_became_active":"2023-09-23T22:43:11.830843+0000","last_became_peered":"2023-09-23T22:43:11.830843+0000","last_unstale":"2023-09-23T22:43:53.579076+0000","last_undegraded":"2023-09-23T22:43:53.579076+0000","last_fullsized":"2023-09-23T22:43:53.579076+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:26.690562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.f","version":"607'6052","reported_seq":6142,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.882192+0000","last_change":"2023-09-23T22:43:56.882192+0000","last_active":"2023-09-23T22:43:56.882192+0000","last_peered":"2023-09-23T22:43:56.882192+0000","last_clean":"2023-09-23T22:43:56.882192+0000","last_became_active":"2023-09-23T22:43:56.855820+0000","last_became_peered":"2023-09-23T22:43:56.855820+0000","last_unstale":"2023-09-23T22:43:56.882192+0000","last_undegraded":"2023-09-23T22:43:56.882192+0000","last_fullsized":"2023-09-23T22:43:56.882192+0000","mapping_epoch":610,"log_start":"590'3357","ondisk_log_start":"590'3357","created":549,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:41:03.442376+0000","last_clean_scrub_stamp":"2023-09-23T22:41:03.442376+0000","objects_scrubbed":0,"log_size":2695,"log_dups_size":0,"ondisk_log_size":2695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:00.564578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6052,"num_write_kb":193664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":768,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579377+0000","last_change":"2023-09-23T22:43:14.436289+0000","last_active":"2023-09-23T22:43:53.579377+0000","last_peered":"2023-09-23T22:43:53.579377+0000","last_clean":"2023-09-23T22:43:53.579377+0000","last_became_active":"2023-09-23T22:43:14.436041+0000","last_became_peered":"2023-09-23T22:43:14.436041+0000","last_unstale":"2023-09-23T22:43:53.579377+0000","last_undegraded":"2023-09-23T22:43:53.579377+0000","last_fullsized":"2023-09-23T22:43:53.579377+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:03.084589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:03.084589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:03.001839+0000","scrub_duration":0.013840414000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":281,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.846369+0000","last_change":"2023-09-23T22:42:57.606062+0000","last_active":"2023-09-23T22:43:55.846369+0000","last_peered":"2023-09-23T22:43:55.846369+0000","last_clean":"2023-09-23T22:43:55.846369+0000","last_became_active":"2023-09-23T22:42:57.605639+0000","last_became_peered":"2023-09-23T22:42:57.605639+0000","last_unstale":"2023-09-23T22:43:55.846369+0000","last_undegraded":"2023-09-23T22:43:55.846369+0000","last_fullsized":"2023-09-23T22:43:55.846369+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:13.114886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":415,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.408168+0000","last_change":"2023-09-23T22:43:11.913285+0000","last_active":"2023-09-23T22:43:54.408168+0000","last_peered":"2023-09-23T22:43:54.408168+0000","last_clean":"2023-09-23T22:43:54.408168+0000","last_became_active":"2023-09-23T22:43:11.841729+0000","last_became_peered":"2023-09-23T22:43:11.841729+0000","last_unstale":"2023-09-23T22:43:54.408168+0000","last_undegraded":"2023-09-23T22:43:54.408168+0000","last_fullsized":"2023-09-23T22:43:54.408168+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:45.396946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":21,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579620+0000","last_change":"2023-09-23T22:43:11.808955+0000","last_active":"2023-09-23T22:43:53.579620+0000","last_peered":"2023-09-23T22:43:53.579620+0000","last_clean":"2023-09-23T22:43:53.579620+0000","last_became_active":"2023-09-23T22:43:11.808676+0000","last_became_peered":"2023-09-23T22:43:11.808676+0000","last_unstale":"2023-09-23T22:43:53.579620+0000","last_undegraded":"2023-09-23T22:43:53.579620+0000","last_fullsized":"2023-09-23T22:43:53.579620+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:38.659893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":953,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.842694+0000","last_change":"2023-09-23T22:42:59.396177+0000","last_active":"2023-09-23T22:43:55.842694+0000","last_peered":"2023-09-23T22:43:55.842694+0000","last_clean":"2023-09-23T22:43:55.842694+0000","last_became_active":"2023-09-23T22:42:59.395852+0000","last_became_peered":"2023-09-23T22:42:59.395852+0000","last_unstale":"2023-09-23T22:43:55.842694+0000","last_undegraded":"2023-09-23T22:43:55.842694+0000","last_fullsized":"2023-09-23T22:43:55.842694+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:40.859228+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:40.859228+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":112,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.409636+0000","last_change":"2023-09-23T22:42:57.598277+0000","last_active":"2023-09-23T22:43:54.409636+0000","last_peered":"2023-09-23T22:43:54.409636+0000","last_clean":"2023-09-23T22:43:54.409636+0000","last_became_active":"2023-09-23T22:42:57.597389+0000","last_became_peered":"2023-09-23T22:42:57.597389+0000","last_unstale":"2023-09-23T22:43:54.409636+0000","last_undegraded":"2023-09-23T22:43:54.409636+0000","last_fullsized":"2023-09-23T22:43:54.409636+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:43.074446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":278,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.849425+0000","last_change":"2023-09-23T22:43:11.749040+0000","last_active":"2023-09-23T22:43:55.849425+0000","last_peered":"2023-09-23T22:43:55.849425+0000","last_clean":"2023-09-23T22:43:55.849425+0000","last_became_active":"2023-09-23T22:43:11.748649+0000","last_became_peered":"2023-09-23T22:43:11.748649+0000","last_unstale":"2023-09-23T22:43:55.849425+0000","last_undegraded":"2023-09-23T22:43:55.849425+0000","last_fullsized":"2023-09-23T22:43:55.849425+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:52.013195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":927,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.847789+0000","last_change":"2023-09-23T22:43:14.445759+0000","last_active":"2023-09-23T22:43:55.847789+0000","last_peered":"2023-09-23T22:43:55.847789+0000","last_clean":"2023-09-23T22:43:55.847789+0000","last_became_active":"2023-09-23T22:43:14.445444+0000","last_became_peered":"2023-09-23T22:43:14.445444+0000","last_unstale":"2023-09-23T22:43:55.847789+0000","last_undegraded":"2023-09-23T22:43:55.847789+0000","last_fullsized":"2023-09-23T22:43:55.847789+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":661,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.578498+0000","last_change":"2023-09-23T22:42:57.577409+0000","last_active":"2023-09-23T22:43:53.578498+0000","last_peered":"2023-09-23T22:43:53.578498+0000","last_clean":"2023-09-23T22:43:53.578498+0000","last_became_active":"2023-09-23T22:42:57.551354+0000","last_became_peered":"2023-09-23T22:42:57.551354+0000","last_unstale":"2023-09-23T22:43:53.578498+0000","last_undegraded":"2023-09-23T22:43:53.578498+0000","last_fullsized":"2023-09-23T22:43:53.578498+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:14.859975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":639,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T22:43:54.408433+0000","last_change":"2023-09-23T22:42:57.597909+0000","last_active":"2023-09-23T22:43:54.408433+0000","last_peered":"2023-09-23T22:43:54.408433+0000","last_clean":"2023-09-23T22:43:54.408433+0000","last_became_active":"2023-09-23T22:42:57.596972+0000","last_became_peered":"2023-09-23T22:42:57.596972+0000","last_unstale":"2023-09-23T22:43:54.408433+0000","last_undegraded":"2023-09-23T22:43:54.408433+0000","last_fullsized":"2023-09-23T22:43:54.408433+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:01.735792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":676,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.578601+0000","last_change":"2023-09-23T22:42:57.481811+0000","last_active":"2023-09-23T22:43:53.578601+0000","last_peered":"2023-09-23T22:43:53.578601+0000","last_clean":"2023-09-23T22:43:53.578601+0000","last_became_active":"2023-09-23T22:42:57.481461+0000","last_became_peered":"2023-09-23T22:42:57.481461+0000","last_unstale":"2023-09-23T22:43:53.578601+0000","last_undegraded":"2023-09-23T22:43:53.578601+0000","last_fullsized":"2023-09-23T22:43:53.578601+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:25.977092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":104,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.842104+0000","last_change":"2023-09-23T22:42:57.645578+0000","last_active":"2023-09-23T22:43:55.842104+0000","last_peered":"2023-09-23T22:43:55.842104+0000","last_clean":"2023-09-23T22:43:55.842104+0000","last_became_active":"2023-09-23T22:42:57.644610+0000","last_became_peered":"2023-09-23T22:42:57.644610+0000","last_unstale":"2023-09-23T22:43:55.842104+0000","last_undegraded":"2023-09-23T22:43:55.842104+0000","last_fullsized":"2023-09-23T22:43:55.842104+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:51.348779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":773,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.579536+0000","last_change":"2023-09-23T22:43:14.435958+0000","last_active":"2023-09-23T22:43:53.579536+0000","last_peered":"2023-09-23T22:43:53.579536+0000","last_clean":"2023-09-23T22:43:53.579536+0000","last_became_active":"2023-09-23T22:43:14.435625+0000","last_became_peered":"2023-09-23T22:43:14.435625+0000","last_unstale":"2023-09-23T22:43:53.579536+0000","last_undegraded":"2023-09-23T22:43:53.579536+0000","last_fullsized":"2023-09-23T22:43:53.579536+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:56.024308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:56.024308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:25.586420+0000","scrub_duration":0.0045490360000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":207,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.847467+0000","last_change":"2023-09-23T22:42:57.720541+0000","last_active":"2023-09-23T22:43:55.847467+0000","last_peered":"2023-09-23T22:43:55.847467+0000","last_clean":"2023-09-23T22:43:55.847467+0000","last_became_active":"2023-09-23T22:42:57.717881+0000","last_became_peered":"2023-09-23T22:42:57.717881+0000","last_unstale":"2023-09-23T22:43:55.847467+0000","last_undegraded":"2023-09-23T22:43:55.847467+0000","last_fullsized":"2023-09-23T22:43:55.847467+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:16.534160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":222,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.843534+0000","last_change":"2023-09-23T22:43:11.853883+0000","last_active":"2023-09-23T22:43:56.843534+0000","last_peered":"2023-09-23T22:43:56.843534+0000","last_clean":"2023-09-23T22:43:56.843534+0000","last_became_active":"2023-09-23T22:43:11.840934+0000","last_became_peered":"2023-09-23T22:43:11.840934+0000","last_unstale":"2023-09-23T22:43:56.843534+0000","last_undegraded":"2023-09-23T22:43:56.843534+0000","last_fullsized":"2023-09-23T22:43:56.843534+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:27.566762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":956,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T22:43:56.847592+0000","last_change":"2023-09-23T22:43:14.458301+0000","last_active":"2023-09-23T22:43:56.847592+0000","last_peered":"2023-09-23T22:43:56.847592+0000","last_clean":"2023-09-23T22:43:56.847592+0000","last_became_active":"2023-09-23T22:43:14.444367+0000","last_became_peered":"2023-09-23T22:43:14.444367+0000","last_unstale":"2023-09-23T22:43:56.847592+0000","last_undegraded":"2023-09-23T22:43:56.847592+0000","last_fullsized":"2023-09-23T22:43:56.847592+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:53.343498+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:53.343498+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:06.952790+0000","scrub_duration":0.0041572029999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":27,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T22:43:53.580188+0000","last_change":"2023-09-23T22:43:11.877374+0000","last_active":"2023-09-23T22:43:53.580188+0000","last_peered":"2023-09-23T22:43:53.580188+0000","last_clean":"2023-09-23T22:43:53.580188+0000","last_became_active":"2023-09-23T22:43:11.805693+0000","last_became_peered":"2023-09-23T22:43:11.805693+0000","last_unstale":"2023-09-23T22:43:53.580188+0000","last_undegraded":"2023-09-23T22:43:53.580188+0000","last_fullsized":"2023-09-23T22:43:53.580188+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:06.140025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":594,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.848750+0000","last_change":"2023-09-23T22:42:57.601834+0000","last_active":"2023-09-23T22:43:55.848750+0000","last_peered":"2023-09-23T22:43:55.848750+0000","last_clean":"2023-09-23T22:43:55.848750+0000","last_became_active":"2023-09-23T22:42:57.600569+0000","last_became_peered":"2023-09-23T22:42:57.600569+0000","last_unstale":"2023-09-23T22:43:55.848750+0000","last_undegraded":"2023-09-23T22:43:55.848750+0000","last_fullsized":"2023-09-23T22:43:55.848750+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:13.910515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6291456,"num_objects":96,"num_object_clones":0,"num_object_copies":288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":96,"num_objects_misplaced":96,"num_objects_unfound":0,"num_objects_dirty":96,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":94692,"num_write_kb":3033153,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16526,"num_bytes_recovered":843579392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":383582208,"data_stored":383582208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":44358,"ondisk_log_size":44358,"up":48,"acting":48,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":47046,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":26924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62789,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1474450,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2314240,"data_stored":2296480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":407,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234474,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":592,"seq":2542620639246,"num_pgs":105,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":766152,"kb_used_data":659200,"kb_used_omap":8,"kb_used_meta":106935,"kb_avail":92979000,"statfs":{"total":95995035648,"available":95210496000,"internally_reserved":0,"allocated":675020800,"data_stored":673031900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":109502306},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":3.996,"5min":3.996,"15min":3.996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":47.795000000000002,"5min":47.795000000000002,"15min":47.795000000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":3.3039999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":47.682000000000002,"5min":47.682000000000002,"15min":47.682000000000002},"last":0.48199999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":7.4900000000000002,"5min":7.4900000000000002,"15min":7.4900000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":115.848,"5min":115.848,"15min":115.848},"last":0.40100000000000002},{"interface":"front","average":{"1min":7.7279999999999998,"5min":7.7279999999999998,"15min":7.7279999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":116.517,"5min":116.517,"15min":116.517},"last":0.54800000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.6379999999999999,"5min":3.6379999999999999,"15min":3.6379999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.637,"5min":2.637,"15min":2.637},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.8490000000000002,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.2,"5min":1.2,"15min":1.2},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.6260000000000003,"5min":5.6260000000000003,"15min":5.6260000000000003},"last":0.505}]},{"osd":4,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.52400000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.339,"5min":1.339,"15min":1.339},"last":0.59599999999999997}]}]},{"osd":1,"up_from":495,"seq":2126008811596,"num_pgs":94,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":436588,"kb_used_data":12580,"kb_used_omap":12,"kb_used_meta":423987,"kb_avail":93308564,"statfs":{"total":95995035648,"available":95547969536,"internally_reserved":0,"allocated":12881920,"data_stored":9791704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":434163374},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":2.2389999999999999,"15min":1.099},"min":{"1min":0.20200000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.0249999999999999,"5min":64.507000000000005,"15min":64.507000000000005},"last":0.24299999999999999},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.323,"15min":1.375},"min":{"1min":0.24399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.8460000000000001,"5min":64.352000000000004,"15min":64.352000000000004},"last":0.30399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":12.957000000000001,"5min":3.7730000000000001,"15min":1.7050000000000001},"min":{"1min":0.254,"5min":0.192,"15min":0.192},"max":{"1min":124.447,"5min":124.447,"15min":124.447},"last":0.33000000000000002},{"interface":"front","average":{"1min":12.526,"5min":3.3849999999999998,"15min":1.6419999999999999},"min":{"1min":0.29399999999999998,"5min":0.218,"15min":0.218},"max":{"1min":124.03,"5min":124.03,"15min":124.03},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:43:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.7949999999999999,"15min":1.7949999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.800000000000001,"5min":11.800000000000001,"15min":11.800000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.7030000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":6.4530000000000003,"5min":6.4530000000000003,"15min":6.4530000000000003},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.0249999999999999,"15min":0.70899999999999996},"min":{"1min":0.38,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.0750000000000002,"5min":5.5439999999999996,"15min":5.5439999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.85299999999999998,"5min":1.01,"15min":1.004},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.4510000000000001,"5min":5.524,"15min":5.6740000000000004},"last":0.40200000000000002}]},{"osd":5,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":1.167,"15min":0.95899999999999996},"min":{"1min":0.42399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":11.734,"5min":11.734,"15min":11.734},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.0600000000000001,"15min":0.755},"min":{"1min":0.439,"5min":0.31,"15min":0.31},"max":{"1min":3.1389999999999998,"5min":5.5839999999999996,"15min":5.5839999999999996},"last":0.57299999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:43:55 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":1.736,"15min":1.736},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":9.3759999999999994,"5min":9.3759999999999994,"15min":9.3759999999999994},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.877,"5min":0.877,"15min":0.877},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.9540000000000002,"5min":2.9540000000000002,"15min":2.9540000000000002},"last":0.50900000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0059999999999998,"5min":1.2809999999999999,"15min":1.0049999999999999},"min":{"1min":0.42999999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.768000000000001,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.119,"15min":0.80100000000000005},"min":{"1min":0.496,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.169,"5min":5.6120000000000001,"15min":5.6120000000000001},"last":0.55300000000000005}]}]},{"osd":0,"up_from":0,"seq":399431958912,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":349,"seq":1498943586475,"num_pgs":89,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":517392,"kb_used_data":8200,"kb_used_omap":15,"kb_used_meta":509168,"kb_avail":93227760,"statfs":{"total":95995035648,"available":95465226240,"internally_reserved":0,"allocated":8396800,"data_stored":5287046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":521388790},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.77,"15min":6.024},"min":{"1min":0.22600000000000001,"5min":0.182,"15min":0.182},"max":{"1min":10.539999999999999,"5min":80.126000000000005,"15min":194.06299999999999},"last":0.27400000000000002},{"interface":"front","average":{"1min":4.0439999999999996,"5min":2.9689999999999999,"15min":6.7220000000000004},"min":{"1min":0.29099999999999998,"5min":0.215,"15min":0.215},"max":{"1min":15.439,"5min":83.899000000000001,"15min":196.947},"last":0.36499999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:43:49 2023","interfaces":[{"interface":"back","average":{"1min":5.2510000000000003,"5min":2.831,"15min":1.371},"min":{"1min":0.224,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":47.018000000000001,"5min":70.042000000000002,"15min":70.042000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":7.2069999999999999,"5min":4.9139999999999997,"15min":2.3690000000000002},"min":{"1min":0.23599999999999999,"5min":0.184,"15min":0.184},"max":{"1min":47.671999999999997,"5min":127.327,"15min":127.327},"last":0.29799999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:43:32 2023","interfaces":[{"interface":"back","average":{"1min":3.9169999999999998,"5min":3.9169999999999998,"15min":3.9169999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.561,"5min":14.561,"15min":14.561},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.8959999999999999,"5min":1.8959999999999999,"15min":1.8959999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.7720000000000002,"5min":4.7720000000000002,"15min":4.7720000000000002},"last":0.34699999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:43:08 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.8340000000000001,"15min":1.919},"min":{"1min":0.44800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.8710000000000004,"5min":45.576999999999998,"15min":45.576999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.913,"5min":1.518,"15min":1.3120000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.157,"5min":45.420999999999999,"15min":45.420999999999999},"last":0.622}]},{"osd":5,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.4810000000000001,"15min":3.3980000000000001},"min":{"1min":0.41499999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.8119999999999994,"5min":8.8119999999999994,"15min":163.56399999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":8.3710000000000004,"5min":2.4199999999999999,"15min":3.4119999999999999},"min":{"1min":0.39900000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":126.81100000000001,"5min":126.81100000000001,"15min":163.541},"last":0.57299999999999995}]},{"osd":6,"last update":"Sat Sep 23 22:38:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":2.851,"15min":1.958},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":2.0979999999999999,"5min":169.798,"15min":169.798},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.169,"5min":3.7669999999999999,"15min":2.411},"min":{"1min":0.42399999999999999,"5min":0.373,"15min":0.373},"max":{"1min":7.1130000000000004,"5min":168.44200000000001,"15min":168.44200000000001},"last":0.55400000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9620000000000002,"5min":1.9550000000000001,"15min":1.919},"min":{"1min":0.52400000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3979999999999997,"5min":75.513999999999996,"15min":108.113},"last":0.63900000000000001},{"interface":"front","average":{"1min":2.5619999999999998,"5min":2.1629999999999998,"15min":2.2949999999999999},"min":{"1min":0.45400000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.8480000000000008,"5min":74.007999999999996,"15min":108.093},"last":0.50900000000000001}]}]},{"osd":3,"up_from":0,"seq":2508260900883,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":0,"seq":1997159792738,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757747,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":557556,"kb_used_data":10796,"kb_used_omap":0,"kb_used_meta":546752,"kb_avail":93187596,"statfs":{"total":95995035648,"available":95424098304,"internally_reserved":0,"allocated":11055104,"data_stored":9070098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":559874048},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:40 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.1799999999999999,"15min":3.6709999999999998},"min":{"1min":0.24099999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.1739999999999995,"5min":26.925000000000001,"15min":145.46600000000001},"last":0.309},{"interface":"front","average":{"1min":1.135,"5min":1.179,"15min":3.5649999999999999},"min":{"1min":0.245,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.4740000000000002,"5min":30.609000000000002,"15min":144.429},"last":0.36599999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:43:45 2023","interfaces":[{"interface":"back","average":{"1min":3.4910000000000001,"5min":1.8700000000000001,"15min":1.0800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":35.643999999999998,"5min":35.643999999999998,"15min":35.643999999999998},"last":0.34499999999999997},{"interface":"front","average":{"1min":3.6949999999999998,"5min":1.869,"15min":0.98199999999999998},"min":{"1min":0.29399999999999998,"5min":0.25,"15min":0.249},"max":{"1min":35.316000000000003,"5min":35.316000000000003,"15min":35.316000000000003},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 22:43:12 2023","interfaces":[{"interface":"back","average":{"1min":3.8620000000000001,"5min":1.7410000000000001,"15min":2.2829999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.26200000000000001},"max":{"1min":25.899000000000001,"5min":25.899000000000001,"15min":70.153000000000006},"last":0.434},{"interface":"front","average":{"1min":2.9079999999999999,"5min":1.3440000000000001,"15min":2.1749999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":25.992000000000001,"5min":25.992000000000001,"15min":70.078999999999994},"last":0.44900000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:43:33 2023","interfaces":[{"interface":"back","average":{"1min":1.532,"5min":1.532,"15min":1.532},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.484,"5min":1.484,"15min":1.484},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.8959999999999999,"5min":7.8959999999999999,"15min":7.8959999999999999},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:43:08 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.5,"15min":2.6909999999999998},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.60299999999999998,"5min":4.0750000000000002,"15min":61.173999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.434,"5min":0.58499999999999996,"15min":2.7599999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":0.66500000000000004,"5min":4.157,"15min":61.203000000000003},"last":0.54600000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:38:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.9319999999999999,"15min":1.4990000000000001},"min":{"1min":0.27000000000000002,"5min":0.26800000000000002,"15min":0.25600000000000001},"max":{"1min":2.375,"5min":56.667000000000002,"15min":70.581999999999994},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.71,"15min":1.478},"min":{"1min":0.35399999999999998,"5min":0.29099999999999998,"15min":0.248},"max":{"1min":1.1240000000000001,"5min":57.185000000000002,"15min":71.093000000000004},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 22:43:40 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.83499999999999996,"15min":1.3160000000000001},"min":{"1min":0.34399999999999997,"5min":0.249,"15min":0.249},"max":{"1min":1.4550000000000001,"5min":31.042999999999999,"15min":59.101999999999997},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.002,"15min":1.538},"min":{"1min":0.33600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.1950000000000001,"5min":30.882999999999999,"15min":59.151000000000003},"last":0.51700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8798,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":98828288,"data_stored":98828288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":284753920,"data_stored":284753920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:44:03.186 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:44:03.858 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:03.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:03.963 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:03.993 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:04.004 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T22:44:04.004 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T22:44:04.005 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T22:44:04.005 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T22:44:04.005 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T22:44:04.005 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:44:04.040 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:44:04.047 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T22:44:04.047 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:44:04.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:04.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:03 smithi070 ceph-mon[192652]: 1.6 deep-scrub starts 2023-09-23T22:44:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:03 smithi070 ceph-mon[192652]: 1.6 deep-scrub ok 2023-09-23T22:44:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:03 smithi070 ceph-mon[192652]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:03 smithi070 ceph-mon[192652]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:03 smithi070 ceph-mon[192652]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3518600092' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:44:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:03 smithi070 ceph-mon[192652]: osdmap e613: 8 total, 8 up, 4 in 2023-09-23T22:44:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:03 smithi070 ceph-mon[192652]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:03 smithi070 ceph-mon[192652]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.125 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T22:44:04.125 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T22:44:04.129 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:04.147 INFO:tasks.workunit:timeout=3h 2023-09-23T22:44:04.147 INFO:tasks.workunit:cleanup=True 2023-09-23T22:44:04.148 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T22:44:04.149 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:04.215 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[223014]: 1.6 deep-scrub starts 2023-09-23T22:44:04.215 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[223014]: 1.6 deep-scrub ok 2023-09-23T22:44:04.215 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[223014]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[223014]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[223014]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3518600092' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[223014]: osdmap e613: 8 total, 8 up, 4 in 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[223014]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[223014]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[215775]: 1.6 deep-scrub starts 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[215775]: 1.6 deep-scrub ok 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[215775]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.216 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[215775]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[215775]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3518600092' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T22:44:04.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[215775]: osdmap e613: 8 total, 8 up, 4 in 2023-09-23T22:44:04.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[215775]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.217 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:03 smithi026 ceph-mon[215775]: failed to encode map e613 with expected crc 2023-09-23T22:44:04.218 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:04.237 INFO:tasks.workunit.client.0.smithi026.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T22:44:04.491 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:04.544 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:44:04.545 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:44:04.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:44:04.545 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [] 2023-09-23T22:44:04.545 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:44:04.572 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:04.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:04.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:04.676 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:04.706 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:04.858 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:04.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:04.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:04.962 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:04.995 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:05.044 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:05.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:05.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: 1.7 scrub starts 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: 1.7 scrub ok 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: pgmap v1590: 103 pgs: 4 remapped+peering, 99 active+clean; 4.5 MiB data, 2.2 GiB used, 355 GiB / 358 GiB avail 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: osdmap e614: 8 total, 8 up, 4 in 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:04 smithi070 ceph-mon[192652]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:44:05.150 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:05.176 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: 1.7 scrub starts 2023-09-23T22:44:05.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: 1.7 scrub ok 2023-09-23T22:44:05.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: pgmap v1590: 103 pgs: 4 remapped+peering, 99 active+clean; 4.5 MiB data, 2.2 GiB used, 355 GiB / 358 GiB avail 2023-09-23T22:44:05.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: osdmap e614: 8 total, 8 up, 4 in 2023-09-23T22:44:05.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.177 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[215775]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: 1.7 scrub starts 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: 1.7 scrub ok 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: pgmap v1590: 103 pgs: 4 remapped+peering, 99 active+clean; 4.5 MiB data, 2.2 GiB used, 355 GiB / 358 GiB avail 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: osdmap e614: 8 total, 8 up, 4 in 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: failed to encode map e614 with expected crc 2023-09-23T22:44:05.178 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:04 smithi026 ceph-mon[223014]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T22:44:05.179 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:05.689 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:44:05.982 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:44:06.058 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:44:06.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:05 smithi070 ceph-mon[192652]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:05 smithi070 ceph-mon[192652]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:05 smithi070 ceph-mon[192652]: osdmap e615: 8 total, 8 up, 4 in 2023-09-23T22:44:06.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:05 smithi070 ceph-mon[192652]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:05 smithi070 ceph-mon[192652]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:05 smithi070 ceph-mon[192652]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:05 smithi070 ceph-mon[192652]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:05 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1922532687' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:06.155 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[215775]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[215775]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[215775]: osdmap e615: 8 total, 8 up, 4 in 2023-09-23T22:44:06.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[215775]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[215775]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[215775]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[215775]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.156 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1922532687' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:06.156 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[223014]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.157 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[223014]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.157 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[223014]: osdmap e615: 8 total, 8 up, 4 in 2023-09-23T22:44:06.157 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[223014]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.157 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[223014]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.157 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[223014]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.157 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[223014]: failed to encode map e615 with expected crc 2023-09-23T22:44:06.157 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:05 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1922532687' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:06.442 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:06.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:06.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:06.548 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:06.579 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:06.724 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:06.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:06.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:06.828 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:06.854 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:06.897 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:06.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:06.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:07.001 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:07.032 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: pgmap v1593: 103 pgs: 7 peering, 2 active+undersized+remapped, 4 remapped+peering, 90 active+clean; 4.6 MiB data, 2.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1922532687' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: osdmap e616: 8 total, 8 up, 4 in 2023-09-23T22:44:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:06 smithi070 ceph-mon[192652]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: pgmap v1593: 103 pgs: 7 peering, 2 active+undersized+remapped, 4 remapped+peering, 90 active+clean; 4.6 MiB data, 2.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1922532687' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: osdmap e616: 8 total, 8 up, 4 in 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[215775]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: pgmap v1593: 103 pgs: 7 peering, 2 active+undersized+remapped, 4 remapped+peering, 90 active+clean; 4.6 MiB data, 2.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1922532687' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: osdmap e616: 8 total, 8 up, 4 in 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:06 smithi026 ceph-mon[223014]: failed to encode map e616 with expected crc 2023-09-23T22:44:07.632 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:07.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:07.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:07.736 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:08.048 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:44:08.048 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:44:08.048 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:44:08.048 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:44:08.117 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:08.264 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:08.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:08.352 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:44:08.368 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:08.372 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:44:08.397 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:44:08.399 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:08.418 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:44:08.437 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:44:08.442 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:44:08.461 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:44:08.489 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:44:08.500 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:08.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:08.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:08.520 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:44:08.521 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:44:08.522 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:44:08.522 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:44:08.522 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:44:08.522 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:44:08.522 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:44:08.522 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:44:08.522 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:44:08.609 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:08.636 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:08.682 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:08.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:08.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:08.788 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:44:08.790 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:08.822 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:08.879 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:08.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:08.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:08.974 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:44:08.976 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:44:08.983 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:08.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:08 smithi070 ceph-mon[192652]: pgmap v1595: 103 pgs: 7 peering, 2 active+undersized+remapped, 2 remapped+peering, 92 active+clean; 4.6 MiB data, 2.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:08.996 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:08 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1511958931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:09.018 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:09.061 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:09.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:09.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:09.164 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:09.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:08 smithi026 ceph-mon[215775]: pgmap v1595: 103 pgs: 7 peering, 2 active+undersized+remapped, 2 remapped+peering, 92 active+clean; 4.6 MiB data, 2.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:09.190 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:08 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1511958931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:09.190 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:08 smithi026 ceph-mon[223014]: pgmap v1595: 103 pgs: 7 peering, 2 active+undersized+remapped, 2 remapped+peering, 92 active+clean; 4.6 MiB data, 2.1 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:09.191 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:08 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1511958931' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:09.193 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:09.238 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:09.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:09.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:09.342 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:09.369 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:09.413 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:09.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:09.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:09.517 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:09.544 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:09.827 INFO:tasks.workunit.client.0.smithi026.stderr:Updating files: 86% (9966/11502) Updating files: 87% (10007/11502) Updating files: 88% (10122/11502) Updating files: 89% (10237/11502) Updating files: 90% (10352/11502) Updating files: 91% (10467/11502) Updating files: 92% (10582/11502) Updating files: 93% (10697/11502) Updating files: 94% (10812/11502) Updating files: 95% (10927/11502) Updating files: 96% (11042/11502) Updating files: 97% (11157/11502) Updating files: 98% (11272/11502) Updating files: 99% (11387/11502) Updating files: 100% (11502/11502) Updating files: 100% (11502/11502), done. 2023-09-23T22:44:09.843 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T22:44:09.930 INFO:tasks.workunit.client.0.smithi026.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T22:44:09.932 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T22:44:09.932 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T22:44:09.949 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:09.951 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:44:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:44:09] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2023-09-23T22:44:09.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:09.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:10.048 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:44:10.053 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:10.082 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:10.132 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:10.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:10.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:10 smithi026 ceph-mon[215775]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:10.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:44:10.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:10 smithi026 ceph-mon[223014]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:10.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:10.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:10.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:44:10.237 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:10.264 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:10.312 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:10.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:10.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:10.352 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:44:10.352 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:44:10.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:10 smithi070 ceph-mon[192652]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:10.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:10.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:10.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:44:10.416 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:10.444 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:10.777 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T22:44:10.986 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T22:44:10.991 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:44:11.238 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T22:44:11.239 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T22:44:11.240 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T22:44:11.331 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T22:44:11.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:11 smithi070 ceph-mon[192652]: pgmap v1596: 103 pgs: 7 peering, 96 active+clean; 4.6 MiB data, 2.0 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:11.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:11 smithi070 ceph-mon[192652]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:44:11.387 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T22:44:11.388 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T22:44:11.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:11 smithi026 ceph-mon[223014]: pgmap v1596: 103 pgs: 7 peering, 96 active+clean; 4.6 MiB data, 2.0 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:11.388 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:11 smithi026 ceph-mon[223014]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:44:11.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:11 smithi026 ceph-mon[215775]: pgmap v1596: 103 pgs: 7 peering, 96 active+clean; 4.6 MiB data, 2.0 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:11.389 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:11 smithi026 ceph-mon[215775]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T22:44:11.489 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:44:11.705 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T22:44:11.707 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T22:44:11.707 DEBUG:teuthology.orchestra.run.smithi026:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T22:44:11.802 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:11.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:11.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:11.906 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:11.934 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:12.091 INFO:tasks.workunit.client.0.smithi026.stdout:Running main() from gmock_main.cc 2023-09-23T22:44:12.091 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T22:44:12.091 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment set-up. 2023-09-23T22:44:12.091 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T22:44:12.246 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:12.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:12.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:12.350 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:12.380 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:12.397 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1595,"stamp":"2023-09-23T22:44:07.754929+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4825975,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6011,"num_read_kb":12239,"num_write":922,"num_write_kb":12872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":568,"num_bytes_recovered":21388689,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":305,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":355,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":2203868,"kb_used_data":605116,"kb_used_omap":35,"kb_used_meta":1596764,"kb_avail":372776740,"statfs":{"total":383980142592,"available":381723381760,"internally_reserved":0,"allocated":619638784,"data_stored":609422208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36602,"internal_metadata":1635086598},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":26,"apply_latency_ms":26,"commit_latency_ns":26000000,"apply_latency_ns":26000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":113454,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-15,"num_read_kb":-28,"num_write":-7,"num_write_kb":-73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":-1169414,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.893476"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":1086,"reported_epoch":615,"state":"peering","last_fresh":"2023-09-23T22:44:04.879789+0000","last_change":"2023-09-23T22:44:04.880092+0000","last_active":"2023-09-23T22:44:04.879749+0000","last_peered":"2023-09-23T22:44:03.884331+0000","last_clean":"2023-09-23T22:44:01.209858+0000","last_became_active":"2023-09-23T22:44:03.881194+0000","last_became_peered":"2023-09-23T22:44:03.881194+0000","last_unstale":"2023-09-23T22:44:04.879789+0000","last_undegraded":"2023-09-23T22:44:04.879789+0000","last_fullsized":"2023-09-23T22:44:04.879789+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.209739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.209739+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:35.913845+0000","scrub_duration":0.0016266950000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":571,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.859628+0000","last_change":"2023-09-23T22:43:11.791371+0000","last_active":"2023-09-23T22:44:01.859628+0000","last_peered":"2023-09-23T22:44:01.859628+0000","last_clean":"2023-09-23T22:44:01.859628+0000","last_became_active":"2023-09-23T22:43:11.783461+0000","last_became_peered":"2023-09-23T22:43:11.783461+0000","last_unstale":"2023-09-23T22:44:01.859628+0000","last_undegraded":"2023-09-23T22:44:01.859628+0000","last_fullsized":"2023-09-23T22:44:01.859628+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:19.460031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":353,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T22:44:05.884515+0000","last_change":"2023-09-23T22:43:11.733047+0000","last_active":"2023-09-23T22:44:05.884515+0000","last_peered":"2023-09-23T22:44:05.884515+0000","last_clean":"2023-09-23T22:44:05.884515+0000","last_became_active":"2023-09-23T22:43:11.731356+0000","last_became_peered":"2023-09-23T22:43:11.731356+0000","last_unstale":"2023-09-23T22:44:05.884515+0000","last_undegraded":"2023-09-23T22:44:05.884515+0000","last_fullsized":"2023-09-23T22:44:05.884515+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:09.953038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":809,"reported_epoch":615,"state":"peering","last_fresh":"2023-09-23T22:44:04.885172+0000","last_change":"2023-09-23T22:44:04.885495+0000","last_active":"2023-09-23T22:44:04.885135+0000","last_peered":"2023-09-23T22:44:03.880482+0000","last_clean":"2023-09-23T22:42:59.388987+0000","last_became_active":"2023-09-23T22:44:03.880044+0000","last_became_peered":"2023-09-23T22:44:03.880044+0000","last_unstale":"2023-09-23T22:44:04.885172+0000","last_undegraded":"2023-09-23T22:44:04.885172+0000","last_fullsized":"2023-09-23T22:44:04.885172+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:29.970215+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:29.970215+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":388,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.879369+0000","last_change":"2023-09-23T22:42:57.702640+0000","last_active":"2023-09-23T22:44:04.879369+0000","last_peered":"2023-09-23T22:44:04.879369+0000","last_clean":"2023-09-23T22:44:04.879369+0000","last_became_active":"2023-09-23T22:42:57.701938+0000","last_became_peered":"2023-09-23T22:42:57.701938+0000","last_unstale":"2023-09-23T22:44:04.879369+0000","last_undegraded":"2023-09-23T22:44:04.879369+0000","last_fullsized":"2023-09-23T22:44:04.879369+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:04.282382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":95,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.880274+0000","last_change":"2023-09-23T22:43:11.805439+0000","last_active":"2023-09-23T22:44:04.880274+0000","last_peered":"2023-09-23T22:44:04.880274+0000","last_clean":"2023-09-23T22:44:04.880274+0000","last_became_active":"2023-09-23T22:43:11.804518+0000","last_became_peered":"2023-09-23T22:43:11.804518+0000","last_unstale":"2023-09-23T22:44:04.880274+0000","last_undegraded":"2023-09-23T22:44:04.880274+0000","last_fullsized":"2023-09-23T22:44:04.880274+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:32.668394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":790,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T22:44:05.894762+0000","last_change":"2023-09-23T22:44:05.893969+0000","last_active":"2023-09-23T22:44:05.894762+0000","last_peered":"2023-09-23T22:44:05.894762+0000","last_clean":"2023-09-23T22:44:05.894762+0000","last_became_active":"2023-09-23T22:44:05.893507+0000","last_became_peered":"2023-09-23T22:44:05.893507+0000","last_unstale":"2023-09-23T22:44:05.894762+0000","last_undegraded":"2023-09-23T22:44:05.894762+0000","last_fullsized":"2023-09-23T22:44:05.894762+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:43:58.171880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:43:58.171880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":680,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.876851+0000","last_change":"2023-09-23T22:43:11.861828+0000","last_active":"2023-09-23T22:44:04.876851+0000","last_peered":"2023-09-23T22:44:04.876851+0000","last_clean":"2023-09-23T22:44:04.876851+0000","last_became_active":"2023-09-23T22:43:11.860602+0000","last_became_peered":"2023-09-23T22:43:11.860602+0000","last_unstale":"2023-09-23T22:44:04.876851+0000","last_undegraded":"2023-09-23T22:44:04.876851+0000","last_fullsized":"2023-09-23T22:44:04.876851+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:43.850844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":229,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.870613+0000","last_change":"2023-09-23T22:42:56.509390+0000","last_active":"2023-09-23T22:44:02.870613+0000","last_peered":"2023-09-23T22:44:02.870613+0000","last_clean":"2023-09-23T22:44:02.870613+0000","last_became_active":"2023-09-23T22:42:56.509066+0000","last_became_peered":"2023-09-23T22:42:56.509066+0000","last_unstale":"2023-09-23T22:44:02.870613+0000","last_undegraded":"2023-09-23T22:44:02.870613+0000","last_fullsized":"2023-09-23T22:44:02.870613+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:36.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":944,"reported_epoch":615,"state":"peering","last_fresh":"2023-09-23T22:44:04.884615+0000","last_change":"2023-09-23T22:44:04.884880+0000","last_active":"2023-09-23T22:44:04.884555+0000","last_peered":"2023-09-23T22:44:03.880215+0000","last_clean":"2023-09-23T22:42:59.388972+0000","last_became_active":"2023-09-23T22:44:03.879841+0000","last_became_peered":"2023-09-23T22:44:03.879841+0000","last_unstale":"2023-09-23T22:44:04.884615+0000","last_undegraded":"2023-09-23T22:44:04.884615+0000","last_fullsized":"2023-09-23T22:44:04.884615+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:57.273632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:57.273632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:54:04.329266+0000","scrub_duration":0.018058204000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.873556+0000","last_change":"2023-09-23T22:43:11.732475+0000","last_active":"2023-09-23T22:44:03.873556+0000","last_peered":"2023-09-23T22:44:03.873556+0000","last_clean":"2023-09-23T22:44:03.873556+0000","last_became_active":"2023-09-23T22:43:11.732134+0000","last_became_peered":"2023-09-23T22:43:11.732134+0000","last_unstale":"2023-09-23T22:44:03.873556+0000","last_undegraded":"2023-09-23T22:44:03.873556+0000","last_fullsized":"2023-09-23T22:44:03.873556+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:22.683794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":207,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.873900+0000","last_change":"2023-09-23T22:42:57.555024+0000","last_active":"2023-09-23T22:44:02.873900+0000","last_peered":"2023-09-23T22:44:02.873900+0000","last_clean":"2023-09-23T22:44:02.873900+0000","last_became_active":"2023-09-23T22:42:57.554656+0000","last_became_peered":"2023-09-23T22:42:57.554656+0000","last_unstale":"2023-09-23T22:44:02.873900+0000","last_undegraded":"2023-09-23T22:44:02.873900+0000","last_fullsized":"2023-09-23T22:44:02.873900+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:56.648868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":770,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T22:43:55.843385+0000","last_change":"2023-09-23T22:43:14.436119+0000","last_active":"2023-09-23T22:43:55.843385+0000","last_peered":"2023-09-23T22:43:55.843385+0000","last_clean":"2023-09-23T22:43:55.843385+0000","last_became_active":"2023-09-23T22:43:14.435613+0000","last_became_peered":"2023-09-23T22:43:14.435613+0000","last_unstale":"2023-09-23T22:43:55.843385+0000","last_undegraded":"2023-09-23T22:43:55.843385+0000","last_fullsized":"2023-09-23T22:43:55.843385+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:02.056099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:02.056099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:12.546310+0000","scrub_duration":0.012145022,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.873630+0000","last_change":"2023-09-23T22:43:11.828937+0000","last_active":"2023-09-23T22:44:03.873630+0000","last_peered":"2023-09-23T22:44:03.873630+0000","last_clean":"2023-09-23T22:44:03.873630+0000","last_became_active":"2023-09-23T22:43:11.828512+0000","last_became_peered":"2023-09-23T22:43:11.828512+0000","last_unstale":"2023-09-23T22:44:03.873630+0000","last_undegraded":"2023-09-23T22:44:03.873630+0000","last_fullsized":"2023-09-23T22:44:03.873630+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:56.701943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":210,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.872577+0000","last_change":"2023-09-23T22:42:57.699399+0000","last_active":"2023-09-23T22:44:02.872577+0000","last_peered":"2023-09-23T22:44:02.872577+0000","last_clean":"2023-09-23T22:44:02.872577+0000","last_became_active":"2023-09-23T22:42:57.698650+0000","last_became_peered":"2023-09-23T22:42:57.698650+0000","last_unstale":"2023-09-23T22:44:02.872577+0000","last_undegraded":"2023-09-23T22:44:02.872577+0000","last_fullsized":"2023-09-23T22:44:02.872577+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:17.884820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":882,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872936+0000","last_change":"2023-09-23T22:44:02.962008+0000","last_active":"2023-09-23T22:44:03.872936+0000","last_peered":"2023-09-23T22:44:03.872936+0000","last_clean":"2023-09-23T22:44:03.872936+0000","last_became_active":"2023-09-23T22:44:02.961671+0000","last_became_peered":"2023-09-23T22:44:02.961671+0000","last_unstale":"2023-09-23T22:44:03.872936+0000","last_undegraded":"2023-09-23T22:44:03.872936+0000","last_fullsized":"2023-09-23T22:44:03.872936+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":52,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.872225+0000","last_change":"2023-09-23T22:43:11.882816+0000","last_active":"2023-09-23T22:44:02.872225+0000","last_peered":"2023-09-23T22:44:02.872225+0000","last_clean":"2023-09-23T22:44:02.872225+0000","last_became_active":"2023-09-23T22:43:11.822292+0000","last_became_peered":"2023-09-23T22:43:11.822292+0000","last_unstale":"2023-09-23T22:44:02.872225+0000","last_undegraded":"2023-09-23T22:44:02.872225+0000","last_fullsized":"2023-09-23T22:44:02.872225+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:05.717891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":267,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.878741+0000","last_change":"2023-09-23T22:42:56.567330+0000","last_active":"2023-09-23T22:44:04.878741+0000","last_peered":"2023-09-23T22:44:04.878741+0000","last_clean":"2023-09-23T22:44:04.878741+0000","last_became_active":"2023-09-23T22:42:56.567016+0000","last_became_peered":"2023-09-23T22:42:56.567016+0000","last_unstale":"2023-09-23T22:44:04.878741+0000","last_undegraded":"2023-09-23T22:44:04.878741+0000","last_fullsized":"2023-09-23T22:44:04.878741+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:00.456335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":799,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T22:44:05.892488+0000","last_change":"2023-09-23T22:44:05.892244+0000","last_active":"2023-09-23T22:44:05.892488+0000","last_peered":"2023-09-23T22:44:05.892488+0000","last_clean":"2023-09-23T22:44:05.892488+0000","last_became_active":"2023-09-23T22:44:05.891879+0000","last_became_peered":"2023-09-23T22:44:05.891879+0000","last_unstale":"2023-09-23T22:44:05.892488+0000","last_undegraded":"2023-09-23T22:44:05.892488+0000","last_fullsized":"2023-09-23T22:44:05.892488+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":604,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.871065+0000","last_change":"2023-09-23T22:43:11.747332+0000","last_active":"2023-09-23T22:44:02.871065+0000","last_peered":"2023-09-23T22:44:02.871065+0000","last_clean":"2023-09-23T22:44:02.871065+0000","last_became_active":"2023-09-23T22:43:11.733061+0000","last_became_peered":"2023-09-23T22:43:11.733061+0000","last_unstale":"2023-09-23T22:44:02.871065+0000","last_undegraded":"2023-09-23T22:44:02.871065+0000","last_fullsized":"2023-09-23T22:44:02.871065+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:49:43.986136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":42606,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":537,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.869365+0000","last_change":"2023-09-23T22:42:57.485105+0000","last_active":"2023-09-23T22:44:02.869365+0000","last_peered":"2023-09-23T22:44:02.869365+0000","last_clean":"2023-09-23T22:44:02.869365+0000","last_became_active":"2023-09-23T22:42:57.484320+0000","last_became_peered":"2023-09-23T22:42:57.484320+0000","last_unstale":"2023-09-23T22:44:02.869365+0000","last_undegraded":"2023-09-23T22:44:02.869365+0000","last_fullsized":"2023-09-23T22:44:02.869365+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:36.390910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":1038,"reported_epoch":615,"state":"peering","last_fresh":"2023-09-23T22:44:04.881152+0000","last_change":"2023-09-23T22:44:04.881460+0000","last_active":"2023-09-23T22:44:04.881117+0000","last_peered":"2023-09-23T22:44:03.878040+0000","last_clean":"2023-09-23T22:43:56.847811+0000","last_became_active":"2023-09-23T22:44:03.877705+0000","last_became_peered":"2023-09-23T22:44:03.877705+0000","last_unstale":"2023-09-23T22:44:04.881152+0000","last_undegraded":"2023-09-23T22:44:04.881152+0000","last_fullsized":"2023-09-23T22:44:04.881152+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:19.077318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:19.077318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:55.175174+0000","scrub_duration":0.0080980949999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":301,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.859256+0000","last_change":"2023-09-23T22:42:57.736915+0000","last_active":"2023-09-23T22:44:01.859256+0000","last_peered":"2023-09-23T22:44:01.859256+0000","last_clean":"2023-09-23T22:44:01.859256+0000","last_became_active":"2023-09-23T22:42:57.629094+0000","last_became_peered":"2023-09-23T22:42:57.629094+0000","last_unstale":"2023-09-23T22:44:01.859256+0000","last_undegraded":"2023-09-23T22:44:01.859256+0000","last_fullsized":"2023-09-23T22:44:01.859256+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:03.304992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.860665+0000","last_change":"2023-09-23T22:41:52.385773+0000","last_active":"2023-09-23T22:44:01.860665+0000","last_peered":"2023-09-23T22:44:01.860665+0000","last_clean":"2023-09-23T22:44:01.860665+0000","last_became_active":"2023-09-23T22:41:52.385433+0000","last_became_peered":"2023-09-23T22:41:52.385433+0000","last_unstale":"2023-09-23T22:44:01.860665+0000","last_undegraded":"2023-09-23T22:44:01.860665+0000","last_fullsized":"2023-09-23T22:44:01.860665+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:15.934387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":884,"reported_epoch":615,"state":"peering","last_fresh":"2023-09-23T22:44:04.880024+0000","last_change":"2023-09-23T22:44:04.880024+0000","last_active":"2023-09-23T22:42:55.485732+0000","last_peered":"2023-09-23T22:42:54.457149+0000","last_clean":"2023-09-23T22:42:54.457149+0000","last_became_active":"2023-09-23T22:42:08.564391+0000","last_became_peered":"2023-09-23T22:42:08.564391+0000","last_unstale":"2023-09-23T22:44:04.880024+0000","last_undegraded":"2023-09-23T22:44:04.880024+0000","last_fullsized":"2023-09-23T22:44:04.880024+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.556720+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:04.062174+0000","scrub_duration":0.0022349409999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":82,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.857700+0000","last_change":"2023-09-23T22:43:11.791005+0000","last_active":"2023-09-23T22:44:01.857700+0000","last_peered":"2023-09-23T22:44:01.857700+0000","last_clean":"2023-09-23T22:44:01.857700+0000","last_became_active":"2023-09-23T22:43:11.783005+0000","last_became_peered":"2023-09-23T22:43:11.783005+0000","last_unstale":"2023-09-23T22:44:01.857700+0000","last_undegraded":"2023-09-23T22:44:01.857700+0000","last_fullsized":"2023-09-23T22:44:01.857700+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:23.783502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.875493+0000","last_change":"2023-09-23T22:43:11.725543+0000","last_active":"2023-09-23T22:44:03.875493+0000","last_peered":"2023-09-23T22:44:03.875493+0000","last_clean":"2023-09-23T22:44:03.875493+0000","last_became_active":"2023-09-23T22:43:11.725152+0000","last_became_peered":"2023-09-23T22:43:11.725152+0000","last_unstale":"2023-09-23T22:44:03.875493+0000","last_undegraded":"2023-09-23T22:44:03.875493+0000","last_fullsized":"2023-09-23T22:44:03.875493+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:31.160745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":636,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T22:44:05.883095+0000","last_change":"2023-09-23T22:44:02.925951+0000","last_active":"2023-09-23T22:44:05.883095+0000","last_peered":"2023-09-23T22:44:05.883095+0000","last_clean":"2023-09-23T22:44:05.883095+0000","last_became_active":"2023-09-23T22:44:02.922183+0000","last_became_peered":"2023-09-23T22:44:02.922183+0000","last_unstale":"2023-09-23T22:44:05.883095+0000","last_undegraded":"2023-09-23T22:44:05.883095+0000","last_fullsized":"2023-09-23T22:44:05.883095+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:57.922489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:57.922489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:25.336046+0000","scrub_duration":0.001544351,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":457,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.872408+0000","last_change":"2023-09-23T22:43:11.802960+0000","last_active":"2023-09-23T22:44:02.872408+0000","last_peered":"2023-09-23T22:44:02.872408+0000","last_clean":"2023-09-23T22:44:02.872408+0000","last_became_active":"2023-09-23T22:43:11.802684+0000","last_became_peered":"2023-09-23T22:43:11.802684+0000","last_unstale":"2023-09-23T22:44:02.872408+0000","last_undegraded":"2023-09-23T22:44:02.872408+0000","last_fullsized":"2023-09-23T22:44:02.872408+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:21.148096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":92,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.862091+0000","last_change":"2023-09-23T22:43:11.807494+0000","last_active":"2023-09-23T22:44:01.862091+0000","last_peered":"2023-09-23T22:44:01.862091+0000","last_clean":"2023-09-23T22:44:01.862091+0000","last_became_active":"2023-09-23T22:43:11.807103+0000","last_became_peered":"2023-09-23T22:43:11.807103+0000","last_unstale":"2023-09-23T22:44:01.862091+0000","last_undegraded":"2023-09-23T22:44:01.862091+0000","last_fullsized":"2023-09-23T22:44:01.862091+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:50.696866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":943,"reported_epoch":615,"state":"peering","last_fresh":"2023-09-23T22:44:04.885271+0000","last_change":"2023-09-23T22:44:04.885462+0000","last_active":"2023-09-23T22:44:04.885243+0000","last_peered":"2023-09-23T22:44:03.878516+0000","last_clean":"2023-09-23T22:42:59.396276+0000","last_became_active":"2023-09-23T22:44:03.878216+0000","last_became_peered":"2023-09-23T22:44:03.878216+0000","last_unstale":"2023-09-23T22:44:04.885271+0000","last_undegraded":"2023-09-23T22:44:04.885271+0000","last_fullsized":"2023-09-23T22:44:04.885271+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:26.313042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:26.313042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:33.628451+0000","scrub_duration":0.0017143830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":273,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.875215+0000","last_change":"2023-09-23T22:43:11.844157+0000","last_active":"2023-09-23T22:44:02.875215+0000","last_peered":"2023-09-23T22:44:02.875215+0000","last_clean":"2023-09-23T22:44:02.875215+0000","last_became_active":"2023-09-23T22:43:11.751401+0000","last_became_peered":"2023-09-23T22:43:11.751401+0000","last_unstale":"2023-09-23T22:44:02.875215+0000","last_undegraded":"2023-09-23T22:44:02.875215+0000","last_fullsized":"2023-09-23T22:44:02.875215+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:37.797785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.859443+0000","last_change":"2023-09-23T22:42:57.618059+0000","last_active":"2023-09-23T22:44:01.859443+0000","last_peered":"2023-09-23T22:44:01.859443+0000","last_clean":"2023-09-23T22:44:01.859443+0000","last_became_active":"2023-09-23T22:42:57.617619+0000","last_became_peered":"2023-09-23T22:42:57.617619+0000","last_unstale":"2023-09-23T22:44:01.859443+0000","last_undegraded":"2023-09-23T22:44:01.859443+0000","last_fullsized":"2023-09-23T22:44:01.859443+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:00.845651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":959,"reported_epoch":613,"state":"remapped+peering","last_fresh":"2023-09-23T22:44:02.870595+0000","last_change":"2023-09-23T22:44:02.870767+0000","last_active":"2023-09-23T22:44:01.861476+0000","last_peered":"2023-09-23T22:43:56.847826+0000","last_clean":"2023-09-23T22:43:56.847826+0000","last_became_active":"2023-09-23T22:43:14.443078+0000","last_became_peered":"2023-09-23T22:43:14.443078+0000","last_unstale":"2023-09-23T22:44:02.870595+0000","last_undegraded":"2023-09-23T22:44:02.870595+0000","last_fullsized":"2023-09-23T22:44:02.870595+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:34.147486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:34.147486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:37.391713+0000","scrub_duration":0.0018320190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,5,6],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":156,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.877937+0000","last_change":"2023-09-23T22:43:11.920516+0000","last_active":"2023-09-23T22:44:04.877937+0000","last_peered":"2023-09-23T22:44:04.877937+0000","last_clean":"2023-09-23T22:44:04.877937+0000","last_became_active":"2023-09-23T22:43:11.755782+0000","last_became_peered":"2023-09-23T22:43:11.755782+0000","last_unstale":"2023-09-23T22:44:04.877937+0000","last_undegraded":"2023-09-23T22:44:04.877937+0000","last_fullsized":"2023-09-23T22:44:04.877937+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:16.670619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872269+0000","last_change":"2023-09-23T22:43:11.831473+0000","last_active":"2023-09-23T22:44:03.872269+0000","last_peered":"2023-09-23T22:44:03.872269+0000","last_clean":"2023-09-23T22:44:03.872269+0000","last_became_active":"2023-09-23T22:43:11.831153+0000","last_became_peered":"2023-09-23T22:43:11.831153+0000","last_unstale":"2023-09-23T22:44:03.872269+0000","last_undegraded":"2023-09-23T22:44:03.872269+0000","last_fullsized":"2023-09-23T22:44:03.872269+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:00:24.678192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":722,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.908326+0000","last_change":"2023-09-23T22:44:02.893918+0000","last_active":"2023-09-23T22:44:02.908326+0000","last_peered":"2023-09-23T22:44:02.908326+0000","last_clean":"2023-09-23T22:44:02.908326+0000","last_became_active":"2023-09-23T22:44:02.893292+0000","last_became_peered":"2023-09-23T22:44:02.893292+0000","last_unstale":"2023-09-23T22:44:02.908326+0000","last_undegraded":"2023-09-23T22:44:02.908326+0000","last_fullsized":"2023-09-23T22:44:02.908326+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:04.048606+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:04.048606+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:36.525412+0000","scrub_duration":0.0064015249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":533,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.882541+0000","last_change":"2023-09-23T22:43:11.899388+0000","last_active":"2023-09-23T22:44:04.882541+0000","last_peered":"2023-09-23T22:44:04.882541+0000","last_clean":"2023-09-23T22:44:04.882541+0000","last_became_active":"2023-09-23T22:43:11.841184+0000","last_became_peered":"2023-09-23T22:43:11.841184+0000","last_unstale":"2023-09-23T22:44:04.882541+0000","last_undegraded":"2023-09-23T22:44:04.882541+0000","last_fullsized":"2023-09-23T22:44:04.882541+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:01.020339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":628,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.879283+0000","last_change":"2023-09-23T22:42:57.702798+0000","last_active":"2023-09-23T22:44:04.879283+0000","last_peered":"2023-09-23T22:44:04.879283+0000","last_clean":"2023-09-23T22:44:04.879283+0000","last_became_active":"2023-09-23T22:42:57.702317+0000","last_became_peered":"2023-09-23T22:42:57.702317+0000","last_unstale":"2023-09-23T22:44:04.879283+0000","last_undegraded":"2023-09-23T22:44:04.879283+0000","last_fullsized":"2023-09-23T22:44:04.879283+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:12.697584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":898,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.870800+0000","last_change":"2023-09-23T22:43:14.445750+0000","last_active":"2023-09-23T22:44:02.870800+0000","last_peered":"2023-09-23T22:44:02.870800+0000","last_clean":"2023-09-23T22:44:02.870800+0000","last_became_active":"2023-09-23T22:43:14.445359+0000","last_became_peered":"2023-09-23T22:43:14.445359+0000","last_unstale":"2023-09-23T22:44:02.870800+0000","last_undegraded":"2023-09-23T22:44:02.870800+0000","last_fullsized":"2023-09-23T22:44:02.870800+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":31,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.875412+0000","last_change":"2023-09-23T22:43:11.918943+0000","last_active":"2023-09-23T22:44:03.875412+0000","last_peered":"2023-09-23T22:44:03.875412+0000","last_clean":"2023-09-23T22:44:03.875412+0000","last_became_active":"2023-09-23T22:43:11.827406+0000","last_became_peered":"2023-09-23T22:43:11.827406+0000","last_unstale":"2023-09-23T22:44:03.875412+0000","last_undegraded":"2023-09-23T22:44:03.875412+0000","last_fullsized":"2023-09-23T22:44:03.875412+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:57.335665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":247,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.869846+0000","last_change":"2023-09-23T22:42:57.480776+0000","last_active":"2023-09-23T22:44:02.869846+0000","last_peered":"2023-09-23T22:44:02.869846+0000","last_clean":"2023-09-23T22:44:02.869846+0000","last_became_active":"2023-09-23T22:42:57.480378+0000","last_became_peered":"2023-09-23T22:42:57.480378+0000","last_unstale":"2023-09-23T22:44:02.869846+0000","last_undegraded":"2023-09-23T22:44:02.869846+0000","last_fullsized":"2023-09-23T22:44:02.869846+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.689275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":788,"reported_epoch":613,"state":"remapped+peering","last_fresh":"2023-09-23T22:44:02.866521+0000","last_change":"2023-09-23T22:44:02.866717+0000","last_active":"2023-09-23T22:42:55.516961+0000","last_peered":"2023-09-23T22:42:54.465200+0000","last_clean":"2023-09-23T22:42:54.465200+0000","last_became_active":"2023-09-23T22:42:08.585797+0000","last_became_peered":"2023-09-23T22:42:08.585797+0000","last_unstale":"2023-09-23T22:44:02.866521+0000","last_undegraded":"2023-09-23T22:44:02.866521+0000","last_fullsized":"2023-09-23T22:44:02.866521+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,5,6],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872772+0000","last_change":"2023-09-23T22:43:11.732060+0000","last_active":"2023-09-23T22:44:03.872772+0000","last_peered":"2023-09-23T22:44:03.872772+0000","last_clean":"2023-09-23T22:44:03.872772+0000","last_became_active":"2023-09-23T22:43:11.731696+0000","last_became_peered":"2023-09-23T22:43:11.731696+0000","last_unstale":"2023-09-23T22:44:03.872772+0000","last_undegraded":"2023-09-23T22:44:03.872772+0000","last_fullsized":"2023-09-23T22:44:03.872772+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:41.169298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":421,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.877236+0000","last_change":"2023-09-23T22:43:11.733183+0000","last_active":"2023-09-23T22:44:04.877236+0000","last_peered":"2023-09-23T22:44:04.877236+0000","last_clean":"2023-09-23T22:44:04.877236+0000","last_became_active":"2023-09-23T22:43:11.731600+0000","last_became_peered":"2023-09-23T22:43:11.731600+0000","last_unstale":"2023-09-23T22:44:04.877236+0000","last_undegraded":"2023-09-23T22:44:04.877236+0000","last_fullsized":"2023-09-23T22:44:04.877236+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:17:17.223104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":967,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.858038+0000","last_change":"2023-09-23T22:42:59.395878+0000","last_active":"2023-09-23T22:44:01.858038+0000","last_peered":"2023-09-23T22:44:01.858038+0000","last_clean":"2023-09-23T22:44:01.858038+0000","last_became_active":"2023-09-23T22:42:59.395538+0000","last_became_peered":"2023-09-23T22:42:59.395538+0000","last_unstale":"2023-09-23T22:44:01.858038+0000","last_undegraded":"2023-09-23T22:44:01.858038+0000","last_fullsized":"2023-09-23T22:44:01.858038+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:58.211060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:58.211060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:50.018093+0000","scrub_duration":0.0035013240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":94,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.859558+0000","last_change":"2023-09-23T22:43:11.840050+0000","last_active":"2023-09-23T22:44:01.859558+0000","last_peered":"2023-09-23T22:44:01.859558+0000","last_clean":"2023-09-23T22:44:01.859558+0000","last_became_active":"2023-09-23T22:43:11.780186+0000","last_became_peered":"2023-09-23T22:43:11.780186+0000","last_unstale":"2023-09-23T22:44:01.859558+0000","last_undegraded":"2023-09-23T22:44:01.859558+0000","last_fullsized":"2023-09-23T22:44:01.859558+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:00.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":455,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.869006+0000","last_change":"2023-09-23T22:42:57.606366+0000","last_active":"2023-09-23T22:44:02.869006+0000","last_peered":"2023-09-23T22:44:02.869006+0000","last_clean":"2023-09-23T22:44:02.869006+0000","last_became_active":"2023-09-23T22:42:57.604437+0000","last_became_peered":"2023-09-23T22:42:57.604437+0000","last_unstale":"2023-09-23T22:44:02.869006+0000","last_undegraded":"2023-09-23T22:44:02.869006+0000","last_fullsized":"2023-09-23T22:44:02.869006+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:34.648244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":1074,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872387+0000","last_change":"2023-09-23T22:44:00.183260+0000","last_active":"2023-09-23T22:44:03.872387+0000","last_peered":"2023-09-23T22:44:03.872387+0000","last_clean":"2023-09-23T22:44:03.872387+0000","last_became_active":"2023-09-23T22:43:14.434417+0000","last_became_peered":"2023-09-23T22:43:14.434417+0000","last_unstale":"2023-09-23T22:44:03.872387+0000","last_undegraded":"2023-09-23T22:44:03.872387+0000","last_fullsized":"2023-09-23T22:44:03.872387+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.183183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.183183+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:35.744336+0000","scrub_duration":0.0015656730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":244,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.870930+0000","last_change":"2023-09-23T22:42:57.480722+0000","last_active":"2023-09-23T22:44:02.870930+0000","last_peered":"2023-09-23T22:44:02.870930+0000","last_clean":"2023-09-23T22:44:02.870930+0000","last_became_active":"2023-09-23T22:42:57.480180+0000","last_became_peered":"2023-09-23T22:42:57.480180+0000","last_unstale":"2023-09-23T22:44:02.870930+0000","last_undegraded":"2023-09-23T22:44:02.870930+0000","last_fullsized":"2023-09-23T22:44:02.870930+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:00.944581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":262,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.860452+0000","last_change":"2023-09-23T22:42:57.671849+0000","last_active":"2023-09-23T22:44:01.860452+0000","last_peered":"2023-09-23T22:44:01.860452+0000","last_clean":"2023-09-23T22:44:01.860452+0000","last_became_active":"2023-09-23T22:42:57.667120+0000","last_became_peered":"2023-09-23T22:42:57.667120+0000","last_unstale":"2023-09-23T22:44:01.860452+0000","last_undegraded":"2023-09-23T22:44:01.860452+0000","last_fullsized":"2023-09-23T22:44:01.860452+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:11.794628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":809,"reported_epoch":615,"state":"peering","last_fresh":"2023-09-23T22:44:04.888566+0000","last_change":"2023-09-23T22:44:04.888750+0000","last_active":"2023-09-23T22:44:04.888530+0000","last_peered":"2023-09-23T22:44:03.881114+0000","last_clean":"2023-09-23T22:42:59.389403+0000","last_became_active":"2023-09-23T22:44:03.880848+0000","last_became_peered":"2023-09-23T22:44:03.880848+0000","last_unstale":"2023-09-23T22:44:04.888566+0000","last_undegraded":"2023-09-23T22:44:04.888566+0000","last_fullsized":"2023-09-23T22:44:04.888566+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:26.963124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:26.963124+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":586,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.858256+0000","last_change":"2023-09-23T22:43:11.854114+0000","last_active":"2023-09-23T22:44:01.858256+0000","last_peered":"2023-09-23T22:44:01.858256+0000","last_clean":"2023-09-23T22:44:01.858256+0000","last_became_active":"2023-09-23T22:43:11.841381+0000","last_became_peered":"2023-09-23T22:43:11.841381+0000","last_unstale":"2023-09-23T22:44:01.858256+0000","last_undegraded":"2023-09-23T22:44:01.858256+0000","last_fullsized":"2023-09-23T22:44:01.858256+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:46.359977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":544,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.871310+0000","last_change":"2023-09-23T22:43:11.817112+0000","last_active":"2023-09-23T22:44:02.871310+0000","last_peered":"2023-09-23T22:44:02.871310+0000","last_clean":"2023-09-23T22:44:02.871310+0000","last_became_active":"2023-09-23T22:43:11.816563+0000","last_became_peered":"2023-09-23T22:43:11.816563+0000","last_unstale":"2023-09-23T22:44:02.871310+0000","last_undegraded":"2023-09-23T22:44:02.871310+0000","last_fullsized":"2023-09-23T22:44:02.871310+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:56.365553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":1023,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.874388+0000","last_change":"2023-09-23T22:43:57.157649+0000","last_active":"2023-09-23T22:44:03.874388+0000","last_peered":"2023-09-23T22:44:03.874388+0000","last_clean":"2023-09-23T22:44:03.874388+0000","last_became_active":"2023-09-23T22:42:56.530915+0000","last_became_peered":"2023-09-23T22:42:56.530915+0000","last_unstale":"2023-09-23T22:44:03.874388+0000","last_undegraded":"2023-09-23T22:44:03.874388+0000","last_fullsized":"2023-09-23T22:44:03.874388+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_clean_scrub_stamp":"2023-09-23T22:43:57.157560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:20.159228+0000","scrub_duration":0.001566287,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":372,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.878226+0000","last_change":"2023-09-23T22:43:11.862025+0000","last_active":"2023-09-23T22:44:04.878226+0000","last_peered":"2023-09-23T22:44:04.878226+0000","last_clean":"2023-09-23T22:44:04.878226+0000","last_became_active":"2023-09-23T22:43:11.860993+0000","last_became_peered":"2023-09-23T22:43:11.860993+0000","last_unstale":"2023-09-23T22:44:04.878226+0000","last_undegraded":"2023-09-23T22:44:04.878226+0000","last_fullsized":"2023-09-23T22:44:04.878226+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:37.123461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":438,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.878010+0000","last_change":"2023-09-23T22:43:11.756165+0000","last_active":"2023-09-23T22:44:04.878010+0000","last_peered":"2023-09-23T22:44:04.878010+0000","last_clean":"2023-09-23T22:44:04.878010+0000","last_became_active":"2023-09-23T22:43:11.755617+0000","last_became_peered":"2023-09-23T22:43:11.755617+0000","last_unstale":"2023-09-23T22:44:04.878010+0000","last_undegraded":"2023-09-23T22:44:04.878010+0000","last_fullsized":"2023-09-23T22:44:04.878010+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:21.873764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":909,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.570310+0000","last_change":"2023-09-23T22:44:03.570228+0000","last_active":"2023-09-23T22:44:03.570310+0000","last_peered":"2023-09-23T22:44:03.570310+0000","last_clean":"2023-09-23T22:44:03.570310+0000","last_became_active":"2023-09-23T22:43:14.444896+0000","last_became_peered":"2023-09-23T22:43:14.444896+0000","last_unstale":"2023-09-23T22:44:03.570310+0000","last_undegraded":"2023-09-23T22:44:03.570310+0000","last_fullsized":"2023-09-23T22:44:03.570310+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:03.570110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:03.570110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:13.842237+0000","scrub_duration":0.001987071,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":986,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.860887+0000","last_change":"2023-09-23T22:43:54.413786+0000","last_active":"2023-09-23T22:44:01.860887+0000","last_peered":"2023-09-23T22:44:01.860887+0000","last_clean":"2023-09-23T22:44:01.860887+0000","last_became_active":"2023-09-23T22:43:54.413159+0000","last_became_peered":"2023-09-23T22:43:54.413159+0000","last_unstale":"2023-09-23T22:44:01.860887+0000","last_undegraded":"2023-09-23T22:44:01.860887+0000","last_fullsized":"2023-09-23T22:44:01.860887+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:37.708567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.875317+0000","last_change":"2023-09-23T22:43:11.827683+0000","last_active":"2023-09-23T22:44:03.875317+0000","last_peered":"2023-09-23T22:44:03.875317+0000","last_clean":"2023-09-23T22:44:03.875317+0000","last_became_active":"2023-09-23T22:43:11.827004+0000","last_became_peered":"2023-09-23T22:43:11.827004+0000","last_unstale":"2023-09-23T22:44:03.875317+0000","last_undegraded":"2023-09-23T22:44:03.875317+0000","last_fullsized":"2023-09-23T22:44:03.875317+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:21.277304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":373,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.878354+0000","last_change":"2023-09-23T22:43:11.772491+0000","last_active":"2023-09-23T22:44:04.878354+0000","last_peered":"2023-09-23T22:44:04.878354+0000","last_clean":"2023-09-23T22:44:04.878354+0000","last_became_active":"2023-09-23T22:43:11.772145+0000","last_became_peered":"2023-09-23T22:43:11.772145+0000","last_unstale":"2023-09-23T22:44:04.878354+0000","last_undegraded":"2023-09-23T22:44:04.878354+0000","last_fullsized":"2023-09-23T22:44:04.878354+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:22.248227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":1027,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872344+0000","last_change":"2023-09-23T22:43:14.435063+0000","last_active":"2023-09-23T22:44:03.872344+0000","last_peered":"2023-09-23T22:44:03.872344+0000","last_clean":"2023-09-23T22:44:03.872344+0000","last_became_active":"2023-09-23T22:43:14.434698+0000","last_became_peered":"2023-09-23T22:43:14.434698+0000","last_unstale":"2023-09-23T22:44:03.872344+0000","last_undegraded":"2023-09-23T22:44:03.872344+0000","last_fullsized":"2023-09-23T22:44:03.872344+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:16.067396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:16.067396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:58.817933+0000","scrub_duration":0.0072300120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1527,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.874214+0000","last_change":"2023-09-23T22:42:56.548561+0000","last_active":"2023-09-23T22:44:03.874214+0000","last_peered":"2023-09-23T22:44:03.874214+0000","last_clean":"2023-09-23T22:44:03.874214+0000","last_became_active":"2023-09-23T22:42:56.548134+0000","last_became_peered":"2023-09-23T22:42:56.548134+0000","last_unstale":"2023-09-23T22:44:03.874214+0000","last_undegraded":"2023-09-23T22:44:03.874214+0000","last_fullsized":"2023-09-23T22:44:03.874214+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:36.511009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":507,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.869649+0000","last_change":"2023-09-23T22:42:57.574455+0000","last_active":"2023-09-23T22:44:02.869649+0000","last_peered":"2023-09-23T22:44:02.869649+0000","last_clean":"2023-09-23T22:44:02.869649+0000","last_became_active":"2023-09-23T22:42:57.573786+0000","last_became_peered":"2023-09-23T22:42:57.573786+0000","last_unstale":"2023-09-23T22:44:02.869649+0000","last_undegraded":"2023-09-23T22:44:02.869649+0000","last_fullsized":"2023-09-23T22:44:02.869649+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:09.464699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":261,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.861000+0000","last_change":"2023-09-23T22:42:57.591303+0000","last_active":"2023-09-23T22:44:01.861000+0000","last_peered":"2023-09-23T22:44:01.861000+0000","last_clean":"2023-09-23T22:44:01.861000+0000","last_became_active":"2023-09-23T22:42:57.587701+0000","last_became_peered":"2023-09-23T22:42:57.587701+0000","last_unstale":"2023-09-23T22:44:01.861000+0000","last_undegraded":"2023-09-23T22:44:01.861000+0000","last_fullsized":"2023-09-23T22:44:01.861000+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:32.621790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":957,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.859106+0000","last_change":"2023-09-23T22:43:14.448779+0000","last_active":"2023-09-23T22:44:01.859106+0000","last_peered":"2023-09-23T22:44:01.859106+0000","last_clean":"2023-09-23T22:44:01.859106+0000","last_became_active":"2023-09-23T22:43:14.448360+0000","last_became_peered":"2023-09-23T22:43:14.448360+0000","last_unstale":"2023-09-23T22:44:01.859106+0000","last_undegraded":"2023-09-23T22:44:01.859106+0000","last_fullsized":"2023-09-23T22:44:01.859106+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:07.291196+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:07.291196+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:38.426551+0000","scrub_duration":0.0016895390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":372,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.858176+0000","last_change":"2023-09-23T22:43:54.199739+0000","last_active":"2023-09-23T22:44:01.858176+0000","last_peered":"2023-09-23T22:44:01.858176+0000","last_clean":"2023-09-23T22:44:01.858176+0000","last_became_active":"2023-09-23T22:42:57.672992+0000","last_became_peered":"2023-09-23T22:42:57.672992+0000","last_unstale":"2023-09-23T22:44:01.858176+0000","last_undegraded":"2023-09-23T22:44:01.858176+0000","last_fullsized":"2023-09-23T22:44:01.858176+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:41.870977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":243,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.858178+0000","last_change":"2023-09-23T22:42:56.548974+0000","last_active":"2023-09-23T22:44:01.858178+0000","last_peered":"2023-09-23T22:44:01.858178+0000","last_clean":"2023-09-23T22:44:01.858178+0000","last_became_active":"2023-09-23T22:42:56.548663+0000","last_became_peered":"2023-09-23T22:42:56.548663+0000","last_unstale":"2023-09-23T22:44:01.858178+0000","last_undegraded":"2023-09-23T22:44:01.858178+0000","last_fullsized":"2023-09-23T22:44:01.858178+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:06.786917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":307,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.879886+0000","last_change":"2023-09-23T22:43:11.756366+0000","last_active":"2023-09-23T22:44:04.879886+0000","last_peered":"2023-09-23T22:44:04.879886+0000","last_clean":"2023-09-23T22:44:04.879886+0000","last_became_active":"2023-09-23T22:43:11.756048+0000","last_became_peered":"2023-09-23T22:43:11.756048+0000","last_unstale":"2023-09-23T22:44:04.879886+0000","last_undegraded":"2023-09-23T22:44:04.879886+0000","last_fullsized":"2023-09-23T22:44:04.879886+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:30.924506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1522,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.874094+0000","last_change":"2023-09-23T22:43:56.860133+0000","last_active":"2023-09-23T22:44:03.874094+0000","last_peered":"2023-09-23T22:44:03.874094+0000","last_clean":"2023-09-23T22:44:03.874094+0000","last_became_active":"2023-09-23T22:43:56.859594+0000","last_became_peered":"2023-09-23T22:43:56.859594+0000","last_unstale":"2023-09-23T22:44:03.874094+0000","last_undegraded":"2023-09-23T22:44:03.874094+0000","last_fullsized":"2023-09-23T22:44:03.874094+0000","mapping_epoch":610,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:54.531616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":813,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.873801+0000","last_change":"2023-09-23T22:44:00.574883+0000","last_active":"2023-09-23T22:44:02.873801+0000","last_peered":"2023-09-23T22:44:02.873801+0000","last_clean":"2023-09-23T22:44:02.873801+0000","last_became_active":"2023-09-23T22:42:57.600816+0000","last_became_peered":"2023-09-23T22:42:57.600816+0000","last_unstale":"2023-09-23T22:44:02.873801+0000","last_undegraded":"2023-09-23T22:44:02.873801+0000","last_fullsized":"2023-09-23T22:44:02.873801+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.574795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.574795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:22:03.052349+0000","scrub_duration":0.0023357170000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872251+0000","last_change":"2023-09-23T22:43:11.806599+0000","last_active":"2023-09-23T22:44:03.872251+0000","last_peered":"2023-09-23T22:44:03.872251+0000","last_clean":"2023-09-23T22:44:03.872251+0000","last_became_active":"2023-09-23T22:43:11.806297+0000","last_became_peered":"2023-09-23T22:43:11.806297+0000","last_unstale":"2023-09-23T22:44:03.872251+0000","last_undegraded":"2023-09-23T22:44:03.872251+0000","last_fullsized":"2023-09-23T22:44:03.872251+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:11:17.339834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":259,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.873333+0000","last_change":"2023-09-23T22:42:57.493464+0000","last_active":"2023-09-23T22:44:02.873333+0000","last_peered":"2023-09-23T22:44:02.873333+0000","last_clean":"2023-09-23T22:44:02.873333+0000","last_became_active":"2023-09-23T22:42:57.492614+0000","last_became_peered":"2023-09-23T22:42:57.492614+0000","last_unstale":"2023-09-23T22:44:02.873333+0000","last_undegraded":"2023-09-23T22:44:02.873333+0000","last_fullsized":"2023-09-23T22:44:02.873333+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:13.562710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":825,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.869542+0000","last_change":"2023-09-23T22:43:14.428011+0000","last_active":"2023-09-23T22:44:02.869542+0000","last_peered":"2023-09-23T22:44:02.869542+0000","last_clean":"2023-09-23T22:44:02.869542+0000","last_became_active":"2023-09-23T22:43:14.427691+0000","last_became_peered":"2023-09-23T22:43:14.427691+0000","last_unstale":"2023-09-23T22:44:02.869542+0000","last_undegraded":"2023-09-23T22:44:02.869542+0000","last_fullsized":"2023-09-23T22:44:02.869542+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:56.928006+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:56.928006+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":1084,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.877140+0000","last_change":"2023-09-23T22:43:47.341779+0000","last_active":"2023-09-23T22:44:04.877140+0000","last_peered":"2023-09-23T22:44:04.877140+0000","last_clean":"2023-09-23T22:44:04.877140+0000","last_became_active":"2023-09-23T22:43:11.808371+0000","last_became_peered":"2023-09-23T22:43:11.808371+0000","last_unstale":"2023-09-23T22:44:04.877140+0000","last_undegraded":"2023-09-23T22:44:04.877140+0000","last_fullsized":"2023-09-23T22:44:04.877140+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:38.132844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":123,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872930+0000","last_change":"2023-09-23T22:42:57.480877+0000","last_active":"2023-09-23T22:44:03.872930+0000","last_peered":"2023-09-23T22:44:03.872930+0000","last_clean":"2023-09-23T22:44:03.872930+0000","last_became_active":"2023-09-23T22:42:57.480576+0000","last_became_peered":"2023-09-23T22:42:57.480576+0000","last_unstale":"2023-09-23T22:44:03.872930+0000","last_undegraded":"2023-09-23T22:44:03.872930+0000","last_fullsized":"2023-09-23T22:44:03.872930+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:39.895915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":446,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.862016+0000","last_change":"2023-09-23T22:43:11.809271+0000","last_active":"2023-09-23T22:44:01.862016+0000","last_peered":"2023-09-23T22:44:01.862016+0000","last_clean":"2023-09-23T22:44:01.862016+0000","last_became_active":"2023-09-23T22:43:11.809001+0000","last_became_peered":"2023-09-23T22:43:11.809001+0000","last_unstale":"2023-09-23T22:44:01.862016+0000","last_undegraded":"2023-09-23T22:44:01.862016+0000","last_fullsized":"2023-09-23T22:44:01.862016+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:18.092293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":941,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.869249+0000","last_change":"2023-09-23T22:44:02.567423+0000","last_active":"2023-09-23T22:44:02.869249+0000","last_peered":"2023-09-23T22:44:02.869249+0000","last_clean":"2023-09-23T22:44:02.869249+0000","last_became_active":"2023-09-23T22:43:14.445311+0000","last_became_peered":"2023-09-23T22:43:14.445311+0000","last_unstale":"2023-09-23T22:44:02.869249+0000","last_undegraded":"2023-09-23T22:44:02.869249+0000","last_fullsized":"2023-09-23T22:44:02.869249+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_clean_scrub_stamp":"2023-09-23T22:44:02.567345+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:03.924259+0000","scrub_duration":0.0019136540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":53,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.872061+0000","last_change":"2023-09-23T22:43:27.513607+0000","last_active":"2023-09-23T22:44:02.872061+0000","last_peered":"2023-09-23T22:44:02.872061+0000","last_clean":"2023-09-23T22:44:02.872061+0000","last_became_active":"2023-09-23T22:42:57.674485+0000","last_became_peered":"2023-09-23T22:42:57.674485+0000","last_unstale":"2023-09-23T22:44:02.872061+0000","last_undegraded":"2023-09-23T22:44:02.872061+0000","last_fullsized":"2023-09-23T22:44:02.872061+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:57.916453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":265,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.861044+0000","last_change":"2023-09-23T22:43:11.749267+0000","last_active":"2023-09-23T22:44:01.861044+0000","last_peered":"2023-09-23T22:44:01.861044+0000","last_clean":"2023-09-23T22:44:01.861044+0000","last_became_active":"2023-09-23T22:43:11.748938+0000","last_became_peered":"2023-09-23T22:43:11.748938+0000","last_unstale":"2023-09-23T22:44:01.861044+0000","last_undegraded":"2023-09-23T22:44:01.861044+0000","last_fullsized":"2023-09-23T22:44:01.861044+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:38.249466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.874173+0000","last_change":"2023-09-23T22:43:11.794701+0000","last_active":"2023-09-23T22:44:03.874173+0000","last_peered":"2023-09-23T22:44:03.874173+0000","last_clean":"2023-09-23T22:44:03.874173+0000","last_became_active":"2023-09-23T22:43:11.794419+0000","last_became_peered":"2023-09-23T22:43:11.794419+0000","last_unstale":"2023-09-23T22:44:03.874173+0000","last_undegraded":"2023-09-23T22:44:03.874173+0000","last_fullsized":"2023-09-23T22:44:03.874173+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:21.897331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":957,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.860544+0000","last_change":"2023-09-23T22:43:14.458068+0000","last_active":"2023-09-23T22:44:01.860544+0000","last_peered":"2023-09-23T22:44:01.860544+0000","last_clean":"2023-09-23T22:44:01.860544+0000","last_became_active":"2023-09-23T22:43:14.444011+0000","last_became_peered":"2023-09-23T22:43:14.444011+0000","last_unstale":"2023-09-23T22:44:01.860544+0000","last_undegraded":"2023-09-23T22:44:01.860544+0000","last_fullsized":"2023-09-23T22:44:01.860544+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:27.290379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:27.290379+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:55.282838+0000","scrub_duration":0.0016473779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":98,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.873665+0000","last_change":"2023-09-23T22:42:05.533737+0000","last_active":"2023-09-23T22:44:03.873665+0000","last_peered":"2023-09-23T22:44:03.873665+0000","last_clean":"2023-09-23T22:44:03.873665+0000","last_became_active":"2023-09-23T22:42:05.525900+0000","last_became_peered":"2023-09-23T22:42:05.525900+0000","last_unstale":"2023-09-23T22:44:03.873665+0000","last_undegraded":"2023-09-23T22:44:03.873665+0000","last_fullsized":"2023-09-23T22:44:03.873665+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:31.248241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":1215,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.882441+0000","last_change":"2023-09-23T22:43:39.799350+0000","last_active":"2023-09-23T22:44:04.882441+0000","last_peered":"2023-09-23T22:44:04.882441+0000","last_clean":"2023-09-23T22:44:04.882441+0000","last_became_active":"2023-09-23T22:42:57.585398+0000","last_became_peered":"2023-09-23T22:42:57.585398+0000","last_unstale":"2023-09-23T22:44:04.882441+0000","last_undegraded":"2023-09-23T22:44:04.882441+0000","last_fullsized":"2023-09-23T22:44:04.882441+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:06.542366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872484+0000","last_change":"2023-09-23T22:43:11.831381+0000","last_active":"2023-09-23T22:44:03.872484+0000","last_peered":"2023-09-23T22:44:03.872484+0000","last_clean":"2023-09-23T22:44:03.872484+0000","last_became_active":"2023-09-23T22:43:11.830843+0000","last_became_peered":"2023-09-23T22:43:11.830843+0000","last_unstale":"2023-09-23T22:44:03.872484+0000","last_undegraded":"2023-09-23T22:44:03.872484+0000","last_fullsized":"2023-09-23T22:44:03.872484+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:26.690562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":774,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872677+0000","last_change":"2023-09-23T22:43:14.436289+0000","last_active":"2023-09-23T22:44:03.872677+0000","last_peered":"2023-09-23T22:44:03.872677+0000","last_clean":"2023-09-23T22:44:03.872677+0000","last_became_active":"2023-09-23T22:43:14.436041+0000","last_became_peered":"2023-09-23T22:43:14.436041+0000","last_unstale":"2023-09-23T22:44:03.872677+0000","last_undegraded":"2023-09-23T22:44:03.872677+0000","last_fullsized":"2023-09-23T22:44:03.872677+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:03.084589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:03.084589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:03.001839+0000","scrub_duration":0.013840414000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":284,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.871717+0000","last_change":"2023-09-23T22:42:57.606062+0000","last_active":"2023-09-23T22:44:02.871717+0000","last_peered":"2023-09-23T22:44:02.871717+0000","last_clean":"2023-09-23T22:44:02.871717+0000","last_became_active":"2023-09-23T22:42:57.605639+0000","last_became_peered":"2023-09-23T22:42:57.605639+0000","last_unstale":"2023-09-23T22:44:02.871717+0000","last_undegraded":"2023-09-23T22:44:02.871717+0000","last_fullsized":"2023-09-23T22:44:02.871717+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:13.114886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":421,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.882594+0000","last_change":"2023-09-23T22:43:11.913285+0000","last_active":"2023-09-23T22:44:04.882594+0000","last_peered":"2023-09-23T22:44:04.882594+0000","last_clean":"2023-09-23T22:44:04.882594+0000","last_became_active":"2023-09-23T22:43:11.841729+0000","last_became_peered":"2023-09-23T22:43:11.841729+0000","last_unstale":"2023-09-23T22:44:04.882594+0000","last_undegraded":"2023-09-23T22:44:04.882594+0000","last_fullsized":"2023-09-23T22:44:04.882594+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:45.396946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":27,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.872461+0000","last_change":"2023-09-23T22:43:11.808955+0000","last_active":"2023-09-23T22:44:03.872461+0000","last_peered":"2023-09-23T22:44:03.872461+0000","last_clean":"2023-09-23T22:44:03.872461+0000","last_became_active":"2023-09-23T22:43:11.808676+0000","last_became_peered":"2023-09-23T22:43:11.808676+0000","last_unstale":"2023-09-23T22:44:03.872461+0000","last_undegraded":"2023-09-23T22:44:03.872461+0000","last_fullsized":"2023-09-23T22:44:03.872461+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:38.659893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":933,"reported_epoch":614,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:44:03.879134+0000","last_change":"2023-09-23T22:44:03.879053+0000","last_active":"2023-09-23T22:44:03.879134+0000","last_peered":"2023-09-23T22:44:03.879134+0000","last_clean":"2023-09-23T22:42:54.464183+0000","last_became_active":"2023-09-23T22:44:03.879053+0000","last_became_peered":"2023-09-23T22:44:03.879053+0000","last_unstale":"2023-09-23T22:44:03.879134+0000","last_undegraded":"2023-09-23T22:44:03.879134+0000","last_fullsized":"2023-09-23T22:44:03.873004+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.397050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.397050+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5],"avail_no_missing":["6","1","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":118,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T22:44:04.879525+0000","last_change":"2023-09-23T22:42:57.598277+0000","last_active":"2023-09-23T22:44:04.879525+0000","last_peered":"2023-09-23T22:44:04.879525+0000","last_clean":"2023-09-23T22:44:04.879525+0000","last_became_active":"2023-09-23T22:42:57.597389+0000","last_became_peered":"2023-09-23T22:42:57.597389+0000","last_unstale":"2023-09-23T22:44:04.879525+0000","last_undegraded":"2023-09-23T22:44:04.879525+0000","last_fullsized":"2023-09-23T22:44:04.879525+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:43.074446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":280,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.861090+0000","last_change":"2023-09-23T22:43:11.749040+0000","last_active":"2023-09-23T22:44:01.861090+0000","last_peered":"2023-09-23T22:44:01.861090+0000","last_clean":"2023-09-23T22:44:01.861090+0000","last_became_active":"2023-09-23T22:43:11.748649+0000","last_became_peered":"2023-09-23T22:43:11.748649+0000","last_unstale":"2023-09-23T22:44:01.861090+0000","last_undegraded":"2023-09-23T22:44:01.861090+0000","last_fullsized":"2023-09-23T22:44:01.861090+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:52.013195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":940,"reported_epoch":614,"state":"active+undersized+remapped","last_fresh":"2023-09-23T22:44:03.880963+0000","last_change":"2023-09-23T22:44:03.880809+0000","last_active":"2023-09-23T22:44:03.880963+0000","last_peered":"2023-09-23T22:44:03.880963+0000","last_clean":"2023-09-23T22:43:56.854197+0000","last_became_active":"2023-09-23T22:44:03.880809+0000","last_became_peered":"2023-09-23T22:44:03.880809+0000","last_unstale":"2023-09-23T22:44:03.880963+0000","last_undegraded":"2023-09-23T22:44:03.880963+0000","last_fullsized":"2023-09-23T22:44:03.875914+0000","mapping_epoch":613,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5],"avail_no_missing":["6","1","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":667,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.873505+0000","last_change":"2023-09-23T22:42:57.577409+0000","last_active":"2023-09-23T22:44:03.873505+0000","last_peered":"2023-09-23T22:44:03.873505+0000","last_clean":"2023-09-23T22:44:03.873505+0000","last_became_active":"2023-09-23T22:42:57.551354+0000","last_became_peered":"2023-09-23T22:42:57.551354+0000","last_unstale":"2023-09-23T22:44:03.873505+0000","last_undegraded":"2023-09-23T22:44:03.873505+0000","last_fullsized":"2023-09-23T22:44:03.873505+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:14.859975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":646,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T22:44:05.883180+0000","last_change":"2023-09-23T22:42:57.597909+0000","last_active":"2023-09-23T22:44:05.883180+0000","last_peered":"2023-09-23T22:44:05.883180+0000","last_clean":"2023-09-23T22:44:05.883180+0000","last_became_active":"2023-09-23T22:42:57.596972+0000","last_became_peered":"2023-09-23T22:42:57.596972+0000","last_unstale":"2023-09-23T22:44:05.883180+0000","last_undegraded":"2023-09-23T22:44:05.883180+0000","last_fullsized":"2023-09-23T22:44:05.883180+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:01.735792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":682,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.875650+0000","last_change":"2023-09-23T22:42:57.481811+0000","last_active":"2023-09-23T22:44:03.875650+0000","last_peered":"2023-09-23T22:44:03.875650+0000","last_clean":"2023-09-23T22:44:03.875650+0000","last_became_active":"2023-09-23T22:42:57.481461+0000","last_became_peered":"2023-09-23T22:42:57.481461+0000","last_unstale":"2023-09-23T22:44:03.875650+0000","last_undegraded":"2023-09-23T22:44:03.875650+0000","last_fullsized":"2023-09-23T22:44:03.875650+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:25.977092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":106,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.858992+0000","last_change":"2023-09-23T22:42:57.645578+0000","last_active":"2023-09-23T22:44:01.858992+0000","last_peered":"2023-09-23T22:44:01.858992+0000","last_clean":"2023-09-23T22:44:01.858992+0000","last_became_active":"2023-09-23T22:42:57.644610+0000","last_became_peered":"2023-09-23T22:42:57.644610+0000","last_unstale":"2023-09-23T22:44:01.858992+0000","last_undegraded":"2023-09-23T22:44:01.858992+0000","last_fullsized":"2023-09-23T22:44:01.858992+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:51.348779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":722,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.908298+0000","last_change":"2023-09-23T22:44:02.893662+0000","last_active":"2023-09-23T22:44:02.908298+0000","last_peered":"2023-09-23T22:44:02.908298+0000","last_clean":"2023-09-23T22:44:02.908298+0000","last_became_active":"2023-09-23T22:44:02.892760+0000","last_became_peered":"2023-09-23T22:44:02.892760+0000","last_unstale":"2023-09-23T22:44:02.908298+0000","last_undegraded":"2023-09-23T22:44:02.908298+0000","last_fullsized":"2023-09-23T22:44:02.908298+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:56.024308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:56.024308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:30.183672+0000","scrub_duration":0.0045490360000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":210,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.870194+0000","last_change":"2023-09-23T22:42:57.720541+0000","last_active":"2023-09-23T22:44:02.870194+0000","last_peered":"2023-09-23T22:44:02.870194+0000","last_clean":"2023-09-23T22:44:02.870194+0000","last_became_active":"2023-09-23T22:42:57.717881+0000","last_became_peered":"2023-09-23T22:42:57.717881+0000","last_unstale":"2023-09-23T22:44:02.870194+0000","last_undegraded":"2023-09-23T22:44:02.870194+0000","last_fullsized":"2023-09-23T22:44:02.870194+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:16.534160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":224,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T22:44:01.857943+0000","last_change":"2023-09-23T22:43:11.853883+0000","last_active":"2023-09-23T22:44:01.857943+0000","last_peered":"2023-09-23T22:44:01.857943+0000","last_clean":"2023-09-23T22:44:01.857943+0000","last_became_active":"2023-09-23T22:43:11.840934+0000","last_became_peered":"2023-09-23T22:43:11.840934+0000","last_unstale":"2023-09-23T22:44:01.857943+0000","last_undegraded":"2023-09-23T22:44:01.857943+0000","last_fullsized":"2023-09-23T22:44:01.857943+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:27.566762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":970,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T22:44:05.894742+0000","last_change":"2023-09-23T22:44:05.893864+0000","last_active":"2023-09-23T22:44:05.894742+0000","last_peered":"2023-09-23T22:44:05.894742+0000","last_clean":"2023-09-23T22:44:05.894742+0000","last_became_active":"2023-09-23T22:44:05.893517+0000","last_became_peered":"2023-09-23T22:44:05.893517+0000","last_unstale":"2023-09-23T22:44:05.894742+0000","last_undegraded":"2023-09-23T22:44:05.894742+0000","last_fullsized":"2023-09-23T22:44:05.894742+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:53.343498+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:53.343498+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:19.183253+0000","scrub_duration":0.0041572029999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":33,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T22:44:03.873372+0000","last_change":"2023-09-23T22:43:11.877374+0000","last_active":"2023-09-23T22:44:03.873372+0000","last_peered":"2023-09-23T22:44:03.873372+0000","last_clean":"2023-09-23T22:44:03.873372+0000","last_became_active":"2023-09-23T22:43:11.805693+0000","last_became_peered":"2023-09-23T22:43:11.805693+0000","last_unstale":"2023-09-23T22:44:03.873372+0000","last_undegraded":"2023-09-23T22:44:03.873372+0000","last_fullsized":"2023-09-23T22:44:03.873372+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:06.140025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":597,"reported_epoch":613,"state":"active+clean","last_fresh":"2023-09-23T22:44:02.873798+0000","last_change":"2023-09-23T22:42:57.601834+0000","last_active":"2023-09-23T22:44:02.873798+0000","last_peered":"2023-09-23T22:44:02.873798+0000","last_clean":"2023-09-23T22:44:02.873798+0000","last_became_active":"2023-09-23T22:42:57.600569+0000","last_became_peered":"2023-09-23T22:42:57.600569+0000","last_unstale":"2023-09-23T22:44:02.873798+0000","last_undegraded":"2023-09-23T22:44:02.873798+0000","last_fullsized":"2023-09-23T22:44:02.873798+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:13.910515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":47046,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":26924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62789,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":29,"num_read_kb":55,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":370028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2908160,"data_stored":2886848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":89,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":417,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234476,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":592,"seq":2542620639247,"num_pgs":105,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":684636,"kb_used_data":572820,"kb_used_omap":8,"kb_used_meta":109879,"kb_avail":93060516,"statfs":{"total":95995035648,"available":95293968384,"internally_reserved":0,"allocated":586567680,"data_stored":584574769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":112516962},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":3.996,"5min":3.996,"15min":3.996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":47.795000000000002,"5min":47.795000000000002,"15min":47.795000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":3.3039999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":47.682000000000002,"5min":47.682000000000002,"15min":47.682000000000002},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":7.4900000000000002,"5min":7.4900000000000002,"15min":7.4900000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":115.848,"5min":115.848,"15min":115.848},"last":0.42099999999999999},{"interface":"front","average":{"1min":7.7279999999999998,"5min":7.7279999999999998,"15min":7.7279999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":116.517,"5min":116.517,"15min":116.517},"last":0.371}]},{"osd":2,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.6379999999999999,"5min":3.6379999999999999,"15min":3.6379999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.637,"5min":2.637,"15min":2.637},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.8490000000000002,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.2,"5min":1.2,"15min":1.2},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.6260000000000003,"5min":5.6260000000000003,"15min":5.6260000000000003},"last":0.53000000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.66400000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.339,"5min":1.339,"15min":1.339},"last":0.67600000000000005}]}]},{"osd":1,"up_from":495,"seq":2126008811598,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":436664,"kb_used_data":12656,"kb_used_omap":12,"kb_used_meta":423987,"kb_avail":93308488,"statfs":{"total":95995035648,"available":95547891712,"internally_reserved":0,"allocated":12959744,"data_stored":9843060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":434163374},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":2.2389999999999999,"15min":1.099},"min":{"1min":0.20200000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.0249999999999999,"5min":64.507000000000005,"15min":64.507000000000005},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.323,"15min":1.375},"min":{"1min":0.24399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.8460000000000001,"5min":64.352000000000004,"15min":64.352000000000004},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":12.957000000000001,"5min":3.7730000000000001,"15min":1.7050000000000001},"min":{"1min":0.254,"5min":0.192,"15min":0.192},"max":{"1min":124.447,"5min":124.447,"15min":124.447},"last":0.60899999999999999},{"interface":"front","average":{"1min":12.526,"5min":3.3849999999999998,"15min":1.6419999999999999},"min":{"1min":0.29399999999999998,"5min":0.218,"15min":0.218},"max":{"1min":124.03,"5min":124.03,"15min":124.03},"last":0.44500000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:43:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.7949999999999999,"15min":1.7949999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.800000000000001,"5min":11.800000000000001,"15min":11.800000000000001},"last":0.501},{"interface":"front","average":{"1min":1.7030000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":6.4530000000000003,"5min":6.4530000000000003,"15min":6.4530000000000003},"last":0.58799999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.0249999999999999,"15min":0.70899999999999996},"min":{"1min":0.38,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.0750000000000002,"5min":5.5439999999999996,"15min":5.5439999999999996},"last":0.747},{"interface":"front","average":{"1min":0.85299999999999998,"5min":1.01,"15min":1.004},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.4510000000000001,"5min":5.524,"15min":5.6740000000000004},"last":0.70099999999999996}]},{"osd":5,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":1.167,"15min":0.95899999999999996},"min":{"1min":0.42399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":11.734,"5min":11.734,"15min":11.734},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.0600000000000001,"15min":0.755},"min":{"1min":0.439,"5min":0.31,"15min":0.31},"max":{"1min":3.1389999999999998,"5min":5.5839999999999996,"15min":5.5839999999999996},"last":0.79600000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:43:55 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":1.736,"15min":1.736},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":9.3759999999999994,"5min":9.3759999999999994,"15min":9.3759999999999994},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.877,"5min":0.877,"15min":0.877},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.9540000000000002,"5min":2.9540000000000002,"15min":2.9540000000000002},"last":0.81200000000000006}]},{"osd":7,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0059999999999998,"5min":1.2809999999999999,"15min":1.0049999999999999},"min":{"1min":0.42999999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.768000000000001,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.119,"15min":0.80100000000000005},"min":{"1min":0.496,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.169,"5min":5.6120000000000001,"15min":5.6120000000000001},"last":0.68799999999999994}]}]},{"osd":0,"up_from":0,"seq":399431958914,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":349,"seq":1498943586476,"num_pgs":84,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":517404,"kb_used_data":8212,"kb_used_omap":15,"kb_used_meta":509168,"kb_avail":93227748,"statfs":{"total":95995035648,"available":95465213952,"internally_reserved":0,"allocated":8409088,"data_stored":5295675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":521388790},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.77,"15min":6.024},"min":{"1min":0.22600000000000001,"5min":0.182,"15min":0.182},"max":{"1min":10.539999999999999,"5min":80.126000000000005,"15min":194.06299999999999},"last":0.25600000000000001},{"interface":"front","average":{"1min":4.0439999999999996,"5min":2.9689999999999999,"15min":6.7220000000000004},"min":{"1min":0.29099999999999998,"5min":0.215,"15min":0.215},"max":{"1min":15.439,"5min":83.899000000000001,"15min":196.947},"last":0.31}]},{"osd":1,"last update":"Sat Sep 23 22:43:49 2023","interfaces":[{"interface":"back","average":{"1min":5.2510000000000003,"5min":2.831,"15min":1.371},"min":{"1min":0.224,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":47.018000000000001,"5min":70.042000000000002,"15min":70.042000000000002},"last":0.35499999999999998},{"interface":"front","average":{"1min":7.2069999999999999,"5min":4.9139999999999997,"15min":2.3690000000000002},"min":{"1min":0.23599999999999999,"5min":0.184,"15min":0.184},"max":{"1min":47.671999999999997,"5min":127.327,"15min":127.327},"last":0.27900000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:43:32 2023","interfaces":[{"interface":"back","average":{"1min":3.9169999999999998,"5min":3.9169999999999998,"15min":3.9169999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.561,"5min":14.561,"15min":14.561},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.8959999999999999,"5min":1.8959999999999999,"15min":1.8959999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.7720000000000002,"5min":4.7720000000000002,"15min":4.7720000000000002},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:43:08 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.8340000000000001,"15min":1.919},"min":{"1min":0.44800000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.8710000000000004,"5min":45.576999999999998,"15min":45.576999999999998},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.913,"5min":1.518,"15min":1.3120000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.157,"5min":45.420999999999999,"15min":45.420999999999999},"last":0.76300000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":2.504,"5min":1.4810000000000001,"15min":3.3980000000000001},"min":{"1min":0.41499999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.8119999999999994,"5min":8.8119999999999994,"15min":163.56399999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":8.3710000000000004,"5min":2.4199999999999999,"15min":3.4119999999999999},"min":{"1min":0.39900000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":126.81100000000001,"5min":126.81100000000001,"15min":163.541},"last":0.72099999999999997}]},{"osd":6,"last update":"Sat Sep 23 22:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2629999999999999,"5min":2.2629999999999999,"15min":2.2629999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":7.351,"5min":7.351,"15min":7.351},"last":0.73799999999999999},{"interface":"front","average":{"1min":1.476,"5min":1.476,"15min":1.476},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":3.895,"5min":3.895,"15min":3.895},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:43:13 2023","interfaces":[{"interface":"back","average":{"1min":2.9620000000000002,"5min":1.9550000000000001,"15min":1.919},"min":{"1min":0.52400000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.3979999999999997,"5min":75.513999999999996,"15min":108.113},"last":0.79000000000000004},{"interface":"front","average":{"1min":2.5619999999999998,"5min":2.1629999999999998,"15min":2.2949999999999999},"min":{"1min":0.45400000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.8480000000000008,"5min":74.007999999999996,"15min":108.093},"last":0.627}]}]},{"osd":3,"up_from":0,"seq":2508260900884,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":0,"seq":1997159792740,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757749,"num_pgs":84,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":565164,"kb_used_data":11428,"kb_used_omap":0,"kb_used_meta":553728,"kb_avail":93179988,"statfs":{"total":95995035648,"available":95416307712,"internally_reserved":0,"allocated":11702272,"data_stored":9708704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":567017472},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:40 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.1799999999999999,"15min":3.6709999999999998},"min":{"1min":0.24099999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.1739999999999995,"5min":26.925000000000001,"15min":145.46600000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.135,"5min":1.179,"15min":3.5649999999999999},"min":{"1min":0.245,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.4740000000000002,"5min":30.609000000000002,"15min":144.429},"last":0.79600000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:43:45 2023","interfaces":[{"interface":"back","average":{"1min":3.4910000000000001,"5min":1.8700000000000001,"15min":1.0800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":35.643999999999998,"5min":35.643999999999998,"15min":35.643999999999998},"last":0.81599999999999995},{"interface":"front","average":{"1min":3.6949999999999998,"5min":1.869,"15min":0.98199999999999998},"min":{"1min":0.29399999999999998,"5min":0.25,"15min":0.249},"max":{"1min":35.316000000000003,"5min":35.316000000000003,"15min":35.316000000000003},"last":0.63900000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:43:12 2023","interfaces":[{"interface":"back","average":{"1min":3.8620000000000001,"5min":1.7410000000000001,"15min":2.2829999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.26200000000000001},"max":{"1min":25.899000000000001,"5min":25.899000000000001,"15min":70.153000000000006},"last":0.76900000000000002},{"interface":"front","average":{"1min":2.9079999999999999,"5min":1.3440000000000001,"15min":2.1749999999999998},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":25.992000000000001,"5min":25.992000000000001,"15min":70.078999999999994},"last":0.75}]},{"osd":3,"last update":"Sat Sep 23 22:43:33 2023","interfaces":[{"interface":"back","average":{"1min":1.532,"5min":1.532,"15min":1.532},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.484,"5min":1.484,"15min":1.484},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.8959999999999999,"5min":7.8959999999999999,"15min":7.8959999999999999},"last":0.91700000000000004}]},{"osd":4,"last update":"Sat Sep 23 22:43:08 2023","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.5,"15min":2.6909999999999998},"min":{"1min":0.24399999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.60299999999999998,"5min":4.0750000000000002,"15min":61.173999999999999},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.434,"5min":0.58499999999999996,"15min":2.7599999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":0.66500000000000004,"5min":4.157,"15min":61.203000000000003},"last":0.93700000000000006}]},{"osd":6,"last update":"Sat Sep 23 22:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.72399999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:43:40 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.83499999999999996,"15min":1.3160000000000001},"min":{"1min":0.34399999999999997,"5min":0.249,"15min":0.249},"max":{"1min":1.4550000000000001,"5min":31.042999999999999,"15min":59.101999999999997},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.002,"15min":1.538},"min":{"1min":0.33600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.1950000000000001,"5min":30.882999999999999,"15min":59.151000000000003},"last":0.89000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8798,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:44:12.403 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T22:44:12.527 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:12.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:12.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:12.553 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:12.631 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:12.658 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:12.760 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:12.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:12.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:12.869 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:12.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:12 smithi070 ceph-mon[192652]: pgmap v1597: 103 pgs: 103 active+clean; 4.6 MiB data, 2.0 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:12 smithi070 ceph-mon[192652]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:44:12.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:12 smithi026 ceph-mon[223014]: pgmap v1597: 103 pgs: 103 active+clean; 4.6 MiB data, 2.0 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:12.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:12 smithi026 ceph-mon[223014]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:44:12.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:12 smithi026 ceph-mon[215775]: pgmap v1597: 103 pgs: 103 active+clean; 4.6 MiB data, 2.0 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:12.898 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:12 smithi026 ceph-mon[215775]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T22:44:12.900 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:13.266 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:44:13.266 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:44:13.352 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:13.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:13.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:13.456 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:13.486 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:13.628 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T22:44:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[215775]: failed to encode map e617 with expected crc 2023-09-23T22:44:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[215775]: failed to encode map e617 with expected crc 2023-09-23T22:44:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[215775]: osdmap e617: 8 total, 8 up, 4 in 2023-09-23T22:44:13.632 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3760498777' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-682481-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:44:13.633 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-682481-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:44:13.728 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_all_features (100 ms) 2023-09-23T22:44:13.728 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T22:44:13.905 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.copyup (177 ms) 2023-09-23T22:44:13.905 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T22:44:13.917 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.sparse_copyup (12 ms) 2023-09-23T22:44:13.917 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T22:44:13.923 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_and_set_id (7 ms) 2023-09-23T22:44:13.924 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T22:44:13.933 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.add_remove_child (9 ms) 2023-09-23T22:44:13.933 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T22:44:13.958 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.directory_methods (26 ms) 2023-09-23T22:44:13.958 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.create 2023-09-23T22:44:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[223014]: failed to encode map e617 with expected crc 2023-09-23T22:44:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[223014]: failed to encode map e617 with expected crc 2023-09-23T22:44:13.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[223014]: osdmap e617: 8 total, 8 up, 4 in 2023-09-23T22:44:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3760498777' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-682481-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:44:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:13 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-682481-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:44:14.002 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.create (44 ms) 2023-09-23T22:44:14.002 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T22:44:14.004 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-09-23T22:44:14.004 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T22:44:14.032 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_object_prefix (29 ms) 2023-09-23T22:44:14.033 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T22:44:14.034 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-09-23T22:44:14.034 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T22:44:14.035 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2023-09-23T22:44:14.035 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T22:44:14.036 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-09-23T22:44:14.036 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T22:44:14.039 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-09-23T22:44:14.039 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T22:44:14.042 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-09-23T22:44:14.043 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T22:44:14.047 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.set_size (4 ms) 2023-09-23T22:44:14.047 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T22:44:14.064 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2023-09-23T22:44:14.064 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T22:44:14.071 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2023-09-23T22:44:14.071 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T22:44:14.099 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:14.101 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:13 smithi070 ceph-mon[192652]: failed to encode map e617 with expected crc 2023-09-23T22:44:14.101 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:13 smithi070 ceph-mon[192652]: failed to encode map e617 with expected crc 2023-09-23T22:44:14.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:13 smithi070 ceph-mon[192652]: osdmap e617: 8 total, 8 up, 4 in 2023-09-23T22:44:14.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3760498777' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-682481-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:44:14.102 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:13 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-682481-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T22:44:14.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:14.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:14.111 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.parents_v1 (41 ms) 2023-09-23T22:44:14.111 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T22:44:14.139 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.parents_v2 (27 ms) 2023-09-23T22:44:14.139 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T22:44:14.160 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapshots (21 ms) 2023-09-23T22:44:14.160 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T22:44:14.164 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.snapid_race (4 ms) 2023-09-23T22:44:14.164 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T22:44:14.174 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2023-09-23T22:44:14.174 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T22:44:14.175 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2023-09-23T22:44:14.175 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T22:44:14.180 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-09-23T22:44:14.180 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T22:44:14.185 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_update (4 ms) 2023-09-23T22:44:14.185 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T22:44:14.185 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T22:44:14.186 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T22:44:14.188 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_snap_add (4 ms) 2023-09-23T22:44:14.189 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T22:44:14.192 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.object_map_snap_remove (3 ms) 2023-09-23T22:44:14.192 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T22:44:14.194 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T22:44:14.197 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.flags (6 ms) 2023-09-23T22:44:14.197 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T22:44:14.204 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:14.228 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.metadata (30 ms) 2023-09-23T22:44:14.228 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T22:44:14.232 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:14.239 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.set_features (11 ms) 2023-09-23T22:44:14.239 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T22:44:14.257 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:14.282 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:14.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:14.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:14.298 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:44:14.318 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror (80 ms) 2023-09-23T22:44:14.318 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T22:44:14.347 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image (29 ms) 2023-09-23T22:44:14.348 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T22:44:14.389 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:14.436 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:14.601 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:14.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:14.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:14.705 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:14.734 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:14.839 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:14.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:14.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:14.943 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:14.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:14 smithi070 ceph-mon[192652]: failed to encode map e617 with expected crc 2023-09-23T22:44:14.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:14 smithi070 ceph-mon[192652]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:14 smithi070 ceph-mon[192652]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.968 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:14 smithi070 ceph-mon[192652]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.969 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:14 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-682481-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T22:44:14.969 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:14 smithi070 ceph-mon[192652]: osdmap e618: 8 total, 8 up, 4 in 2023-09-23T22:44:14.969 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:14 smithi070 ceph-mon[192652]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.969 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:14 smithi070 ceph-mon[192652]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.969 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:14 smithi070 ceph-mon[192652]: pgmap v1600: 135 pgs: 32 unknown, 103 active+clean; 4.6 MiB data, 2.0 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:14.970 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[215775]: failed to encode map e617 with expected crc 2023-09-23T22:44:14.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[215775]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[215775]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[215775]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-682481-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T22:44:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[215775]: osdmap e618: 8 total, 8 up, 4 in 2023-09-23T22:44:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[215775]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[215775]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[215775]: pgmap v1600: 135 pgs: 32 unknown, 103 active+clean; 4.6 MiB data, 2.0 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[223014]: failed to encode map e617 with expected crc 2023-09-23T22:44:14.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[223014]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[223014]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[223014]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi026-682481-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T22:44:14.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[223014]: osdmap e618: 8 total, 8 up, 4 in 2023-09-23T22:44:14.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[223014]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[223014]: failed to encode map e618 with expected crc 2023-09-23T22:44:14.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:14 smithi026 ceph-mon[223014]: pgmap v1600: 135 pgs: 32 unknown, 103 active+clean; 4.6 MiB data, 2.0 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:15.022 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:15.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:15.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:15.128 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:15.156 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:15.203 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:15.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:15.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:15.309 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:15.336 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:15.382 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:15.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:15.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:15.486 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:15.512 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:15.557 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:15.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:15.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:15.663 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:15.706 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:15.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[215775]: failed to encode map e619 with expected crc 2023-09-23T22:44:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[215775]: failed to encode map e619 with expected crc 2023-09-23T22:44:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[215775]: osdmap e619: 8 total, 8 up, 4 in 2023-09-23T22:44:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[215775]: failed to encode map e619 with expected crc 2023-09-23T22:44:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:15.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:15.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[223014]: failed to encode map e619 with expected crc 2023-09-23T22:44:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[223014]: failed to encode map e619 with expected crc 2023-09-23T22:44:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[223014]: osdmap e619: 8 total, 8 up, 4 in 2023-09-23T22:44:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[223014]: failed to encode map e619 with expected crc 2023-09-23T22:44:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:15.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:15 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:15.995 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:15.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:16.099 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:16.122 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:16.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:15 smithi070 ceph-mon[192652]: failed to encode map e619 with expected crc 2023-09-23T22:44:16.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:15 smithi070 ceph-mon[192652]: failed to encode map e619 with expected crc 2023-09-23T22:44:16.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:15 smithi070 ceph-mon[192652]: osdmap e619: 8 total, 8 up, 4 in 2023-09-23T22:44:16.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:15 smithi070 ceph-mon[192652]: failed to encode map e619 with expected crc 2023-09-23T22:44:16.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:15 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:16.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:15 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:16.147 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:16.176 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:16.245 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:16.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:16.349 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:16.376 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:16.420 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:16.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:16.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:16.525 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:16.551 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:16.596 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:16.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:16.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:16 smithi026 ceph-mon[215775]: pgmap v1602: 135 pgs: 15 unknown, 120 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail; 31 KiB/s rd, 10 KiB/s wr, 42 op/s 2023-09-23T22:44:16.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2888173568' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:16.643 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:16 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:16.653 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:44:16.711 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:16.736 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:16.779 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:16.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:16.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:16.884 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:16.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:16 smithi026 ceph-mon[223014]: pgmap v1602: 135 pgs: 15 unknown, 120 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail; 31 KiB/s rd, 10 KiB/s wr, 42 op/s 2023-09-23T22:44:16.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2888173568' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:16.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:16 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:16.926 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:17.037 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:44:17.038 INFO:teuthology.orchestra.run.smithi026.stdout:{"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2041,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":620,"num_osds":8,"num_up_osds":8,"osd_up_since":1695508975,"num_in_osds":4,"osd_in_since":1695508924,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":120},{"state_name":"unknown","count":15}],"num_pgs":135,"num_pools":5,"num_objects":42,"data_bytes":4826151,"bytes_used":2048233472,"bytes_avail":381931909120,"bytes_total":383980142592,"unknown_pgs_ratio":0.1111111119389534,"read_bytes_sec":31906,"write_bytes_sec":10578,"read_op_per_sec":30,"write_op_per_sec":12},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi026.rancfk","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.26:8443/","prometheus":"http://172.21.15.26:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T22:43:55.751353+0000","services":{}},"progress_events":{"77f01214-fbdf-4eb1-b58e-606a4fceccf4":{"message":"Global Recovery Event (9m)\n [=======================.....] (remaining: 93s)","progress":0.85185188055038452,"add_to_ceph_s":true},"f7291679-7ac9-4049-9fd0-3ee51dcd8c90":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T22:44:17.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:16 smithi070 ceph-mon[192652]: pgmap v1602: 135 pgs: 15 unknown, 120 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail; 31 KiB/s rd, 10 KiB/s wr, 42 op/s 2023-09-23T22:44:17.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2888173568' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:17.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:16 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:17.606 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image_status (3258 ms) 2023-09-23T22:44:17.606 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T22:44:17.694 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_image_map (89 ms) 2023-09-23T22:44:17.695 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T22:44:17.703 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_instances (8 ms) 2023-09-23T22:44:17.703 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T22:44:17.713 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.mirror_snapshot (11 ms) 2023-09-23T22:44:17.713 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T22:44:17.716 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_list (2 ms) 2023-09-23T22:44:17.716 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T22:44:17.719 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_add (3 ms) 2023-09-23T22:44:17.719 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T22:44:17.722 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T22:44:17.722 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T22:44:17.728 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-09-23T22:44:17.728 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T22:44:17.731 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-09-23T22:44:17.731 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T22:44:17.735 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-09-23T22:44:17.735 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T22:44:17.737 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-09-23T22:44:17.737 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T22:44:17.740 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-09-23T22:44:17.740 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T22:44:17.743 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_list (2 ms) 2023-09-23T22:44:17.743 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T22:44:17.747 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_image_clean (5 ms) 2023-09-23T22:44:17.747 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T22:44:17.749 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-09-23T22:44:17.750 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T22:44:17.752 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-09-23T22:44:17.752 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T22:44:17.754 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-09-23T22:44:17.755 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T22:44:17.756 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-09-23T22:44:17.756 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T22:44:17.758 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (1 ms) 2023-09-23T22:44:17.758 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T22:44:17.760 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-09-23T22:44:17.761 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T22:44:17.763 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (2 ms) 2023-09-23T22:44:17.763 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T22:44:17.765 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-09-23T22:44:17.765 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T22:44:17.769 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-09-23T22:44:17.769 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T22:44:17.783 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (14 ms) 2023-09-23T22:44:17.783 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T22:44:17.975 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (190 ms) 2023-09-23T22:44:17.975 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T22:44:17.977 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_remove (3 ms) 2023-09-23T22:44:17.978 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T22:44:17.980 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T22:44:17.981 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T22:44:17.985 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.trash_methods (6 ms) 2023-09-23T22:44:17.985 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T22:44:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[215775]: failed to encode map e620 with expected crc 2023-09-23T22:44:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[215775]: failed to encode map e620 with expected crc 2023-09-23T22:44:17.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:17.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[215775]: osdmap e620: 8 total, 8 up, 4 in 2023-09-23T22:44:17.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/649625778' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:44:17.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:17.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[223014]: failed to encode map e620 with expected crc 2023-09-23T22:44:17.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[223014]: failed to encode map e620 with expected crc 2023-09-23T22:44:17.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:17.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[223014]: osdmap e620: 8 total, 8 up, 4 in 2023-09-23T22:44:17.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:17 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/649625778' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:44:17.996 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.op_features (12 ms) 2023-09-23T22:44:17.996 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T22:44:18.018 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_parent (20 ms) 2023-09-23T22:44:18.018 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T22:44:18.028 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2023-09-23T22:44:18.028 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T22:44:18.036 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2023-09-23T22:44:18.036 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T22:44:18.042 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-09-23T22:44:18.043 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T22:44:18.054 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-09-23T22:44:18.054 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T22:44:18.062 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.migration_v1 (8 ms) 2023-09-23T22:44:18.062 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T22:44:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:17 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:17 smithi070 ceph-mon[192652]: failed to encode map e620 with expected crc 2023-09-23T22:44:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:17 smithi070 ceph-mon[192652]: failed to encode map e620 with expected crc 2023-09-23T22:44:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:17 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:17 smithi070 ceph-mon[192652]: osdmap e620: 8 total, 8 up, 4 in 2023-09-23T22:44:18.127 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:17 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/649625778' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T22:44:18.290 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:18.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:18.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:18.394 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:18.424 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:18.467 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:18.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:18.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:18.575 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:18.604 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:18 smithi070 ceph-mon[192652]: failed to encode map e620 with expected crc 2023-09-23T22:44:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:18 smithi070 ceph-mon[192652]: failed to encode map e620 with expected crc 2023-09-23T22:44:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:18 smithi070 ceph-mon[192652]: pgmap v1604: 135 pgs: 11 unknown, 124 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail; 80 KiB/s rd, 27 KiB/s wr, 108 op/s 2023-09-23T22:44:18.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:18 smithi026 ceph-mon[223014]: failed to encode map e620 with expected crc 2023-09-23T22:44:18.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:18 smithi026 ceph-mon[223014]: failed to encode map e620 with expected crc 2023-09-23T22:44:18.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:18 smithi026 ceph-mon[223014]: pgmap v1604: 135 pgs: 11 unknown, 124 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail; 80 KiB/s rd, 27 KiB/s wr, 108 op/s 2023-09-23T22:44:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:18 smithi026 ceph-mon[215775]: failed to encode map e620 with expected crc 2023-09-23T22:44:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:18 smithi026 ceph-mon[215775]: failed to encode map e620 with expected crc 2023-09-23T22:44:18.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:18 smithi026 ceph-mon[215775]: pgmap v1604: 135 pgs: 11 unknown, 124 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail; 80 KiB/s rd, 27 KiB/s wr, 108 op/s 2023-09-23T22:44:19.504 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:19.507 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:19.608 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:19.638 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:19.654 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1594 ms) 2023-09-23T22:44:19.654 INFO:tasks.workunit.client.0.smithi026.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T22:44:19.691 INFO:tasks.workunit.client.0.smithi026.stdout:[ OK ] TestClsRbd.sparsify (37 ms) 2023-09-23T22:44:19.699 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:19.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:19.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:19.805 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:19.833 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:44:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:44:19] "GET /metrics HTTP/1.1" 200 34090 "" "Prometheus/2.43.0" 2023-09-23T22:44:19.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[215775]: failed to encode map e621 with expected crc 2023-09-23T22:44:19.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[215775]: failed to encode map e621 with expected crc 2023-09-23T22:44:19.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[215775]: osdmap e621: 8 total, 8 up, 4 in 2023-09-23T22:44:19.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[215775]: failed to encode map e621 with expected crc 2023-09-23T22:44:19.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[215775]: failed to encode map e621 with expected crc 2023-09-23T22:44:19.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[223014]: failed to encode map e621 with expected crc 2023-09-23T22:44:19.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[223014]: failed to encode map e621 with expected crc 2023-09-23T22:44:19.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[223014]: osdmap e621: 8 total, 8 up, 4 in 2023-09-23T22:44:19.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[223014]: failed to encode map e621 with expected crc 2023-09-23T22:44:19.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:19 smithi026 ceph-mon[223014]: failed to encode map e621 with expected crc 2023-09-23T22:44:20.065 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:20.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:20.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:20.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:19 smithi070 ceph-mon[192652]: failed to encode map e621 with expected crc 2023-09-23T22:44:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:19 smithi070 ceph-mon[192652]: failed to encode map e621 with expected crc 2023-09-23T22:44:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:19 smithi070 ceph-mon[192652]: osdmap e621: 8 total, 8 up, 4 in 2023-09-23T22:44:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:19 smithi070 ceph-mon[192652]: failed to encode map e621 with expected crc 2023-09-23T22:44:20.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:19 smithi070 ceph-mon[192652]: failed to encode map e621 with expected crc 2023-09-23T22:44:20.147 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T22:44:20.147 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:44:20.172 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:20.221 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:20.257 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:44:20.435 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:20.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:20.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:20.539 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:20.571 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:20.621 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:20.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:20.662 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] 70 tests from TestClsRbd (6063 ms total) 2023-09-23T22:44:20.662 INFO:tasks.workunit.client.0.smithi026.stdout: 2023-09-23T22:44:20.663 INFO:tasks.workunit.client.0.smithi026.stdout:[----------] Global test environment tear-down 2023-09-23T22:44:20.663 INFO:tasks.workunit.client.0.smithi026.stdout:[==========] 70 tests from 1 test suite ran. (8572 ms total) 2023-09-23T22:44:20.663 INFO:tasks.workunit.client.0.smithi026.stdout:[ PASSED ] 70 tests. 2023-09-23T22:44:20.665 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T22:44:20.665 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T22:44:20.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[215775]: failed to encode map e622 with expected crc 2023-09-23T22:44:20.705 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[215775]: failed to encode map e622 with expected crc 2023-09-23T22:44:20.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[215775]: osdmap e622: 8 total, 8 up, 4 in 2023-09-23T22:44:20.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[215775]: failed to encode map e622 with expected crc 2023-09-23T22:44:20.706 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[215775]: pgmap v1607: 135 pgs: 135 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail; 3.6 MiB/s rd, 1.6 MiB/s wr, 2.76k op/s 2023-09-23T22:44:20.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[223014]: failed to encode map e622 with expected crc 2023-09-23T22:44:20.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[223014]: failed to encode map e622 with expected crc 2023-09-23T22:44:20.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[223014]: osdmap e622: 8 total, 8 up, 4 in 2023-09-23T22:44:20.706 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[223014]: failed to encode map e622 with expected crc 2023-09-23T22:44:20.707 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:20 smithi026 ceph-mon[223014]: pgmap v1607: 135 pgs: 135 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail; 3.6 MiB/s rd, 1.6 MiB/s wr, 2.76k op/s 2023-09-23T22:44:20.711 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:44:20.711 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:44:20.712 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T22:44:20.713 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T22:44:20.727 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:20.783 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:20 smithi070 ceph-mon[192652]: failed to encode map e622 with expected crc 2023-09-23T22:44:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:20 smithi070 ceph-mon[192652]: failed to encode map e622 with expected crc 2023-09-23T22:44:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:20 smithi070 ceph-mon[192652]: osdmap e622: 8 total, 8 up, 4 in 2023-09-23T22:44:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:20 smithi070 ceph-mon[192652]: failed to encode map e622 with expected crc 2023-09-23T22:44:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:20 smithi070 ceph-mon[192652]: pgmap v1607: 135 pgs: 135 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail; 3.6 MiB/s rd, 1.6 MiB/s wr, 2.76k op/s 2023-09-23T22:44:20.952 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:20.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:20.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:21.056 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:21.083 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:21.389 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:44:21.389 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:44:21.389 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:44:21.389 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:44:21.389 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:44:21.389 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:44:21.390 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:44:21.393 DEBUG:teuthology.parallel:result is None 2023-09-23T22:44:21.393 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:44:21.407 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:44:21.409 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:44:21.409 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:44:21.409 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:44:21.409 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:44:21.409 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:44:21.409 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:44:21.409 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:44:21.409 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:44:21.613 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:44:21.613 DEBUG:teuthology.orchestra.run.smithi026:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:44:21.630 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:44:21.630 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T22:44:21.630 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T22:44:21.630 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T22:44:21.631 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T22:44:21.631 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T22:44:21.631 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:44:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:44:21.689 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T22:44:21.690 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:44:21.747 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T22:44:21.748 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T22:44:21.806 INFO:tasks.workunit:timeout=3h 2023-09-23T22:44:21.806 INFO:tasks.workunit:cleanup=True 2023-09-23T22:44:21.807 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T22:44:21.865 INFO:tasks.workunit.client.0.smithi026.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T22:44:21.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[223014]: failed to encode map e623 with expected crc 2023-09-23T22:44:21.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[223014]: failed to encode map e623 with expected crc 2023-09-23T22:44:21.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[223014]: failed to encode map e623 with expected crc 2023-09-23T22:44:21.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[223014]: osdmap e623: 8 total, 8 up, 4 in 2023-09-23T22:44:21.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3352859484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[215775]: failed to encode map e623 with expected crc 2023-09-23T22:44:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[215775]: failed to encode map e623 with expected crc 2023-09-23T22:44:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[215775]: failed to encode map e623 with expected crc 2023-09-23T22:44:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[215775]: osdmap e623: 8 total, 8 up, 4 in 2023-09-23T22:44:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:21 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3352859484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:22.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:21 smithi070 ceph-mon[192652]: failed to encode map e623 with expected crc 2023-09-23T22:44:22.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:21 smithi070 ceph-mon[192652]: failed to encode map e623 with expected crc 2023-09-23T22:44:22.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:21 smithi070 ceph-mon[192652]: failed to encode map e623 with expected crc 2023-09-23T22:44:22.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:21 smithi070 ceph-mon[192652]: osdmap e623: 8 total, 8 up, 4 in 2023-09-23T22:44:22.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:21 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3352859484' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:22.340 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T22:44:22.434 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:22.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:22.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:22.539 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:22.568 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:22.676 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:22.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:22.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:22.731 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:22.787 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:22.789 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:22.820 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:22.870 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:22.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:22.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:22.913 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:22.975 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:22.982 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:22 smithi026 ceph-mon[215775]: pgmap v1609: 103 pgs: 103 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:22.982 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:22 smithi026 ceph-mon[223014]: pgmap v1609: 103 pgs: 103 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:23.001 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:22 smithi070 ceph-mon[192652]: pgmap v1609: 103 pgs: 103 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:23.003 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:23.047 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:23.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:23.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:23.153 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:23.185 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:23.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:23.671 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:23 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:23.942 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:23.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:23.944 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:23 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:23.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:23.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:24.046 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:24.075 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:24.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:24.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:23 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:24.338 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:24.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:24.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:24.442 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:24.470 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:24.686 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:44:25.080 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:25.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:25.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:25.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:25.219 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:25.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[223014]: pgmap v1610: 103 pgs: 103 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:25.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:25.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[215775]: pgmap v1610: 103 pgs: 103 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:25 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.268 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:25.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:25.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:25.373 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:25 smithi070 ceph-mon[192652]: pgmap v1610: 103 pgs: 103 active+clean; 4.6 MiB data, 1.9 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:25 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:25.402 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:25.448 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:25.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:25.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:25.555 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:25.581 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:25.626 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:25.632 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:25.733 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:25.767 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:26.016 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:26.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:26.122 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:26.149 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:26.191 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:26.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:26.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:26.296 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:26.334 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:26.496 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:26.625 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:26.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:26.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:26.730 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:26.752 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:44:26.753 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:44:26.767 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:26.960 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:44:26.961 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:44:26.961 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:44:26.961 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:44:26.961 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:44:27.045 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T22:44:27.104 INFO:tasks.workunit.client.0.smithi026.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T22:44:27.106 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T22:44:27.106 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T22:44:27.125 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:44:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:27 smithi026 ceph-mon[223014]: pgmap v1611: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:27 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1052251594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:27.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:27 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:27 smithi026 ceph-mon[215775]: pgmap v1611: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:27 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1052251594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:27.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:27 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:27 smithi070 ceph-mon[192652]: pgmap v1611: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:27 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1052251594' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:27.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:27 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:44:27.460 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T22:44:27.554 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T22:44:27.676 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T22:44:27.677 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T22:44:27.677 INFO:tasks.workunit.client.0.smithi026.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T22:44:27.806 INFO:tasks.workunit.client.0.smithi026.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T22:44:27.809 DEBUG:teuthology.orchestra.run.smithi026:> set -ex 2023-09-23T22:44:27.809 DEBUG:teuthology.orchestra.run.smithi026:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T22:44:27.867 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T22:44:27.868 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T22:44:27.868 DEBUG:teuthology.orchestra.run.smithi026:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T22:44:27.929 INFO:tasks.workunit.client.0.smithi026.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T22:44:27.929 INFO:tasks.workunit.client.0.smithi026.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T22:44:27.929 INFO:tasks.workunit.client.0.smithi026.stderr:+ TMPDIR=/tmp/rbd_import_export_687972 2023-09-23T22:44:27.929 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm -rf /tmp/rbd_import_export_687972 2023-09-23T22:44:27.929 INFO:tasks.workunit.client.0.smithi026.stderr:+ mkdir /tmp/rbd_import_export_687972 2023-09-23T22:44:27.930 INFO:tasks.workunit.client.0.smithi026.stderr:+ trap 'rm -rf /tmp/rbd_import_export_687972' INT TERM EXIT 2023-09-23T22:44:27.931 INFO:tasks.workunit.client.0.smithi026.stderr:+ mkdir foo.687972 2023-09-23T22:44:27.931 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import foo.687972 foo.dir 2023-09-23T22:44:28.120 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: cannot import a directory 2023-09-23T22:44:28.120 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 0% complete...failed. 2023-09-23T22:44:28.120 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: import failed: (21) Is a directory 2023-09-23T22:44:28.124 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:44:28.124 INFO:tasks.workunit.client.0.smithi026.stderr:+ rmdir foo.687972 2023-09-23T22:44:28.125 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_687972/img bs=1k count=1 seek=10 2023-09-23T22:44:28.126 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:44:28.126 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:44:28.126 INFO:tasks.workunit.client.0.smithi026.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.8251e-05 s, 21.2 MB/s 2023-09-23T22:44:28.126 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_687972/img bs=1k count=10 seek=100 2023-09-23T22:44:28.126 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records in 2023-09-23T22:44:28.126 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records out 2023-09-23T22:44:28.127 INFO:tasks.workunit.client.0.smithi026.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.2763e-05 s, 163 MB/s 2023-09-23T22:44:28.127 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_687972/img bs=1k count=100 seek=1000 2023-09-23T22:44:28.128 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records in 2023-09-23T22:44:28.128 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records out 2023-09-23T22:44:28.128 INFO:tasks.workunit.client.0.smithi026.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000210601 s, 342 MB/s 2023-09-23T22:44:28.128 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_687972/img bs=1k seek=10000 2023-09-23T22:44:28.128 INFO:tasks.workunit.client.0.smithi026.stderr:139+1 records in 2023-09-23T22:44:28.129 INFO:tasks.workunit.client.0.smithi026.stderr:139+1 records out 2023-09-23T22:44:28.129 INFO:tasks.workunit.client.0.smithi026.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000357504 s, 401 MB/s 2023-09-23T22:44:28.129 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_687972/img bs=1k seek=100000 2023-09-23T22:44:28.248 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records in 2023-09-23T22:44:28.248 INFO:tasks.workunit.client.0.smithi026.stderr:70+1 records out 2023-09-23T22:44:28.248 INFO:tasks.workunit.client.0.smithi026.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.118326 s, 610 kB/s 2023-09-23T22:44:28.248 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_687972/img bs=1k seek=1000000 2023-09-23T22:44:28.249 INFO:tasks.workunit.client.0.smithi026.stderr:189+1 records in 2023-09-23T22:44:28.249 INFO:tasks.workunit.client.0.smithi026.stderr:189+1 records out 2023-09-23T22:44:28.249 INFO:tasks.workunit.client.0.smithi026.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000508191 s, 381 MB/s 2023-09-23T22:44:28.249 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T22:44:28.289 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 0% complete...failed. 2023-09-23T22:44:28.289 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T22:44:28.292 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:44:28.292 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format /tmp/rbd_import_export_687972/img testimg 2023-09-23T22:44:28.373 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:28 smithi070 ceph-mon[192652]: from='client.46913 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:28 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:28 smithi070 ceph-mon[192652]: failed to encode map e624 with expected crc 2023-09-23T22:44:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:28 smithi070 ceph-mon[192652]: failed to encode map e624 with expected crc 2023-09-23T22:44:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:28 smithi070 ceph-mon[192652]: failed to encode map e624 with expected crc 2023-09-23T22:44:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:28 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:44:28.374 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:28 smithi070 ceph-mon[192652]: osdmap e624: 8 total, 8 up, 4 in 2023-09-23T22:44:28.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[223014]: from='client.46913 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:28.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:28.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[223014]: failed to encode map e624 with expected crc 2023-09-23T22:44:28.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[223014]: failed to encode map e624 with expected crc 2023-09-23T22:44:28.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[223014]: failed to encode map e624 with expected crc 2023-09-23T22:44:28.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:44:28.403 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[223014]: osdmap e624: 8 total, 8 up, 4 in 2023-09-23T22:44:28.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[215775]: from='client.46913 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:28.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:28.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[215775]: failed to encode map e624 with expected crc 2023-09-23T22:44:28.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[215775]: failed to encode map e624 with expected crc 2023-09-23T22:44:28.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[215775]: failed to encode map e624 with expected crc 2023-09-23T22:44:28.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:44:28.404 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:28 smithi026 ceph-mon[215775]: osdmap e624: 8 total, 8 up, 4 in 2023-09-23T22:44:28.637 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:28.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:28.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:28.742 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:28.776 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:28.886 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T22:44:28.887 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:44:28.889 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg /tmp/rbd_import_export_687972/img2 2023-09-23T22:44:29.268 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:29.271 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg - 2023-09-23T22:44:29.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:44:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:44:29] "GET /metrics HTTP/1.1" 200 34089 "" "Prometheus/2.43.0" 2023-09-23T22:44:29.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:29 smithi026 ceph-mon[215775]: pgmap v1613: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:29.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:29 smithi026 ceph-mon[215775]: 1.e scrub starts 2023-09-23T22:44:29.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:29 smithi026 ceph-mon[215775]: 1.e scrub ok 2023-09-23T22:44:29.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:29 smithi026 ceph-mon[223014]: pgmap v1613: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:29.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:29 smithi026 ceph-mon[223014]: 1.e scrub starts 2023-09-23T22:44:29.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:29 smithi026 ceph-mon[223014]: 1.e scrub ok 2023-09-23T22:44:29.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:29 smithi070 ceph-mon[192652]: pgmap v1613: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail 2023-09-23T22:44:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:29 smithi070 ceph-mon[192652]: 1.e scrub starts 2023-09-23T22:44:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:29 smithi070 ceph-mon[192652]: 1.e scrub ok 2023-09-23T22:44:30.117 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:30.120 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T22:44:30.210 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T22:44:30.213 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_687972/img /tmp/rbd_import_export_687972/img2 2023-09-23T22:44:30.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:30 smithi026 ceph-mon[215775]: 1.17 scrub starts 2023-09-23T22:44:30.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:30 smithi026 ceph-mon[215775]: 1.17 scrub ok 2023-09-23T22:44:30.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:30 smithi026 ceph-mon[223014]: 1.17 scrub starts 2023-09-23T22:44:30.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:30 smithi026 ceph-mon[223014]: 1.17 scrub ok 2023-09-23T22:44:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:30 smithi070 ceph-mon[192652]: 1.17 scrub starts 2023-09-23T22:44:30.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:30 smithi070 ceph-mon[192652]: 1.17 scrub ok 2023-09-23T22:44:31.019 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_687972/img /tmp/rbd_import_export_687972/img3 2023-09-23T22:44:31.545 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/img2 /tmp/rbd_import_export_687972/img3 2023-09-23T22:44:31.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:31 smithi026 ceph-mon[223014]: pgmap v1614: 103 pgs: 103 active+clean; 7.3 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 52 KiB/s rd, 36 KiB/s wr, 0 op/s 2023-09-23T22:44:31.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:31 smithi026 ceph-mon[215775]: pgmap v1614: 103 pgs: 103 active+clean; 7.3 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 52 KiB/s rd, 36 KiB/s wr, 0 op/s 2023-09-23T22:44:31.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:31 smithi070 ceph-mon[192652]: pgmap v1614: 103 pgs: 103 active+clean; 7.3 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 52 KiB/s rd, 36 KiB/s wr, 0 op/s 2023-09-23T22:44:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:32 smithi026 ceph-mon[223014]: 1.16 scrub starts 2023-09-23T22:44:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:32 smithi026 ceph-mon[223014]: 1.16 scrub ok 2023-09-23T22:44:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:32 smithi026 ceph-mon[223014]: 1.1b scrub starts 2023-09-23T22:44:32.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:32 smithi026 ceph-mon[223014]: 1.1b scrub ok 2023-09-23T22:44:32.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:32 smithi026 ceph-mon[215775]: 1.16 scrub starts 2023-09-23T22:44:32.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:32 smithi026 ceph-mon[215775]: 1.16 scrub ok 2023-09-23T22:44:32.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:32 smithi026 ceph-mon[215775]: 1.1b scrub starts 2023-09-23T22:44:32.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:32 smithi026 ceph-mon[215775]: 1.1b scrub ok 2023-09-23T22:44:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:32 smithi070 ceph-mon[192652]: 1.16 scrub starts 2023-09-23T22:44:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:32 smithi070 ceph-mon[192652]: 1.16 scrub ok 2023-09-23T22:44:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:32 smithi070 ceph-mon[192652]: 1.1b scrub starts 2023-09-23T22:44:32.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:32 smithi070 ceph-mon[192652]: 1.1b scrub ok 2023-09-23T22:44:33.568 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format - testimg 2023-09-23T22:44:33.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[215775]: 1.1a scrub starts 2023-09-23T22:44:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[215775]: 1.1a scrub ok 2023-09-23T22:44:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[215775]: pgmap v1615: 103 pgs: 103 active+clean; 7.3 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 81 KiB/s rd, 42 KiB/s wr, 27 op/s 2023-09-23T22:44:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[215775]: 1.b scrub starts 2023-09-23T22:44:33.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[215775]: 1.b scrub ok 2023-09-23T22:44:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[223014]: 1.1a scrub starts 2023-09-23T22:44:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[223014]: 1.1a scrub ok 2023-09-23T22:44:33.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[223014]: pgmap v1615: 103 pgs: 103 active+clean; 7.3 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 81 KiB/s rd, 42 KiB/s wr, 27 op/s 2023-09-23T22:44:33.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[223014]: 1.b scrub starts 2023-09-23T22:44:33.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:33 smithi026 ceph-mon[223014]: 1.b scrub ok 2023-09-23T22:44:33.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:33 smithi070 ceph-mon[192652]: 1.1a scrub starts 2023-09-23T22:44:33.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:33 smithi070 ceph-mon[192652]: 1.1a scrub ok 2023-09-23T22:44:33.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:33 smithi070 ceph-mon[192652]: pgmap v1615: 103 pgs: 103 active+clean; 7.3 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 81 KiB/s rd, 42 KiB/s wr, 27 op/s 2023-09-23T22:44:33.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:33 smithi070 ceph-mon[192652]: 1.b scrub starts 2023-09-23T22:44:33.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:33 smithi070 ceph-mon[192652]: 1.b scrub ok 2023-09-23T22:44:33.979 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:44:33.980 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:44:33.982 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg /tmp/rbd_import_export_687972/img2 2023-09-23T22:44:34.267 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:34.271 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg - 2023-09-23T22:44:35.169 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:35.171 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T22:44:35.215 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:35.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:35.264 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T22:44:35.267 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_687972/img /tmp/rbd_import_export_687972/img2 2023-09-23T22:44:35.320 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:35.352 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:35.401 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:35.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:35.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:35.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:35 smithi026 ceph-mon[215775]: pgmap v1616: 103 pgs: 103 active+clean; 7.3 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 100 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-09-23T22:44:35.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:35 smithi026 ceph-mon[223014]: pgmap v1616: 103 pgs: 103 active+clean; 7.3 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 100 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-09-23T22:44:35.507 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:35.533 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:35.575 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:35.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:35.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:35.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:35 smithi070 ceph-mon[192652]: pgmap v1616: 103 pgs: 103 active+clean; 7.3 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 100 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-09-23T22:44:35.681 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:35.707 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:35.748 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:35.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:35.754 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:35.855 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:35.871 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T22:44:35.880 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:35.922 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:35.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:35.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:35.967 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp /tmp/rbd_import_export_687972/img /tmp/rbd_import_export_687972/img3 2023-09-23T22:44:36.028 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:36.058 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:36.086 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1611,"stamp":"2023-09-23T22:44:25.759214+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4825975,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6011,"num_read_kb":12239,"num_write":922,"num_write_kb":12872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":568,"num_bytes_recovered":21388689,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":1921672,"kb_used_data":335544,"kb_used_omap":35,"kb_used_meta":1586012,"kb_avail":373058936,"statfs":{"total":383980142592,"available":382012350464,"internally_reserved":0,"allocated":343597056,"data_stored":333285062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36602,"internal_metadata":1624076550},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-374,"num_objects":-36,"num_object_clones":0,"num_object_copies":-108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-36,"num_whiteouts":0,"num_read":-9262,"num_read_kb":-17458,"num_write":-4394,"num_write_kb":-8436,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-28,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.112995"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":1102,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658083+0000","last_change":"2023-09-23T22:44:05.893795+0000","last_active":"2023-09-23T22:44:19.658083+0000","last_peered":"2023-09-23T22:44:19.658083+0000","last_clean":"2023-09-23T22:44:19.658083+0000","last_became_active":"2023-09-23T22:44:05.893431+0000","last_became_peered":"2023-09-23T22:44:05.893431+0000","last_unstale":"2023-09-23T22:44:19.658083+0000","last_undegraded":"2023-09-23T22:44:19.658083+0000","last_fullsized":"2023-09-23T22:44:19.658083+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.209739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.209739+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:55.009700+0000","scrub_duration":0.0016266950000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":581,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666682+0000","last_change":"2023-09-23T22:43:11.791371+0000","last_active":"2023-09-23T22:44:19.666682+0000","last_peered":"2023-09-23T22:44:19.666682+0000","last_clean":"2023-09-23T22:44:19.666682+0000","last_became_active":"2023-09-23T22:43:11.783461+0000","last_became_peered":"2023-09-23T22:43:11.783461+0000","last_unstale":"2023-09-23T22:44:19.666682+0000","last_undegraded":"2023-09-23T22:44:19.666682+0000","last_fullsized":"2023-09-23T22:44:19.666682+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:19.460031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":360,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662601+0000","last_change":"2023-09-23T22:43:11.733047+0000","last_active":"2023-09-23T22:44:19.662601+0000","last_peered":"2023-09-23T22:44:19.662601+0000","last_clean":"2023-09-23T22:44:19.662601+0000","last_became_active":"2023-09-23T22:43:11.731356+0000","last_became_peered":"2023-09-23T22:43:11.731356+0000","last_unstale":"2023-09-23T22:44:19.662601+0000","last_undegraded":"2023-09-23T22:44:19.662601+0000","last_fullsized":"2023-09-23T22:44:19.662601+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:09.953038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":825,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658441+0000","last_change":"2023-09-23T22:44:05.895100+0000","last_active":"2023-09-23T22:44:19.658441+0000","last_peered":"2023-09-23T22:44:19.658441+0000","last_clean":"2023-09-23T22:44:19.658441+0000","last_became_active":"2023-09-23T22:44:05.894732+0000","last_became_peered":"2023-09-23T22:44:05.894732+0000","last_unstale":"2023-09-23T22:44:19.658441+0000","last_undegraded":"2023-09-23T22:44:19.658441+0000","last_fullsized":"2023-09-23T22:44:19.658441+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:29.970215+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:29.970215+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":395,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661022+0000","last_change":"2023-09-23T22:42:57.702640+0000","last_active":"2023-09-23T22:44:19.661022+0000","last_peered":"2023-09-23T22:44:19.661022+0000","last_clean":"2023-09-23T22:44:19.661022+0000","last_became_active":"2023-09-23T22:42:57.701938+0000","last_became_peered":"2023-09-23T22:42:57.701938+0000","last_unstale":"2023-09-23T22:44:19.661022+0000","last_undegraded":"2023-09-23T22:44:19.661022+0000","last_fullsized":"2023-09-23T22:44:19.661022+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:04.282382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":102,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662453+0000","last_change":"2023-09-23T22:43:11.805439+0000","last_active":"2023-09-23T22:44:19.662453+0000","last_peered":"2023-09-23T22:44:19.662453+0000","last_clean":"2023-09-23T22:44:19.662453+0000","last_became_active":"2023-09-23T22:43:11.804518+0000","last_became_peered":"2023-09-23T22:43:11.804518+0000","last_unstale":"2023-09-23T22:44:19.662453+0000","last_undegraded":"2023-09-23T22:44:19.662453+0000","last_fullsized":"2023-09-23T22:44:19.662453+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:32.668394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":797,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661187+0000","last_change":"2023-09-23T22:44:05.893969+0000","last_active":"2023-09-23T22:44:19.661187+0000","last_peered":"2023-09-23T22:44:19.661187+0000","last_clean":"2023-09-23T22:44:19.661187+0000","last_became_active":"2023-09-23T22:44:05.893507+0000","last_became_peered":"2023-09-23T22:44:05.893507+0000","last_unstale":"2023-09-23T22:44:19.661187+0000","last_undegraded":"2023-09-23T22:44:19.661187+0000","last_fullsized":"2023-09-23T22:44:19.661187+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:43:58.171880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:43:58.171880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039200069999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":687,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.660749+0000","last_change":"2023-09-23T22:43:11.861828+0000","last_active":"2023-09-23T22:44:19.660749+0000","last_peered":"2023-09-23T22:44:19.660749+0000","last_clean":"2023-09-23T22:44:19.660749+0000","last_became_active":"2023-09-23T22:43:11.860602+0000","last_became_peered":"2023-09-23T22:43:11.860602+0000","last_unstale":"2023-09-23T22:44:19.660749+0000","last_undegraded":"2023-09-23T22:44:19.660749+0000","last_fullsized":"2023-09-23T22:44:19.660749+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:43.850844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":238,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.663909+0000","last_change":"2023-09-23T22:42:56.509390+0000","last_active":"2023-09-23T22:44:19.663909+0000","last_peered":"2023-09-23T22:44:19.663909+0000","last_clean":"2023-09-23T22:44:19.663909+0000","last_became_active":"2023-09-23T22:42:56.509066+0000","last_became_peered":"2023-09-23T22:42:56.509066+0000","last_unstale":"2023-09-23T22:44:19.663909+0000","last_undegraded":"2023-09-23T22:44:19.663909+0000","last_fullsized":"2023-09-23T22:44:19.663909+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:36.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":960,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658653+0000","last_change":"2023-09-23T22:44:05.895248+0000","last_active":"2023-09-23T22:44:19.658653+0000","last_peered":"2023-09-23T22:44:19.658653+0000","last_clean":"2023-09-23T22:44:19.658653+0000","last_became_active":"2023-09-23T22:44:05.894827+0000","last_became_peered":"2023-09-23T22:44:05.894827+0000","last_unstale":"2023-09-23T22:44:19.658653+0000","last_undegraded":"2023-09-23T22:44:19.658653+0000","last_fullsized":"2023-09-23T22:44:19.658653+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:57.273632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:57.273632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:54.065385+0000","scrub_duration":0.018058204000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.660474+0000","last_change":"2023-09-23T22:43:11.732475+0000","last_active":"2023-09-23T22:44:19.660474+0000","last_peered":"2023-09-23T22:44:19.660474+0000","last_clean":"2023-09-23T22:44:19.660474+0000","last_became_active":"2023-09-23T22:43:11.732134+0000","last_became_peered":"2023-09-23T22:43:11.732134+0000","last_unstale":"2023-09-23T22:44:19.660474+0000","last_undegraded":"2023-09-23T22:44:19.660474+0000","last_fullsized":"2023-09-23T22:44:19.660474+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:22.683794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":216,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661283+0000","last_change":"2023-09-23T22:42:57.555024+0000","last_active":"2023-09-23T22:44:19.661283+0000","last_peered":"2023-09-23T22:44:19.661283+0000","last_clean":"2023-09-23T22:44:19.661283+0000","last_became_active":"2023-09-23T22:42:57.554656+0000","last_became_peered":"2023-09-23T22:42:57.554656+0000","last_unstale":"2023-09-23T22:44:19.661283+0000","last_undegraded":"2023-09-23T22:44:19.661283+0000","last_fullsized":"2023-09-23T22:44:19.661283+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:56.648868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":774,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666480+0000","last_change":"2023-09-23T22:44:05.887689+0000","last_active":"2023-09-23T22:44:19.666480+0000","last_peered":"2023-09-23T22:44:19.666480+0000","last_clean":"2023-09-23T22:44:19.666480+0000","last_became_active":"2023-09-23T22:44:05.887374+0000","last_became_peered":"2023-09-23T22:44:05.887374+0000","last_unstale":"2023-09-23T22:44:19.666480+0000","last_undegraded":"2023-09-23T22:44:19.666480+0000","last_fullsized":"2023-09-23T22:44:19.666480+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:02.056099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:02.056099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:42.640691+0000","scrub_duration":0.012145022,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.659360+0000","last_change":"2023-09-23T22:43:11.828937+0000","last_active":"2023-09-23T22:44:19.659360+0000","last_peered":"2023-09-23T22:44:19.659360+0000","last_clean":"2023-09-23T22:44:19.659360+0000","last_became_active":"2023-09-23T22:43:11.828512+0000","last_became_peered":"2023-09-23T22:43:11.828512+0000","last_unstale":"2023-09-23T22:44:19.659360+0000","last_undegraded":"2023-09-23T22:44:19.659360+0000","last_fullsized":"2023-09-23T22:44:19.659360+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:56.701943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":219,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662999+0000","last_change":"2023-09-23T22:42:57.699399+0000","last_active":"2023-09-23T22:44:19.662999+0000","last_peered":"2023-09-23T22:44:19.662999+0000","last_clean":"2023-09-23T22:44:19.662999+0000","last_became_active":"2023-09-23T22:42:57.698650+0000","last_became_peered":"2023-09-23T22:42:57.698650+0000","last_unstale":"2023-09-23T22:44:19.662999+0000","last_undegraded":"2023-09-23T22:44:19.662999+0000","last_fullsized":"2023-09-23T22:44:19.662999+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:17.884820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":890,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.659008+0000","last_change":"2023-09-23T22:44:02.962008+0000","last_active":"2023-09-23T22:44:19.659008+0000","last_peered":"2023-09-23T22:44:19.659008+0000","last_clean":"2023-09-23T22:44:19.659008+0000","last_became_active":"2023-09-23T22:44:02.961671+0000","last_became_peered":"2023-09-23T22:44:02.961671+0000","last_unstale":"2023-09-23T22:44:19.659008+0000","last_undegraded":"2023-09-23T22:44:19.659008+0000","last_fullsized":"2023-09-23T22:44:19.659008+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":61,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662309+0000","last_change":"2023-09-23T22:43:11.882816+0000","last_active":"2023-09-23T22:44:19.662309+0000","last_peered":"2023-09-23T22:44:19.662309+0000","last_clean":"2023-09-23T22:44:19.662309+0000","last_became_active":"2023-09-23T22:43:11.822292+0000","last_became_peered":"2023-09-23T22:43:11.822292+0000","last_unstale":"2023-09-23T22:44:19.662309+0000","last_undegraded":"2023-09-23T22:44:19.662309+0000","last_fullsized":"2023-09-23T22:44:19.662309+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:05.717891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":274,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661528+0000","last_change":"2023-09-23T22:42:56.567330+0000","last_active":"2023-09-23T22:44:19.661528+0000","last_peered":"2023-09-23T22:44:19.661528+0000","last_clean":"2023-09-23T22:44:19.661528+0000","last_became_active":"2023-09-23T22:42:56.567016+0000","last_became_peered":"2023-09-23T22:42:56.567016+0000","last_unstale":"2023-09-23T22:44:19.661528+0000","last_undegraded":"2023-09-23T22:44:19.661528+0000","last_fullsized":"2023-09-23T22:44:19.661528+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:00.456335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":806,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662390+0000","last_change":"2023-09-23T22:44:05.892244+0000","last_active":"2023-09-23T22:44:19.662390+0000","last_peered":"2023-09-23T22:44:19.662390+0000","last_clean":"2023-09-23T22:44:19.662390+0000","last_became_active":"2023-09-23T22:44:05.891879+0000","last_became_peered":"2023-09-23T22:44:05.891879+0000","last_unstale":"2023-09-23T22:44:19.662390+0000","last_undegraded":"2023-09-23T22:44:19.662390+0000","last_fullsized":"2023-09-23T22:44:19.662390+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":613,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.664109+0000","last_change":"2023-09-23T22:43:11.747332+0000","last_active":"2023-09-23T22:44:19.664109+0000","last_peered":"2023-09-23T22:44:19.664109+0000","last_clean":"2023-09-23T22:44:19.664109+0000","last_became_active":"2023-09-23T22:43:11.733061+0000","last_became_peered":"2023-09-23T22:43:11.733061+0000","last_unstale":"2023-09-23T22:44:19.664109+0000","last_undegraded":"2023-09-23T22:44:19.664109+0000","last_fullsized":"2023-09-23T22:44:19.664109+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:49:43.986136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":42606,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":546,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662206+0000","last_change":"2023-09-23T22:42:57.485105+0000","last_active":"2023-09-23T22:44:19.662206+0000","last_peered":"2023-09-23T22:44:19.662206+0000","last_clean":"2023-09-23T22:44:19.662206+0000","last_became_active":"2023-09-23T22:42:57.484320+0000","last_became_peered":"2023-09-23T22:42:57.484320+0000","last_unstale":"2023-09-23T22:44:19.662206+0000","last_undegraded":"2023-09-23T22:44:19.662206+0000","last_fullsized":"2023-09-23T22:44:19.662206+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:36.390910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":1054,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658126+0000","last_change":"2023-09-23T22:44:05.895162+0000","last_active":"2023-09-23T22:44:19.658126+0000","last_peered":"2023-09-23T22:44:19.658126+0000","last_clean":"2023-09-23T22:44:19.658126+0000","last_became_active":"2023-09-23T22:44:05.894753+0000","last_became_peered":"2023-09-23T22:44:05.894753+0000","last_unstale":"2023-09-23T22:44:19.658126+0000","last_undegraded":"2023-09-23T22:44:19.658126+0000","last_fullsized":"2023-09-23T22:44:19.658126+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:19.077318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:19.077318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:47.206519+0000","scrub_duration":0.0080980949999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":311,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.665970+0000","last_change":"2023-09-23T22:42:57.736915+0000","last_active":"2023-09-23T22:44:19.665970+0000","last_peered":"2023-09-23T22:44:19.665970+0000","last_clean":"2023-09-23T22:44:19.665970+0000","last_became_active":"2023-09-23T22:42:57.629094+0000","last_became_peered":"2023-09-23T22:42:57.629094+0000","last_unstale":"2023-09-23T22:44:19.665970+0000","last_undegraded":"2023-09-23T22:44:19.665970+0000","last_fullsized":"2023-09-23T22:44:19.665970+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:03.304992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":83,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666604+0000","last_change":"2023-09-23T22:41:52.385773+0000","last_active":"2023-09-23T22:44:19.666604+0000","last_peered":"2023-09-23T22:44:19.666604+0000","last_clean":"2023-09-23T22:44:19.666604+0000","last_became_active":"2023-09-23T22:41:52.385433+0000","last_became_peered":"2023-09-23T22:41:52.385433+0000","last_unstale":"2023-09-23T22:44:19.666604+0000","last_undegraded":"2023-09-23T22:44:19.666604+0000","last_fullsized":"2023-09-23T22:44:19.666604+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:15.934387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":900,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.657782+0000","last_change":"2023-09-23T22:44:05.895570+0000","last_active":"2023-09-23T22:44:19.657782+0000","last_peered":"2023-09-23T22:44:19.657782+0000","last_clean":"2023-09-23T22:44:19.657782+0000","last_became_active":"2023-09-23T22:44:05.895214+0000","last_became_peered":"2023-09-23T22:44:05.895214+0000","last_unstale":"2023-09-23T22:44:19.657782+0000","last_undegraded":"2023-09-23T22:44:19.657782+0000","last_fullsized":"2023-09-23T22:44:19.657782+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.556720+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:11.670386+0000","scrub_duration":0.0022349409999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":93,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.665950+0000","last_change":"2023-09-23T22:43:11.791005+0000","last_active":"2023-09-23T22:44:19.665950+0000","last_peered":"2023-09-23T22:44:19.665950+0000","last_clean":"2023-09-23T22:44:19.665950+0000","last_became_active":"2023-09-23T22:43:11.783005+0000","last_became_peered":"2023-09-23T22:43:11.783005+0000","last_unstale":"2023-09-23T22:44:19.665950+0000","last_undegraded":"2023-09-23T22:44:19.665950+0000","last_fullsized":"2023-09-23T22:44:19.665950+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:23.783502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.659360+0000","last_change":"2023-09-23T22:43:11.725543+0000","last_active":"2023-09-23T22:44:19.659360+0000","last_peered":"2023-09-23T22:44:19.659360+0000","last_clean":"2023-09-23T22:44:19.659360+0000","last_became_active":"2023-09-23T22:43:11.725152+0000","last_became_peered":"2023-09-23T22:43:11.725152+0000","last_unstale":"2023-09-23T22:44:19.659360+0000","last_undegraded":"2023-09-23T22:44:19.659360+0000","last_fullsized":"2023-09-23T22:44:19.659360+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:31.160745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":643,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661538+0000","last_change":"2023-09-23T22:44:02.925951+0000","last_active":"2023-09-23T22:44:19.661538+0000","last_peered":"2023-09-23T22:44:19.661538+0000","last_clean":"2023-09-23T22:44:19.661538+0000","last_became_active":"2023-09-23T22:44:02.922183+0000","last_became_peered":"2023-09-23T22:44:02.922183+0000","last_unstale":"2023-09-23T22:44:19.661538+0000","last_undegraded":"2023-09-23T22:44:19.661538+0000","last_fullsized":"2023-09-23T22:44:19.661538+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:57.922489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:57.922489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:25.336046+0000","scrub_duration":0.001544351,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":466,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662866+0000","last_change":"2023-09-23T22:43:11.802960+0000","last_active":"2023-09-23T22:44:19.662866+0000","last_peered":"2023-09-23T22:44:19.662866+0000","last_clean":"2023-09-23T22:44:19.662866+0000","last_became_active":"2023-09-23T22:43:11.802684+0000","last_became_peered":"2023-09-23T22:43:11.802684+0000","last_unstale":"2023-09-23T22:44:19.662866+0000","last_undegraded":"2023-09-23T22:44:19.662866+0000","last_fullsized":"2023-09-23T22:44:19.662866+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:21.148096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":102,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666807+0000","last_change":"2023-09-23T22:43:11.807494+0000","last_active":"2023-09-23T22:44:19.666807+0000","last_peered":"2023-09-23T22:44:19.666807+0000","last_clean":"2023-09-23T22:44:19.666807+0000","last_became_active":"2023-09-23T22:43:11.807103+0000","last_became_peered":"2023-09-23T22:43:11.807103+0000","last_unstale":"2023-09-23T22:44:19.666807+0000","last_undegraded":"2023-09-23T22:44:19.666807+0000","last_fullsized":"2023-09-23T22:44:19.666807+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:50.696866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":959,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658126+0000","last_change":"2023-09-23T22:44:05.892047+0000","last_active":"2023-09-23T22:44:19.658126+0000","last_peered":"2023-09-23T22:44:19.658126+0000","last_clean":"2023-09-23T22:44:19.658126+0000","last_became_active":"2023-09-23T22:44:05.891675+0000","last_became_peered":"2023-09-23T22:44:05.891675+0000","last_unstale":"2023-09-23T22:44:19.658126+0000","last_undegraded":"2023-09-23T22:44:19.658126+0000","last_fullsized":"2023-09-23T22:44:19.658126+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:26.313042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:26.313042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:06.388598+0000","scrub_duration":0.0017143830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":282,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661569+0000","last_change":"2023-09-23T22:43:11.844157+0000","last_active":"2023-09-23T22:44:19.661569+0000","last_peered":"2023-09-23T22:44:19.661569+0000","last_clean":"2023-09-23T22:44:19.661569+0000","last_became_active":"2023-09-23T22:43:11.751401+0000","last_became_peered":"2023-09-23T22:43:11.751401+0000","last_unstale":"2023-09-23T22:44:19.661569+0000","last_undegraded":"2023-09-23T22:44:19.661569+0000","last_fullsized":"2023-09-23T22:44:19.661569+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:37.797785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":83,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666617+0000","last_change":"2023-09-23T22:42:57.618059+0000","last_active":"2023-09-23T22:44:19.666617+0000","last_peered":"2023-09-23T22:44:19.666617+0000","last_clean":"2023-09-23T22:44:19.666617+0000","last_became_active":"2023-09-23T22:42:57.617619+0000","last_became_peered":"2023-09-23T22:42:57.617619+0000","last_unstale":"2023-09-23T22:44:19.666617+0000","last_undegraded":"2023-09-23T22:44:19.666617+0000","last_fullsized":"2023-09-23T22:44:19.666617+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:00.845651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":984,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.667141+0000","last_change":"2023-09-23T22:44:05.887684+0000","last_active":"2023-09-23T22:44:19.667141+0000","last_peered":"2023-09-23T22:44:19.667141+0000","last_clean":"2023-09-23T22:44:19.667141+0000","last_became_active":"2023-09-23T22:44:05.887349+0000","last_became_peered":"2023-09-23T22:44:05.887349+0000","last_unstale":"2023-09-23T22:44:19.667141+0000","last_undegraded":"2023-09-23T22:44:19.667141+0000","last_fullsized":"2023-09-23T22:44:19.667141+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:34.147486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:34.147486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.306225+0000","scrub_duration":0.0018320190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":163,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662689+0000","last_change":"2023-09-23T22:43:11.920516+0000","last_active":"2023-09-23T22:44:19.662689+0000","last_peered":"2023-09-23T22:44:19.662689+0000","last_clean":"2023-09-23T22:44:19.662689+0000","last_became_active":"2023-09-23T22:43:11.755782+0000","last_became_peered":"2023-09-23T22:43:11.755782+0000","last_unstale":"2023-09-23T22:44:19.662689+0000","last_undegraded":"2023-09-23T22:44:19.662689+0000","last_fullsized":"2023-09-23T22:44:19.662689+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:16.670619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658152+0000","last_change":"2023-09-23T22:43:11.831473+0000","last_active":"2023-09-23T22:44:19.658152+0000","last_peered":"2023-09-23T22:44:19.658152+0000","last_clean":"2023-09-23T22:44:19.658152+0000","last_became_active":"2023-09-23T22:43:11.831153+0000","last_became_peered":"2023-09-23T22:43:11.831153+0000","last_unstale":"2023-09-23T22:44:19.658152+0000","last_undegraded":"2023-09-23T22:44:19.658152+0000","last_fullsized":"2023-09-23T22:44:19.658152+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:00:24.678192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":731,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.663016+0000","last_change":"2023-09-23T22:44:02.893918+0000","last_active":"2023-09-23T22:44:19.663016+0000","last_peered":"2023-09-23T22:44:19.663016+0000","last_clean":"2023-09-23T22:44:19.663016+0000","last_became_active":"2023-09-23T22:44:02.893292+0000","last_became_peered":"2023-09-23T22:44:02.893292+0000","last_unstale":"2023-09-23T22:44:19.663016+0000","last_undegraded":"2023-09-23T22:44:19.663016+0000","last_fullsized":"2023-09-23T22:44:19.663016+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:04.048606+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:04.048606+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:36.525412+0000","scrub_duration":0.0064015249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":540,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662766+0000","last_change":"2023-09-23T22:43:11.899388+0000","last_active":"2023-09-23T22:44:19.662766+0000","last_peered":"2023-09-23T22:44:19.662766+0000","last_clean":"2023-09-23T22:44:19.662766+0000","last_became_active":"2023-09-23T22:43:11.841184+0000","last_became_peered":"2023-09-23T22:43:11.841184+0000","last_unstale":"2023-09-23T22:44:19.662766+0000","last_undegraded":"2023-09-23T22:44:19.662766+0000","last_fullsized":"2023-09-23T22:44:19.662766+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:01.020339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":636,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.660967+0000","last_change":"2023-09-23T22:42:57.702798+0000","last_active":"2023-09-23T22:44:19.660967+0000","last_peered":"2023-09-23T22:44:19.660967+0000","last_clean":"2023-09-23T22:44:19.660967+0000","last_became_active":"2023-09-23T22:42:57.702317+0000","last_became_peered":"2023-09-23T22:42:57.702317+0000","last_unstale":"2023-09-23T22:44:19.660967+0000","last_undegraded":"2023-09-23T22:44:19.660967+0000","last_fullsized":"2023-09-23T22:44:19.660967+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:12.697584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":907,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.664106+0000","last_change":"2023-09-23T22:43:14.445750+0000","last_active":"2023-09-23T22:44:19.664106+0000","last_peered":"2023-09-23T22:44:19.664106+0000","last_clean":"2023-09-23T22:44:19.664106+0000","last_became_active":"2023-09-23T22:43:14.445359+0000","last_became_peered":"2023-09-23T22:43:14.445359+0000","last_unstale":"2023-09-23T22:44:19.664106+0000","last_undegraded":"2023-09-23T22:44:19.664106+0000","last_fullsized":"2023-09-23T22:44:19.664106+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:31:19.430538+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:31:19.430538+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0039698270000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":39,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.659052+0000","last_change":"2023-09-23T22:43:11.918943+0000","last_active":"2023-09-23T22:44:19.659052+0000","last_peered":"2023-09-23T22:44:19.659052+0000","last_clean":"2023-09-23T22:44:19.659052+0000","last_became_active":"2023-09-23T22:43:11.827406+0000","last_became_peered":"2023-09-23T22:43:11.827406+0000","last_unstale":"2023-09-23T22:44:19.659052+0000","last_undegraded":"2023-09-23T22:44:19.659052+0000","last_fullsized":"2023-09-23T22:44:19.659052+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:57.335665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":256,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661543+0000","last_change":"2023-09-23T22:42:57.480776+0000","last_active":"2023-09-23T22:44:19.661543+0000","last_peered":"2023-09-23T22:44:19.661543+0000","last_clean":"2023-09-23T22:44:19.661543+0000","last_became_active":"2023-09-23T22:42:57.480378+0000","last_became_peered":"2023-09-23T22:42:57.480378+0000","last_unstale":"2023-09-23T22:44:19.661543+0000","last_undegraded":"2023-09-23T22:44:19.661543+0000","last_fullsized":"2023-09-23T22:44:19.661543+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.689275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":814,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666505+0000","last_change":"2023-09-23T22:44:05.889031+0000","last_active":"2023-09-23T22:44:19.666505+0000","last_peered":"2023-09-23T22:44:19.666505+0000","last_clean":"2023-09-23T22:44:19.666505+0000","last_became_active":"2023-09-23T22:44:05.888659+0000","last_became_peered":"2023-09-23T22:44:05.888659+0000","last_unstale":"2023-09-23T22:44:19.666505+0000","last_undegraded":"2023-09-23T22:44:19.666505+0000","last_fullsized":"2023-09-23T22:44:19.666505+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658668+0000","last_change":"2023-09-23T22:43:11.732060+0000","last_active":"2023-09-23T22:44:19.658668+0000","last_peered":"2023-09-23T22:44:19.658668+0000","last_clean":"2023-09-23T22:44:19.658668+0000","last_became_active":"2023-09-23T22:43:11.731696+0000","last_became_peered":"2023-09-23T22:43:11.731696+0000","last_unstale":"2023-09-23T22:44:19.658668+0000","last_undegraded":"2023-09-23T22:44:19.658668+0000","last_fullsized":"2023-09-23T22:44:19.658668+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:41.169298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":428,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661428+0000","last_change":"2023-09-23T22:43:11.733183+0000","last_active":"2023-09-23T22:44:19.661428+0000","last_peered":"2023-09-23T22:44:19.661428+0000","last_clean":"2023-09-23T22:44:19.661428+0000","last_became_active":"2023-09-23T22:43:11.731600+0000","last_became_peered":"2023-09-23T22:43:11.731600+0000","last_unstale":"2023-09-23T22:44:19.661428+0000","last_undegraded":"2023-09-23T22:44:19.661428+0000","last_fullsized":"2023-09-23T22:44:19.661428+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:17:17.223104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":977,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.665641+0000","last_change":"2023-09-23T22:42:59.395878+0000","last_active":"2023-09-23T22:44:19.665641+0000","last_peered":"2023-09-23T22:44:19.665641+0000","last_clean":"2023-09-23T22:44:19.665641+0000","last_became_active":"2023-09-23T22:42:59.395538+0000","last_became_peered":"2023-09-23T22:42:59.395538+0000","last_unstale":"2023-09-23T22:44:19.665641+0000","last_undegraded":"2023-09-23T22:44:19.665641+0000","last_fullsized":"2023-09-23T22:44:19.665641+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:58.211060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:58.211060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:50.018093+0000","scrub_duration":0.0035013240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":105,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666118+0000","last_change":"2023-09-23T22:43:11.840050+0000","last_active":"2023-09-23T22:44:19.666118+0000","last_peered":"2023-09-23T22:44:19.666118+0000","last_clean":"2023-09-23T22:44:19.666118+0000","last_became_active":"2023-09-23T22:43:11.780186+0000","last_became_peered":"2023-09-23T22:43:11.780186+0000","last_unstale":"2023-09-23T22:44:19.666118+0000","last_undegraded":"2023-09-23T22:44:19.666118+0000","last_fullsized":"2023-09-23T22:44:19.666118+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:00.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":464,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.663112+0000","last_change":"2023-09-23T22:42:57.606366+0000","last_active":"2023-09-23T22:44:19.663112+0000","last_peered":"2023-09-23T22:44:19.663112+0000","last_clean":"2023-09-23T22:44:19.663112+0000","last_became_active":"2023-09-23T22:42:57.604437+0000","last_became_peered":"2023-09-23T22:42:57.604437+0000","last_unstale":"2023-09-23T22:44:19.663112+0000","last_undegraded":"2023-09-23T22:44:19.663112+0000","last_fullsized":"2023-09-23T22:44:19.663112+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:34.648244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":1082,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658187+0000","last_change":"2023-09-23T22:44:00.183260+0000","last_active":"2023-09-23T22:44:19.658187+0000","last_peered":"2023-09-23T22:44:19.658187+0000","last_clean":"2023-09-23T22:44:19.658187+0000","last_became_active":"2023-09-23T22:43:14.434417+0000","last_became_peered":"2023-09-23T22:43:14.434417+0000","last_unstale":"2023-09-23T22:44:19.658187+0000","last_undegraded":"2023-09-23T22:44:19.658187+0000","last_fullsized":"2023-09-23T22:44:19.658187+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.183183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.183183+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:35.744336+0000","scrub_duration":0.0015656730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":253,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661678+0000","last_change":"2023-09-23T22:42:57.480722+0000","last_active":"2023-09-23T22:44:19.661678+0000","last_peered":"2023-09-23T22:44:19.661678+0000","last_clean":"2023-09-23T22:44:19.661678+0000","last_became_active":"2023-09-23T22:42:57.480180+0000","last_became_peered":"2023-09-23T22:42:57.480180+0000","last_unstale":"2023-09-23T22:44:19.661678+0000","last_undegraded":"2023-09-23T22:44:19.661678+0000","last_fullsized":"2023-09-23T22:44:19.661678+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:00.944581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":272,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666497+0000","last_change":"2023-09-23T22:42:57.671849+0000","last_active":"2023-09-23T22:44:19.666497+0000","last_peered":"2023-09-23T22:44:19.666497+0000","last_clean":"2023-09-23T22:44:19.666497+0000","last_became_active":"2023-09-23T22:42:57.667120+0000","last_became_peered":"2023-09-23T22:42:57.667120+0000","last_unstale":"2023-09-23T22:44:19.666497+0000","last_undegraded":"2023-09-23T22:44:19.666497+0000","last_fullsized":"2023-09-23T22:44:19.666497+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:11.794628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":825,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658547+0000","last_change":"2023-09-23T22:44:05.893879+0000","last_active":"2023-09-23T22:44:19.658547+0000","last_peered":"2023-09-23T22:44:19.658547+0000","last_clean":"2023-09-23T22:44:19.658547+0000","last_became_active":"2023-09-23T22:44:05.893424+0000","last_became_peered":"2023-09-23T22:44:05.893424+0000","last_unstale":"2023-09-23T22:44:19.658547+0000","last_undegraded":"2023-09-23T22:44:19.658547+0000","last_fullsized":"2023-09-23T22:44:19.658547+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:26.963124+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:26.963124+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":596,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.665748+0000","last_change":"2023-09-23T22:43:11.854114+0000","last_active":"2023-09-23T22:44:19.665748+0000","last_peered":"2023-09-23T22:44:19.665748+0000","last_clean":"2023-09-23T22:44:19.665748+0000","last_became_active":"2023-09-23T22:43:11.841381+0000","last_became_peered":"2023-09-23T22:43:11.841381+0000","last_unstale":"2023-09-23T22:44:19.665748+0000","last_undegraded":"2023-09-23T22:44:19.665748+0000","last_fullsized":"2023-09-23T22:44:19.665748+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:46.359977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":553,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662908+0000","last_change":"2023-09-23T22:43:11.817112+0000","last_active":"2023-09-23T22:44:19.662908+0000","last_peered":"2023-09-23T22:44:19.662908+0000","last_clean":"2023-09-23T22:44:19.662908+0000","last_became_active":"2023-09-23T22:43:11.816563+0000","last_became_peered":"2023-09-23T22:43:11.816563+0000","last_unstale":"2023-09-23T22:44:19.662908+0000","last_undegraded":"2023-09-23T22:44:19.662908+0000","last_fullsized":"2023-09-23T22:44:19.662908+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:56.365553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":1031,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658558+0000","last_change":"2023-09-23T22:43:57.157649+0000","last_active":"2023-09-23T22:44:19.658558+0000","last_peered":"2023-09-23T22:44:19.658558+0000","last_clean":"2023-09-23T22:44:19.658558+0000","last_became_active":"2023-09-23T22:42:56.530915+0000","last_became_peered":"2023-09-23T22:42:56.530915+0000","last_unstale":"2023-09-23T22:44:19.658558+0000","last_undegraded":"2023-09-23T22:44:19.658558+0000","last_fullsized":"2023-09-23T22:44:19.658558+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_clean_scrub_stamp":"2023-09-23T22:43:57.157560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:20.159228+0000","scrub_duration":0.001566287,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":379,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661042+0000","last_change":"2023-09-23T22:43:11.862025+0000","last_active":"2023-09-23T22:44:19.661042+0000","last_peered":"2023-09-23T22:44:19.661042+0000","last_clean":"2023-09-23T22:44:19.661042+0000","last_became_active":"2023-09-23T22:43:11.860993+0000","last_became_peered":"2023-09-23T22:43:11.860993+0000","last_unstale":"2023-09-23T22:44:19.661042+0000","last_undegraded":"2023-09-23T22:44:19.661042+0000","last_fullsized":"2023-09-23T22:44:19.661042+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:37.123461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":445,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662875+0000","last_change":"2023-09-23T22:43:11.756165+0000","last_active":"2023-09-23T22:44:19.662875+0000","last_peered":"2023-09-23T22:44:19.662875+0000","last_clean":"2023-09-23T22:44:19.662875+0000","last_became_active":"2023-09-23T22:43:11.755617+0000","last_became_peered":"2023-09-23T22:43:11.755617+0000","last_unstale":"2023-09-23T22:44:19.662875+0000","last_undegraded":"2023-09-23T22:44:19.662875+0000","last_fullsized":"2023-09-23T22:44:19.662875+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:21.873764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":918,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.663412+0000","last_change":"2023-09-23T22:44:03.570228+0000","last_active":"2023-09-23T22:44:19.663412+0000","last_peered":"2023-09-23T22:44:19.663412+0000","last_clean":"2023-09-23T22:44:19.663412+0000","last_became_active":"2023-09-23T22:43:14.444896+0000","last_became_peered":"2023-09-23T22:43:14.444896+0000","last_unstale":"2023-09-23T22:44:19.663412+0000","last_undegraded":"2023-09-23T22:44:19.663412+0000","last_fullsized":"2023-09-23T22:44:19.663412+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:03.570110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:03.570110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:13.842237+0000","scrub_duration":0.001987071,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"84'154","reported_seq":996,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666725+0000","last_change":"2023-09-23T22:43:54.413786+0000","last_active":"2023-09-23T22:44:19.666725+0000","last_peered":"2023-09-23T22:44:19.666725+0000","last_clean":"2023-09-23T22:44:19.666725+0000","last_became_active":"2023-09-23T22:43:54.413159+0000","last_became_peered":"2023-09-23T22:43:54.413159+0000","last_unstale":"2023-09-23T22:44:19.666725+0000","last_undegraded":"2023-09-23T22:44:19.666725+0000","last_fullsized":"2023-09-23T22:44:19.666725+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:37.708567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":707,"num_read_kb":2984,"num_write":159,"num_write_kb":3307,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658923+0000","last_change":"2023-09-23T22:43:11.827683+0000","last_active":"2023-09-23T22:44:19.658923+0000","last_peered":"2023-09-23T22:44:19.658923+0000","last_clean":"2023-09-23T22:44:19.658923+0000","last_became_active":"2023-09-23T22:43:11.827004+0000","last_became_peered":"2023-09-23T22:43:11.827004+0000","last_unstale":"2023-09-23T22:44:19.658923+0000","last_undegraded":"2023-09-23T22:44:19.658923+0000","last_fullsized":"2023-09-23T22:44:19.658923+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:21.277304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":380,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661193+0000","last_change":"2023-09-23T22:43:11.772491+0000","last_active":"2023-09-23T22:44:19.661193+0000","last_peered":"2023-09-23T22:44:19.661193+0000","last_clean":"2023-09-23T22:44:19.661193+0000","last_became_active":"2023-09-23T22:43:11.772145+0000","last_became_peered":"2023-09-23T22:43:11.772145+0000","last_unstale":"2023-09-23T22:44:19.661193+0000","last_undegraded":"2023-09-23T22:44:19.661193+0000","last_fullsized":"2023-09-23T22:44:19.661193+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:22.248227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":1035,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658146+0000","last_change":"2023-09-23T22:43:14.435063+0000","last_active":"2023-09-23T22:44:19.658146+0000","last_peered":"2023-09-23T22:44:19.658146+0000","last_clean":"2023-09-23T22:44:19.658146+0000","last_became_active":"2023-09-23T22:43:14.434698+0000","last_became_peered":"2023-09-23T22:43:14.434698+0000","last_unstale":"2023-09-23T22:44:19.658146+0000","last_undegraded":"2023-09-23T22:44:19.658146+0000","last_fullsized":"2023-09-23T22:44:19.658146+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:16.067396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:16.067396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:58.817933+0000","scrub_duration":0.0072300120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'121","reported_seq":1535,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658733+0000","last_change":"2023-09-23T22:42:56.548561+0000","last_active":"2023-09-23T22:44:19.658733+0000","last_peered":"2023-09-23T22:44:19.658733+0000","last_clean":"2023-09-23T22:44:19.658733+0000","last_became_active":"2023-09-23T22:42:56.548134+0000","last_became_peered":"2023-09-23T22:42:56.548134+0000","last_unstale":"2023-09-23T22:44:19.658733+0000","last_undegraded":"2023-09-23T22:44:19.658733+0000","last_fullsized":"2023-09-23T22:44:19.658733+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":121,"log_dups_size":0,"ondisk_log_size":121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:36.511009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048995500000000003,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1044,"num_read_kb":2976,"num_write":101,"num_write_kb":6204,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":516,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662632+0000","last_change":"2023-09-23T22:42:57.574455+0000","last_active":"2023-09-23T22:44:19.662632+0000","last_peered":"2023-09-23T22:44:19.662632+0000","last_clean":"2023-09-23T22:44:19.662632+0000","last_became_active":"2023-09-23T22:42:57.573786+0000","last_became_peered":"2023-09-23T22:42:57.573786+0000","last_unstale":"2023-09-23T22:44:19.662632+0000","last_undegraded":"2023-09-23T22:44:19.662632+0000","last_fullsized":"2023-09-23T22:44:19.662632+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:09.464699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":272,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666819+0000","last_change":"2023-09-23T22:42:57.591303+0000","last_active":"2023-09-23T22:44:19.666819+0000","last_peered":"2023-09-23T22:44:19.666819+0000","last_clean":"2023-09-23T22:44:19.666819+0000","last_became_active":"2023-09-23T22:42:57.587701+0000","last_became_peered":"2023-09-23T22:42:57.587701+0000","last_unstale":"2023-09-23T22:44:19.666819+0000","last_undegraded":"2023-09-23T22:44:19.666819+0000","last_fullsized":"2023-09-23T22:44:19.666819+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:32.621790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":967,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666281+0000","last_change":"2023-09-23T22:43:14.448779+0000","last_active":"2023-09-23T22:44:19.666281+0000","last_peered":"2023-09-23T22:44:19.666281+0000","last_clean":"2023-09-23T22:44:19.666281+0000","last_became_active":"2023-09-23T22:43:14.448360+0000","last_became_peered":"2023-09-23T22:43:14.448360+0000","last_unstale":"2023-09-23T22:44:19.666281+0000","last_undegraded":"2023-09-23T22:44:19.666281+0000","last_fullsized":"2023-09-23T22:44:19.666281+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:07.291196+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:07.291196+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:38.426551+0000","scrub_duration":0.0016895390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"84'93","reported_seq":383,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.665718+0000","last_change":"2023-09-23T22:43:54.199739+0000","last_active":"2023-09-23T22:44:19.665718+0000","last_peered":"2023-09-23T22:44:19.665718+0000","last_clean":"2023-09-23T22:44:19.665718+0000","last_became_active":"2023-09-23T22:42:57.672992+0000","last_became_peered":"2023-09-23T22:42:57.672992+0000","last_unstale":"2023-09-23T22:44:19.665718+0000","last_undegraded":"2023-09-23T22:44:19.665718+0000","last_fullsized":"2023-09-23T22:44:19.665718+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:41.870977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":408,"num_read_kb":1671,"num_write":92,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":253,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.665248+0000","last_change":"2023-09-23T22:42:56.548974+0000","last_active":"2023-09-23T22:44:19.665248+0000","last_peered":"2023-09-23T22:44:19.665248+0000","last_clean":"2023-09-23T22:44:19.665248+0000","last_became_active":"2023-09-23T22:42:56.548663+0000","last_became_peered":"2023-09-23T22:42:56.548663+0000","last_unstale":"2023-09-23T22:44:19.665248+0000","last_undegraded":"2023-09-23T22:44:19.665248+0000","last_fullsized":"2023-09-23T22:44:19.665248+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:06.786917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":315,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.663046+0000","last_change":"2023-09-23T22:43:11.756366+0000","last_active":"2023-09-23T22:44:19.663046+0000","last_peered":"2023-09-23T22:44:19.663046+0000","last_clean":"2023-09-23T22:44:19.663046+0000","last_became_active":"2023-09-23T22:43:11.756048+0000","last_became_peered":"2023-09-23T22:43:11.756048+0000","last_unstale":"2023-09-23T22:44:19.663046+0000","last_undegraded":"2023-09-23T22:44:19.663046+0000","last_fullsized":"2023-09-23T22:44:19.663046+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:30.924506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"84'202","reported_seq":1530,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.659070+0000","last_change":"2023-09-23T22:43:56.860133+0000","last_active":"2023-09-23T22:44:19.659070+0000","last_peered":"2023-09-23T22:44:19.659070+0000","last_clean":"2023-09-23T22:44:19.659070+0000","last_became_active":"2023-09-23T22:43:56.859594+0000","last_became_peered":"2023-09-23T22:43:56.859594+0000","last_unstale":"2023-09-23T22:44:19.659070+0000","last_undegraded":"2023-09-23T22:44:19.659070+0000","last_fullsized":"2023-09-23T22:44:19.659070+0000","mapping_epoch":610,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:54.531616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072439599999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":928,"num_read_kb":1221,"num_write":183,"num_write_kb":301,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":822,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661296+0000","last_change":"2023-09-23T22:44:00.574883+0000","last_active":"2023-09-23T22:44:19.661296+0000","last_peered":"2023-09-23T22:44:19.661296+0000","last_clean":"2023-09-23T22:44:19.661296+0000","last_became_active":"2023-09-23T22:42:57.600816+0000","last_became_peered":"2023-09-23T22:42:57.600816+0000","last_unstale":"2023-09-23T22:44:19.661296+0000","last_undegraded":"2023-09-23T22:44:19.661296+0000","last_fullsized":"2023-09-23T22:44:19.661296+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.574795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.574795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:22:03.052349+0000","scrub_duration":0.0023357170000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658326+0000","last_change":"2023-09-23T22:43:11.806599+0000","last_active":"2023-09-23T22:44:19.658326+0000","last_peered":"2023-09-23T22:44:19.658326+0000","last_clean":"2023-09-23T22:44:19.658326+0000","last_became_active":"2023-09-23T22:43:11.806297+0000","last_became_peered":"2023-09-23T22:43:11.806297+0000","last_unstale":"2023-09-23T22:44:19.658326+0000","last_undegraded":"2023-09-23T22:44:19.658326+0000","last_fullsized":"2023-09-23T22:44:19.658326+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:11:17.339834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":268,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662534+0000","last_change":"2023-09-23T22:42:57.493464+0000","last_active":"2023-09-23T22:44:19.662534+0000","last_peered":"2023-09-23T22:44:19.662534+0000","last_clean":"2023-09-23T22:44:19.662534+0000","last_became_active":"2023-09-23T22:42:57.492614+0000","last_became_peered":"2023-09-23T22:42:57.492614+0000","last_unstale":"2023-09-23T22:44:19.662534+0000","last_undegraded":"2023-09-23T22:44:19.662534+0000","last_fullsized":"2023-09-23T22:44:19.662534+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:13.562710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":834,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662369+0000","last_change":"2023-09-23T22:43:14.428011+0000","last_active":"2023-09-23T22:44:19.662369+0000","last_peered":"2023-09-23T22:44:19.662369+0000","last_clean":"2023-09-23T22:44:19.662369+0000","last_became_active":"2023-09-23T22:43:14.427691+0000","last_became_peered":"2023-09-23T22:43:14.427691+0000","last_unstale":"2023-09-23T22:44:19.662369+0000","last_undegraded":"2023-09-23T22:44:19.662369+0000","last_fullsized":"2023-09-23T22:44:19.662369+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:56.928006+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:56.928006+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'97","reported_seq":1091,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.660866+0000","last_change":"2023-09-23T22:43:47.341779+0000","last_active":"2023-09-23T22:44:19.660866+0000","last_peered":"2023-09-23T22:44:19.660866+0000","last_clean":"2023-09-23T22:44:19.660866+0000","last_became_active":"2023-09-23T22:43:11.808371+0000","last_became_peered":"2023-09-23T22:43:11.808371+0000","last_unstale":"2023-09-23T22:44:19.660866+0000","last_undegraded":"2023-09-23T22:44:19.660866+0000","last_fullsized":"2023-09-23T22:44:19.660866+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":97,"log_dups_size":0,"ondisk_log_size":97,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:38.132844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000361923,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":629,"num_read_kb":683,"num_write":81,"num_write_kb":108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":131,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658476+0000","last_change":"2023-09-23T22:42:57.480877+0000","last_active":"2023-09-23T22:44:19.658476+0000","last_peered":"2023-09-23T22:44:19.658476+0000","last_clean":"2023-09-23T22:44:19.658476+0000","last_became_active":"2023-09-23T22:42:57.480576+0000","last_became_peered":"2023-09-23T22:42:57.480576+0000","last_unstale":"2023-09-23T22:44:19.658476+0000","last_undegraded":"2023-09-23T22:44:19.658476+0000","last_fullsized":"2023-09-23T22:44:19.658476+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:39.895915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":456,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666767+0000","last_change":"2023-09-23T22:43:11.809271+0000","last_active":"2023-09-23T22:44:19.666767+0000","last_peered":"2023-09-23T22:44:19.666767+0000","last_clean":"2023-09-23T22:44:19.666767+0000","last_became_active":"2023-09-23T22:43:11.809001+0000","last_became_peered":"2023-09-23T22:43:11.809001+0000","last_unstale":"2023-09-23T22:44:19.666767+0000","last_undegraded":"2023-09-23T22:44:19.666767+0000","last_fullsized":"2023-09-23T22:44:19.666767+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:18.092293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":950,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662559+0000","last_change":"2023-09-23T22:44:02.567423+0000","last_active":"2023-09-23T22:44:19.662559+0000","last_peered":"2023-09-23T22:44:19.662559+0000","last_clean":"2023-09-23T22:44:19.662559+0000","last_became_active":"2023-09-23T22:43:14.445311+0000","last_became_peered":"2023-09-23T22:43:14.445311+0000","last_unstale":"2023-09-23T22:44:19.662559+0000","last_undegraded":"2023-09-23T22:44:19.662559+0000","last_fullsized":"2023-09-23T22:44:19.662559+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_clean_scrub_stamp":"2023-09-23T22:44:02.567345+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:03.924259+0000","scrub_duration":0.0019136540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"83'100","reported_seq":62,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662178+0000","last_change":"2023-09-23T22:43:27.513607+0000","last_active":"2023-09-23T22:44:19.662178+0000","last_peered":"2023-09-23T22:44:19.662178+0000","last_clean":"2023-09-23T22:44:19.662178+0000","last_became_active":"2023-09-23T22:42:57.674485+0000","last_became_peered":"2023-09-23T22:42:57.674485+0000","last_unstale":"2023-09-23T22:44:19.662178+0000","last_undegraded":"2023-09-23T22:44:19.662178+0000","last_fullsized":"2023-09-23T22:44:19.662178+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":100,"log_dups_size":0,"ondisk_log_size":100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:57.916453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00036643400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":822,"num_read_kb":1139,"num_write":71,"num_write_kb":257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":276,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666847+0000","last_change":"2023-09-23T22:43:11.749267+0000","last_active":"2023-09-23T22:44:19.666847+0000","last_peered":"2023-09-23T22:44:19.666847+0000","last_clean":"2023-09-23T22:44:19.666847+0000","last_became_active":"2023-09-23T22:43:11.748938+0000","last_became_peered":"2023-09-23T22:43:11.748938+0000","last_unstale":"2023-09-23T22:44:19.666847+0000","last_undegraded":"2023-09-23T22:44:19.666847+0000","last_fullsized":"2023-09-23T22:44:19.666847+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:38.249466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.660251+0000","last_change":"2023-09-23T22:43:11.794701+0000","last_active":"2023-09-23T22:44:19.660251+0000","last_peered":"2023-09-23T22:44:19.660251+0000","last_clean":"2023-09-23T22:44:19.660251+0000","last_became_active":"2023-09-23T22:43:11.794419+0000","last_became_peered":"2023-09-23T22:43:11.794419+0000","last_unstale":"2023-09-23T22:44:19.660251+0000","last_undegraded":"2023-09-23T22:44:19.660251+0000","last_fullsized":"2023-09-23T22:44:19.660251+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:21.897331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":967,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666591+0000","last_change":"2023-09-23T22:43:14.458068+0000","last_active":"2023-09-23T22:44:19.666591+0000","last_peered":"2023-09-23T22:44:19.666591+0000","last_clean":"2023-09-23T22:44:19.666591+0000","last_became_active":"2023-09-23T22:43:14.444011+0000","last_became_peered":"2023-09-23T22:43:14.444011+0000","last_unstale":"2023-09-23T22:44:19.666591+0000","last_undegraded":"2023-09-23T22:44:19.666591+0000","last_fullsized":"2023-09-23T22:44:19.666591+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:27.290379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:27.290379+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:55.282838+0000","scrub_duration":0.0016473779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":106,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.660494+0000","last_change":"2023-09-23T22:42:05.533737+0000","last_active":"2023-09-23T22:44:19.660494+0000","last_peered":"2023-09-23T22:44:19.660494+0000","last_clean":"2023-09-23T22:44:19.660494+0000","last_became_active":"2023-09-23T22:42:05.525900+0000","last_became_peered":"2023-09-23T22:42:05.525900+0000","last_unstale":"2023-09-23T22:44:19.660494+0000","last_undegraded":"2023-09-23T22:44:19.660494+0000","last_fullsized":"2023-09-23T22:44:19.660494+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:31.248241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"84'164","reported_seq":1223,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662616+0000","last_change":"2023-09-23T22:43:39.799350+0000","last_active":"2023-09-23T22:44:19.662616+0000","last_peered":"2023-09-23T22:44:19.662616+0000","last_clean":"2023-09-23T22:44:19.662616+0000","last_became_active":"2023-09-23T22:42:57.585398+0000","last_became_peered":"2023-09-23T22:42:57.585398+0000","last_unstale":"2023-09-23T22:44:19.662616+0000","last_undegraded":"2023-09-23T22:44:19.662616+0000","last_fullsized":"2023-09-23T22:44:19.662616+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":164,"log_dups_size":0,"ondisk_log_size":164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:06.542366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00097638200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1223,"num_read_kb":1321,"num_write":121,"num_write_kb":209,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658257+0000","last_change":"2023-09-23T22:43:11.831381+0000","last_active":"2023-09-23T22:44:19.658257+0000","last_peered":"2023-09-23T22:44:19.658257+0000","last_clean":"2023-09-23T22:44:19.658257+0000","last_became_active":"2023-09-23T22:43:11.830843+0000","last_became_peered":"2023-09-23T22:43:11.830843+0000","last_unstale":"2023-09-23T22:44:19.658257+0000","last_undegraded":"2023-09-23T22:44:19.658257+0000","last_fullsized":"2023-09-23T22:44:19.658257+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:26.690562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":782,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658657+0000","last_change":"2023-09-23T22:43:14.436289+0000","last_active":"2023-09-23T22:44:19.658657+0000","last_peered":"2023-09-23T22:44:19.658657+0000","last_clean":"2023-09-23T22:44:19.658657+0000","last_became_active":"2023-09-23T22:43:14.436041+0000","last_became_peered":"2023-09-23T22:43:14.436041+0000","last_unstale":"2023-09-23T22:44:19.658657+0000","last_undegraded":"2023-09-23T22:44:19.658657+0000","last_fullsized":"2023-09-23T22:44:19.658657+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:03.084589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:03.084589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:03.001839+0000","scrub_duration":0.013840414000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"83'45","reported_seq":293,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.664270+0000","last_change":"2023-09-23T22:42:57.606062+0000","last_active":"2023-09-23T22:44:19.664270+0000","last_peered":"2023-09-23T22:44:19.664270+0000","last_clean":"2023-09-23T22:44:19.664270+0000","last_became_active":"2023-09-23T22:42:57.605639+0000","last_became_peered":"2023-09-23T22:42:57.605639+0000","last_unstale":"2023-09-23T22:44:19.664270+0000","last_undegraded":"2023-09-23T22:44:19.664270+0000","last_fullsized":"2023-09-23T22:44:19.664270+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":45,"log_dups_size":0,"ondisk_log_size":45,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:13.114886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043554599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":221,"num_read_kb":189,"num_write":38,"num_write_kb":1036,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":429,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662874+0000","last_change":"2023-09-23T22:43:11.913285+0000","last_active":"2023-09-23T22:44:19.662874+0000","last_peered":"2023-09-23T22:44:19.662874+0000","last_clean":"2023-09-23T22:44:19.662874+0000","last_became_active":"2023-09-23T22:43:11.841729+0000","last_became_peered":"2023-09-23T22:43:11.841729+0000","last_unstale":"2023-09-23T22:44:19.662874+0000","last_undegraded":"2023-09-23T22:44:19.662874+0000","last_fullsized":"2023-09-23T22:44:19.662874+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:45.396946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":35,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.658240+0000","last_change":"2023-09-23T22:43:11.808955+0000","last_active":"2023-09-23T22:44:19.658240+0000","last_peered":"2023-09-23T22:44:19.658240+0000","last_clean":"2023-09-23T22:44:19.658240+0000","last_became_active":"2023-09-23T22:43:11.808676+0000","last_became_peered":"2023-09-23T22:43:11.808676+0000","last_unstale":"2023-09-23T22:44:19.658240+0000","last_undegraded":"2023-09-23T22:44:19.658240+0000","last_fullsized":"2023-09-23T22:44:19.658240+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:38.659893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":952,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662736+0000","last_change":"2023-09-23T22:44:05.891619+0000","last_active":"2023-09-23T22:44:19.662736+0000","last_peered":"2023-09-23T22:44:19.662736+0000","last_clean":"2023-09-23T22:44:19.662736+0000","last_became_active":"2023-09-23T22:44:05.891208+0000","last_became_peered":"2023-09-23T22:44:05.891208+0000","last_unstale":"2023-09-23T22:44:19.662736+0000","last_undegraded":"2023-09-23T22:44:19.662736+0000","last_fullsized":"2023-09-23T22:44:19.662736+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.397050+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.397050+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.018671509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":125,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661739+0000","last_change":"2023-09-23T22:42:57.598277+0000","last_active":"2023-09-23T22:44:19.661739+0000","last_peered":"2023-09-23T22:44:19.661739+0000","last_clean":"2023-09-23T22:44:19.661739+0000","last_became_active":"2023-09-23T22:42:57.597389+0000","last_became_peered":"2023-09-23T22:42:57.597389+0000","last_unstale":"2023-09-23T22:44:19.661739+0000","last_undegraded":"2023-09-23T22:44:19.661739+0000","last_fullsized":"2023-09-23T22:44:19.661739+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:43.074446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":290,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666903+0000","last_change":"2023-09-23T22:43:11.749040+0000","last_active":"2023-09-23T22:44:19.666903+0000","last_peered":"2023-09-23T22:44:19.666903+0000","last_clean":"2023-09-23T22:44:19.666903+0000","last_became_active":"2023-09-23T22:43:11.748649+0000","last_became_peered":"2023-09-23T22:43:11.748649+0000","last_unstale":"2023-09-23T22:44:19.666903+0000","last_undegraded":"2023-09-23T22:44:19.666903+0000","last_fullsized":"2023-09-23T22:44:19.666903+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:52.013195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":959,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.663009+0000","last_change":"2023-09-23T22:44:05.892528+0000","last_active":"2023-09-23T22:44:19.663009+0000","last_peered":"2023-09-23T22:44:19.663009+0000","last_clean":"2023-09-23T22:44:19.663009+0000","last_became_active":"2023-09-23T22:44:05.892153+0000","last_became_peered":"2023-09-23T22:44:05.892153+0000","last_unstale":"2023-09-23T22:44:19.663009+0000","last_undegraded":"2023-09-23T22:44:19.663009+0000","last_fullsized":"2023-09-23T22:44:19.663009+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:20:45.626257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:20:45.626257+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0025635290000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":675,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.659111+0000","last_change":"2023-09-23T22:42:57.577409+0000","last_active":"2023-09-23T22:44:19.659111+0000","last_peered":"2023-09-23T22:44:19.659111+0000","last_clean":"2023-09-23T22:44:19.659111+0000","last_became_active":"2023-09-23T22:42:57.551354+0000","last_became_peered":"2023-09-23T22:42:57.551354+0000","last_unstale":"2023-09-23T22:44:19.659111+0000","last_undegraded":"2023-09-23T22:44:19.659111+0000","last_fullsized":"2023-09-23T22:44:19.659111+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:14.859975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":653,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661578+0000","last_change":"2023-09-23T22:42:57.597909+0000","last_active":"2023-09-23T22:44:19.661578+0000","last_peered":"2023-09-23T22:44:19.661578+0000","last_clean":"2023-09-23T22:44:19.661578+0000","last_became_active":"2023-09-23T22:42:57.596972+0000","last_became_peered":"2023-09-23T22:42:57.596972+0000","last_unstale":"2023-09-23T22:44:19.661578+0000","last_undegraded":"2023-09-23T22:44:19.661578+0000","last_fullsized":"2023-09-23T22:44:19.661578+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:01.735792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":690,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.659523+0000","last_change":"2023-09-23T22:42:57.481811+0000","last_active":"2023-09-23T22:44:19.659523+0000","last_peered":"2023-09-23T22:44:19.659523+0000","last_clean":"2023-09-23T22:44:19.659523+0000","last_became_active":"2023-09-23T22:42:57.481461+0000","last_became_peered":"2023-09-23T22:42:57.481461+0000","last_unstale":"2023-09-23T22:44:19.659523+0000","last_undegraded":"2023-09-23T22:44:19.659523+0000","last_fullsized":"2023-09-23T22:44:19.659523+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:25.977092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":116,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.666177+0000","last_change":"2023-09-23T22:42:57.645578+0000","last_active":"2023-09-23T22:44:19.666177+0000","last_peered":"2023-09-23T22:44:19.666177+0000","last_clean":"2023-09-23T22:44:19.666177+0000","last_became_active":"2023-09-23T22:42:57.644610+0000","last_became_peered":"2023-09-23T22:42:57.644610+0000","last_unstale":"2023-09-23T22:44:19.666177+0000","last_undegraded":"2023-09-23T22:44:19.666177+0000","last_fullsized":"2023-09-23T22:44:19.666177+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:51.348779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":731,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662441+0000","last_change":"2023-09-23T22:44:02.893662+0000","last_active":"2023-09-23T22:44:19.662441+0000","last_peered":"2023-09-23T22:44:19.662441+0000","last_clean":"2023-09-23T22:44:19.662441+0000","last_became_active":"2023-09-23T22:44:02.892760+0000","last_became_peered":"2023-09-23T22:44:02.892760+0000","last_unstale":"2023-09-23T22:44:19.662441+0000","last_undegraded":"2023-09-23T22:44:19.662441+0000","last_fullsized":"2023-09-23T22:44:19.662441+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:56.024308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:56.024308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:30.183672+0000","scrub_duration":0.0045490360000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":219,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662720+0000","last_change":"2023-09-23T22:42:57.720541+0000","last_active":"2023-09-23T22:44:19.662720+0000","last_peered":"2023-09-23T22:44:19.662720+0000","last_clean":"2023-09-23T22:44:19.662720+0000","last_became_active":"2023-09-23T22:42:57.717881+0000","last_became_peered":"2023-09-23T22:42:57.717881+0000","last_unstale":"2023-09-23T22:44:19.662720+0000","last_undegraded":"2023-09-23T22:44:19.662720+0000","last_fullsized":"2023-09-23T22:44:19.662720+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:16.534160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":234,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.665637+0000","last_change":"2023-09-23T22:43:11.853883+0000","last_active":"2023-09-23T22:44:19.665637+0000","last_peered":"2023-09-23T22:44:19.665637+0000","last_clean":"2023-09-23T22:44:19.665637+0000","last_became_active":"2023-09-23T22:43:11.840934+0000","last_became_peered":"2023-09-23T22:43:11.840934+0000","last_unstale":"2023-09-23T22:44:19.665637+0000","last_undegraded":"2023-09-23T22:44:19.665637+0000","last_fullsized":"2023-09-23T22:44:19.665637+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:27.566762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":977,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.662319+0000","last_change":"2023-09-23T22:44:05.893864+0000","last_active":"2023-09-23T22:44:19.662319+0000","last_peered":"2023-09-23T22:44:19.662319+0000","last_clean":"2023-09-23T22:44:19.662319+0000","last_became_active":"2023-09-23T22:44:05.893517+0000","last_became_peered":"2023-09-23T22:44:05.893517+0000","last_unstale":"2023-09-23T22:44:19.662319+0000","last_undegraded":"2023-09-23T22:44:19.662319+0000","last_fullsized":"2023-09-23T22:44:19.662319+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:53.343498+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:53.343498+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:19.183253+0000","scrub_duration":0.0041572029999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":41,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.660378+0000","last_change":"2023-09-23T22:43:11.877374+0000","last_active":"2023-09-23T22:44:19.660378+0000","last_peered":"2023-09-23T22:44:19.660378+0000","last_clean":"2023-09-23T22:44:19.660378+0000","last_became_active":"2023-09-23T22:43:11.805693+0000","last_became_peered":"2023-09-23T22:43:11.805693+0000","last_unstale":"2023-09-23T22:44:19.660378+0000","last_undegraded":"2023-09-23T22:44:19.660378+0000","last_fullsized":"2023-09-23T22:44:19.660378+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:06.140025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":606,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T22:44:19.661205+0000","last_change":"2023-09-23T22:42:57.601834+0000","last_active":"2023-09-23T22:44:19.661205+0000","last_peered":"2023-09-23T22:44:19.661205+0000","last_clean":"2023-09-23T22:44:19.661205+0000","last_became_active":"2023-09-23T22:42:57.600569+0000","last_became_peered":"2023-09-23T22:42:57.600569+0000","last_unstale":"2023-09-23T22:44:19.661205+0000","last_undegraded":"2023-09-23T22:44:19.661205+0000","last_fullsized":"2023-09-23T22:44:19.661205+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:13.910515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":47046,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":26924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62789,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":626148,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":29,"num_read_kb":55,"num_write":15,"num_write_kb":147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":370028,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5982,"num_read_kb":12184,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":417,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234480,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":592,"seq":2542620639251,"num_pgs":103,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":419308,"kb_used_data":302948,"kb_used_omap":8,"kb_used_meta":116343,"kb_avail":93325844,"statfs":{"total":95995035648,"available":95565664256,"internally_reserved":0,"allocated":310218752,"data_stored":308223102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":119136098},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":3.996,"5min":3.996,"15min":3.996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":47.795000000000002,"5min":47.795000000000002,"15min":47.795000000000002},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":3.3039999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":47.682000000000002,"5min":47.682000000000002,"15min":47.682000000000002},"last":0.61799999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":7.4900000000000002,"5min":7.4900000000000002,"15min":7.4900000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":115.848,"5min":115.848,"15min":115.848},"last":0.48899999999999999},{"interface":"front","average":{"1min":7.7279999999999998,"5min":7.7279999999999998,"15min":7.7279999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":116.517,"5min":116.517,"15min":116.517},"last":0.56799999999999995}]},{"osd":2,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.6379999999999999,"5min":3.6379999999999999,"15min":3.6379999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.637,"5min":2.637,"15min":2.637},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.8490000000000002,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.2,"5min":1.2,"15min":1.2},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.6260000000000003,"5min":5.6260000000000003,"15min":5.6260000000000003},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.66800000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.59899999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.339,"5min":1.339,"15min":1.339},"last":0.64300000000000002}]}]},{"osd":1,"up_from":495,"seq":2126008811601,"num_pgs":104,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":432720,"kb_used_data":12760,"kb_used_omap":12,"kb_used_meta":419891,"kb_avail":93312432,"statfs":{"total":95995035648,"available":95551930368,"internally_reserved":0,"allocated":13066240,"data_stored":9901522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":429969070},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":2.2389999999999999,"15min":1.099},"min":{"1min":0.20200000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.0249999999999999,"5min":64.507000000000005,"15min":64.507000000000005},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.323,"15min":1.375},"min":{"1min":0.24399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.8460000000000001,"5min":64.352000000000004,"15min":64.352000000000004},"last":0.48399999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":12.957000000000001,"5min":3.7730000000000001,"15min":1.7050000000000001},"min":{"1min":0.254,"5min":0.192,"15min":0.192},"max":{"1min":124.447,"5min":124.447,"15min":124.447},"last":0.54400000000000004},{"interface":"front","average":{"1min":12.526,"5min":3.3849999999999998,"15min":1.6419999999999999},"min":{"1min":0.29399999999999998,"5min":0.218,"15min":0.218},"max":{"1min":124.03,"5min":124.03,"15min":124.03},"last":0.57299999999999995}]},{"osd":3,"last update":"Sat Sep 23 22:43:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7949999999999999,"5min":1.7949999999999999,"15min":1.7949999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.800000000000001,"5min":11.800000000000001,"15min":11.800000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.7030000000000001,"5min":1.7030000000000001,"15min":1.7030000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":6.4530000000000003,"5min":6.4530000000000003,"15min":6.4530000000000003},"last":0.71699999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.0249999999999999,"15min":0.70899999999999996},"min":{"1min":0.38,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.0750000000000002,"5min":5.5439999999999996,"15min":5.5439999999999996},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.85299999999999998,"5min":1.01,"15min":1.004},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.4510000000000001,"5min":5.524,"15min":5.6740000000000004},"last":0.58699999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":1.167,"15min":0.95899999999999996},"min":{"1min":0.42399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":11.734,"5min":11.734,"15min":11.734},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.0600000000000001,"15min":0.755},"min":{"1min":0.439,"5min":0.31,"15min":0.31},"max":{"1min":3.1389999999999998,"5min":5.5839999999999996,"15min":5.5839999999999996},"last":0.82899999999999996}]},{"osd":6,"last update":"Sat Sep 23 22:43:55 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":1.736,"15min":1.736},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":9.3759999999999994,"5min":9.3759999999999994,"15min":9.3759999999999994},"last":0.79800000000000004},{"interface":"front","average":{"1min":0.877,"5min":0.877,"15min":0.877},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.9540000000000002,"5min":2.9540000000000002,"15min":2.9540000000000002},"last":0.55400000000000005}]},{"osd":7,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0059999999999998,"5min":1.2809999999999999,"15min":1.0049999999999999},"min":{"1min":0.42999999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.768000000000001,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.119,"15min":0.80100000000000005},"min":{"1min":0.496,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.169,"5min":5.6120000000000001,"15min":5.6120000000000001},"last":0.86099999999999999}]}]},{"osd":0,"up_from":0,"seq":399431958917,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":349,"seq":1498943586480,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":509340,"kb_used_data":8340,"kb_used_omap":15,"kb_used_meta":500976,"kb_avail":93235812,"statfs":{"total":95995035648,"available":95473471488,"internally_reserved":0,"allocated":8540160,"data_stored":5387748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":513000182},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":2.9289999999999998,"15min":4.8280000000000003},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":4.4370000000000003,"5min":80.126000000000005,"15min":194.06299999999999},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.909,"5min":3.2050000000000001,"15min":5.4169999999999998},"min":{"1min":0.29499999999999998,"5min":0.215,"15min":0.215},"max":{"1min":7.5999999999999996,"5min":83.899000000000001,"15min":196.947},"last":0.35699999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:43:49 2023","interfaces":[{"interface":"back","average":{"1min":5.2510000000000003,"5min":2.831,"15min":1.371},"min":{"1min":0.224,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":47.018000000000001,"5min":70.042000000000002,"15min":70.042000000000002},"last":0.30599999999999999},{"interface":"front","average":{"1min":7.2069999999999999,"5min":4.9139999999999997,"15min":2.3690000000000002},"min":{"1min":0.23599999999999999,"5min":0.184,"15min":0.184},"max":{"1min":47.671999999999997,"5min":127.327,"15min":127.327},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:43:32 2023","interfaces":[{"interface":"back","average":{"1min":3.9169999999999998,"5min":3.9169999999999998,"15min":3.9169999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":14.561,"5min":14.561,"15min":14.561},"last":0.443},{"interface":"front","average":{"1min":1.8959999999999999,"5min":1.8959999999999999,"15min":1.8959999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":4.7720000000000002,"5min":4.7720000000000002,"15min":4.7720000000000002},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:44:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.8380000000000001,"15min":1.865},"min":{"1min":0.432,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.359,"5min":45.576999999999998,"15min":45.576999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.119,"5min":1.6180000000000001,"15min":1.3049999999999999},"min":{"1min":0.36599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.0620000000000003,"5min":45.420999999999999,"15min":45.420999999999999},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:44:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.4650000000000001,"15min":3.3730000000000002},"min":{"1min":0.39300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.419,"5min":8.8119999999999994,"15min":163.56399999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.1479999999999999,"5min":2.5350000000000001,"15min":3.423},"min":{"1min":0.34699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5499999999999998,"5min":126.81100000000001,"15min":163.541},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2629999999999999,"5min":2.2629999999999999,"15min":2.2629999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":7.351,"5min":7.351,"15min":7.351},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.476,"5min":1.476,"15min":1.476},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":3.895,"5min":3.895,"15min":3.895},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:44:15 2023","interfaces":[{"interface":"back","average":{"1min":1.704,"5min":2.1269999999999998,"15min":1.9450000000000001},"min":{"1min":0.46600000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.601,"5min":75.513999999999996,"15min":108.113},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.198,"5min":2.1589999999999998,"15min":2.2810000000000001},"min":{"1min":0.48999999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.4529999999999998,"5min":74.007999999999996,"15min":108.093},"last":0.50800000000000001}]}]},{"osd":3,"up_from":0,"seq":2508260900888,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":0,"seq":1997159792743,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757753,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":560304,"kb_used_data":11496,"kb_used_omap":0,"kb_used_meta":548800,"kb_avail":93184848,"statfs":{"total":95995035648,"available":95421284352,"internally_reserved":0,"allocated":11771904,"data_stored":9772690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":561971200},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:40 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":1.1799999999999999,"15min":3.6709999999999998},"min":{"1min":0.24099999999999999,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.1739999999999995,"5min":26.925000000000001,"15min":145.46600000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.135,"5min":1.179,"15min":3.5649999999999999},"min":{"1min":0.245,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.4740000000000002,"5min":30.609000000000002,"15min":144.429},"last":0.629}]},{"osd":1,"last update":"Sat Sep 23 22:43:45 2023","interfaces":[{"interface":"back","average":{"1min":3.4910000000000001,"5min":1.8700000000000001,"15min":1.0800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":35.643999999999998,"5min":35.643999999999998,"15min":35.643999999999998},"last":0.61099999999999999},{"interface":"front","average":{"1min":3.6949999999999998,"5min":1.869,"15min":0.98199999999999998},"min":{"1min":0.29399999999999998,"5min":0.25,"15min":0.249},"max":{"1min":35.316000000000003,"5min":35.316000000000003,"15min":35.316000000000003},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Sep 23 22:44:14 2023","interfaces":[{"interface":"back","average":{"1min":5.1829999999999998,"5min":2.556,"15min":2.468},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":77.813999999999993,"5min":77.813999999999993,"15min":77.813999999999993},"last":0.52000000000000002},{"interface":"front","average":{"1min":4.9790000000000001,"5min":2.2400000000000002,"15min":2.3380000000000001},"min":{"1min":0.36599999999999999,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":76.265000000000001,"5min":76.265000000000001,"15min":76.265000000000001},"last":0.64400000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:43:33 2023","interfaces":[{"interface":"back","average":{"1min":1.532,"5min":1.532,"15min":1.532},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.0919999999999996,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.484,"5min":1.484,"15min":1.484},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.8959999999999999,"5min":7.8959999999999999,"15min":7.8959999999999999},"last":0.57099999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51900000000000002,"15min":2.4470000000000001},"min":{"1min":0.27200000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.476,"5min":4.0750000000000002,"15min":61.173999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.60499999999999998,"15min":2.5179999999999998},"min":{"1min":0.35799999999999998,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":1.5409999999999999,"5min":4.157,"15min":61.203000000000003},"last":0.72799999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.69499999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:43:40 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.83499999999999996,"15min":1.3160000000000001},"min":{"1min":0.34399999999999997,"5min":0.249,"15min":0.249},"max":{"1min":1.4550000000000001,"5min":31.042999999999999,"15min":59.101999999999997},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.50900000000000001,"5min":1.002,"15min":1.538},"min":{"1min":0.33600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.1950000000000001,"5min":30.882999999999999,"15min":59.151000000000003},"last":0.76500000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8798,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:44:36.165 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:36.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:36.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:36.270 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:36.304 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:36.541 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/img /tmp/rbd_import_export_687972/img2 /tmp/rbd_import_export_687972/img3 2023-09-23T22:44:37.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:37 smithi026 ceph-mon[223014]: pgmap v1617: 103 pgs: 103 active+clean; 6.5 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 151 KiB/s rd, 60 KiB/s wr, 38 op/s 2023-09-23T22:44:37.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:37 smithi026 ceph-mon[223014]: 1.f scrub starts 2023-09-23T22:44:37.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:37 smithi026 ceph-mon[223014]: 1.f scrub ok 2023-09-23T22:44:37.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:37 smithi026 ceph-mon[215775]: pgmap v1617: 103 pgs: 103 active+clean; 6.5 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 151 KiB/s rd, 60 KiB/s wr, 38 op/s 2023-09-23T22:44:37.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:37 smithi026 ceph-mon[215775]: 1.f scrub starts 2023-09-23T22:44:37.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:37 smithi026 ceph-mon[215775]: 1.f scrub ok 2023-09-23T22:44:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:37 smithi070 ceph-mon[192652]: pgmap v1617: 103 pgs: 103 active+clean; 6.5 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 151 KiB/s rd, 60 KiB/s wr, 38 op/s 2023-09-23T22:44:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:37 smithi070 ceph-mon[192652]: 1.f scrub starts 2023-09-23T22:44:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:37 smithi070 ceph-mon[192652]: 1.f scrub ok 2023-09-23T22:44:38.245 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:44:39.094 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:44:39.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:44:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:44:39] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2023-09-23T22:44:39.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:39 smithi026 ceph-mon[223014]: pgmap v1618: 103 pgs: 103 active+clean; 6.5 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 183 KiB/s rd, 65 KiB/s wr, 67 op/s 2023-09-23T22:44:39.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:39 smithi026 ceph-mon[223014]: 1.3 scrub starts 2023-09-23T22:44:39.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:39 smithi026 ceph-mon[223014]: 1.3 scrub ok 2023-09-23T22:44:39.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:39.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:39 smithi026 ceph-mon[215775]: pgmap v1618: 103 pgs: 103 active+clean; 6.5 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 183 KiB/s rd, 65 KiB/s wr, 67 op/s 2023-09-23T22:44:39.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:39 smithi026 ceph-mon[215775]: 1.3 scrub starts 2023-09-23T22:44:39.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:39 smithi026 ceph-mon[215775]: 1.3 scrub ok 2023-09-23T22:44:39.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:39 smithi070 ceph-mon[192652]: pgmap v1618: 103 pgs: 103 active+clean; 6.5 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 183 KiB/s rd, 65 KiB/s wr, 67 op/s 2023-09-23T22:44:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:39 smithi070 ceph-mon[192652]: 1.3 scrub starts 2023-09-23T22:44:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:39 smithi070 ceph-mon[192652]: 1.3 scrub ok 2023-09-23T22:44:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:40.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:40 smithi026 ceph-mon[223014]: 1.19 scrub starts 2023-09-23T22:44:40.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:40 smithi026 ceph-mon[223014]: 1.19 scrub ok 2023-09-23T22:44:40.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:40 smithi026 ceph-mon[215775]: 1.19 scrub starts 2023-09-23T22:44:40.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:40 smithi026 ceph-mon[215775]: 1.19 scrub ok 2023-09-23T22:44:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:40 smithi070 ceph-mon[192652]: 1.19 scrub starts 2023-09-23T22:44:40.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:40 smithi070 ceph-mon[192652]: 1.19 scrub ok 2023-09-23T22:44:41.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:41 smithi026 ceph-mon[215775]: pgmap v1619: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 206 KiB/s rd, 74 KiB/s wr, 63 op/s 2023-09-23T22:44:41.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:41 smithi026 ceph-mon[223014]: pgmap v1619: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 206 KiB/s rd, 74 KiB/s wr, 63 op/s 2023-09-23T22:44:41.749 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd help export 2023-09-23T22:44:41.749 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -q export-format 2023-09-23T22:44:41.770 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_687972/img bs=1k count=10 seek=100 2023-09-23T22:44:41.847 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records in 2023-09-23T22:44:41.847 INFO:tasks.workunit.client.0.smithi026.stderr:10+0 records out 2023-09-23T22:44:41.847 INFO:tasks.workunit.client.0.smithi026.stderr:10240 bytes (10 kB, 10 KiB) copied, 8.0968e-05 s, 126 MB/s 2023-09-23T22:44:41.847 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format /tmp/rbd_import_export_687972/img testimg 2023-09-23T22:44:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:41 smithi070 ceph-mon[192652]: pgmap v1619: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 206 KiB/s rd, 74 KiB/s wr, 63 op/s 2023-09-23T22:44:41.904 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:44:41.904 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:44:41.906 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap create testimg@snap 2023-09-23T22:44:42.449 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:42.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:42.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:42.458 INFO:tasks.workunit.client.0.smithi026.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T22:44:42.466 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T22:44:42.519 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd image-meta list testimg 2023-09-23T22:44:42.555 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:42.562 INFO:tasks.workunit.client.0.smithi026.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T22:44:42.562 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T22:44:42.562 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T22:44:42.562 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' 2023-09-23T22:44:42.562 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_687972/img_v2 2023-09-23T22:44:42.589 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:42.623 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:42.629 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_687972/img_v2 testimg_import 2023-09-23T22:44:42.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:42 smithi026 ceph-mon[223014]: 1.1e scrub starts 2023-09-23T22:44:42.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:42 smithi026 ceph-mon[223014]: 1.1e scrub ok 2023-09-23T22:44:42.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:42 smithi026 ceph-mon[215775]: 1.1e scrub starts 2023-09-23T22:44:42.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:42 smithi026 ceph-mon[215775]: 1.1e scrub ok 2023-09-23T22:44:42.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:42 smithi070 ceph-mon[192652]: 1.1e scrub starts 2023-09-23T22:44:42.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:42 smithi070 ceph-mon[192652]: 1.1e scrub ok 2023-09-23T22:44:42.988 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:42.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:42.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:43.092 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:43.125 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:43.173 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:43.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:43.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:43.207 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:43.215 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:43.266 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:43.280 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:43.314 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:43.366 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:44:43.385 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:44:43.408 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:43.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:43.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:43.413 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:44:43.430 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:44:43.460 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:44:43.471 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:44:43.474 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T22:44:43.482 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:44:43.511 INFO:tasks.workunit.client.0.smithi026.stdout:rbd image 'testimg_import': 2023-09-23T22:44:43.511 INFO:tasks.workunit.client.0.smithi026.stdout: size 110 KiB in 1 objects 2023-09-23T22:44:43.511 INFO:tasks.workunit.client.0.smithi026.stdout: order 22 (4 MiB objects) 2023-09-23T22:44:43.511 INFO:tasks.workunit.client.0.smithi026.stdout: snapshot_count: 1 2023-09-23T22:44:43.511 INFO:tasks.workunit.client.0.smithi026.stdout: id: 90e4cc1a1b1c 2023-09-23T22:44:43.511 INFO:tasks.workunit.client.0.smithi026.stdout: block_name_prefix: rbd_data.90e4cc1a1b1c 2023-09-23T22:44:43.511 INFO:tasks.workunit.client.0.smithi026.stdout: format: 2 2023-09-23T22:44:43.511 INFO:tasks.workunit.client.0.smithi026.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T22:44:43.511 INFO:tasks.workunit.client.0.smithi026.stdout: op_features: 2023-09-23T22:44:43.512 INFO:tasks.workunit.client.0.smithi026.stdout: flags: 2023-09-23T22:44:43.512 INFO:tasks.workunit.client.0.smithi026.stdout: create_timestamp: Sat Sep 23 22:44:42 2023 2023-09-23T22:44:43.512 INFO:tasks.workunit.client.0.smithi026.stdout: access_timestamp: Sat Sep 23 22:44:42 2023 2023-09-23T22:44:43.512 INFO:tasks.workunit.client.0.smithi026.stdout: modify_timestamp: Sat Sep 23 22:44:42 2023 2023-09-23T22:44:43.513 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:43.514 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import@snap 2023-09-23T22:44:43.517 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:44:43.547 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:44:43.552 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:44:43.552 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:44:43.552 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:44:43.552 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:44:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:44:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:44:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:44:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:44:43.553 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:43.562 INFO:tasks.workunit.client.0.smithi026.stdout:rbd image 'testimg_import': 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: size 110 KiB in 1 objects 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: order 22 (4 MiB objects) 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: snapshot_count: 1 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: id: 90e4cc1a1b1c 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: block_name_prefix: rbd_data.90e4cc1a1b1c 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: format: 2 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: op_features: 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: flags: 2023-09-23T22:44:43.563 INFO:tasks.workunit.client.0.smithi026.stdout: create_timestamp: Sat Sep 23 22:44:42 2023 2023-09-23T22:44:43.564 INFO:tasks.workunit.client.0.smithi026.stdout: access_timestamp: Sat Sep 23 22:44:42 2023 2023-09-23T22:44:43.564 INFO:tasks.workunit.client.0.smithi026.stdout: modify_timestamp: Sat Sep 23 22:44:42 2023 2023-09-23T22:44:43.564 INFO:tasks.workunit.client.0.smithi026.stdout: protected: False 2023-09-23T22:44:43.567 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd image-meta list testimg_import 2023-09-23T22:44:43.613 INFO:tasks.workunit.client.0.smithi026.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T22:44:43.613 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T22:44:43.613 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T22:44:43.613 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' 2023-09-23T22:44:43.614 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T22:44:43.614 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T22:44:43.614 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T22:44:43.614 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T22:44:43.614 INFO:tasks.workunit.client.0.smithi026.stderr: 2023-09-23T22:44:43.614 INFO:tasks.workunit.client.0.smithi026.stderr:Key Value 2023-09-23T22:44:43.614 INFO:tasks.workunit.client.0.smithi026.stderr:key1 value1' ']' 2023-09-23T22:44:43.614 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg_import /tmp/rbd_import_export_687972/img_import 2023-09-23T22:44:43.651 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T22:44:43.654 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_687972/img /tmp/rbd_import_export_687972/img_import 2023-09-23T22:44:43.654 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_687972/img /tmp/rbd_import_export_687972/img_import 2023-09-23T22:44:43.655 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/img --format %b 2023-09-23T22:44:43.656 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=24 2023-09-23T22:44:43.657 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/img_import --format %b 2023-09-23T22:44:43.658 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=224 2023-09-23T22:44:43.658 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=200 2023-09-23T22:44:43.658 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=200 2023-09-23T22:44:43.658 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T22:44:43.659 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_687972/img_snap 2023-09-23T22:44:43.705 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T22:44:43.708 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_687972/img_snap_import 2023-09-23T22:44:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[223014]: pgmap v1620: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 167 KiB/s rd, 47 KiB/s wr, 62 op/s 2023-09-23T22:44:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[223014]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[223014]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[223014]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[223014]: osdmap e625: 8 total, 8 up, 4 in 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[223014]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1007510719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[215775]: pgmap v1620: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 167 KiB/s rd, 47 KiB/s wr, 62 op/s 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[215775]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[215775]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[215775]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[215775]: osdmap e625: 8 total, 8 up, 4 in 2023-09-23T22:44:43.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[215775]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:43 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1007510719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:43.744 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T22:44:43.747 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_687972/img_snap /tmp/rbd_import_export_687972/img_snap_import 2023-09-23T22:44:43.747 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_687972/img_snap /tmp/rbd_import_export_687972/img_snap_import 2023-09-23T22:44:43.748 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/img_snap --format %b 2023-09-23T22:44:43.749 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=224 2023-09-23T22:44:43.749 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/img_snap_import --format %b 2023-09-23T22:44:43.750 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=224 2023-09-23T22:44:43.750 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:43.750 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:43.750 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:44:43.751 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/img_v2 2023-09-23T22:44:43.751 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/img_import 2023-09-23T22:44:43.751 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/img_snap 2023-09-23T22:44:43.752 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/img_snap_import 2023-09-23T22:44:43.753 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T22:44:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:43 smithi070 ceph-mon[192652]: pgmap v1620: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 167 KiB/s rd, 47 KiB/s wr, 62 op/s 2023-09-23T22:44:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:43 smithi070 ceph-mon[192652]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:43 smithi070 ceph-mon[192652]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:43 smithi070 ceph-mon[192652]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:43 smithi070 ceph-mon[192652]: osdmap e625: 8 total, 8 up, 4 in 2023-09-23T22:44:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:43 smithi070 ceph-mon[192652]: failed to encode map e625 with expected crc 2023-09-23T22:44:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:43 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1007510719' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:43.898 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:43.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:44.002 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:44.029 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:44.071 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:44.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:44.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:44.180 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:44.215 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:44.465 INFO:tasks.workunit.client.0.smithi026.stderr: Removing snap: 100% complete...done. 2023-09-23T22:44:44.474 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:44:44.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:44 smithi026 ceph-mon[215775]: failed to encode map e626 with expected crc 2023-09-23T22:44:44.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:44 smithi026 ceph-mon[215775]: failed to encode map e626 with expected crc 2023-09-23T22:44:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:44 smithi026 ceph-mon[215775]: osdmap e626: 8 total, 8 up, 4 in 2023-09-23T22:44:44.486 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:44 smithi026 ceph-mon[215775]: failed to encode map e626 with expected crc 2023-09-23T22:44:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:44 smithi026 ceph-mon[223014]: failed to encode map e626 with expected crc 2023-09-23T22:44:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:44 smithi026 ceph-mon[223014]: failed to encode map e626 with expected crc 2023-09-23T22:44:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:44 smithi026 ceph-mon[223014]: osdmap e626: 8 total, 8 up, 4 in 2023-09-23T22:44:44.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:44 smithi026 ceph-mon[223014]: failed to encode map e626 with expected crc 2023-09-23T22:44:44.656 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:44:44.658 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap rm testimg@snap 2023-09-23T22:44:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:44 smithi070 ceph-mon[192652]: failed to encode map e626 with expected crc 2023-09-23T22:44:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:44 smithi070 ceph-mon[192652]: failed to encode map e626 with expected crc 2023-09-23T22:44:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:44 smithi070 ceph-mon[192652]: osdmap e626: 8 total, 8 up, 4 in 2023-09-23T22:44:44.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:44 smithi070 ceph-mon[192652]: failed to encode map e626 with expected crc 2023-09-23T22:44:45.081 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:45.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:45.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:45.186 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:45.219 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:45.267 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:45.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:45.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:45.374 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:45.402 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:45.447 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:45.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:45.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:45.465 INFO:tasks.workunit.client.0.smithi026.stderr: Removing snap: 100% complete...done. 2023-09-23T22:44:45.473 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm testimg 2023-09-23T22:44:45.553 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:45.565 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:44:45.569 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --order 20 /tmp/rbd_import_export_687972/img testimg 2023-09-23T22:44:45.588 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:45 smithi070 ceph-mon[192652]: pgmap v1623: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 125 KiB/s rd, 38 KiB/s wr, 55 op/s 2023-09-23T22:44:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:45 smithi070 ceph-mon[192652]: failed to encode map e627 with expected crc 2023-09-23T22:44:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:45 smithi070 ceph-mon[192652]: failed to encode map e627 with expected crc 2023-09-23T22:44:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:45 smithi070 ceph-mon[192652]: osdmap e627: 8 total, 8 up, 4 in 2023-09-23T22:44:45.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:45 smithi070 ceph-mon[192652]: failed to encode map e627 with expected crc 2023-09-23T22:44:45.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[215775]: pgmap v1623: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 125 KiB/s rd, 38 KiB/s wr, 55 op/s 2023-09-23T22:44:45.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[215775]: failed to encode map e627 with expected crc 2023-09-23T22:44:45.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[215775]: failed to encode map e627 with expected crc 2023-09-23T22:44:45.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[215775]: osdmap e627: 8 total, 8 up, 4 in 2023-09-23T22:44:45.627 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[215775]: failed to encode map e627 with expected crc 2023-09-23T22:44:45.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[223014]: pgmap v1623: 103 pgs: 103 active+clean; 4.6 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 125 KiB/s rd, 38 KiB/s wr, 55 op/s 2023-09-23T22:44:45.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[223014]: failed to encode map e627 with expected crc 2023-09-23T22:44:45.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[223014]: failed to encode map e627 with expected crc 2023-09-23T22:44:45.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[223014]: osdmap e627: 8 total, 8 up, 4 in 2023-09-23T22:44:45.628 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:45 smithi026 ceph-mon[223014]: failed to encode map e627 with expected crc 2023-09-23T22:44:45.634 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:44:45.635 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:44:45.637 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_687972/img_v2 2023-09-23T22:44:45.676 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:45.679 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_687972/img_v2 testimg_import 2023-09-23T22:44:45.742 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:44:45.745 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T22:44:45.746 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep order 2023-09-23T22:44:45.746 INFO:tasks.workunit.client.0.smithi026.stderr:+ awk '{print $2}' 2023-09-23T22:44:45.746 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 20 2023-09-23T22:44:45.781 INFO:tasks.workunit.client.0.smithi026.stdout:20 2023-09-23T22:44:45.781 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/img_v2 2023-09-23T22:44:45.825 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:44:45.950 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:44:45.952 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T22:44:46.037 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:44:46.040 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_687972/img testimg 2023-09-23T22:44:46.086 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:44:46.089 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd info testimg 2023-09-23T22:44:46.089 INFO:tasks.workunit.client.0.smithi026.stderr:++ grep features 2023-09-23T22:44:46.125 INFO:tasks.workunit.client.0.smithi026.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T22:44:46.125 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' 2023-09-23T22:44:46.125 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_687972/img_v2 2023-09-23T22:44:46.161 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:46.164 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_687972/img_v2 testimg_import 2023-09-23T22:44:46.218 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:44:46.221 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd info testimg_import 2023-09-23T22:44:46.221 INFO:tasks.workunit.client.0.smithi026.stderr:++ grep features 2023-09-23T22:44:46.257 INFO:tasks.workunit.client.0.smithi026.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T22:44:46.258 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' 2023-09-23T22:44:46.258 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' ' features: layering 2023-09-23T22:44:46.258 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' '!=' ' features: layering 2023-09-23T22:44:46.258 INFO:tasks.workunit.client.0.smithi026.stderr: op_features: ' ']' 2023-09-23T22:44:46.258 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/img_v2 2023-09-23T22:44:46.258 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:44:46.326 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:44:46.329 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T22:44:46.399 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:44:46.402 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_687972/img testimg 2023-09-23T22:44:46.460 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:44:46.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:46 smithi026 ceph-mon[215775]: failed to encode map e628 with expected crc 2023-09-23T22:44:46.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:46 smithi026 ceph-mon[215775]: failed to encode map e628 with expected crc 2023-09-23T22:44:46.464 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_687972/img_v2 2023-09-23T22:44:46.502 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:46.505 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_687972/img_v2 testimg_import 2023-09-23T22:44:46.568 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:44:46.571 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T22:44:46.571 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'stripe unit' 2023-09-23T22:44:46.571 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T22:44:46.607 INFO:tasks.workunit.client.0.smithi026.stdout: stripe unit: 4 KiB 2023-09-23T22:44:46.608 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import 2023-09-23T22:44:46.608 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'stripe count' 2023-09-23T22:44:46.608 INFO:tasks.workunit.client.0.smithi026.stderr:+ awk '{print $3}' 2023-09-23T22:44:46.608 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 1000 2023-09-23T22:44:46.643 INFO:tasks.workunit.client.0.smithi026.stdout:1000 2023-09-23T22:44:46.643 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/img_v2 2023-09-23T22:44:46.644 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:44:46.730 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T22:44:46.734 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T22:44:46.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:46 smithi026 ceph-mon[215775]: failed to encode map e628 with expected crc 2023-09-23T22:44:46.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:46 smithi026 ceph-mon[215775]: osdmap e628: 8 total, 8 up, 4 in 2023-09-23T22:44:46.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:46 smithi026 ceph-mon[223014]: failed to encode map e628 with expected crc 2023-09-23T22:44:46.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:46 smithi026 ceph-mon[223014]: failed to encode map e628 with expected crc 2023-09-23T22:44:46.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:46 smithi026 ceph-mon[223014]: failed to encode map e628 with expected crc 2023-09-23T22:44:46.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:46 smithi026 ceph-mon[223014]: osdmap e628: 8 total, 8 up, 4 in 2023-09-23T22:44:46.822 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T22:44:46.825 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_687972/img testimg 2023-09-23T22:44:46.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:46 smithi070 ceph-mon[192652]: failed to encode map e628 with expected crc 2023-09-23T22:44:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:46 smithi070 ceph-mon[192652]: failed to encode map e628 with expected crc 2023-09-23T22:44:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:46 smithi070 ceph-mon[192652]: failed to encode map e628 with expected crc 2023-09-23T22:44:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:46 smithi070 ceph-mon[192652]: osdmap e628: 8 total, 8 up, 4 in 2023-09-23T22:44:46.877 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:44:46.880 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap create testimg@snap1 2023-09-23T22:44:47.021 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:44:47.480 INFO:tasks.workunit.client.0.smithi026.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T22:44:47.486 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap create testimg@snap2 2023-09-23T22:44:47.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:47 smithi026 ceph-mon[223014]: pgmap v1626: 103 pgs: 103 active+clean; 4.8 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 109 KiB/s rd, 11 KiB/s wr, 130 op/s 2023-09-23T22:44:47.615 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:47 smithi026 ceph-mon[223014]: failed to encode map e628 with expected crc 2023-09-23T22:44:47.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:47 smithi026 ceph-mon[215775]: pgmap v1626: 103 pgs: 103 active+clean; 4.8 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 109 KiB/s rd, 11 KiB/s wr, 130 op/s 2023-09-23T22:44:47.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:47 smithi026 ceph-mon[215775]: failed to encode map e628 with expected crc 2023-09-23T22:44:47.735 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:47.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:47.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:47.842 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:47.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:47 smithi070 ceph-mon[192652]: pgmap v1626: 103 pgs: 103 active+clean; 4.8 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 109 KiB/s rd, 11 KiB/s wr, 130 op/s 2023-09-23T22:44:47.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:47 smithi070 ceph-mon[192652]: failed to encode map e628 with expected crc 2023-09-23T22:44:47.867 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:47.910 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:47.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:47.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:48.016 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:48.017 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:48.054 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:48.195 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:44:48.197 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:44:48.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: failed to encode map e629 with expected crc 2023-09-23T22:44:48.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: failed to encode map e629 with expected crc 2023-09-23T22:44:48.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: osdmap e629: 8 total, 8 up, 4 in 2023-09-23T22:44:48.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: failed to encode map e629 with expected crc 2023-09-23T22:44:48.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:48.475 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:48.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:44:48.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:44:48.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:48.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4285700797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:48.476 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:48.477 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:44:48.510 INFO:teuthology.orchestra.run.smithi026.stdout:ERROR: (22) Invalid argument 2023-09-23T22:44:48.510 INFO:teuthology.orchestra.run.smithi026.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:44:48.515 INFO:tasks.workunit.client.0.smithi026.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T22:44:48.522 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap protect testimg@snap2 2023-09-23T22:44:48.566 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_687972/snap_protect 2023-09-23T22:44:48.606 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:48.610 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_687972/snap_protect testimg_import 2023-09-23T22:44:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: failed to encode map e629 with expected crc 2023-09-23T22:44:48.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: failed to encode map e629 with expected crc 2023-09-23T22:44:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: osdmap e629: 8 total, 8 up, 4 in 2023-09-23T22:44:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: failed to encode map e629 with expected crc 2023-09-23T22:44:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:44:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:44:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4285700797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:48.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:48 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: failed to encode map e629 with expected crc 2023-09-23T22:44:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: failed to encode map e629 with expected crc 2023-09-23T22:44:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: osdmap e629: 8 total, 8 up, 4 in 2023-09-23T22:44:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: failed to encode map e629 with expected crc 2023-09-23T22:44:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:44:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:44:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4285700797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:48.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:48 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:44:49.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[223014]: pgmap v1628: 103 pgs: 103 active+clean; 4.8 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 209 KiB/s rd, 13 KiB/s wr, 248 op/s 2023-09-23T22:44:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[223014]: from='client.37302 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:44:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:44:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[223014]: failed to encode map e630 with expected crc 2023-09-23T22:44:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[223014]: failed to encode map e630 with expected crc 2023-09-23T22:44:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[223014]: osdmap e630: 8 total, 8 up, 4 in 2023-09-23T22:44:49.614 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[223014]: failed to encode map e630 with expected crc 2023-09-23T22:44:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[215775]: pgmap v1628: 103 pgs: 103 active+clean; 4.8 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 209 KiB/s rd, 13 KiB/s wr, 248 op/s 2023-09-23T22:44:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[215775]: from='client.37302 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:44:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:44:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[215775]: failed to encode map e630 with expected crc 2023-09-23T22:44:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[215775]: failed to encode map e630 with expected crc 2023-09-23T22:44:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:49.615 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[215775]: osdmap e630: 8 total, 8 up, 4 in 2023-09-23T22:44:49.616 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:49 smithi026 ceph-mon[215775]: failed to encode map e630 with expected crc 2023-09-23T22:44:49.842 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:49.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:49.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:49 smithi070 ceph-mon[192652]: pgmap v1628: 103 pgs: 103 active+clean; 4.8 MiB data, 1.8 GiB used, 356 GiB / 358 GiB avail; 209 KiB/s rd, 13 KiB/s wr, 248 op/s 2023-09-23T22:44:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:49 smithi070 ceph-mon[192652]: from='client.37302 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:44:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:49 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:44:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:49 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:44:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:49 smithi070 ceph-mon[192652]: failed to encode map e630 with expected crc 2023-09-23T22:44:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:49 smithi070 ceph-mon[192652]: failed to encode map e630 with expected crc 2023-09-23T22:44:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:49 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:44:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:49 smithi070 ceph-mon[192652]: osdmap e630: 8 total, 8 up, 4 in 2023-09-23T22:44:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:49 smithi070 ceph-mon[192652]: failed to encode map e630 with expected crc 2023-09-23T22:44:49.947 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:49.979 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:49.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:44:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:44:49] "GET /metrics HTTP/1.1" 200 34088 "" "Prometheus/2.43.0" 2023-09-23T22:44:50.028 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:50.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:50.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:50.132 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:50.169 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:50.174 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1628,"stamp":"2023-09-23T22:44:47.764682+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5015596,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":7721,"num_read_kb":15648,"num_write":1118,"num_write_kb":13828,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":567,"num_bytes_recovered":21281349,"num_keys_recovered":80,"num_objects_omap":17,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6278,"ondisk_log_size":6278,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":4,"num_per_pool_osds":4,"num_per_pool_omap_osds":4,"kb":374980608,"kb_used":1836696,"kb_used_data":249704,"kb_used_omap":35,"kb_used_meta":1586908,"kb_avail":373143912,"statfs":{"total":383980142592,"available":382099365888,"internally_reserved":0,"allocated":255696896,"data_stored":245352883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":36602,"internal_metadata":1624994054},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":225315,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":998,"num_read_kb":902,"num_write":72,"num_write_kb":56,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.311857"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":1106,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.456742+0000","last_change":"2023-09-23T22:44:05.893795+0000","last_active":"2023-09-23T22:44:43.456742+0000","last_peered":"2023-09-23T22:44:43.456742+0000","last_clean":"2023-09-23T22:44:43.456742+0000","last_became_active":"2023-09-23T22:44:05.893431+0000","last_became_peered":"2023-09-23T22:44:05.893431+0000","last_unstale":"2023-09-23T22:44:43.456742+0000","last_undegraded":"2023-09-23T22:44:43.456742+0000","last_fullsized":"2023-09-23T22:44:43.456742+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.209739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.209739+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:55.009700+0000","scrub_duration":0.0016266950000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":583,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688301+0000","last_change":"2023-09-23T22:43:11.791371+0000","last_active":"2023-09-23T22:44:27.688301+0000","last_peered":"2023-09-23T22:44:27.688301+0000","last_clean":"2023-09-23T22:44:27.688301+0000","last_became_active":"2023-09-23T22:43:11.783461+0000","last_became_peered":"2023-09-23T22:43:11.783461+0000","last_unstale":"2023-09-23T22:44:27.688301+0000","last_undegraded":"2023-09-23T22:44:27.688301+0000","last_fullsized":"2023-09-23T22:44:27.688301+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:46:19.460031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":365,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464673+0000","last_change":"2023-09-23T22:43:11.733047+0000","last_active":"2023-09-23T22:44:44.464673+0000","last_peered":"2023-09-23T22:44:44.464673+0000","last_clean":"2023-09-23T22:44:44.464673+0000","last_became_active":"2023-09-23T22:43:11.731356+0000","last_became_peered":"2023-09-23T22:43:11.731356+0000","last_unstale":"2023-09-23T22:44:44.464673+0000","last_undegraded":"2023-09-23T22:44:44.464673+0000","last_fullsized":"2023-09-23T22:44:44.464673+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:21:09.953038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":839,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457089+0000","last_change":"2023-09-23T22:44:31.112066+0000","last_active":"2023-09-23T22:44:43.457089+0000","last_peered":"2023-09-23T22:44:43.457089+0000","last_clean":"2023-09-23T22:44:43.457089+0000","last_became_active":"2023-09-23T22:44:05.894732+0000","last_became_peered":"2023-09-23T22:44:05.894732+0000","last_unstale":"2023-09-23T22:44:43.457089+0000","last_undegraded":"2023-09-23T22:44:43.457089+0000","last_fullsized":"2023-09-23T22:44:43.457089+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:31.111983+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:31.111983+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:44.255632+0000","scrub_duration":0.001731486,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":400,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.463986+0000","last_change":"2023-09-23T22:42:57.702640+0000","last_active":"2023-09-23T22:44:44.463986+0000","last_peered":"2023-09-23T22:44:44.463986+0000","last_clean":"2023-09-23T22:44:44.463986+0000","last_became_active":"2023-09-23T22:42:57.701938+0000","last_became_peered":"2023-09-23T22:42:57.701938+0000","last_unstale":"2023-09-23T22:44:44.463986+0000","last_undegraded":"2023-09-23T22:44:44.463986+0000","last_fullsized":"2023-09-23T22:44:44.463986+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:04.282382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":107,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464751+0000","last_change":"2023-09-23T22:43:11.805439+0000","last_active":"2023-09-23T22:44:44.464751+0000","last_peered":"2023-09-23T22:44:44.464751+0000","last_clean":"2023-09-23T22:44:44.464751+0000","last_became_active":"2023-09-23T22:43:11.804518+0000","last_became_peered":"2023-09-23T22:43:11.804518+0000","last_unstale":"2023-09-23T22:44:44.464751+0000","last_undegraded":"2023-09-23T22:44:44.464751+0000","last_fullsized":"2023-09-23T22:44:44.464751+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:32.668394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":814,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.463857+0000","last_change":"2023-09-23T22:44:31.625881+0000","last_active":"2023-09-23T22:44:44.463857+0000","last_peered":"2023-09-23T22:44:44.463857+0000","last_clean":"2023-09-23T22:44:44.463857+0000","last_became_active":"2023-09-23T22:44:05.893507+0000","last_became_peered":"2023-09-23T22:44:05.893507+0000","last_unstale":"2023-09-23T22:44:44.463857+0000","last_undegraded":"2023-09-23T22:44:44.463857+0000","last_fullsized":"2023-09-23T22:44:44.463857+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:31.625786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:31.625786+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:32.831447+0000","scrub_duration":0.0014301240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":692,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.463770+0000","last_change":"2023-09-23T22:43:11.861828+0000","last_active":"2023-09-23T22:44:44.463770+0000","last_peered":"2023-09-23T22:44:44.463770+0000","last_clean":"2023-09-23T22:44:44.463770+0000","last_became_active":"2023-09-23T22:43:11.860602+0000","last_became_peered":"2023-09-23T22:43:11.860602+0000","last_unstale":"2023-09-23T22:44:44.463770+0000","last_undegraded":"2023-09-23T22:44:44.463770+0000","last_fullsized":"2023-09-23T22:44:44.463770+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:43.850844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":242,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460316+0000","last_change":"2023-09-23T22:42:56.509390+0000","last_active":"2023-09-23T22:44:43.460316+0000","last_peered":"2023-09-23T22:44:43.460316+0000","last_clean":"2023-09-23T22:44:43.460316+0000","last_became_active":"2023-09-23T22:42:56.509066+0000","last_became_peered":"2023-09-23T22:42:56.509066+0000","last_unstale":"2023-09-23T22:44:43.460316+0000","last_undegraded":"2023-09-23T22:44:43.460316+0000","last_fullsized":"2023-09-23T22:44:43.460316+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:36.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":964,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457084+0000","last_change":"2023-09-23T22:44:05.895248+0000","last_active":"2023-09-23T22:44:43.457084+0000","last_peered":"2023-09-23T22:44:43.457084+0000","last_clean":"2023-09-23T22:44:43.457084+0000","last_became_active":"2023-09-23T22:44:05.894827+0000","last_became_peered":"2023-09-23T22:44:05.894827+0000","last_unstale":"2023-09-23T22:44:43.457084+0000","last_undegraded":"2023-09-23T22:44:43.457084+0000","last_fullsized":"2023-09-23T22:44:43.457084+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:57.273632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:57.273632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:54.065385+0000","scrub_duration":0.018058204000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457970+0000","last_change":"2023-09-23T22:43:11.732475+0000","last_active":"2023-09-23T22:44:43.457970+0000","last_peered":"2023-09-23T22:44:43.457970+0000","last_clean":"2023-09-23T22:44:43.457970+0000","last_became_active":"2023-09-23T22:43:11.732134+0000","last_became_peered":"2023-09-23T22:43:11.732134+0000","last_unstale":"2023-09-23T22:44:43.457970+0000","last_undegraded":"2023-09-23T22:44:43.457970+0000","last_fullsized":"2023-09-23T22:44:43.457970+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:22.683794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":220,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.459694+0000","last_change":"2023-09-23T22:42:57.555024+0000","last_active":"2023-09-23T22:44:43.459694+0000","last_peered":"2023-09-23T22:44:43.459694+0000","last_clean":"2023-09-23T22:44:43.459694+0000","last_became_active":"2023-09-23T22:42:57.554656+0000","last_became_peered":"2023-09-23T22:42:57.554656+0000","last_unstale":"2023-09-23T22:44:43.459694+0000","last_undegraded":"2023-09-23T22:44:43.459694+0000","last_fullsized":"2023-09-23T22:44:43.459694+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:32:56.648868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":776,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688670+0000","last_change":"2023-09-23T22:44:05.887689+0000","last_active":"2023-09-23T22:44:27.688670+0000","last_peered":"2023-09-23T22:44:27.688670+0000","last_clean":"2023-09-23T22:44:27.688670+0000","last_became_active":"2023-09-23T22:44:05.887374+0000","last_became_peered":"2023-09-23T22:44:05.887374+0000","last_unstale":"2023-09-23T22:44:27.688670+0000","last_undegraded":"2023-09-23T22:44:27.688670+0000","last_fullsized":"2023-09-23T22:44:27.688670+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:02.056099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:02.056099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:42.640691+0000","scrub_duration":0.012145022,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460056+0000","last_change":"2023-09-23T22:43:11.828937+0000","last_active":"2023-09-23T22:44:43.460056+0000","last_peered":"2023-09-23T22:44:43.460056+0000","last_clean":"2023-09-23T22:44:43.460056+0000","last_became_active":"2023-09-23T22:43:11.828512+0000","last_became_peered":"2023-09-23T22:43:11.828512+0000","last_unstale":"2023-09-23T22:44:43.460056+0000","last_undegraded":"2023-09-23T22:44:43.460056+0000","last_fullsized":"2023-09-23T22:44:43.460056+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:48:56.701943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":223,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.459969+0000","last_change":"2023-09-23T22:42:57.699399+0000","last_active":"2023-09-23T22:44:43.459969+0000","last_peered":"2023-09-23T22:44:43.459969+0000","last_clean":"2023-09-23T22:44:43.459969+0000","last_became_active":"2023-09-23T22:42:57.698650+0000","last_became_peered":"2023-09-23T22:42:57.698650+0000","last_unstale":"2023-09-23T22:44:43.459969+0000","last_undegraded":"2023-09-23T22:44:43.459969+0000","last_fullsized":"2023-09-23T22:44:43.459969+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:17.884820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":904,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457628+0000","last_change":"2023-09-23T22:44:29.054451+0000","last_active":"2023-09-23T22:44:43.457628+0000","last_peered":"2023-09-23T22:44:43.457628+0000","last_clean":"2023-09-23T22:44:43.457628+0000","last_became_active":"2023-09-23T22:44:02.961671+0000","last_became_peered":"2023-09-23T22:44:02.961671+0000","last_unstale":"2023-09-23T22:44:43.457628+0000","last_undegraded":"2023-09-23T22:44:43.457628+0000","last_fullsized":"2023-09-23T22:44:43.457628+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:29.054342+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:29.054342+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:46.467722+0000","scrub_duration":0.001695039,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":65,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.461131+0000","last_change":"2023-09-23T22:43:11.882816+0000","last_active":"2023-09-23T22:44:43.461131+0000","last_peered":"2023-09-23T22:44:43.461131+0000","last_clean":"2023-09-23T22:44:43.461131+0000","last_became_active":"2023-09-23T22:43:11.822292+0000","last_became_peered":"2023-09-23T22:43:11.822292+0000","last_unstale":"2023-09-23T22:44:43.461131+0000","last_undegraded":"2023-09-23T22:44:43.461131+0000","last_fullsized":"2023-09-23T22:44:43.461131+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:05.717891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":280,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464573+0000","last_change":"2023-09-23T22:42:56.567330+0000","last_active":"2023-09-23T22:44:44.464573+0000","last_peered":"2023-09-23T22:44:44.464573+0000","last_clean":"2023-09-23T22:44:44.464573+0000","last_became_active":"2023-09-23T22:42:56.567016+0000","last_became_peered":"2023-09-23T22:42:56.567016+0000","last_unstale":"2023-09-23T22:44:44.464573+0000","last_undegraded":"2023-09-23T22:44:44.464573+0000","last_fullsized":"2023-09-23T22:44:44.464573+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:00.456335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":823,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464661+0000","last_change":"2023-09-23T22:44:30.598140+0000","last_active":"2023-09-23T22:44:44.464661+0000","last_peered":"2023-09-23T22:44:44.464661+0000","last_clean":"2023-09-23T22:44:44.464661+0000","last_became_active":"2023-09-23T22:44:05.891879+0000","last_became_peered":"2023-09-23T22:44:05.891879+0000","last_unstale":"2023-09-23T22:44:44.464661+0000","last_undegraded":"2023-09-23T22:44:44.464661+0000","last_fullsized":"2023-09-23T22:44:44.464661+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:30.597926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:30.597926+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:07.847159+0000","scrub_duration":0.001666528,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":617,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460480+0000","last_change":"2023-09-23T22:43:11.747332+0000","last_active":"2023-09-23T22:44:43.460480+0000","last_peered":"2023-09-23T22:44:43.460480+0000","last_clean":"2023-09-23T22:44:43.460480+0000","last_became_active":"2023-09-23T22:43:11.733061+0000","last_became_peered":"2023-09-23T22:43:11.733061+0000","last_unstale":"2023-09-23T22:44:43.460480+0000","last_undegraded":"2023-09-23T22:44:43.460480+0000","last_fullsized":"2023-09-23T22:44:43.460480+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:49:43.986136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":42606,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":550,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460590+0000","last_change":"2023-09-23T22:42:57.485105+0000","last_active":"2023-09-23T22:44:43.460590+0000","last_peered":"2023-09-23T22:44:43.460590+0000","last_clean":"2023-09-23T22:44:43.460590+0000","last_became_active":"2023-09-23T22:42:57.484320+0000","last_became_peered":"2023-09-23T22:42:57.484320+0000","last_unstale":"2023-09-23T22:44:43.460590+0000","last_undegraded":"2023-09-23T22:44:43.460590+0000","last_fullsized":"2023-09-23T22:44:43.460590+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:02:36.390910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":1058,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.456795+0000","last_change":"2023-09-23T22:44:05.895162+0000","last_active":"2023-09-23T22:44:43.456795+0000","last_peered":"2023-09-23T22:44:43.456795+0000","last_clean":"2023-09-23T22:44:43.456795+0000","last_became_active":"2023-09-23T22:44:05.894753+0000","last_became_peered":"2023-09-23T22:44:05.894753+0000","last_unstale":"2023-09-23T22:44:43.456795+0000","last_undegraded":"2023-09-23T22:44:43.456795+0000","last_fullsized":"2023-09-23T22:44:43.456795+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:19.077318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:19.077318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:47.206519+0000","scrub_duration":0.0080980949999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":313,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688461+0000","last_change":"2023-09-23T22:42:57.736915+0000","last_active":"2023-09-23T22:44:27.688461+0000","last_peered":"2023-09-23T22:44:27.688461+0000","last_clean":"2023-09-23T22:44:27.688461+0000","last_became_active":"2023-09-23T22:42:57.629094+0000","last_became_peered":"2023-09-23T22:42:57.629094+0000","last_unstale":"2023-09-23T22:44:27.688461+0000","last_undegraded":"2023-09-23T22:44:27.688461+0000","last_fullsized":"2023-09-23T22:44:27.688461+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:13:03.304992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688762+0000","last_change":"2023-09-23T22:41:52.385773+0000","last_active":"2023-09-23T22:44:27.688762+0000","last_peered":"2023-09-23T22:44:27.688762+0000","last_clean":"2023-09-23T22:44:27.688762+0000","last_became_active":"2023-09-23T22:41:52.385433+0000","last_became_peered":"2023-09-23T22:41:52.385433+0000","last_unstale":"2023-09-23T22:44:27.688762+0000","last_undegraded":"2023-09-23T22:44:27.688762+0000","last_fullsized":"2023-09-23T22:44:27.688762+0000","mapping_epoch":570,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":571,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:57:15.934387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":904,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.456559+0000","last_change":"2023-09-23T22:44:05.895570+0000","last_active":"2023-09-23T22:44:43.456559+0000","last_peered":"2023-09-23T22:44:43.456559+0000","last_clean":"2023-09-23T22:44:43.456559+0000","last_became_active":"2023-09-23T22:44:05.895214+0000","last_became_peered":"2023-09-23T22:44:05.895214+0000","last_unstale":"2023-09-23T22:44:43.456559+0000","last_undegraded":"2023-09-23T22:44:43.456559+0000","last_fullsized":"2023-09-23T22:44:43.456559+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.556720+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:11.670386+0000","scrub_duration":0.0022349409999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":95,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.687810+0000","last_change":"2023-09-23T22:43:11.791005+0000","last_active":"2023-09-23T22:44:27.687810+0000","last_peered":"2023-09-23T22:44:27.687810+0000","last_clean":"2023-09-23T22:44:27.687810+0000","last_became_active":"2023-09-23T22:43:11.783005+0000","last_became_peered":"2023-09-23T22:43:11.783005+0000","last_unstale":"2023-09-23T22:44:27.687810+0000","last_undegraded":"2023-09-23T22:44:27.687810+0000","last_fullsized":"2023-09-23T22:44:27.687810+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:03:23.783502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457570+0000","last_change":"2023-09-23T22:43:11.725543+0000","last_active":"2023-09-23T22:44:43.457570+0000","last_peered":"2023-09-23T22:44:43.457570+0000","last_clean":"2023-09-23T22:44:43.457570+0000","last_became_active":"2023-09-23T22:43:11.725152+0000","last_became_peered":"2023-09-23T22:43:11.725152+0000","last_unstale":"2023-09-23T22:44:43.457570+0000","last_undegraded":"2023-09-23T22:44:43.457570+0000","last_fullsized":"2023-09-23T22:44:43.457570+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:31.160745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":648,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464095+0000","last_change":"2023-09-23T22:44:02.925951+0000","last_active":"2023-09-23T22:44:44.464095+0000","last_peered":"2023-09-23T22:44:44.464095+0000","last_clean":"2023-09-23T22:44:44.464095+0000","last_became_active":"2023-09-23T22:44:02.922183+0000","last_became_peered":"2023-09-23T22:44:02.922183+0000","last_unstale":"2023-09-23T22:44:44.464095+0000","last_undegraded":"2023-09-23T22:44:44.464095+0000","last_fullsized":"2023-09-23T22:44:44.464095+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:57.922489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:57.922489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:25.336046+0000","scrub_duration":0.001544351,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":470,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.459881+0000","last_change":"2023-09-23T22:43:11.802960+0000","last_active":"2023-09-23T22:44:43.459881+0000","last_peered":"2023-09-23T22:44:43.459881+0000","last_clean":"2023-09-23T22:44:43.459881+0000","last_became_active":"2023-09-23T22:43:11.802684+0000","last_became_peered":"2023-09-23T22:43:11.802684+0000","last_unstale":"2023-09-23T22:44:43.459881+0000","last_undegraded":"2023-09-23T22:44:43.459881+0000","last_fullsized":"2023-09-23T22:44:43.459881+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:21.148096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":104,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.689326+0000","last_change":"2023-09-23T22:43:11.807494+0000","last_active":"2023-09-23T22:44:27.689326+0000","last_peered":"2023-09-23T22:44:27.689326+0000","last_clean":"2023-09-23T22:44:27.689326+0000","last_became_active":"2023-09-23T22:43:11.807103+0000","last_became_peered":"2023-09-23T22:43:11.807103+0000","last_unstale":"2023-09-23T22:44:27.689326+0000","last_undegraded":"2023-09-23T22:44:27.689326+0000","last_fullsized":"2023-09-23T22:44:27.689326+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:50.696866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":963,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.456796+0000","last_change":"2023-09-23T22:44:05.892047+0000","last_active":"2023-09-23T22:44:43.456796+0000","last_peered":"2023-09-23T22:44:43.456796+0000","last_clean":"2023-09-23T22:44:43.456796+0000","last_became_active":"2023-09-23T22:44:05.891675+0000","last_became_peered":"2023-09-23T22:44:05.891675+0000","last_unstale":"2023-09-23T22:44:43.456796+0000","last_undegraded":"2023-09-23T22:44:43.456796+0000","last_fullsized":"2023-09-23T22:44:43.456796+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:26.313042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:26.313042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:06.388598+0000","scrub_duration":0.0017143830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":286,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460495+0000","last_change":"2023-09-23T22:43:11.844157+0000","last_active":"2023-09-23T22:44:43.460495+0000","last_peered":"2023-09-23T22:44:43.460495+0000","last_clean":"2023-09-23T22:44:43.460495+0000","last_became_active":"2023-09-23T22:43:11.751401+0000","last_became_peered":"2023-09-23T22:43:11.751401+0000","last_unstale":"2023-09-23T22:44:43.460495+0000","last_undegraded":"2023-09-23T22:44:43.460495+0000","last_fullsized":"2023-09-23T22:44:43.460495+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:25:37.797785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.689328+0000","last_change":"2023-09-23T22:42:57.618059+0000","last_active":"2023-09-23T22:44:27.689328+0000","last_peered":"2023-09-23T22:44:27.689328+0000","last_clean":"2023-09-23T22:44:27.689328+0000","last_became_active":"2023-09-23T22:42:57.617619+0000","last_became_peered":"2023-09-23T22:42:57.617619+0000","last_unstale":"2023-09-23T22:44:27.689328+0000","last_undegraded":"2023-09-23T22:44:27.689328+0000","last_fullsized":"2023-09-23T22:44:27.689328+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:35:00.845651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":986,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.689389+0000","last_change":"2023-09-23T22:44:05.887684+0000","last_active":"2023-09-23T22:44:27.689389+0000","last_peered":"2023-09-23T22:44:27.689389+0000","last_clean":"2023-09-23T22:44:27.689389+0000","last_became_active":"2023-09-23T22:44:05.887349+0000","last_became_peered":"2023-09-23T22:44:05.887349+0000","last_unstale":"2023-09-23T22:44:27.689389+0000","last_undegraded":"2023-09-23T22:44:27.689389+0000","last_fullsized":"2023-09-23T22:44:27.689389+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:34.147486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:34.147486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.306225+0000","scrub_duration":0.0018320190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":168,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464682+0000","last_change":"2023-09-23T22:43:11.920516+0000","last_active":"2023-09-23T22:44:44.464682+0000","last_peered":"2023-09-23T22:44:44.464682+0000","last_clean":"2023-09-23T22:44:44.464682+0000","last_became_active":"2023-09-23T22:43:11.755782+0000","last_became_peered":"2023-09-23T22:43:11.755782+0000","last_unstale":"2023-09-23T22:44:44.464682+0000","last_undegraded":"2023-09-23T22:44:44.464682+0000","last_fullsized":"2023-09-23T22:44:44.464682+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:16.670619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.456793+0000","last_change":"2023-09-23T22:43:11.831473+0000","last_active":"2023-09-23T22:44:43.456793+0000","last_peered":"2023-09-23T22:44:43.456793+0000","last_clean":"2023-09-23T22:44:43.456793+0000","last_became_active":"2023-09-23T22:43:11.831153+0000","last_became_peered":"2023-09-23T22:43:11.831153+0000","last_unstale":"2023-09-23T22:44:43.456793+0000","last_undegraded":"2023-09-23T22:44:43.456793+0000","last_fullsized":"2023-09-23T22:44:43.456793+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:00:24.678192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":735,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460171+0000","last_change":"2023-09-23T22:44:02.893918+0000","last_active":"2023-09-23T22:44:43.460171+0000","last_peered":"2023-09-23T22:44:43.460171+0000","last_clean":"2023-09-23T22:44:43.460171+0000","last_became_active":"2023-09-23T22:44:02.893292+0000","last_became_peered":"2023-09-23T22:44:02.893292+0000","last_unstale":"2023-09-23T22:44:43.460171+0000","last_undegraded":"2023-09-23T22:44:43.460171+0000","last_fullsized":"2023-09-23T22:44:43.460171+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:04.048606+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:04.048606+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:53:36.525412+0000","scrub_duration":0.0064015249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":546,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464926+0000","last_change":"2023-09-23T22:43:11.899388+0000","last_active":"2023-09-23T22:44:44.464926+0000","last_peered":"2023-09-23T22:44:44.464926+0000","last_clean":"2023-09-23T22:44:44.464926+0000","last_became_active":"2023-09-23T22:43:11.841184+0000","last_became_peered":"2023-09-23T22:43:11.841184+0000","last_unstale":"2023-09-23T22:44:44.464926+0000","last_undegraded":"2023-09-23T22:44:44.464926+0000","last_fullsized":"2023-09-23T22:44:44.464926+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:01.020339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":641,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.463927+0000","last_change":"2023-09-23T22:42:57.702798+0000","last_active":"2023-09-23T22:44:44.463927+0000","last_peered":"2023-09-23T22:44:44.463927+0000","last_clean":"2023-09-23T22:44:44.463927+0000","last_became_active":"2023-09-23T22:42:57.702317+0000","last_became_peered":"2023-09-23T22:42:57.702317+0000","last_unstale":"2023-09-23T22:44:44.463927+0000","last_undegraded":"2023-09-23T22:44:44.463927+0000","last_fullsized":"2023-09-23T22:44:44.463927+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:33:12.697584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":921,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460428+0000","last_change":"2023-09-23T22:44:36.397228+0000","last_active":"2023-09-23T22:44:43.460428+0000","last_peered":"2023-09-23T22:44:43.460428+0000","last_clean":"2023-09-23T22:44:43.460428+0000","last_became_active":"2023-09-23T22:43:14.445359+0000","last_became_peered":"2023-09-23T22:43:14.445359+0000","last_unstale":"2023-09-23T22:44:43.460428+0000","last_undegraded":"2023-09-23T22:44:43.460428+0000","last_fullsized":"2023-09-23T22:44:43.460428+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:36.397150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:36.397150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:14:30.377208+0000","scrub_duration":0.001754996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":43,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457431+0000","last_change":"2023-09-23T22:43:11.918943+0000","last_active":"2023-09-23T22:44:43.457431+0000","last_peered":"2023-09-23T22:44:43.457431+0000","last_clean":"2023-09-23T22:44:43.457431+0000","last_became_active":"2023-09-23T22:43:11.827406+0000","last_became_peered":"2023-09-23T22:43:11.827406+0000","last_unstale":"2023-09-23T22:44:43.457431+0000","last_undegraded":"2023-09-23T22:44:43.457431+0000","last_fullsized":"2023-09-23T22:44:43.457431+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:57.335665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":260,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.459935+0000","last_change":"2023-09-23T22:42:57.480776+0000","last_active":"2023-09-23T22:44:43.459935+0000","last_peered":"2023-09-23T22:44:43.459935+0000","last_clean":"2023-09-23T22:44:43.459935+0000","last_became_active":"2023-09-23T22:42:57.480378+0000","last_became_peered":"2023-09-23T22:42:57.480378+0000","last_unstale":"2023-09-23T22:44:43.459935+0000","last_undegraded":"2023-09-23T22:44:43.459935+0000","last_fullsized":"2023-09-23T22:44:43.459935+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:36:42.689275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":828,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:28.459260+0000","last_change":"2023-09-23T22:44:28.459260+0000","last_active":"2023-09-23T22:44:28.459260+0000","last_peered":"2023-09-23T22:44:28.459260+0000","last_clean":"2023-09-23T22:44:28.459260+0000","last_became_active":"2023-09-23T22:44:05.888659+0000","last_became_peered":"2023-09-23T22:44:05.888659+0000","last_unstale":"2023-09-23T22:44:28.459260+0000","last_undegraded":"2023-09-23T22:44:28.459260+0000","last_fullsized":"2023-09-23T22:44:28.459260+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:28.459103+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:28.459103+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:16.622668+0000","scrub_duration":0.0021866989999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457206+0000","last_change":"2023-09-23T22:43:11.732060+0000","last_active":"2023-09-23T22:44:43.457206+0000","last_peered":"2023-09-23T22:44:43.457206+0000","last_clean":"2023-09-23T22:44:43.457206+0000","last_became_active":"2023-09-23T22:43:11.731696+0000","last_became_peered":"2023-09-23T22:43:11.731696+0000","last_unstale":"2023-09-23T22:44:43.457206+0000","last_undegraded":"2023-09-23T22:44:43.457206+0000","last_fullsized":"2023-09-23T22:44:43.457206+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:41.169298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":434,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464066+0000","last_change":"2023-09-23T22:43:11.733183+0000","last_active":"2023-09-23T22:44:44.464066+0000","last_peered":"2023-09-23T22:44:44.464066+0000","last_clean":"2023-09-23T22:44:44.464066+0000","last_became_active":"2023-09-23T22:43:11.731600+0000","last_became_peered":"2023-09-23T22:43:11.731600+0000","last_unstale":"2023-09-23T22:44:44.464066+0000","last_undegraded":"2023-09-23T22:44:44.464066+0000","last_fullsized":"2023-09-23T22:44:44.464066+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:17:17.223104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":979,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.687490+0000","last_change":"2023-09-23T22:42:59.395878+0000","last_active":"2023-09-23T22:44:27.687490+0000","last_peered":"2023-09-23T22:44:27.687490+0000","last_clean":"2023-09-23T22:44:27.687490+0000","last_became_active":"2023-09-23T22:42:59.395538+0000","last_became_peered":"2023-09-23T22:42:59.395538+0000","last_unstale":"2023-09-23T22:44:27.687490+0000","last_undegraded":"2023-09-23T22:44:27.687490+0000","last_fullsized":"2023-09-23T22:44:27.687490+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:58.211060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:58.211060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:50.018093+0000","scrub_duration":0.0035013240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":107,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688566+0000","last_change":"2023-09-23T22:43:11.840050+0000","last_active":"2023-09-23T22:44:27.688566+0000","last_peered":"2023-09-23T22:44:27.688566+0000","last_clean":"2023-09-23T22:44:27.688566+0000","last_became_active":"2023-09-23T22:43:11.780186+0000","last_became_peered":"2023-09-23T22:43:11.780186+0000","last_unstale":"2023-09-23T22:44:27.688566+0000","last_undegraded":"2023-09-23T22:44:27.688566+0000","last_fullsized":"2023-09-23T22:44:27.688566+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:53:00.136350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":468,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460281+0000","last_change":"2023-09-23T22:42:57.606366+0000","last_active":"2023-09-23T22:44:43.460281+0000","last_peered":"2023-09-23T22:44:43.460281+0000","last_clean":"2023-09-23T22:44:43.460281+0000","last_became_active":"2023-09-23T22:42:57.604437+0000","last_became_peered":"2023-09-23T22:42:57.604437+0000","last_unstale":"2023-09-23T22:44:43.460281+0000","last_undegraded":"2023-09-23T22:44:43.460281+0000","last_fullsized":"2023-09-23T22:44:43.460281+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:34.648244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":1086,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.456864+0000","last_change":"2023-09-23T22:44:00.183260+0000","last_active":"2023-09-23T22:44:43.456864+0000","last_peered":"2023-09-23T22:44:43.456864+0000","last_clean":"2023-09-23T22:44:43.456864+0000","last_became_active":"2023-09-23T22:43:14.434417+0000","last_became_peered":"2023-09-23T22:43:14.434417+0000","last_unstale":"2023-09-23T22:44:43.456864+0000","last_undegraded":"2023-09-23T22:44:43.456864+0000","last_fullsized":"2023-09-23T22:44:43.456864+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.183183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.183183+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:35.744336+0000","scrub_duration":0.0015656730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":257,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460341+0000","last_change":"2023-09-23T22:42:57.480722+0000","last_active":"2023-09-23T22:44:43.460341+0000","last_peered":"2023-09-23T22:44:43.460341+0000","last_clean":"2023-09-23T22:44:43.460341+0000","last_became_active":"2023-09-23T22:42:57.480180+0000","last_became_peered":"2023-09-23T22:42:57.480180+0000","last_unstale":"2023-09-23T22:44:43.460341+0000","last_undegraded":"2023-09-23T22:44:43.460341+0000","last_fullsized":"2023-09-23T22:44:43.460341+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:00.944581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":274,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.689073+0000","last_change":"2023-09-23T22:42:57.671849+0000","last_active":"2023-09-23T22:44:27.689073+0000","last_peered":"2023-09-23T22:44:27.689073+0000","last_clean":"2023-09-23T22:44:27.689073+0000","last_became_active":"2023-09-23T22:42:57.667120+0000","last_became_peered":"2023-09-23T22:42:57.667120+0000","last_unstale":"2023-09-23T22:44:27.689073+0000","last_undegraded":"2023-09-23T22:44:27.689073+0000","last_fullsized":"2023-09-23T22:44:27.689073+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:11:11.794628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":839,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457220+0000","last_change":"2023-09-23T22:44:32.087262+0000","last_active":"2023-09-23T22:44:43.457220+0000","last_peered":"2023-09-23T22:44:43.457220+0000","last_clean":"2023-09-23T22:44:43.457220+0000","last_became_active":"2023-09-23T22:44:05.893424+0000","last_became_peered":"2023-09-23T22:44:05.893424+0000","last_unstale":"2023-09-23T22:44:43.457220+0000","last_undegraded":"2023-09-23T22:44:43.457220+0000","last_fullsized":"2023-09-23T22:44:43.457220+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:32.087146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:32.087146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:21.928909+0000","scrub_duration":0.0021954880000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":598,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688182+0000","last_change":"2023-09-23T22:43:11.854114+0000","last_active":"2023-09-23T22:44:27.688182+0000","last_peered":"2023-09-23T22:44:27.688182+0000","last_clean":"2023-09-23T22:44:27.688182+0000","last_became_active":"2023-09-23T22:43:11.841381+0000","last_became_peered":"2023-09-23T22:43:11.841381+0000","last_unstale":"2023-09-23T22:44:27.688182+0000","last_undegraded":"2023-09-23T22:44:27.688182+0000","last_fullsized":"2023-09-23T22:44:27.688182+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:46.359977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":557,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460629+0000","last_change":"2023-09-23T22:43:11.817112+0000","last_active":"2023-09-23T22:44:43.460629+0000","last_peered":"2023-09-23T22:44:43.460629+0000","last_clean":"2023-09-23T22:44:43.460629+0000","last_became_active":"2023-09-23T22:43:11.816563+0000","last_became_peered":"2023-09-23T22:43:11.816563+0000","last_unstale":"2023-09-23T22:44:43.460629+0000","last_undegraded":"2023-09-23T22:44:43.460629+0000","last_fullsized":"2023-09-23T22:44:43.460629+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:56.365553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":1035,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457468+0000","last_change":"2023-09-23T22:43:57.157649+0000","last_active":"2023-09-23T22:44:43.457468+0000","last_peered":"2023-09-23T22:44:43.457468+0000","last_clean":"2023-09-23T22:44:43.457468+0000","last_became_active":"2023-09-23T22:42:56.530915+0000","last_became_peered":"2023-09-23T22:42:56.530915+0000","last_unstale":"2023-09-23T22:44:43.457468+0000","last_undegraded":"2023-09-23T22:44:43.457468+0000","last_fullsized":"2023-09-23T22:44:43.457468+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_clean_scrub_stamp":"2023-09-23T22:43:57.157560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:20.159228+0000","scrub_duration":0.001566287,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":384,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.463920+0000","last_change":"2023-09-23T22:43:11.862025+0000","last_active":"2023-09-23T22:44:44.463920+0000","last_peered":"2023-09-23T22:44:44.463920+0000","last_clean":"2023-09-23T22:44:44.463920+0000","last_became_active":"2023-09-23T22:43:11.860993+0000","last_became_peered":"2023-09-23T22:43:11.860993+0000","last_unstale":"2023-09-23T22:44:44.463920+0000","last_undegraded":"2023-09-23T22:44:44.463920+0000","last_fullsized":"2023-09-23T22:44:44.463920+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:37.123461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":451,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464787+0000","last_change":"2023-09-23T22:43:11.756165+0000","last_active":"2023-09-23T22:44:44.464787+0000","last_peered":"2023-09-23T22:44:44.464787+0000","last_clean":"2023-09-23T22:44:44.464787+0000","last_became_active":"2023-09-23T22:43:11.755617+0000","last_became_peered":"2023-09-23T22:43:11.755617+0000","last_unstale":"2023-09-23T22:44:44.464787+0000","last_undegraded":"2023-09-23T22:44:44.464787+0000","last_fullsized":"2023-09-23T22:44:44.464787+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:21.873764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":922,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460230+0000","last_change":"2023-09-23T22:44:03.570228+0000","last_active":"2023-09-23T22:44:43.460230+0000","last_peered":"2023-09-23T22:44:43.460230+0000","last_clean":"2023-09-23T22:44:43.460230+0000","last_became_active":"2023-09-23T22:43:14.444896+0000","last_became_peered":"2023-09-23T22:43:14.444896+0000","last_unstale":"2023-09-23T22:44:43.460230+0000","last_undegraded":"2023-09-23T22:44:43.460230+0000","last_fullsized":"2023-09-23T22:44:43.460230+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:03.570110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:03.570110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:07:13.842237+0000","scrub_duration":0.001987071,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"625'171","reported_seq":1029,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T22:44:42.696312+0000","last_change":"2023-09-23T22:43:54.413786+0000","last_active":"2023-09-23T22:44:42.696312+0000","last_peered":"2023-09-23T22:44:42.696312+0000","last_clean":"2023-09-23T22:44:42.696312+0000","last_became_active":"2023-09-23T22:43:54.413159+0000","last_became_peered":"2023-09-23T22:43:54.413159+0000","last_unstale":"2023-09-23T22:44:42.696312+0000","last_undegraded":"2023-09-23T22:44:42.696312+0000","last_fullsized":"2023-09-23T22:44:42.696312+0000","mapping_epoch":608,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":171,"log_dups_size":0,"ondisk_log_size":171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:37.708567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000336325,"stat_sum":{"num_bytes":35,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":742,"num_read_kb":3534,"num_write":177,"num_write_kb":3577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457422+0000","last_change":"2023-09-23T22:43:11.827683+0000","last_active":"2023-09-23T22:44:43.457422+0000","last_peered":"2023-09-23T22:44:43.457422+0000","last_clean":"2023-09-23T22:44:43.457422+0000","last_became_active":"2023-09-23T22:43:11.827004+0000","last_became_peered":"2023-09-23T22:43:11.827004+0000","last_unstale":"2023-09-23T22:44:43.457422+0000","last_undegraded":"2023-09-23T22:44:43.457422+0000","last_fullsized":"2023-09-23T22:44:43.457422+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:21.277304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":385,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464228+0000","last_change":"2023-09-23T22:43:11.772491+0000","last_active":"2023-09-23T22:44:44.464228+0000","last_peered":"2023-09-23T22:44:44.464228+0000","last_clean":"2023-09-23T22:44:44.464228+0000","last_became_active":"2023-09-23T22:43:11.772145+0000","last_became_peered":"2023-09-23T22:43:11.772145+0000","last_unstale":"2023-09-23T22:44:44.464228+0000","last_undegraded":"2023-09-23T22:44:44.464228+0000","last_fullsized":"2023-09-23T22:44:44.464228+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:22.248227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":1039,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.456842+0000","last_change":"2023-09-23T22:43:14.435063+0000","last_active":"2023-09-23T22:44:43.456842+0000","last_peered":"2023-09-23T22:44:43.456842+0000","last_clean":"2023-09-23T22:44:43.456842+0000","last_became_active":"2023-09-23T22:43:14.434698+0000","last_became_peered":"2023-09-23T22:43:14.434698+0000","last_unstale":"2023-09-23T22:44:43.456842+0000","last_undegraded":"2023-09-23T22:44:43.456842+0000","last_fullsized":"2023-09-23T22:44:43.456842+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:16.067396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:16.067396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:58.817933+0000","scrub_duration":0.0072300120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"627'151","reported_seq":1768,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.652287+0000","last_change":"2023-09-23T22:44:44.463017+0000","last_active":"2023-09-23T22:44:44.652287+0000","last_peered":"2023-09-23T22:44:44.652287+0000","last_clean":"2023-09-23T22:44:44.652287+0000","last_became_active":"2023-09-23T22:42:56.548134+0000","last_became_peered":"2023-09-23T22:42:56.548134+0000","last_unstale":"2023-09-23T22:44:44.652287+0000","last_undegraded":"2023-09-23T22:44:44.652287+0000","last_fullsized":"2023-09-23T22:44:44.652287+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":151,"log_dups_size":0,"ondisk_log_size":151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:36.511009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000841196,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1472,"num_read_kb":3359,"num_write":123,"num_write_kb":6213,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"11","length":"1"}]},{"pgid":"3.3","version":"0'0","reported_seq":520,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460836+0000","last_change":"2023-09-23T22:42:57.574455+0000","last_active":"2023-09-23T22:44:43.460836+0000","last_peered":"2023-09-23T22:44:43.460836+0000","last_clean":"2023-09-23T22:44:43.460836+0000","last_became_active":"2023-09-23T22:42:57.573786+0000","last_became_peered":"2023-09-23T22:42:57.573786+0000","last_unstale":"2023-09-23T22:44:43.460836+0000","last_undegraded":"2023-09-23T22:44:43.460836+0000","last_fullsized":"2023-09-23T22:44:43.460836+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:09.464699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":274,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688891+0000","last_change":"2023-09-23T22:42:57.591303+0000","last_active":"2023-09-23T22:44:27.688891+0000","last_peered":"2023-09-23T22:44:27.688891+0000","last_clean":"2023-09-23T22:44:27.688891+0000","last_became_active":"2023-09-23T22:42:57.587701+0000","last_became_peered":"2023-09-23T22:42:57.587701+0000","last_unstale":"2023-09-23T22:44:27.688891+0000","last_undegraded":"2023-09-23T22:44:27.688891+0000","last_fullsized":"2023-09-23T22:44:27.688891+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:36:32.621790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":969,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688103+0000","last_change":"2023-09-23T22:43:14.448779+0000","last_active":"2023-09-23T22:44:27.688103+0000","last_peered":"2023-09-23T22:44:27.688103+0000","last_clean":"2023-09-23T22:44:27.688103+0000","last_became_active":"2023-09-23T22:43:14.448360+0000","last_became_peered":"2023-09-23T22:43:14.448360+0000","last_unstale":"2023-09-23T22:44:27.688103+0000","last_undegraded":"2023-09-23T22:44:27.688103+0000","last_fullsized":"2023-09-23T22:44:27.688103+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:07.291196+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:07.291196+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:38.426551+0000","scrub_duration":0.0016895390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"625'112","reported_seq":432,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T22:44:42.613303+0000","last_change":"2023-09-23T22:43:54.199739+0000","last_active":"2023-09-23T22:44:42.613303+0000","last_peered":"2023-09-23T22:44:42.613303+0000","last_clean":"2023-09-23T22:44:42.613303+0000","last_became_active":"2023-09-23T22:42:57.672992+0000","last_became_peered":"2023-09-23T22:42:57.672992+0000","last_unstale":"2023-09-23T22:44:42.613303+0000","last_undegraded":"2023-09-23T22:44:42.613303+0000","last_fullsized":"2023-09-23T22:44:42.613303+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":112,"log_dups_size":0,"ondisk_log_size":112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:41.870977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038084799999999998,"stat_sum":{"num_bytes":51,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":472,"num_read_kb":1874,"num_write":111,"num_write_kb":1369,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":255,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.687962+0000","last_change":"2023-09-23T22:42:56.548974+0000","last_active":"2023-09-23T22:44:27.687962+0000","last_peered":"2023-09-23T22:44:27.687962+0000","last_clean":"2023-09-23T22:44:27.687962+0000","last_became_active":"2023-09-23T22:42:56.548663+0000","last_became_peered":"2023-09-23T22:42:56.548663+0000","last_unstale":"2023-09-23T22:44:27.687962+0000","last_undegraded":"2023-09-23T22:44:27.687962+0000","last_fullsized":"2023-09-23T22:44:27.687962+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:06.786917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":320,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464904+0000","last_change":"2023-09-23T22:43:11.756366+0000","last_active":"2023-09-23T22:44:44.464904+0000","last_peered":"2023-09-23T22:44:44.464904+0000","last_clean":"2023-09-23T22:44:44.464904+0000","last_became_active":"2023-09-23T22:43:11.756048+0000","last_became_peered":"2023-09-23T22:43:11.756048+0000","last_unstale":"2023-09-23T22:44:44.464904+0000","last_undegraded":"2023-09-23T22:44:44.464904+0000","last_fullsized":"2023-09-23T22:44:44.464904+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:27:30.924506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"627'231","reported_seq":1584,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.655810+0000","last_change":"2023-09-23T22:44:44.462075+0000","last_active":"2023-09-23T22:44:44.655810+0000","last_peered":"2023-09-23T22:44:44.655810+0000","last_clean":"2023-09-23T22:44:44.655810+0000","last_became_active":"2023-09-23T22:43:56.859594+0000","last_became_peered":"2023-09-23T22:43:56.859594+0000","last_unstale":"2023-09-23T22:44:44.655810+0000","last_undegraded":"2023-09-23T22:44:44.655810+0000","last_fullsized":"2023-09-23T22:44:44.655810+0000","mapping_epoch":610,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":611,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":231,"log_dups_size":0,"ondisk_log_size":231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:54.531616+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040732700000000003,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1000,"num_read_kb":1281,"num_write":217,"num_write_kb":323,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[{"start":"11","length":"1"}]},{"pgid":"1.0","version":"89'161","reported_seq":826,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.459625+0000","last_change":"2023-09-23T22:44:00.574883+0000","last_active":"2023-09-23T22:44:43.459625+0000","last_peered":"2023-09-23T22:44:43.459625+0000","last_clean":"2023-09-23T22:44:43.459625+0000","last_became_active":"2023-09-23T22:42:57.600816+0000","last_became_peered":"2023-09-23T22:42:57.600816+0000","last_unstale":"2023-09-23T22:44:43.459625+0000","last_undegraded":"2023-09-23T22:44:43.459625+0000","last_fullsized":"2023-09-23T22:44:43.459625+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.574795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.574795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:22:03.052349+0000","scrub_duration":0.0023357170000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457148+0000","last_change":"2023-09-23T22:43:11.806599+0000","last_active":"2023-09-23T22:44:43.457148+0000","last_peered":"2023-09-23T22:44:43.457148+0000","last_clean":"2023-09-23T22:44:43.457148+0000","last_became_active":"2023-09-23T22:43:11.806297+0000","last_became_peered":"2023-09-23T22:43:11.806297+0000","last_unstale":"2023-09-23T22:44:43.457148+0000","last_undegraded":"2023-09-23T22:44:43.457148+0000","last_fullsized":"2023-09-23T22:44:43.457148+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:11:17.339834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":272,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.461224+0000","last_change":"2023-09-23T22:42:57.493464+0000","last_active":"2023-09-23T22:44:43.461224+0000","last_peered":"2023-09-23T22:44:43.461224+0000","last_clean":"2023-09-23T22:44:43.461224+0000","last_became_active":"2023-09-23T22:42:57.492614+0000","last_became_peered":"2023-09-23T22:42:57.492614+0000","last_unstale":"2023-09-23T22:44:43.461224+0000","last_undegraded":"2023-09-23T22:44:43.461224+0000","last_fullsized":"2023-09-23T22:44:43.461224+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:13.562710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":848,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460749+0000","last_change":"2023-09-23T22:44:38.452047+0000","last_active":"2023-09-23T22:44:43.460749+0000","last_peered":"2023-09-23T22:44:43.460749+0000","last_clean":"2023-09-23T22:44:43.460749+0000","last_became_active":"2023-09-23T22:43:14.427691+0000","last_became_peered":"2023-09-23T22:43:14.427691+0000","last_unstale":"2023-09-23T22:44:43.460749+0000","last_undegraded":"2023-09-23T22:44:43.460749+0000","last_fullsized":"2023-09-23T22:44:43.460749+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:38.451955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:38.451955+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:31:39.840197+0000","scrub_duration":0.0017243600000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"628'160","reported_seq":1378,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T22:44:45.739544+0000","last_change":"2023-09-23T22:44:45.463796+0000","last_active":"2023-09-23T22:44:45.739544+0000","last_peered":"2023-09-23T22:44:45.739544+0000","last_clean":"2023-09-23T22:44:45.739544+0000","last_became_active":"2023-09-23T22:43:11.808371+0000","last_became_peered":"2023-09-23T22:43:11.808371+0000","last_unstale":"2023-09-23T22:44:45.739544+0000","last_undegraded":"2023-09-23T22:44:45.739544+0000","last_fullsized":"2023-09-23T22:44:45.739544+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":160,"log_dups_size":0,"ondisk_log_size":160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:38.132844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001262867,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1110,"num_read_kb":1286,"num_write":122,"num_write_kb":215,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"3.1","version":"0'0","reported_seq":135,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457129+0000","last_change":"2023-09-23T22:42:57.480877+0000","last_active":"2023-09-23T22:44:43.457129+0000","last_peered":"2023-09-23T22:44:43.457129+0000","last_clean":"2023-09-23T22:44:43.457129+0000","last_became_active":"2023-09-23T22:42:57.480576+0000","last_became_peered":"2023-09-23T22:42:57.480576+0000","last_unstale":"2023-09-23T22:44:43.457129+0000","last_undegraded":"2023-09-23T22:44:43.457129+0000","last_fullsized":"2023-09-23T22:44:43.457129+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:41:39.895915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":458,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.689228+0000","last_change":"2023-09-23T22:43:11.809271+0000","last_active":"2023-09-23T22:44:27.689228+0000","last_peered":"2023-09-23T22:44:27.689228+0000","last_clean":"2023-09-23T22:44:27.689228+0000","last_became_active":"2023-09-23T22:43:11.809001+0000","last_became_peered":"2023-09-23T22:43:11.809001+0000","last_unstale":"2023-09-23T22:44:27.689228+0000","last_undegraded":"2023-09-23T22:44:27.689228+0000","last_fullsized":"2023-09-23T22:44:27.689228+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:18.092293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":954,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460447+0000","last_change":"2023-09-23T22:44:02.567423+0000","last_active":"2023-09-23T22:44:43.460447+0000","last_peered":"2023-09-23T22:44:43.460447+0000","last_clean":"2023-09-23T22:44:43.460447+0000","last_became_active":"2023-09-23T22:43:14.445311+0000","last_became_peered":"2023-09-23T22:43:14.445311+0000","last_unstale":"2023-09-23T22:44:43.460447+0000","last_undegraded":"2023-09-23T22:44:43.460447+0000","last_fullsized":"2023-09-23T22:44:43.460447+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_clean_scrub_stamp":"2023-09-23T22:44:02.567345+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:03.924259+0000","scrub_duration":0.0019136540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"625'102","reported_seq":76,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464620+0000","last_change":"2023-09-23T22:44:44.464961+0000","last_active":"2023-09-23T22:44:44.464620+0000","last_peered":"2023-09-23T22:44:44.464620+0000","last_clean":"2023-09-23T22:44:44.464620+0000","last_became_active":"2023-09-23T22:42:57.674485+0000","last_became_peered":"2023-09-23T22:42:57.674485+0000","last_unstale":"2023-09-23T22:44:44.464620+0000","last_undegraded":"2023-09-23T22:44:44.464620+0000","last_fullsized":"2023-09-23T22:44:44.464620+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":602,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:57.916453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00045682900000000001,"stat_sum":{"num_bytes":112675,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":828,"num_read_kb":1161,"num_write":73,"num_write_kb":268,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":27,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"11","length":"1"}]},{"pgid":"3.4","version":"0'0","reported_seq":278,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688931+0000","last_change":"2023-09-23T22:43:11.749267+0000","last_active":"2023-09-23T22:44:27.688931+0000","last_peered":"2023-09-23T22:44:27.688931+0000","last_clean":"2023-09-23T22:44:27.688931+0000","last_became_active":"2023-09-23T22:43:11.748938+0000","last_became_peered":"2023-09-23T22:43:11.748938+0000","last_unstale":"2023-09-23T22:44:27.688931+0000","last_undegraded":"2023-09-23T22:44:27.688931+0000","last_fullsized":"2023-09-23T22:44:27.688931+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:38.249466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457515+0000","last_change":"2023-09-23T22:43:11.794701+0000","last_active":"2023-09-23T22:44:43.457515+0000","last_peered":"2023-09-23T22:44:43.457515+0000","last_clean":"2023-09-23T22:44:43.457515+0000","last_became_active":"2023-09-23T22:43:11.794419+0000","last_became_peered":"2023-09-23T22:43:11.794419+0000","last_unstale":"2023-09-23T22:44:43.457515+0000","last_undegraded":"2023-09-23T22:44:43.457515+0000","last_fullsized":"2023-09-23T22:44:43.457515+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:21.897331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":969,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.689158+0000","last_change":"2023-09-23T22:43:14.458068+0000","last_active":"2023-09-23T22:44:27.689158+0000","last_peered":"2023-09-23T22:44:27.689158+0000","last_clean":"2023-09-23T22:44:27.689158+0000","last_became_active":"2023-09-23T22:43:14.444011+0000","last_became_peered":"2023-09-23T22:43:14.444011+0000","last_unstale":"2023-09-23T22:44:27.689158+0000","last_undegraded":"2023-09-23T22:44:27.689158+0000","last_fullsized":"2023-09-23T22:44:27.689158+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:27.290379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:27.290379+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:55.282838+0000","scrub_duration":0.0016473779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":110,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.458027+0000","last_change":"2023-09-23T22:42:05.533737+0000","last_active":"2023-09-23T22:44:43.458027+0000","last_peered":"2023-09-23T22:44:43.458027+0000","last_clean":"2023-09-23T22:44:43.458027+0000","last_became_active":"2023-09-23T22:42:05.525900+0000","last_became_peered":"2023-09-23T22:42:05.525900+0000","last_unstale":"2023-09-23T22:44:43.458027+0000","last_undegraded":"2023-09-23T22:44:43.458027+0000","last_fullsized":"2023-09-23T22:44:43.458027+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:31.248241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"628'228","reported_seq":1539,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T22:44:46.160502+0000","last_change":"2023-09-23T22:44:45.464069+0000","last_active":"2023-09-23T22:44:46.160502+0000","last_peered":"2023-09-23T22:44:46.160502+0000","last_clean":"2023-09-23T22:44:46.160502+0000","last_became_active":"2023-09-23T22:42:57.585398+0000","last_became_peered":"2023-09-23T22:42:57.585398+0000","last_unstale":"2023-09-23T22:44:46.160502+0000","last_undegraded":"2023-09-23T22:44:46.160502+0000","last_fullsized":"2023-09-23T22:44:46.160502+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":228,"log_dups_size":0,"ondisk_log_size":228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:06.542366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025313700000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1786,"num_read_kb":1945,"num_write":158,"num_write_kb":296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"10","length":"2"}]},{"pgid":"4.0","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.456904+0000","last_change":"2023-09-23T22:43:11.831381+0000","last_active":"2023-09-23T22:44:43.456904+0000","last_peered":"2023-09-23T22:44:43.456904+0000","last_clean":"2023-09-23T22:44:43.456904+0000","last_became_active":"2023-09-23T22:43:11.830843+0000","last_became_peered":"2023-09-23T22:43:11.830843+0000","last_unstale":"2023-09-23T22:44:43.456904+0000","last_undegraded":"2023-09-23T22:44:43.456904+0000","last_fullsized":"2023-09-23T22:44:43.456904+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:26.690562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":786,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457136+0000","last_change":"2023-09-23T22:43:14.436289+0000","last_active":"2023-09-23T22:44:43.457136+0000","last_peered":"2023-09-23T22:44:43.457136+0000","last_clean":"2023-09-23T22:44:43.457136+0000","last_became_active":"2023-09-23T22:43:14.436041+0000","last_became_peered":"2023-09-23T22:43:14.436041+0000","last_unstale":"2023-09-23T22:44:43.457136+0000","last_undegraded":"2023-09-23T22:44:43.457136+0000","last_fullsized":"2023-09-23T22:44:43.457136+0000","mapping_epoch":600,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":601,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:03.084589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:03.084589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:03.001839+0000","scrub_duration":0.013840414000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"624'77","reported_seq":347,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.465368+0000","last_change":"2023-09-23T22:44:44.465487+0000","last_active":"2023-09-23T22:44:44.465368+0000","last_peered":"2023-09-23T22:44:44.465368+0000","last_clean":"2023-09-23T22:44:44.465368+0000","last_became_active":"2023-09-23T22:42:57.605639+0000","last_became_peered":"2023-09-23T22:42:57.605639+0000","last_unstale":"2023-09-23T22:44:44.465368+0000","last_undegraded":"2023-09-23T22:44:44.465368+0000","last_fullsized":"2023-09-23T22:44:44.465368+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":77,"log_dups_size":0,"ondisk_log_size":77,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:48:13.114886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021167700000000001,"stat_sum":{"num_bytes":112640,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":311,"num_read_kb":1208,"num_write":76,"num_write_kb":1538,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[{"start":"11","length":"1"}]},{"pgid":"3.6","version":"67'1","reported_seq":434,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464990+0000","last_change":"2023-09-23T22:43:11.913285+0000","last_active":"2023-09-23T22:44:44.464990+0000","last_peered":"2023-09-23T22:44:44.464990+0000","last_clean":"2023-09-23T22:44:44.464990+0000","last_became_active":"2023-09-23T22:43:11.841729+0000","last_became_peered":"2023-09-23T22:43:11.841729+0000","last_unstale":"2023-09-23T22:44:44.464990+0000","last_undegraded":"2023-09-23T22:44:44.464990+0000","last_fullsized":"2023-09-23T22:44:44.464990+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:45.396946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":39,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.456917+0000","last_change":"2023-09-23T22:43:11.808955+0000","last_active":"2023-09-23T22:44:43.456917+0000","last_peered":"2023-09-23T22:44:43.456917+0000","last_clean":"2023-09-23T22:44:43.456917+0000","last_became_active":"2023-09-23T22:43:11.808676+0000","last_became_peered":"2023-09-23T22:43:11.808676+0000","last_unstale":"2023-09-23T22:44:43.456917+0000","last_undegraded":"2023-09-23T22:44:43.456917+0000","last_fullsized":"2023-09-23T22:44:43.456917+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:38.659893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":966,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.459773+0000","last_change":"2023-09-23T22:44:39.417900+0000","last_active":"2023-09-23T22:44:43.459773+0000","last_peered":"2023-09-23T22:44:43.459773+0000","last_clean":"2023-09-23T22:44:43.459773+0000","last_became_active":"2023-09-23T22:44:05.891208+0000","last_became_peered":"2023-09-23T22:44:05.891208+0000","last_unstale":"2023-09-23T22:44:43.459773+0000","last_undegraded":"2023-09-23T22:44:43.459773+0000","last_fullsized":"2023-09-23T22:44:43.459773+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:39.417809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:39.417809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:57.676055+0000","scrub_duration":0.0016817830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":130,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464553+0000","last_change":"2023-09-23T22:42:57.598277+0000","last_active":"2023-09-23T22:44:44.464553+0000","last_peered":"2023-09-23T22:44:44.464553+0000","last_clean":"2023-09-23T22:44:44.464553+0000","last_became_active":"2023-09-23T22:42:57.597389+0000","last_became_peered":"2023-09-23T22:42:57.597389+0000","last_unstale":"2023-09-23T22:44:44.464553+0000","last_undegraded":"2023-09-23T22:44:44.464553+0000","last_fullsized":"2023-09-23T22:44:44.464553+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:43.074446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":292,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688976+0000","last_change":"2023-09-23T22:43:11.749040+0000","last_active":"2023-09-23T22:44:27.688976+0000","last_peered":"2023-09-23T22:44:27.688976+0000","last_clean":"2023-09-23T22:44:27.688976+0000","last_became_active":"2023-09-23T22:43:11.748649+0000","last_became_peered":"2023-09-23T22:43:11.748649+0000","last_unstale":"2023-09-23T22:44:27.688976+0000","last_undegraded":"2023-09-23T22:44:27.688976+0000","last_fullsized":"2023-09-23T22:44:27.688976+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:52.013195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":973,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460652+0000","last_change":"2023-09-23T22:44:41.386653+0000","last_active":"2023-09-23T22:44:43.460652+0000","last_peered":"2023-09-23T22:44:43.460652+0000","last_clean":"2023-09-23T22:44:43.460652+0000","last_became_active":"2023-09-23T22:44:05.892153+0000","last_became_peered":"2023-09-23T22:44:05.892153+0000","last_unstale":"2023-09-23T22:44:43.460652+0000","last_undegraded":"2023-09-23T22:44:43.460652+0000","last_fullsized":"2023-09-23T22:44:43.460652+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:41.386557+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:41.386557+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.939437+0000","scrub_duration":0.0019422809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":679,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457421+0000","last_change":"2023-09-23T22:42:57.577409+0000","last_active":"2023-09-23T22:44:43.457421+0000","last_peered":"2023-09-23T22:44:43.457421+0000","last_clean":"2023-09-23T22:44:43.457421+0000","last_became_active":"2023-09-23T22:42:57.551354+0000","last_became_peered":"2023-09-23T22:42:57.551354+0000","last_unstale":"2023-09-23T22:44:43.457421+0000","last_undegraded":"2023-09-23T22:44:43.457421+0000","last_fullsized":"2023-09-23T22:44:43.457421+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:14.859975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":658,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464168+0000","last_change":"2023-09-23T22:42:57.597909+0000","last_active":"2023-09-23T22:44:44.464168+0000","last_peered":"2023-09-23T22:44:44.464168+0000","last_clean":"2023-09-23T22:44:44.464168+0000","last_became_active":"2023-09-23T22:42:57.596972+0000","last_became_peered":"2023-09-23T22:42:57.596972+0000","last_unstale":"2023-09-23T22:44:44.464168+0000","last_undegraded":"2023-09-23T22:44:44.464168+0000","last_fullsized":"2023-09-23T22:44:44.464168+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:01.735792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":694,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457617+0000","last_change":"2023-09-23T22:42:57.481811+0000","last_active":"2023-09-23T22:44:43.457617+0000","last_peered":"2023-09-23T22:44:43.457617+0000","last_clean":"2023-09-23T22:44:43.457617+0000","last_became_active":"2023-09-23T22:42:57.481461+0000","last_became_peered":"2023-09-23T22:42:57.481461+0000","last_unstale":"2023-09-23T22:44:43.457617+0000","last_undegraded":"2023-09-23T22:44:43.457617+0000","last_fullsized":"2023-09-23T22:44:43.457617+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:41:25.977092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":118,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688014+0000","last_change":"2023-09-23T22:42:57.645578+0000","last_active":"2023-09-23T22:44:27.688014+0000","last_peered":"2023-09-23T22:44:27.688014+0000","last_clean":"2023-09-23T22:44:27.688014+0000","last_became_active":"2023-09-23T22:42:57.644610+0000","last_became_peered":"2023-09-23T22:42:57.644610+0000","last_unstale":"2023-09-23T22:44:27.688014+0000","last_undegraded":"2023-09-23T22:44:27.688014+0000","last_fullsized":"2023-09-23T22:44:27.688014+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:37:51.348779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":735,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460162+0000","last_change":"2023-09-23T22:44:02.893662+0000","last_active":"2023-09-23T22:44:43.460162+0000","last_peered":"2023-09-23T22:44:43.460162+0000","last_clean":"2023-09-23T22:44:43.460162+0000","last_became_active":"2023-09-23T22:44:02.892760+0000","last_became_peered":"2023-09-23T22:44:02.892760+0000","last_unstale":"2023-09-23T22:44:43.460162+0000","last_undegraded":"2023-09-23T22:44:43.460162+0000","last_fullsized":"2023-09-23T22:44:43.460162+0000","mapping_epoch":612,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:56.024308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:56.024308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:30.183672+0000","scrub_duration":0.0045490360000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":223,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.460558+0000","last_change":"2023-09-23T22:42:57.720541+0000","last_active":"2023-09-23T22:44:43.460558+0000","last_peered":"2023-09-23T22:44:43.460558+0000","last_clean":"2023-09-23T22:44:43.460558+0000","last_became_active":"2023-09-23T22:42:57.717881+0000","last_became_peered":"2023-09-23T22:42:57.717881+0000","last_unstale":"2023-09-23T22:44:43.460558+0000","last_undegraded":"2023-09-23T22:44:43.460558+0000","last_fullsized":"2023-09-23T22:44:43.460558+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:01:16.534160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":236,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T22:44:27.688005+0000","last_change":"2023-09-23T22:43:11.853883+0000","last_active":"2023-09-23T22:44:27.688005+0000","last_peered":"2023-09-23T22:44:27.688005+0000","last_clean":"2023-09-23T22:44:27.688005+0000","last_became_active":"2023-09-23T22:43:11.840934+0000","last_became_peered":"2023-09-23T22:43:11.840934+0000","last_unstale":"2023-09-23T22:44:27.688005+0000","last_undegraded":"2023-09-23T22:44:27.688005+0000","last_fullsized":"2023-09-23T22:44:27.688005+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:27.566762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":982,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T22:44:44.464369+0000","last_change":"2023-09-23T22:44:05.893864+0000","last_active":"2023-09-23T22:44:44.464369+0000","last_peered":"2023-09-23T22:44:44.464369+0000","last_clean":"2023-09-23T22:44:44.464369+0000","last_became_active":"2023-09-23T22:44:05.893517+0000","last_became_peered":"2023-09-23T22:44:05.893517+0000","last_unstale":"2023-09-23T22:44:44.464369+0000","last_undegraded":"2023-09-23T22:44:44.464369+0000","last_fullsized":"2023-09-23T22:44:44.464369+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:53.343498+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:53.343498+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:19.183253+0000","scrub_duration":0.0041572029999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":45,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.457899+0000","last_change":"2023-09-23T22:43:11.877374+0000","last_active":"2023-09-23T22:44:43.457899+0000","last_peered":"2023-09-23T22:44:43.457899+0000","last_clean":"2023-09-23T22:44:43.457899+0000","last_became_active":"2023-09-23T22:43:11.805693+0000","last_became_peered":"2023-09-23T22:43:11.805693+0000","last_unstale":"2023-09-23T22:44:43.457899+0000","last_undegraded":"2023-09-23T22:44:43.457899+0000","last_fullsized":"2023-09-23T22:44:43.457899+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:40:06.140025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":610,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T22:44:43.459607+0000","last_change":"2023-09-23T22:42:57.601834+0000","last_active":"2023-09-23T22:44:43.459607+0000","last_peered":"2023-09-23T22:44:43.459607+0000","last_clean":"2023-09-23T22:44:43.459607+0000","last_became_active":"2023-09-23T22:42:57.600569+0000","last_became_peered":"2023-09-23T22:42:57.600569+0000","last_unstale":"2023-09-23T22:44:43.459607+0000","last_undegraded":"2023-09-23T22:44:43.459607+0000","last_fullsized":"2023-09-23T22:44:43.459607+0000","mapping_epoch":593,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:13.910515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":47046,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":155648,"data_stored":26924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62789,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2641920,"data_stored":2624160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4419724,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":7721,"num_read_kb":15648,"num_write":1057,"num_write_kb":13799,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":417,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":3,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12701696,"data_stored":12634459,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":1232,"ondisk_log_size":1232,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":231928234484,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":592,"seq":2542620639255,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":334004,"kb_used_data":217000,"kb_used_omap":8,"kb_used_meta":116983,"kb_avail":93411148,"statfs":{"total":95995035648,"available":95653015552,"internally_reserved":0,"allocated":222208000,"data_stored":220186369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":119791458},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":3.996,"5min":3.996,"15min":3.996},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":47.795000000000002,"5min":47.795000000000002,"15min":47.795000000000002},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":3.3039999999999998},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":47.682000000000002,"5min":47.682000000000002,"15min":47.682000000000002},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":7.4900000000000002,"5min":7.4900000000000002,"15min":7.4900000000000002},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":115.848,"5min":115.848,"15min":115.848},"last":0.27800000000000002},{"interface":"front","average":{"1min":7.7279999999999998,"5min":7.7279999999999998,"15min":7.7279999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":116.517,"5min":116.517,"15min":116.517},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.1339999999999999,"15min":1.1339999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.6379999999999999,"5min":3.6379999999999999,"15min":3.6379999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":2.637,"5min":2.637,"15min":2.637},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":12.502000000000001,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.40000000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.8490000000000002,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.2,"5min":1.2,"15min":1.2},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.6260000000000003,"5min":5.6260000000000003,"15min":5.6260000000000003},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.5429999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.4590000000000001,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.626}]},{"osd":5,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.1399999999999999,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.58999999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:43:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.5029999999999999,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":1.339,"5min":1.339,"15min":1.339},"last":0.61199999999999999}]}]},{"osd":1,"up_from":495,"seq":2126008811606,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":432716,"kb_used_data":12792,"kb_used_omap":12,"kb_used_meta":419891,"kb_avail":93312436,"statfs":{"total":95995035648,"available":95551934464,"internally_reserved":0,"allocated":13099008,"data_stored":9949044,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":429969070},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":2.2389999999999999,"15min":1.099},"min":{"1min":0.20200000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.0249999999999999,"5min":64.507000000000005,"15min":64.507000000000005},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.8939999999999999,"5min":2.323,"15min":1.375},"min":{"1min":0.24399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":8.8460000000000001,"5min":64.352000000000004,"15min":64.352000000000004},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":12.957000000000001,"5min":3.7730000000000001,"15min":1.7050000000000001},"min":{"1min":0.254,"5min":0.192,"15min":0.192},"max":{"1min":124.447,"5min":124.447,"15min":124.447},"last":0.36499999999999999},{"interface":"front","average":{"1min":12.526,"5min":3.3849999999999998,"15min":1.6419999999999999},"min":{"1min":0.29399999999999998,"5min":0.218,"15min":0.218},"max":{"1min":124.03,"5min":124.03,"15min":124.03},"last":0.33300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:44:32 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.663,"15min":1.7509999999999999},"min":{"1min":0.33600000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.0830000000000002,"5min":11.800000000000001,"15min":11.800000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":1.4730000000000001,"15min":1.6259999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.9550000000000001,"5min":6.4530000000000003,"15min":6.4530000000000003},"last":0.307}]},{"osd":4,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":1.0249999999999999,"15min":0.70899999999999996},"min":{"1min":0.38,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.0750000000000002,"5min":5.5439999999999996,"15min":5.5439999999999996},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.85299999999999998,"5min":1.01,"15min":1.004},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.4510000000000001,"5min":5.524,"15min":5.6740000000000004},"last":0.71099999999999997}]},{"osd":5,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":1.167,"15min":0.95899999999999996},"min":{"1min":0.42399999999999999,"5min":0.307,"15min":0.307},"max":{"1min":11.734,"5min":11.734,"15min":11.734},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.0600000000000001,"15min":0.755},"min":{"1min":0.439,"5min":0.31,"15min":0.31},"max":{"1min":3.1389999999999998,"5min":5.5839999999999996,"15min":5.5839999999999996},"last":0.81100000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:43:55 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":1.736,"15min":1.736},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":9.3759999999999994,"5min":9.3759999999999994,"15min":9.3759999999999994},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.877,"5min":0.877,"15min":0.877},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":2.9540000000000002,"5min":2.9540000000000002,"15min":2.9540000000000002},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:43:48 2023","interfaces":[{"interface":"back","average":{"1min":2.0059999999999998,"5min":1.2809999999999999,"15min":1.0049999999999999},"min":{"1min":0.42999999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.768000000000001,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.2370000000000001,"5min":1.119,"15min":0.80100000000000005},"min":{"1min":0.496,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":3.169,"5min":5.6120000000000001,"15min":5.6120000000000001},"last":0.57499999999999996}]}]},{"osd":0,"up_from":0,"seq":399431958922,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":349,"seq":1498943586484,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":509384,"kb_used_data":8384,"kb_used_omap":15,"kb_used_meta":500976,"kb_avail":93235768,"statfs":{"total":95995035648,"available":95473426432,"internally_reserved":0,"allocated":8585216,"data_stored":5414012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":513000182},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1930000000000001,"5min":2.9289999999999998,"15min":4.8280000000000003},"min":{"1min":0.16900000000000001,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":4.4370000000000003,"5min":80.126000000000005,"15min":194.06299999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.909,"5min":3.2050000000000001,"15min":5.4169999999999998},"min":{"1min":0.29499999999999998,"5min":0.215,"15min":0.215},"max":{"1min":7.5999999999999996,"5min":83.899000000000001,"15min":196.947},"last":0.35199999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:43:49 2023","interfaces":[{"interface":"back","average":{"1min":5.2510000000000003,"5min":2.831,"15min":1.371},"min":{"1min":0.224,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":47.018000000000001,"5min":70.042000000000002,"15min":70.042000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":7.2069999999999999,"5min":4.9139999999999997,"15min":2.3690000000000002},"min":{"1min":0.23599999999999999,"5min":0.184,"15min":0.184},"max":{"1min":47.671999999999997,"5min":127.327,"15min":127.327},"last":0.46300000000000002}]},{"osd":3,"last update":"Sat Sep 23 22:44:35 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":3.4239999999999999,"15min":3.7519999999999998},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.7889999999999997,"5min":14.561,"15min":14.561},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.6579999999999999,"15min":1.8160000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":2.3929999999999998,"5min":4.7720000000000002,"15min":4.7720000000000002},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 22:44:08 2023","interfaces":[{"interface":"back","average":{"1min":1.1830000000000001,"5min":1.8380000000000001,"15min":1.865},"min":{"1min":0.432,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.359,"5min":45.576999999999998,"15min":45.576999999999998},"last":0.79300000000000004},{"interface":"front","average":{"1min":1.119,"5min":1.6180000000000001,"15min":1.3049999999999999},"min":{"1min":0.36599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.0620000000000003,"5min":45.420999999999999,"15min":45.420999999999999},"last":0.82399999999999995}]},{"osd":5,"last update":"Sat Sep 23 22:44:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1339999999999999,"5min":1.4650000000000001,"15min":3.3730000000000002},"min":{"1min":0.39300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":3.419,"5min":8.8119999999999994,"15min":163.56399999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.1479999999999999,"5min":2.5350000000000001,"15min":3.423},"min":{"1min":0.34699999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.5499999999999998,"5min":126.81100000000001,"15min":163.541},"last":0.77100000000000002}]},{"osd":6,"last update":"Sat Sep 23 22:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.2629999999999999,"5min":2.2629999999999999,"15min":2.2629999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":7.351,"5min":7.351,"15min":7.351},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.476,"5min":1.476,"15min":1.476},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":3.895,"5min":3.895,"15min":3.895},"last":0.73999999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:44:15 2023","interfaces":[{"interface":"back","average":{"1min":1.704,"5min":2.1269999999999998,"15min":1.9450000000000001},"min":{"1min":0.46600000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.601,"5min":75.513999999999996,"15min":108.113},"last":0.70899999999999996},{"interface":"front","average":{"1min":1.198,"5min":2.1589999999999998,"15min":2.2810000000000001},"min":{"1min":0.48999999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.4529999999999998,"5min":74.007999999999996,"15min":108.093},"last":0.68999999999999995}]}]},{"osd":3,"up_from":0,"seq":2508260900892,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":4,"up_from":0,"seq":1997159792748,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757757,"num_pgs":81,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":560592,"kb_used_data":11528,"kb_used_omap":0,"kb_used_meta":549056,"kb_avail":93184560,"statfs":{"total":95995035648,"available":95420989440,"internally_reserved":0,"allocated":11804672,"data_stored":9803458,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":562233344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:44:43 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":1.2050000000000001,"15min":3.673},"min":{"1min":0.245,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":1.5349999999999999,"5min":26.925000000000001,"15min":145.46600000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.63,"5min":1.196,"15min":3.5830000000000002},"min":{"1min":0.26900000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.476,"5min":30.609000000000002,"15min":144.429},"last":0.36599999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:43:45 2023","interfaces":[{"interface":"back","average":{"1min":3.4910000000000001,"5min":1.8700000000000001,"15min":1.0800000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":35.643999999999998,"5min":35.643999999999998,"15min":35.643999999999998},"last":0.40300000000000002},{"interface":"front","average":{"1min":3.6949999999999998,"5min":1.869,"15min":0.98199999999999998},"min":{"1min":0.29399999999999998,"5min":0.25,"15min":0.249},"max":{"1min":35.316000000000003,"5min":35.316000000000003,"15min":35.316000000000003},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:44:14 2023","interfaces":[{"interface":"back","average":{"1min":5.1829999999999998,"5min":2.556,"15min":2.468},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":77.813999999999993,"5min":77.813999999999993,"15min":77.813999999999993},"last":0.44600000000000001},{"interface":"front","average":{"1min":4.9790000000000001,"5min":2.2400000000000002,"15min":2.3380000000000001},"min":{"1min":0.36599999999999999,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":76.265000000000001,"5min":76.265000000000001,"15min":76.265000000000001},"last":0.38700000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:44:38 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":1.383,"15min":1.482},"min":{"1min":0.439,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.2469999999999999,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.432},{"interface":"front","average":{"1min":0.78200000000000003,"5min":1.343,"15min":1.4370000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":2.3319999999999999,"5min":7.8959999999999999,"15min":7.8959999999999999},"last":0.46999999999999997}]},{"osd":4,"last update":"Sat Sep 23 22:44:10 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.51900000000000002,"15min":2.4470000000000001},"min":{"1min":0.27200000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":1.476,"5min":4.0750000000000002,"15min":61.173999999999999},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.60499999999999998,"15min":2.5179999999999998},"min":{"1min":0.35799999999999998,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":1.5409999999999999,"5min":4.157,"15min":61.203000000000003},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:44:00 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.97599999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 22:44:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.86199999999999999,"15min":1.3280000000000001},"min":{"1min":0.372,"5min":0.249,"15min":0.249},"max":{"1min":1.343,"5min":31.042999999999999,"15min":59.101999999999997},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.0349999999999999,"15min":1.5529999999999999},"min":{"1min":0.372,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.6499999999999999,"5min":30.882999999999999,"15min":59.151000000000003},"last":0.51300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4214784,"data_stored":4204579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":45056,"data_stored":20617,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4243456,"data_stored":4214940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":56,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8798,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5482,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:44:50.180 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T22:44:50.180 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:44:50.180 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.9054493477498636 2023-09-23T22:44:50.180 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 0.9054493477498636 2023-09-23T22:44:50.217 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T22:44:50.564 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:44:50.628 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:50.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:50.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:50.732 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[215775]: failed to encode map e631 with expected crc 2023-09-23T22:44:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[215775]: failed to encode map e631 with expected crc 2023-09-23T22:44:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[215775]: osdmap e631: 8 total, 8 up, 4 in 2023-09-23T22:44:50.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[215775]: failed to encode map e631 with expected crc 2023-09-23T22:44:50.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[215775]: pgmap v1631: 103 pgs: 103 active+clean; 4.7 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 267 KiB/s rd, 28 KiB/s wr, 358 op/s 2023-09-23T22:44:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[223014]: failed to encode map e631 with expected crc 2023-09-23T22:44:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[223014]: failed to encode map e631 with expected crc 2023-09-23T22:44:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[223014]: osdmap e631: 8 total, 8 up, 4 in 2023-09-23T22:44:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[223014]: failed to encode map e631 with expected crc 2023-09-23T22:44:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:50 smithi026 ceph-mon[223014]: pgmap v1631: 103 pgs: 103 active+clean; 4.7 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 267 KiB/s rd, 28 KiB/s wr, 358 op/s 2023-09-23T22:44:50.762 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:50.768 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T22:44:50.771 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import@snap1 2023-09-23T22:44:50.771 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'protected: False' 2023-09-23T22:44:50.809 INFO:tasks.workunit.client.0.smithi026.stdout: protected: False 2023-09-23T22:44:50.810 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd info testimg_import@snap2 2023-09-23T22:44:50.810 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep 'protected: True' 2023-09-23T22:44:50.811 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:50.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:50.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:50.848 INFO:tasks.workunit.client.0.smithi026.stdout: protected: True 2023-09-23T22:44:50.848 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/snap_protect 2023-09-23T22:44:50.849 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T22:44:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:50 smithi070 ceph-mon[192652]: failed to encode map e631 with expected crc 2023-09-23T22:44:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:50 smithi070 ceph-mon[192652]: failed to encode map e631 with expected crc 2023-09-23T22:44:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:50 smithi070 ceph-mon[192652]: osdmap e631: 8 total, 8 up, 4 in 2023-09-23T22:44:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:50 smithi070 ceph-mon[192652]: failed to encode map e631 with expected crc 2023-09-23T22:44:50.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:50 smithi070 ceph-mon[192652]: pgmap v1631: 103 pgs: 103 active+clean; 4.7 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 267 KiB/s rd, 28 KiB/s wr, 358 op/s 2023-09-23T22:44:50.893 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T22:44:50.915 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:50.937 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap purge testimg 2023-09-23T22:44:50.946 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:51.113 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:51.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:51.154 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:51.218 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:44:51.247 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:51.322 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:44:51.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:51.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:44:51.425 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:51.453 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:51.494 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:51.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:51.598 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:51.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:51 smithi026 ceph-mon[215775]: failed to encode map e632 with expected crc 2023-09-23T22:44:51.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:51 smithi026 ceph-mon[215775]: failed to encode map e632 with expected crc 2023-09-23T22:44:51.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:51 smithi026 ceph-mon[215775]: osdmap e632: 8 total, 8 up, 4 in 2023-09-23T22:44:51.606 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:51 smithi026 ceph-mon[215775]: failed to encode map e632 with expected crc 2023-09-23T22:44:51.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:51 smithi026 ceph-mon[223014]: failed to encode map e632 with expected crc 2023-09-23T22:44:51.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:51 smithi026 ceph-mon[223014]: failed to encode map e632 with expected crc 2023-09-23T22:44:51.606 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:51 smithi026 ceph-mon[223014]: osdmap e632: 8 total, 8 up, 4 in 2023-09-23T22:44:51.607 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:51 smithi026 ceph-mon[223014]: failed to encode map e632 with expected crc 2023-09-23T22:44:51.627 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:51.649 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:44:51.676 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:44:51.695 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:44:51.714 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:44:51.733 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:44:51.752 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:44:51.771 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:44:51.793 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:44:51.795 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:44:51.795 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:44:51.795 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:44:51.795 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:44:51.795 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:44:51.795 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:44:51.795 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:44:51.795 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:44:51.815 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:51.816 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:51 smithi070 ceph-mon[192652]: failed to encode map e632 with expected crc 2023-09-23T22:44:51.817 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:51 smithi070 ceph-mon[192652]: failed to encode map e632 with expected crc 2023-09-23T22:44:51.817 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:51 smithi070 ceph-mon[192652]: osdmap e632: 8 total, 8 up, 4 in 2023-09-23T22:44:51.817 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:51 smithi070 ceph-mon[192652]: failed to encode map e632 with expected crc 2023-09-23T22:44:51.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:51.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:51.919 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:51.945 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:51.988 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:51.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:51.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:52.093 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:52.124 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:52.452 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:44:52.452 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:44:52.452 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:44:52.453 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:44:52.464 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T22:44:52.624 INFO:tasks.workunit.client.0.smithi026.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T22:44:52.626 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:52.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:52.632 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd snap purge testimg_import 2023-09-23T22:44:52.731 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:52.757 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:52 smithi070 ceph-mon[192652]: failed to encode map e632 with expected crc 2023-09-23T22:44:52.758 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:52 smithi070 ceph-mon[192652]: failed to encode map e633 with expected crc 2023-09-23T22:44:52.758 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:52 smithi070 ceph-mon[192652]: failed to encode map e633 with expected crc 2023-09-23T22:44:52.758 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:52 smithi070 ceph-mon[192652]: failed to encode map e633 with expected crc 2023-09-23T22:44:52.758 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:52 smithi070 ceph-mon[192652]: osdmap e633: 8 total, 8 up, 4 in 2023-09-23T22:44:52.758 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4065333288' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:52.758 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:52 smithi070 ceph-mon[192652]: pgmap v1634: 103 pgs: 103 active+clean; 4.8 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 325 KiB/s rd, 46 KiB/s wr, 450 op/s 2023-09-23T22:44:52.760 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:52.804 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:52.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:52.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:52.907 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:52.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[215775]: failed to encode map e632 with expected crc 2023-09-23T22:44:52.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[215775]: failed to encode map e633 with expected crc 2023-09-23T22:44:52.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[215775]: failed to encode map e633 with expected crc 2023-09-23T22:44:52.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[215775]: failed to encode map e633 with expected crc 2023-09-23T22:44:52.933 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[215775]: osdmap e633: 8 total, 8 up, 4 in 2023-09-23T22:44:52.934 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4065333288' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:52.934 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[215775]: pgmap v1634: 103 pgs: 103 active+clean; 4.8 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 325 KiB/s rd, 46 KiB/s wr, 450 op/s 2023-09-23T22:44:52.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[223014]: failed to encode map e632 with expected crc 2023-09-23T22:44:52.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[223014]: failed to encode map e633 with expected crc 2023-09-23T22:44:52.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[223014]: failed to encode map e633 with expected crc 2023-09-23T22:44:52.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[223014]: failed to encode map e633 with expected crc 2023-09-23T22:44:52.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[223014]: osdmap e633: 8 total, 8 up, 4 in 2023-09-23T22:44:52.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4065333288' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:52.934 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:52 smithi026 ceph-mon[223014]: pgmap v1634: 103 pgs: 103 active+clean; 4.8 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 325 KiB/s rd, 46 KiB/s wr, 450 op/s 2023-09-23T22:44:52.936 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:53.085 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:53.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:53.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:53.161 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T22:44:53.190 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:53.225 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:53.225 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:53.323 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:53.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:53.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:53.427 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:53.455 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:53.508 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:44:53.550 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:53.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:53.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:53.662 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:53.688 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:53.704 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:53 smithi070 ceph-mon[192652]: failed to encode map e634 with expected crc 2023-09-23T22:44:53.704 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:53 smithi070 ceph-mon[192652]: failed to encode map e634 with expected crc 2023-09-23T22:44:53.704 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:53 smithi070 ceph-mon[192652]: osdmap e634: 8 total, 8 up, 4 in 2023-09-23T22:44:53.704 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:53 smithi070 ceph-mon[192652]: failed to encode map e634 with expected crc 2023-09-23T22:44:53.705 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:53 smithi070 ceph-mon[192652]: osdmap e635: 8 total, 8 up, 4 in 2023-09-23T22:44:53.737 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:53.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:53.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:53.841 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T22:44:53.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[223014]: failed to encode map e634 with expected crc 2023-09-23T22:44:53.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[223014]: failed to encode map e634 with expected crc 2023-09-23T22:44:53.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[223014]: osdmap e634: 8 total, 8 up, 4 in 2023-09-23T22:44:53.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[223014]: failed to encode map e634 with expected crc 2023-09-23T22:44:53.871 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[223014]: osdmap e635: 8 total, 8 up, 4 in 2023-09-23T22:44:53.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[215775]: failed to encode map e634 with expected crc 2023-09-23T22:44:53.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[215775]: failed to encode map e634 with expected crc 2023-09-23T22:44:53.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[215775]: osdmap e634: 8 total, 8 up, 4 in 2023-09-23T22:44:53.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[215775]: failed to encode map e634 with expected crc 2023-09-23T22:44:53.872 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:53 smithi026 ceph-mon[215775]: osdmap e635: 8 total, 8 up, 4 in 2023-09-23T22:44:53.875 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:54.024 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T22:44:54.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:54.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T22:44:54.128 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:54.155 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:54.197 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:54.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:54.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:54.301 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:54.327 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:54.429 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 0.905449 (e7cb) 2023-09-23T22:44:54.431 INFO:tasks.workunit.client.0.smithi026.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T22:44:54.439 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg 2023-09-23T22:44:54.476 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:54.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:54.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:54.531 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:44:54.534 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd remove testimg_import 2023-09-23T22:44:54.579 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:54.609 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:54.685 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:54.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:54.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:54.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: failed to encode map e635 with expected crc 2023-09-23T22:44:54.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: failed to encode map e635 with expected crc 2023-09-23T22:44:54.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: failed to encode map e635 with expected crc 2023-09-23T22:44:54.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: pgmap v1637: 103 pgs: 103 active+clean; 4.8 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 164 KiB/s rd, 23 KiB/s wr, 228 op/s 2023-09-23T22:44:54.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3945106097' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.9054493477498636}]: dispatch 2023-09-23T22:44:54.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.9054493477498636}]: dispatch 2023-09-23T22:44:54.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:54.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:54.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.9054493477498636}]': finished 2023-09-23T22:44:54.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[223014]: osdmap e636: 8 total, 8 up, 4 in 2023-09-23T22:44:54.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: failed to encode map e635 with expected crc 2023-09-23T22:44:54.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: failed to encode map e635 with expected crc 2023-09-23T22:44:54.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: failed to encode map e635 with expected crc 2023-09-23T22:44:54.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: pgmap v1637: 103 pgs: 103 active+clean; 4.8 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 164 KiB/s rd, 23 KiB/s wr, 228 op/s 2023-09-23T22:44:54.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3945106097' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.9054493477498636}]: dispatch 2023-09-23T22:44:54.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.9054493477498636}]: dispatch 2023-09-23T22:44:54.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:54.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:54.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.9054493477498636}]': finished 2023-09-23T22:44:54.738 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:54 smithi026 ceph-mon[215775]: osdmap e636: 8 total, 8 up, 4 in 2023-09-23T22:44:54.789 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:54.817 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: failed to encode map e635 with expected crc 2023-09-23T22:44:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: failed to encode map e635 with expected crc 2023-09-23T22:44:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: failed to encode map e635 with expected crc 2023-09-23T22:44:54.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: pgmap v1637: 103 pgs: 103 active+clean; 4.8 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 164 KiB/s rd, 23 KiB/s wr, 228 op/s 2023-09-23T22:44:54.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3945106097' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.9054493477498636}]: dispatch 2023-09-23T22:44:54.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.9054493477498636}]: dispatch 2023-09-23T22:44:54.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:44:54.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:44:54.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.9054493477498636}]': finished 2023-09-23T22:44:54.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:54 smithi070 ceph-mon[192652]: osdmap e636: 8 total, 8 up, 4 in 2023-09-23T22:44:54.979 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:44:54.979 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:44:54.980 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:44:54.980 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:44:54.980 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:44:55.091 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:55.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:55.153 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T22:44:55.195 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:55.224 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:55.268 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:55.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:55.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:55.372 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:55.404 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:55.515 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:55.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:55.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:55.597 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:55.619 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:55.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[215775]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[215775]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[215775]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[215775]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[215775]: osdmap e637: 8 total, 8 up, 4 in 2023-09-23T22:44:55.651 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:55.725 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:55.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:55.796 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T22:44:55.828 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:55.865 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:55.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:55 smithi070 ceph-mon[192652]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:55 smithi070 ceph-mon[192652]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:55 smithi070 ceph-mon[192652]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:55 smithi070 ceph-mon[192652]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:55 smithi070 ceph-mon[192652]: osdmap e637: 8 total, 8 up, 4 in 2023-09-23T22:44:55.909 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:55.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:55.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:55.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[223014]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.961 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[223014]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[223014]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[223014]: failed to encode map e636 with expected crc 2023-09-23T22:44:55.962 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:55 smithi026 ceph-mon[223014]: osdmap e637: 8 total, 8 up, 4 in 2023-09-23T22:44:56.015 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:56.042 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:56.138 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:56.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:56.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:56.152 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:44:56.241 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:56.267 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:56.411 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:56.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:56.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:56.429 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is set 2023-09-23T22:44:56.438 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:44:56.442 INFO:tasks.workunit.client.0.smithi026.stderr:+ tiered=0 2023-09-23T22:44:56.442 INFO:tasks.workunit.client.0.smithi026.stderr:+ ceph osd dump 2023-09-23T22:44:56.442 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep '^pool' 2023-09-23T22:44:56.443 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep ''\''rbd'\''' 2023-09-23T22:44:56.443 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep tier 2023-09-23T22:44:56.515 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:56.542 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: pgmap v1640: 103 pgs: 7 peering, 96 active+clean; 4.8 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 140 KiB/s rd, 10 KiB/s wr, 184 op/s 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3160396056' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3160396056' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:56.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[223014]: osdmap e638: 8 total, 8 up, 4 in 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: pgmap v1640: 103 pgs: 7 peering, 96 active+clean; 4.8 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 140 KiB/s rd, 10 KiB/s wr, 184 op/s 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3160396056' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3160396056' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:56.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:56 smithi026 ceph-mon[215775]: osdmap e638: 8 total, 8 up, 4 in 2023-09-23T22:44:56.738 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:56.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:56.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:56.841 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: failed to encode map e637 with expected crc 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: pgmap v1640: 103 pgs: 7 peering, 96 active+clean; 4.8 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 140 KiB/s rd, 10 KiB/s wr, 184 op/s 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3160396056' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:44:56.865 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3160396056' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T22:44:56.866 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:56 smithi070 ceph-mon[192652]: osdmap e638: 8 total, 8 up, 4 in 2023-09-23T22:44:56.867 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:56.908 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:56.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:57.012 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:57.038 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:57.185 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:57.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:57.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:57.212 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_687972/sparse1 2023-09-23T22:44:57.218 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:44:57.218 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:44:57.218 INFO:tasks.workunit.client.0.smithi026.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00503541 s, 208 MB/s 2023-09-23T22:44:57.218 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_687972/sparse2 2023-09-23T22:44:57.224 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:44:57.224 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:44:57.224 INFO:tasks.workunit.client.0.smithi026.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00501505 s, 209 MB/s 2023-09-23T22:44:57.224 INFO:tasks.workunit.client.0.smithi026.stderr:+ truncate /tmp/rbd_import_export_687972/sparse2 -s 2M 2023-09-23T22:44:57.233 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse1 2023-09-23T22:44:57.273 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 0% complete...failed. 2023-09-23T22:44:57.273 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T22:44:57.276 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:44:57.276 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_687972/sparse1 2023-09-23T22:44:57.288 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:57.320 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:57.343 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T22:44:57.343 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:44:57.344 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:44:57.346 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T22:44:57.346 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse1 2023-09-23T22:44:57.346 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T22:44:57.387 INFO:tasks.workunit.client.0.smithi026.stdout:sparse1 2 MiB 2 2023-09-23T22:44:57.387 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse1 2023-09-23T22:44:57.387 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse1 2023-09-23T22:44:57.388 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse1 2023-09-23T22:44:57.388 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:44:57.388 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:44:57.433 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:57.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:57.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:57.441 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.b8d9551bdd60 2023-09-23T22:44:57.442 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.b8d9551bdd60 2023-09-23T22:44:57.442 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse1 2023-09-23T22:44:57.442 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.b8d9551bdd60\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:44:57.442 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse1 2023-09-23T22:44:57.442 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:44:57.443 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse1 2023-09-23T22:44:57.443 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:44:57.443 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:44:57.538 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:57.568 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:57.667 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:57.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:57.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:57.745 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104682 9.f 9:f58eaa2e:::benchmark_data_smithi026_555954_object9953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.265869+0000", 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 463.477004593, 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1940709429999998, 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:44:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104682 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265869+0000", 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265869+0000", 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265868+0000", 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265875+0000", 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.52e-06 2023-09-23T22:44:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265888+0000", 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2493e-05 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265892+0000", 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6199999999999998e-06 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265936+0000", 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3751999999999997e-05 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266080+0000", 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014391999999999999 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266249+0000", 2023-09-23T22:44:57.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000168729 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.269497+0000", 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003247757 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292030+0000", 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022533972999999999 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292040+0000", 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6260000000000007e-06 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276089+0000", 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98404894300000001 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276099+0000", 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8539999999999992e-06 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425242+0000", 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1491429769999999 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425390+0000", 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000148601 2023-09-23T22:44:57.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.459827+0000", 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.034436178999999997 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.459940+0000", 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011315900000000001 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.754 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104687 9.f 9:fb4465c7:::benchmark_data_smithi026_555954_object9958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.279126+0000", 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 463.46374716700001, 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1856558540000002, 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:44:57.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104687 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279126+0000", 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279126+0000", 2023-09-23T22:44:57.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279125+0000", 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279132+0000", 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.968e-06 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279147+0000", 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5252000000000001e-05 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279155+0000", 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4139999999999997e-06 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279205+0000", 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0002e-05 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279383+0000", 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017872 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:44:57.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.279550+0000", 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000166343 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292069+0000", 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012519417 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.292074+0000", 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1499999999999998e-06 2023-09-23T22:44:57.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276124+0000", 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98404924299999996 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276129+0000", 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6949999999999997e-06 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425447+0000", 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1493180839999999 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425628+0000", 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00018074899999999999 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464649+0000", 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039020688999999997 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464782+0000", 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013333600000000001 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.763 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104699 9.f 9:fa6f98c4:::benchmark_data_smithi026_555954_object9970:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.355308+0000", 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 463.38756590000003, 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1458075220000001, 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104699 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.764 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355308+0000", 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355308+0000", 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355306+0000", 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355314+0000", 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2239999999999995e-06 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355332+0000", 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7728999999999999e-05 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355355+0000", 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3218999999999999e-05 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355394+0000", 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8661999999999997e-05 2023-09-23T22:44:57.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.355401+0000", 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9550000000000003e-06 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276149+0000", 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92074822599999995 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.276155+0000", 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5539999999999997e-06 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.425668+0000", 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.149513765 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426016+0000", 2023-09-23T22:44:57.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000347487 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.445038+0000", 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0190222 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.445155+0000", 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011651599999999999 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.445326+0000", 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017155499999999999 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464769+0000", 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019442566000000001 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464804+0000", 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5648999999999997e-05 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.497550+0000", 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.032745511999999997 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.497594+0000", 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4270000000000001e-05 2023-09-23T22:44:57.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.501115+0000", 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035209569999999999 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104701 9.f 9:f9048e3a:::benchmark_data_smithi026_555954_object9972:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.295854+0000", 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 462.44701927900002, 2023-09-23T22:44:57.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.208795673, 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104701 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295854+0000", 2023-09-23T22:44:57.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295854+0000", 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295853+0000", 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295861+0000", 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3720000000000005e-06 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295877+0000", 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5875e-05 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295885+0000", 2023-09-23T22:44:57.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5109999999999997e-06 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295932+0000", 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6916999999999998e-05 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.295940+0000", 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0020000000000006e-06 2023-09-23T22:44:57.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426058+0000", 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1301181819999999 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426486+0000", 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00042859000000000001 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.452638+0000", 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026151917 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.452720+0000", 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1389000000000003e-05 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:44:57.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.452832+0000", 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000112194 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464779+0000", 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011947107 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.464856+0000", 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6678e-05 2023-09-23T22:44:57.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.501167+0000", 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036311450000000002 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.501211+0000", 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4246999999999998e-05 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.504650+0000", 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034385829999999998 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104709 9.f 9:fe76147f:::benchmark_data_smithi026_555954_object9980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.348800+0000", 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 462.39407394599999, 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.175744452, 2023-09-23T22:44:57.782 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104709 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348800+0000", 2023-09-23T22:44:57.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348800+0000", 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348798+0000", 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348807+0000", 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7070000000000004e-06 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348822+0000", 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4944e-05 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.348831+0000", 2023-09-23T22:44:57.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7280000000000001e-06 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.389358+0000", 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.040527173999999999 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.389371+0000", 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2469e-05 2023-09-23T22:44:57.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426523+0000", 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0371526820000001 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for degraded object", 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.426916+0000", 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00039232699999999999 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.493074+0000", 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.066158162000000006 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.493164+0000", 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0309000000000006e-05 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,3", 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.493254+0000", 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0066000000000001e-05 2023-09-23T22:44:57.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.504667+0000", 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011412558999999999 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.504705+0000", 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8373e-05 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.521152+0000", 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016447092999999999 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.521194+0000", 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1467999999999997e-05 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.524544+0000", 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0033503230000000001 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:105990 9.f 9:fc198dae:::benchmark_data_smithi026_555954_object11261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:43.319569+0000", 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 434.423304723, 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4132751570000002, 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.791 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 105990 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319569+0000", 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319569+0000", 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319566+0000", 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999971 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319582+0000", 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5855999999999999e-05 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319622+0000", 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0615999999999999e-05 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319637+0000", 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4922999999999999e-05 2023-09-23T22:44:57.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319766+0000", 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012936299999999999 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319786+0000", 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9723000000000001e-05 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.680894+0000", 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3611081509999998 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.681104+0000", 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020942399999999999 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:44:57.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.681225+0000", 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012102199999999999 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.729953+0000", 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.048728018999999997 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730035+0000", 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2010999999999998e-05 2023-09-23T22:44:57.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730114+0000", 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9276000000000005e-05 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730166+0000", 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1437000000000002e-05 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.732844+0000", 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026783789999999998 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:105991 9.f 9:f5926516:::benchmark_data_smithi026_555954_object11262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 434.42320708900002, 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4159624549999998, 2023-09-23T22:44:57.799 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 105991 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:44:57.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319666+0000", 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319671+0000", 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4060000000000004e-06 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319681+0000", 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0563e-05 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319688+0000", 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8000000000000001e-06 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319843+0000", 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015483200000000001 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:43.319865+0000", 2023-09-23T22:44:57.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1933000000000002e-05 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.684141+0000", 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.364276378 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.684269+0000", 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012795800000000001 2023-09-23T22:44:57.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.684369+0000", 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9584000000000001e-05 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.730094+0000", 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.045724787000000003 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.732880+0000", 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002786186 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.732981+0000", 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100999 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.733032+0000", 2023-09-23T22:44:57.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0701000000000001e-05 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:46.735629+0000", 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0025971940000000002 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.37014.0:27 2.3 2:d4a7e8c6:::rbd_object_map.90964ea96200:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T22:44:57.807 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:28.345828+0000", 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.397045034000001, 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001331727, 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.37014", 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3610649076", 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 27 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.345828+0000", 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.345828+0000", 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.345828+0000", 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.345833+0000", 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5829999999999997e-06 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.345845+0000", 2023-09-23T22:44:57.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2106999999999999e-05 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.345850+0000", 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0030000000000002e-06 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.345896+0000", 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5423999999999997e-05 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.345969+0000", 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3458e-05 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.346200+0000", 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00023098300000000001 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.346702+0000", 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00050202500000000002 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.346993+0000", 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00029074299999999999 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.347070+0000", 2023-09-23T22:44:57.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.6632999999999996e-05 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.347122+0000", 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2803999999999998e-05 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.347160+0000", 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7728999999999997e-05 2023-09-23T22:44:57.814 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.37014.0:32 2.3 2:d4a7e8c6:::rbd_object_map.90964ea96200:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:28.352173+0000", 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.390700211999999, 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001333341, 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.815 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.37014", 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3610649076", 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 32 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.352173+0000", 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.352173+0000", 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.352172+0000", 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.352180+0000", 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5120000000000002e-06 2023-09-23T22:44:57.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.352196+0000", 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5957999999999999e-05 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.352202+0000", 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2740000000000004e-06 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.352251+0000", 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9434000000000001e-05 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.352315+0000", 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3808999999999998e-05 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:44:57.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.352570+0000", 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025444299999999998 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.353042+0000", 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00047214700000000002 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.353406+0000", 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036407099999999999 2023-09-23T22:44:57.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.353439+0000", 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3448000000000002e-05 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.353478+0000", 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8229000000000003e-05 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.353507+0000", 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9026e-05 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.37014.0:34 2.3 2:d4a7e8c6:::rbd_object_map.90964ea96200:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:28.402467+0000", 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 29.34040688, 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017623420000000001, 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.822 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.37014", 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3610649076", 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 34 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.402467+0000", 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.402467+0000", 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.402464+0000", 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.402479+0000", 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4168999999999999e-05 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.402514+0000", 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5332e-05 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.402528+0000", 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3647e-05 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.402635+0000", 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010765 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.826 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.402705+0000", 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9604000000000006e-05 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.402982+0000", 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000277106 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.403456+0000", 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00047417800000000001 2023-09-23T22:44:57.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.403807+0000", 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00035085900000000002 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.404157+0000", 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00034978800000000002 2023-09-23T22:44:57.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.404197+0000", 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0108000000000003e-05 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.404229+0000", 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2004000000000001e-05 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.829 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.37014.0:36 2.3 2:d4a7e8c6:::rbd_object_map.90964ea96200:head [call lock.assert_locked in=17b,call rbd.object_map_update in=20b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:28.878604+0000", 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 28.864269467, 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001565171, 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.37014", 2023-09-23T22:44:57.830 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3610649076", 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 36 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.878604+0000", 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.878604+0000", 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.878601+0000", 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999971 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.878616+0000", 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4674e-05 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.878653+0000", 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6952000000000001e-05 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.878684+0000", 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1158999999999998e-05 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.878776+0000", 2023-09-23T22:44:57.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1978999999999995e-05 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.878876+0000", 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9814000000000004e-05 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.879153+0000", 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00027726699999999999 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.879729+0000", 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00057611200000000004 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.880048+0000", 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000318893 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.835 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.880085+0000", 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6792999999999998e-05 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.880131+0000", 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6428999999999999e-05 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:28.880169+0000", 2023-09-23T22:44:57.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7818999999999998e-05 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.37092.0:49 2.2 2:4e1777b8:::rbd_header.90e4cc1a1b1c:head [call rbd.snapshot_add in=15b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e626)", 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:43.453975+0000", 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 14.288898319999999, 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052688880000000002, 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.837 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.37092", 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937521917", 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 49 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.453975+0000", 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.453975+0000", 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.453973+0000", 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.453983+0000", 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0068000000000001e-05 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.454002+0000", 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9111e-05 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.454010+0000", 2023-09-23T22:44:57.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7589999999999997e-06 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.454079+0000", 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9086999999999998e-05 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op must wait for map", 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.454085+0000", 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1279999999999996e-06 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.457681+0000", 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035957430000000002 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.457791+0000", 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011035399999999999 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.458145+0000", 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000353643 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.458706+0000", 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00056125499999999998 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.459072+0000", 2023-09-23T22:44:57.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00036597799999999998 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.459156+0000", 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3803999999999999e-05 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.459206+0000", 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0361000000000001e-05 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:43.459244+0000", 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7546000000000003e-05 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.37356.0:41 2.3 2:c50c492b:::rbd_header.b86131379f28:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e634)", 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:52.618329+0000", 2023-09-23T22:44:57.845 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 5.1245446729999999, 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050248790000000003, 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.37356", 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3272172288", 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 41 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.618329+0000", 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.618329+0000", 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.618328+0000", 2023-09-23T22:44:57.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.618335+0000", 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0600000000000002e-06 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.618351+0000", 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6078000000000001e-05 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.618360+0000", 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4630000000000007e-06 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.618439+0000", 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8287000000000004e-05 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op must wait for map", 2023-09-23T22:44:57.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.618447+0000", 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7779999999999996e-06 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.622000+0000", 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035524229999999999 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.622100+0000", 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000100249 2023-09-23T22:44:57.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.622402+0000", 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030200000000000002 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.622917+0000", 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000515286 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.623172+0000", 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00025411299999999998 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.623301+0000", 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000129164 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.623330+0000", 2023-09-23T22:44:57.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9652999999999999e-05 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.623354+0000", 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3354e-05 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.37356.0:44 2.3 2:c3e3c0c8:::rbd_object_map.b86131379f28:head [call lock.unlock in=10b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e634)", 2023-09-23T22:44:57.853 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:52.624740+0000", 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 5.1181339670000003, 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019145329999999999, 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.37356", 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/3272172288", 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 44 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.854 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.624740+0000", 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.624740+0000", 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.624739+0000", 2023-09-23T22:44:57.855 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.624745+0000", 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9179999999999999e-06 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.624760+0000", 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.502e-05 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.856 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.624766+0000", 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4590000000000003e-06 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.624808+0000", 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2116000000000001e-05 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.857 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.624899+0000", 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0966999999999998e-05 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.625094+0000", 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019527699999999999 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.625505+0000", 2023-09-23T22:44:57.858 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00041070000000000001 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.625678+0000", 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00017286600000000001 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.626596+0000", 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00091808199999999997 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.859 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.626630+0000", 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3898e-05 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:52.626654+0000", 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.41e-05 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.860 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.57061.0:26 2.2 2:5dab6326:::rbd_header.b888711a537:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e635)", 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:53.417185+0000", 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 4.3256880200000003, 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0080329379999999999, 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.57061", 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1122423865", 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 26 2023-09-23T22:44:57.861 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.417185+0000", 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.417185+0000", 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.862 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.417184+0000", 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.417194+0000", 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0370999999999999e-05 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.863 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.417214+0000", 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9678e-05 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.417223+0000", 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6609999999999995e-06 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.864 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.417293+0000", 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9326000000000007e-05 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op must wait for map", 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.417299+0000", 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4819999999999999e-06 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.865 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.423265+0000", 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.005966173 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.423365+0000", 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9760999999999996e-05 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.423671+0000", 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00030570100000000002 2023-09-23T22:44:57.866 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.424244+0000", 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00057291099999999995 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.424775+0000", 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000531027 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.867 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.425124+0000", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000349701 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.425174+0000", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.003e-05 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:53.425218+0000", 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4056e-05 2023-09-23T22:44:57.868 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.57061.0:41 2.2 2:5dab6326:::rbd_header.b888711a537:head [call rbd.snapshot_remove in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e636)", 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:54.424383+0000", 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 3.3184899799999998, 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073163769999999998, 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.869 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.57061", 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1122423865", 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 41 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.424383+0000", 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.870 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.424383+0000", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.424382+0000", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.871 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.424391+0000", 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4670000000000004e-06 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.424401+0000", 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0437e-05 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.424407+0000", 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3090000000000001e-06 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.872 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.424488+0000", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0568999999999994e-05 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op must wait for map", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.424496+0000", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8429999999999993e-06 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.873 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.428464+0000", 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039684769999999998 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.428569+0000", 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010478 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.428895+0000", 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00032553399999999999 2023-09-23T22:44:57.874 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.429561+0000", 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00066650699999999999 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.431572+0000", 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020106619999999999 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.875 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.431605+0000", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3670000000000001e-05 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.431658+0000", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2311000000000001e-05 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.876 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.431700+0000", 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2216999999999999e-05 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.47297.0:57 2.3 2:c4c92e5a:::rbd_trash:head [call rbd.trash_state_set in=18b] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e636)", 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:54.605063+0000", 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 3.1378108830000002, 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014556479999999999, 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.877 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.47297", 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1815957787", 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 57 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605063+0000", 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.878 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605063+0000", 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605062+0000", 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.879 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605073+0000", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1440000000000001e-05 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605080+0000", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8349999999999998e-06 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605084+0000", 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6119999999999999e-06 2023-09-23T22:44:57.880 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605110+0000", 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6519000000000001e-05 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605153+0000", 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2449e-05 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.881 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605294+0000", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000140851 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.605900+0000", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00060644799999999999 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.606215+0000", 2023-09-23T22:44:57.882 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031465200000000001 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.606454+0000", 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000239337 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.606484+0000", 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0005000000000002e-05 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.883 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.606518+0000", 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.4078999999999997e-05 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.47297.0:69 2.2 2:5dab6326:::rbd_header.b888711a537:head [watch watch cookie 140124046342176] snapc 0=[] ondisk+write+known_if_redirected+full_try+supports_pool_eio e636)", 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:54.612963+0000", 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 3.1299103609999999, 2023-09-23T22:44:57.884 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001666498, 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.47297", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1815957787", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 69 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.612963+0000", 2023-09-23T22:44:57.885 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.612963+0000", 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.612962+0000", 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:44:57.886 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.612966+0000", 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0110000000000002e-06 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.612978+0000", 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1308000000000001e-05 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.887 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.612999+0000", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0752999999999999e-05 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.613030+0000", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1489000000000003e-05 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.613076+0000", 2023-09-23T22:44:57.888 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5741000000000001e-05 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,6", 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.613179+0000", 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103082 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.613631+0000", 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00045242999999999998 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.889 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.614466+0000", 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00083511900000000005 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.614502+0000", 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5493000000000001e-05 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.890 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.614563+0000", 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1080999999999993e-05 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.614630+0000", 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6656999999999996e-05 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.891 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.47321.0:29 2.3 2:d5ad9a37:::rbd_data.b8d9551bdd60.0000000000000001:head [set-alloc-hint object_size 1048576 write_size 1048576,writefull 0~1048576 [fadvise_nocache]] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e638)", 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:57.335268+0000", 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.40760593099999998, 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044815819999999996, 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.47321", 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2051268068", 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 29 2023-09-23T22:44:57.892 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.335268+0000", 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.335268+0000", 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.893 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.335267+0000", 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.336314+0000", 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001047489 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.894 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.336340+0000", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6037000000000002e-05 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.336350+0000", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6350000000000001e-06 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.336414+0000", 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4474999999999995e-05 2023-09-23T22:44:57.895 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.336530+0000", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115747 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,6", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.336633+0000", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103016 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:44:57.896 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.338296+0000", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016633450000000001 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.339302+0000", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001005398 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.339638+0000", 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00033656300000000001 2023-09-23T22:44:57.897 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.339703+0000", 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4405000000000004e-05 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.339749+0000", 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6360000000000003e-05 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.898 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.47333.0:7 2.2 2:5ac68a50:::rbd_header.b8d9551bdd60:head [call rbd.get_size in=11b,call rbd.get_features in=15b,call rbd.get_flags in=12b,call,call lock.get_info in=12b] snapc 0=[] ondisk+read+known_if_redirected+supports_pool_eio e638)", 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:57.430974+0000", 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 0.31189936499999998, 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036829559999999998, 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.47333", 2023-09-23T22:44:57.899 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/2414666950", 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 7 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.430974+0000", 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.430974+0000", 2023-09-23T22:44:57.900 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.430972+0000", 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.430997+0000", 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4244e-05 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.901 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.431018+0000", 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1710999999999999e-05 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.431027+0000", 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6260000000000001e-06 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.902 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.431101+0000", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4351999999999996e-05 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.431204+0000", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010281999999999999 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:57.434657+0000", 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034529280000000001 2023-09-23T22:44:57.903 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:44:57.904 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:44:57.904 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:57.905 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:57 smithi070 ceph-mon[192652]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.905 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:57 smithi070 ceph-mon[192652]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.905 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:57 smithi070 ceph-mon[192652]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.906 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:57 smithi070 ceph-mon[192652]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.906 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:57 smithi070 ceph-mon[192652]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.906 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1319669328' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:44:57.906 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1319669328' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:44:57.906 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:57 smithi070 ceph-mon[192652]: osdmap e639: 8 total, 8 up, 4 in 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:44:57.906 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:44:57.907 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:44:57.916 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:44:57.936 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:44:57.942 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:57.955 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:44:57.973 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:44:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[215775]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[215775]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[215775]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[215775]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[215775]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1319669328' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1319669328' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[215775]: osdmap e639: 8 total, 8 up, 4 in 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[223014]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[223014]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[223014]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[223014]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[223014]: failed to encode map e638 with expected crc 2023-09-23T22:44:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1319669328' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:44:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1319669328' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T22:44:57.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:57 smithi026 ceph-mon[223014]: osdmap e639: 8 total, 8 up, 4 in 2023-09-23T22:44:57.987 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:57.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:57.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:57.993 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T22:44:57.994 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T22:44:57.995 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T22:44:58.089 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:44:58.119 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:58.215 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:44:58.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:58.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:44:58.319 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T22:44:58.346 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:58.456 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T22:44:58.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:58.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T22:44:58.484 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:44:58.485 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:44:58.485 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:44:58.485 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:44:58.485 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:44:58.516 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T22:44:58.559 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:44:58.589 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:58.616 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=1 2023-09-23T22:44:58.617 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 1 2023-09-23T22:44:58.617 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T22:44:58.617 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse1 /tmp/rbd_import_export_687972/sparse1.out 2023-09-23T22:44:58.656 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:58.659 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_687972/sparse1 /tmp/rbd_import_export_687972/sparse1.out 2023-09-23T22:44:58.659 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_687972/sparse1 /tmp/rbd_import_export_687972/sparse1.out 2023-09-23T22:44:58.661 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/sparse1 --format %b 2023-09-23T22:44:58.662 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=2048 2023-09-23T22:44:58.662 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/sparse1.out --format %b 2023-09-23T22:44:58.664 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=2048 2023-09-23T22:44:58.664 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:58.664 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:58.664 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:44:58.664 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/sparse1.out 2023-09-23T22:44:58.664 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse1 2023-09-23T22:44:58.753 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T22:44:58.755 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse2 2023-09-23T22:44:58.793 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 0% complete...failed. 2023-09-23T22:44:58.793 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T22:44:58.797 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:44:58.797 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_687972/sparse2 2023-09-23T22:44:58.805 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:44:58.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:58.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:44:58.859 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T22:44:58.859 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:44:58.859 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:44:58.862 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T22:44:58.862 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse2 2023-09-23T22:44:58.862 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T22:44:58.865 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:44:58.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[215775]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[215775]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[215775]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[215775]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.875 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[215775]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[215775]: pgmap v1643: 103 pgs: 1 remapped+peering, 7 peering, 95 active+clean; 4.7 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 143 KiB/s rd, 10 KiB/s wr, 189 op/s 2023-09-23T22:44:58.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/501259969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:58.876 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[215775]: osdmap e640: 8 total, 8 up, 4 in 2023-09-23T22:44:58.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[223014]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[223014]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[223014]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[223014]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[223014]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.876 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[223014]: pgmap v1643: 103 pgs: 1 remapped+peering, 7 peering, 95 active+clean; 4.7 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 143 KiB/s rd, 10 KiB/s wr, 189 op/s 2023-09-23T22:44:58.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/501259969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:58.877 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:58 smithi026 ceph-mon[223014]: osdmap e640: 8 total, 8 up, 4 in 2023-09-23T22:44:58.902 INFO:tasks.workunit.client.0.smithi026.stdout:sparse2 2 MiB 2 2023-09-23T22:44:58.903 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse2 2023-09-23T22:44:58.903 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse2 2023-09-23T22:44:58.904 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse2 2023-09-23T22:44:58.904 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:44:58.904 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:44:58.909 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T22:44:58.932 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:58 smithi070 ceph-mon[192652]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.933 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:58 smithi070 ceph-mon[192652]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.933 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:58 smithi070 ceph-mon[192652]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.933 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:58 smithi070 ceph-mon[192652]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.933 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:58 smithi070 ceph-mon[192652]: failed to encode map e639 with expected crc 2023-09-23T22:44:58.933 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:58 smithi070 ceph-mon[192652]: pgmap v1643: 103 pgs: 1 remapped+peering, 7 peering, 95 active+clean; 4.7 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 143 KiB/s rd, 10 KiB/s wr, 189 op/s 2023-09-23T22:44:58.933 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/501259969' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:44:58.933 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:58 smithi070 ceph-mon[192652]: osdmap e640: 8 total, 8 up, 4 in 2023-09-23T22:44:58.935 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:58.943 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.9252b1fc4e1b 2023-09-23T22:44:58.943 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.9252b1fc4e1b 2023-09-23T22:44:58.944 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse2 2023-09-23T22:44:58.944 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.9252b1fc4e1b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:44:58.944 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse2 2023-09-23T22:44:58.944 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:44:58.944 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse2 2023-09-23T22:44:58.945 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:44:58.945 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:44:58.982 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:44:58.982 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:44:58.983 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:44:58.983 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:44:58.983 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:44:58.999 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T22:44:59.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:59.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T22:44:59.015 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=0 2023-09-23T22:44:59.015 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 0 2023-09-23T22:44:59.015 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T22:44:59.015 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse2 /tmp/rbd_import_export_687972/sparse2.out 2023-09-23T22:44:59.052 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:59.055 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_687972/sparse2 /tmp/rbd_import_export_687972/sparse2.out 2023-09-23T22:44:59.056 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_687972/sparse2 /tmp/rbd_import_export_687972/sparse2.out 2023-09-23T22:44:59.058 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/sparse2 --format %b 2023-09-23T22:44:59.059 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=2048 2023-09-23T22:44:59.059 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/sparse2.out --format %b 2023-09-23T22:44:59.060 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=2048 2023-09-23T22:44:59.060 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:59.060 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:59.060 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:44:59.060 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/sparse2.out 2023-09-23T22:44:59.061 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse2 2023-09-23T22:44:59.103 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:59.134 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:59.150 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T22:44:59.153 INFO:tasks.workunit.client.0.smithi026.stderr:+ truncate /tmp/rbd_import_export_687972/sparse1 -s 10M 2023-09-23T22:44:59.154 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T22:44:59.177 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:59.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:59.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:59.241 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:44:59.242 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:44:59.242 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:44:59.244 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T22:44:59.244 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse1 2023-09-23T22:44:59.244 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T22:44:59.280 INFO:tasks.workunit.client.0.smithi026.stdout:sparse1 10 MiB 2 2023-09-23T22:44:59.281 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse1 2023-09-23T22:44:59.281 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse1 2023-09-23T22:44:59.281 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse1 2023-09-23T22:44:59.281 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:44:59.281 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:44:59.281 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:44:59.311 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:59.318 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.b93f851ef0ff 2023-09-23T22:44:59.318 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.b93f851ef0ff 2023-09-23T22:44:59.319 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse1 2023-09-23T22:44:59.319 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse1 2023-09-23T22:44:59.319 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.b93f851ef0ff\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:44:59.319 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:44:59.319 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse1 2023-09-23T22:44:59.320 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:44:59.320 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:44:59.356 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:44:59.356 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:44:59.356 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:44:59.356 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:44:59.356 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:44:59.389 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=1 2023-09-23T22:44:59.389 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 1 2023-09-23T22:44:59.389 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T22:44:59.389 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse1 /tmp/rbd_import_export_687972/sparse1.out 2023-09-23T22:44:59.427 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:59.431 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_687972/sparse1 /tmp/rbd_import_export_687972/sparse1.out 2023-09-23T22:44:59.431 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_687972/sparse1 /tmp/rbd_import_export_687972/sparse1.out 2023-09-23T22:44:59.433 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T22:44:59.439 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/sparse1 --format %b 2023-09-23T22:44:59.440 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=2048 2023-09-23T22:44:59.440 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/sparse1.out --format %b 2023-09-23T22:44:59.442 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=2048 2023-09-23T22:44:59.442 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:59.442 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:59.442 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:44:59.442 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/sparse1.out 2023-09-23T22:44:59.443 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse1 2023-09-23T22:44:59.455 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:44:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:59.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:44:59.558 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:59.584 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:59.626 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:59.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:59.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:59.667 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T22:44:59.670 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_687972/sparse2 oflag=append conv=notrunc 2023-09-23T22:44:59.683 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:44:59.683 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:44:59.683 INFO:tasks.workunit.client.0.smithi026.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0120583 s, 174 MB/s 2023-09-23T22:44:59.683 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T22:44:59.729 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T22:44:59.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:44:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:44:59] "GET /metrics HTTP/1.1" 200 34081 "" "Prometheus/2.43.0" 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[215775]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[215775]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[215775]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[215775]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[215775]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[215775]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[215775]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[223014]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[223014]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[223014]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[223014]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[223014]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[223014]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:44:59 smithi026 ceph-mon[223014]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.755 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:44:59.768 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:44:59.768 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:44:59.768 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:44:59.772 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd ls -l 2023-09-23T22:44:59.773 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep sparse2 2023-09-23T22:44:59.773 INFO:tasks.workunit.client.0.smithi026.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T22:44:59.816 INFO:tasks.workunit.client.0.smithi026.stdout:sparse2 4 MiB 2 2023-09-23T22:44:59.816 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse2 2023-09-23T22:44:59.816 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse2 2023-09-23T22:44:59.817 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse2 2023-09-23T22:44:59.817 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:44:59.818 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:44:59.830 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:44:59.858 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.b96c67413d4d 2023-09-23T22:44:59.858 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.b96c67413d4d 2023-09-23T22:44:59.858 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse2 2023-09-23T22:44:59.858 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.b96c67413d4d\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:44:59.858 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse2 2023-09-23T22:44:59.858 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:44:59.859 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse2 2023-09-23T22:44:59.859 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:44:59.859 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:44:59.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:59 smithi070 ceph-mon[192652]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:59 smithi070 ceph-mon[192652]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:59 smithi070 ceph-mon[192652]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:59 smithi070 ceph-mon[192652]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:59 smithi070 ceph-mon[192652]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:59 smithi070 ceph-mon[192652]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:44:59 smithi070 ceph-mon[192652]: failed to encode map e640 with expected crc 2023-09-23T22:44:59.898 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:44:59.898 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:44:59.899 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:44:59.899 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:44:59.899 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:44:59.931 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects='0 2023-09-23T22:44:59.931 INFO:tasks.workunit.client.0.smithi026.stderr:2 2023-09-23T22:44:59.931 INFO:tasks.workunit.client.0.smithi026.stderr:3' 2023-09-23T22:44:59.931 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 0 2 3 2023-09-23T22:44:59.931 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T22:44:59.931 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse2 /tmp/rbd_import_export_687972/sparse2.out 2023-09-23T22:44:59.968 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T22:44:59.969 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T22:44:59.971 INFO:tasks.workunit.client.0.smithi026.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_687972/sparse2 /tmp/rbd_import_export_687972/sparse2.out 2023-09-23T22:44:59.971 INFO:tasks.workunit.client.0.smithi026.stderr:+ cmp -l /tmp/rbd_import_export_687972/sparse2 /tmp/rbd_import_export_687972/sparse2.out 2023-09-23T22:44:59.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:44:59.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T22:44:59.975 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/sparse2 --format %b 2023-09-23T22:44:59.976 INFO:tasks.workunit.client.0.smithi026.stderr:+ origsize=6144 2023-09-23T22:44:59.976 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/sparse2.out --format %b 2023-09-23T22:44:59.977 INFO:tasks.workunit.client.0.smithi026.stderr:+ exportsize=6144 2023-09-23T22:44:59.977 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:59.978 INFO:tasks.workunit.client.0.smithi026.stderr:+ difference=0 2023-09-23T22:44:59.978 INFO:tasks.workunit.client.0.smithi026.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T22:44:59.978 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/sparse2.out 2023-09-23T22:44:59.978 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse2 2023-09-23T22:45:00.068 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T22:45:00.073 DEBUG:teuthology.orchestra.run.smithi070:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T22:45:00.074 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T22:45:00.074 INFO:tasks.workunit.client.0.smithi026.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T22:45:00.075 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_687972/sparse1 sparse 2023-09-23T22:45:00.101 INFO:teuthology.orchestra.run.smithi070.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:00.141 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T22:45:00.142 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:45:00.142 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T22:45:00.145 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse 2023-09-23T22:45:00.145 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T22:45:00.145 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T22:45:00.146 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:45:00.146 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:45:00.167 INFO:tasks.thrashosds.thrasher:in_osds: [5, 1, 6, 2] out_osds: [4, 7, 3, 0] dead_osds: [] live_osds: [0, 5, 7, 2, 4, 1, 3, 6] 2023-09-23T22:45:00.167 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T22:45:00.167 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T22:45:00.168 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T22:45:00.184 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.92d9288479e0 2023-09-23T22:45:00.184 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.92d9288479e0 2023-09-23T22:45:00.184 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.92d9288479e0\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:45:00.185 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse 2023-09-23T22:45:00.185 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse 2023-09-23T22:45:00.185 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:45:00.185 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse 2023-09-23T22:45:00.185 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:45:00.186 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:45:00.195 INFO:teuthology.orchestra.run.smithi070.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T22:45:00.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:00.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi070 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T22:45:00.224 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:45:00.224 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:45:00.225 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:45:00.225 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:45:00.225 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:45:00.261 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=1 2023-09-23T22:45:00.261 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 1 2023-09-23T22:45:00.261 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T22:45:00.262 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse 2023-09-23T22:45:00.299 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:00.338 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:00.363 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T22:45:00.367 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T22:45:00.367 INFO:tasks.workunit.client.0.smithi026.stdout:zeros import through stdin to sparse image 2023-09-23T22:45:00.368 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T22:45:00.368 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd import --new-format - sparse 2023-09-23T22:45:00.420 INFO:tasks.workunit.client.0.smithi026.stderr:4+0 records in 2023-09-23T22:45:00.420 INFO:tasks.workunit.client.0.smithi026.stderr:4+0 records out 2023-09-23T22:45:00.420 INFO:tasks.workunit.client.0.smithi026.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0512129 s, 81.9 MB/s 2023-09-23T22:45:00.430 INFO:tasks.workunit.client.0.smithi026.stderr: Importing image: 100% complete...done. 2023-09-23T22:45:00.431 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:45:00.433 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse 2023-09-23T22:45:00.434 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T22:45:00.434 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T22:45:00.435 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:45:00.435 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:45:00.446 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:00.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:00.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:00.476 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.b9b7c5a49693 2023-09-23T22:45:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.b9b7c5a49693 2023-09-23T22:45:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.b9b7c5a49693\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:45:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse 2023-09-23T22:45:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse 2023-09-23T22:45:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:45:00.477 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse 2023-09-23T22:45:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:45:00.478 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:45:00.482 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:45:00.482 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:45:00.513 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:45:00.513 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:45:00.514 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:45:00.514 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:45:00.514 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:45:00.545 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects= 2023-09-23T22:45:00.546 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 2023-09-23T22:45:00.546 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T22:45:00.546 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse 2023-09-23T22:45:00.551 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:45:00.580 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:00.689 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:45:00.692 INFO:tasks.workunit.client.0.smithi026.stdout:zeros export to sparse file 2023-09-23T22:45:00.692 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo 'zeros export to sparse file' 2023-09-23T22:45:00.693 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T22:45:00.736 INFO:tasks.workunit.client.0.smithi026.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T22:45:00.739 INFO:tasks.workunit.client.0.smithi026.stderr:++ rbd info sparse 2023-09-23T22:45:00.739 INFO:tasks.workunit.client.0.smithi026.stderr:++ grep block_name_prefix 2023-09-23T22:45:00.739 INFO:tasks.workunit.client.0.smithi026.stderr:++ awk '{print $NF;}' 2023-09-23T22:45:00.776 INFO:tasks.workunit.client.0.smithi026.stderr:+ prefix=rbd_data.b9d8b0e8db5d 2023-09-23T22:45:00.776 INFO:tasks.workunit.client.0.smithi026.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T22:45:00.777 INFO:tasks.workunit.client.0.smithi026.stderr:++ get_image_data_pool sparse 2023-09-23T22:45:00.777 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T22:45:00.777 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T22:45:00.777 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep 'data_pool: ' 2023-09-23T22:45:00.777 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T22:45:00.895 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T22:45:00.911 INFO:tasks.workunit.client.0.smithi026.stderr:++ data_pool= 2023-09-23T22:45:00.911 INFO:tasks.workunit.client.0.smithi026.stderr:++ '[' -z ']' 2023-09-23T22:45:00.911 INFO:tasks.workunit.client.0.smithi026.stderr:++ data_pool=rbd 2023-09-23T22:45:00.911 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo rbd 2023-09-23T22:45:00.911 INFO:tasks.workunit.client.0.smithi026.stderr:+ rados -p rbd put rbd_data.b9d8b0e8db5d.000000000000 - 2023-09-23T22:45:00.922 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:00 smithi026 ceph-mon[223014]: pgmap v1645: 103 pgs: 1 remapped+peering, 5 peering, 97 active+clean; 4.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 65 KiB/s rd, 5.5 KiB/s wr, 94 op/s; 0 B/s, 3 objects/s recovering 2023-09-23T22:45:00.922 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:00 smithi026 ceph-mon[215775]: pgmap v1645: 103 pgs: 1 remapped+peering, 5 peering, 97 active+clean; 4.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 65 KiB/s rd, 5.5 KiB/s wr, 94 op/s; 0 B/s, 3 objects/s recovering 2023-09-23T22:45:00.942 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records in 2023-09-23T22:45:00.942 INFO:tasks.workunit.client.0.smithi026.stderr:1+0 records out 2023-09-23T22:45:00.943 INFO:tasks.workunit.client.0.smithi026.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.16578 s, 25.3 MB/s 2023-09-23T22:45:00.958 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects sparse 2023-09-23T22:45:00.959 INFO:tasks.workunit.client.0.smithi026.stderr:++ image=sparse 2023-09-23T22:45:00.959 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rbd info sparse 2023-09-23T22:45:00.959 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep block_name_prefix 2023-09-23T22:45:00.959 INFO:tasks.workunit.client.0.smithi026.stderr:+++ awk '{print $NF;}' 2023-09-23T22:45:00.966 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:45:00.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:00.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:45:00.997 INFO:tasks.workunit.client.0.smithi026.stderr:++ prefix=rbd_data.b9d8b0e8db5d 2023-09-23T22:45:00.998 INFO:tasks.workunit.client.0.smithi026.stderr:+++ grep rbd_data.b9d8b0e8db5d 2023-09-23T22:45:00.998 INFO:tasks.workunit.client.0.smithi026.stderr:++++ get_image_data_pool sparse 2023-09-23T22:45:00.998 INFO:tasks.workunit.client.0.smithi026.stderr:++++ image=sparse 2023-09-23T22:45:00.998 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sed -e 's/rbd_data.b9d8b0e8db5d\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T22:45:00.998 INFO:tasks.workunit.client.0.smithi026.stderr:+++ sort -u 2023-09-23T22:45:00.999 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ rbd info sparse 2023-09-23T22:45:00.999 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ grep 'data_pool: ' 2023-09-23T22:45:00.999 INFO:tasks.workunit.client.0.smithi026.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T22:45:01.051 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool= 2023-09-23T22:45:01.052 INFO:tasks.workunit.client.0.smithi026.stderr:++++ '[' -z ']' 2023-09-23T22:45:01.052 INFO:tasks.workunit.client.0.smithi026.stderr:++++ data_pool=rbd 2023-09-23T22:45:01.052 INFO:tasks.workunit.client.0.smithi026.stderr:++++ echo rbd 2023-09-23T22:45:01.052 INFO:tasks.workunit.client.0.smithi026.stderr:+++ rados ls -p rbd 2023-09-23T22:45:01.070 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:01.098 INFO:tasks.workunit.client.0.smithi026.stderr:++ objects=0 2023-09-23T22:45:01.098 INFO:tasks.workunit.client.0.smithi026.stderr:++ echo 0 2023-09-23T22:45:01.098 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T22:45:01.098 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/sparse 2023-09-23T22:45:01.099 INFO:tasks.workunit.client.0.smithi026.stderr:rm: cannot remove '/tmp/rbd_import_export_687972/sparse': No such file or directory 2023-09-23T22:45:01.099 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:45:01.099 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd export sparse /tmp/rbd_import_export_687972/sparse 2023-09-23T22:45:01.108 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:01.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:00 smithi070 ceph-mon[192652]: pgmap v1645: 103 pgs: 1 remapped+peering, 5 peering, 97 active+clean; 4.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 65 KiB/s rd, 5.5 KiB/s wr, 94 op/s; 0 B/s, 3 objects/s recovering 2023-09-23T22:45:01.152 INFO:tasks.workunit.client.0.smithi026.stderr: Exporting image: 100% complete...done. 2023-09-23T22:45:01.160 INFO:tasks.workunit.client.0.smithi026.stderr:++ stat /tmp/rbd_import_export_687972/sparse --format=%b 2023-09-23T22:45:01.161 INFO:tasks.workunit.client.0.smithi026.stderr:+ '[' 0 = 0 ']' 2023-09-23T22:45:01.161 INFO:tasks.workunit.client.0.smithi026.stderr:+ rbd rm sparse 2023-09-23T22:45:01.174 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:01.181 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:01.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:01.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:01.199 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T22:45:01.270 INFO:tasks.workunit.client.0.smithi026.stderr: Removing image: 100% complete...done. 2023-09-23T22:45:01.274 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm /tmp/rbd_import_export_687972/sparse /tmp/rbd_import_export_687972/sparse1 /tmp/rbd_import_export_687972/sparse2 /tmp/rbd_import_export_687972/sparse3 2023-09-23T22:45:01.276 INFO:tasks.workunit.client.0.smithi026.stderr:rm: cannot remove '/tmp/rbd_import_export_687972/sparse3': No such file or directory 2023-09-23T22:45:01.276 INFO:tasks.workunit.client.0.smithi026.stderr:+ true 2023-09-23T22:45:01.277 INFO:tasks.workunit.client.0.smithi026.stdout:OK 2023-09-23T22:45:01.277 INFO:tasks.workunit.client.0.smithi026.stderr:+ echo OK 2023-09-23T22:45:01.277 INFO:tasks.workunit.client.0.smithi026.stderr:+ rm -rf /tmp/rbd_import_export_687972 2023-09-23T22:45:01.278 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T22:45:01.278 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T22:45:01.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T22:45:01.308 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T22:45:01.308 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T22:45:01.381 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:01.545 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:01.567 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T22:45:01.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:01.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T22:45:01.582 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:01.672 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T22:45:01.700 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:01.753 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T22:45:01.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:01.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T22:45:01.769 DEBUG:teuthology.parallel:result is None 2023-09-23T22:45:01.769 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:45:01.839 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:45:01.840 DEBUG:teuthology.orchestra.run.smithi026:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:45:01.860 DEBUG:teuthology.orchestra.run.smithi026:> sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T22:45:01.907 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T22:45:01.907 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T22:45:01.907 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T22:45:01.907 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T22:45:01.908 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T22:45:01.908 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T22:45:01.908 DEBUG:teuthology.orchestra.run.smithi026:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:45:01.910 INFO:teuthology.orchestra.run.smithi026.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T22:45:01.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T22:45:01.929 INFO:teuthology.orchestra.run.smithi026.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T22:45:01.929 DEBUG:teuthology.orchestra.run.smithi026:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T22:45:01.990 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T22:45:01.990 DEBUG:teuthology.orchestra.run.smithi026:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T22:45:02.048 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T22:45:02.835 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is set 2023-09-23T22:45:02.918 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:02 smithi070 ceph-mon[192652]: pgmap v1646: 103 pgs: 103 active+clean; 5.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 681 KiB/s rd, 692 KiB/s wr, 222 op/s; 92 B/s, 2 objects/s recovering 2023-09-23T22:45:02.919 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1824349747' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:03.130 INFO:teuthology.orchestra.run.smithi070.stdout:ERROR: (22) Invalid argument 2023-09-23T22:45:03.131 INFO:teuthology.orchestra.run.smithi070.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T22:45:03.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:02 smithi026 ceph-mon[223014]: pgmap v1646: 103 pgs: 103 active+clean; 5.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 681 KiB/s rd, 692 KiB/s wr, 222 op/s; 92 B/s, 2 objects/s recovering 2023-09-23T22:45:03.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1824349747' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:02 smithi026 ceph-mon[215775]: pgmap v1646: 103 pgs: 103 active+clean; 5.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 681 KiB/s rd, 692 KiB/s wr, 222 op/s; 92 B/s, 2 objects/s recovering 2023-09-23T22:45:03.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1824349747' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:03.458 INFO:teuthology.orchestra.run.smithi026.stderr:Error: no container with name or ID ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T22:45:03.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T22:45:03.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi026 with status 125: 'sudo docker kill -s 1 ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T22:45:03.766 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T22:45:04.112 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:03 smithi070 ceph-mon[192652]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:03 smithi070 ceph-mon[192652]: failed to encode map e641 with expected crc 2023-09-23T22:45:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:03 smithi070 ceph-mon[192652]: failed to encode map e641 with expected crc 2023-09-23T22:45:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:03 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1824349747' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:03 smithi070 ceph-mon[192652]: osdmap e641: 8 total, 8 up, 4 in 2023-09-23T22:45:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:03 smithi070 ceph-mon[192652]: failed to encode map e642 with expected crc 2023-09-23T22:45:04.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:03 smithi070 ceph-mon[192652]: osdmap e642: 8 total, 8 up, 4 in 2023-09-23T22:45:04.161 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[223014]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:04.162 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[223014]: failed to encode map e641 with expected crc 2023-09-23T22:45:04.162 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[223014]: failed to encode map e641 with expected crc 2023-09-23T22:45:04.162 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1824349747' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:04.162 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[223014]: osdmap e641: 8 total, 8 up, 4 in 2023-09-23T22:45:04.162 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[223014]: failed to encode map e642 with expected crc 2023-09-23T22:45:04.162 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[223014]: osdmap e642: 8 total, 8 up, 4 in 2023-09-23T22:45:04.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[215775]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:04.162 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[215775]: failed to encode map e641 with expected crc 2023-09-23T22:45:04.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[215775]: failed to encode map e641 with expected crc 2023-09-23T22:45:04.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1824349747' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:04.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[215775]: osdmap e641: 8 total, 8 up, 4 in 2023-09-23T22:45:04.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[215775]: failed to encode map e642 with expected crc 2023-09-23T22:45:04.163 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:03 smithi026 ceph-mon[215775]: osdmap e642: 8 total, 8 up, 4 in 2023-09-23T22:45:04.605 INFO:teuthology.orchestra.run.smithi026.stdout:osd.0: {} 2023-09-23T22:45:04.623 INFO:teuthology.orchestra.run.smithi026.stdout:osd.1: {} 2023-09-23T22:45:04.642 INFO:teuthology.orchestra.run.smithi026.stdout:osd.2: {} 2023-09-23T22:45:04.661 INFO:teuthology.orchestra.run.smithi026.stdout:osd.3: {} 2023-09-23T22:45:04.680 INFO:teuthology.orchestra.run.smithi026.stdout:osd.4: {} 2023-09-23T22:45:04.698 INFO:teuthology.orchestra.run.smithi026.stdout:osd.5: {} 2023-09-23T22:45:04.717 INFO:teuthology.orchestra.run.smithi026.stdout:osd.6: {} 2023-09-23T22:45:04.735 INFO:teuthology.orchestra.run.smithi026.stdout:osd.7: {} 2023-09-23T22:45:04.737 INFO:teuthology.orchestra.run.smithi026.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T22:45:04.737 INFO:teuthology.orchestra.run.smithi026.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T22:45:04.737 INFO:teuthology.orchestra.run.smithi026.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T22:45:04.737 INFO:teuthology.orchestra.run.smithi026.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T22:45:04.737 INFO:teuthology.orchestra.run.smithi026.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T22:45:04.737 INFO:teuthology.orchestra.run.smithi026.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T22:45:04.738 INFO:teuthology.orchestra.run.smithi026.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T22:45:04.738 INFO:teuthology.orchestra.run.smithi026.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T22:45:04.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:04 smithi026 ceph-mon[223014]: failed to encode map e642 with expected crc 2023-09-23T22:45:04.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:04 smithi026 ceph-mon[223014]: pgmap v1649: 103 pgs: 103 active+clean; 9.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 284 op/s; 92 B/s, 2 objects/s recovering 2023-09-23T22:45:04.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3449535498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:04.975 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1893428473' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:45:04.976 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:04 smithi026 conmon[215751]: 2023-09-23T22:45:04.586+0000 7f4580f04700 -1 mon.a@0(leader).osd e642 definitely_dead 0 2023-09-23T22:45:04.976 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:04 smithi026 ceph-mon[215775]: failed to encode map e642 with expected crc 2023-09-23T22:45:04.976 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:04 smithi026 ceph-mon[215775]: pgmap v1649: 103 pgs: 103 active+clean; 9.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 284 op/s; 92 B/s, 2 objects/s recovering 2023-09-23T22:45:04.976 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3449535498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:04.976 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1893428473' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:45:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:04 smithi070 ceph-mon[192652]: failed to encode map e642 with expected crc 2023-09-23T22:45:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:04 smithi070 ceph-mon[192652]: pgmap v1649: 103 pgs: 103 active+clean; 9.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 284 op/s; 92 B/s, 2 objects/s recovering 2023-09-23T22:45:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3449535498' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T22:45:05.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1893428473' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T22:45:05.455 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.7. 2023-09-23T22:45:05.664 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:45:05.664 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:45:05.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:45:05.664 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:45:05.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.664 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9495 9.d 9:b6f72c49:::benchmark_data_smithi026_555954_object9494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-09-23T22:45:05.664 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.350545+0000", 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 578.31068721300005, 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.022504189, 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9495 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350545+0000", 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350545+0000", 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350543+0000", 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350613+0000", 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0388000000000004e-05 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350628+0000", 2023-09-23T22:45:05.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4489e-05 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350635+0000", 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6150000000000002e-06 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350683+0000", 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8446000000000003e-05 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350811+0000", 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000128374 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.350977+0000", 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016547600000000001 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.352085+0000", 2023-09-23T22:45:05.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00110797 2023-09-23T22:45:05.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.352551+0000", 2023-09-23T22:45:05.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00046640899999999999 2023-09-23T22:45:05.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370196+0000", 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017644389999999999 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370205+0000", 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4939999999999992e-06 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372976+0000", 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0027705790000001 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373049+0000", 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3143000000000001e-05 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9496 9.c 9:3bf3d114:::benchmark_data_smithi026_555954_object9495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.370618+0000", 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 578.29061397500004, 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.001084182, 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9496 2023-09-23T22:45:05.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370618+0000", 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370618+0000", 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370617+0000", 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370666+0000", 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9101999999999998e-05 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370677+0000", 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1586999999999999e-05 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370684+0000", 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2400000000000004e-06 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370725+0000", 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1288000000000002e-05 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.370731+0000", 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8649999999999998e-06 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371493+0000", 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.000761907 2023-09-23T22:45:05.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371702+0000", 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00020950600000000001 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9509 9.c 9:31b495ae:::benchmark_data_smithi026_555954_object9508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:45:05.678 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.442632+0000", 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 578.21860002200003, 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.93168581900000003, 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9509 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442632+0000", 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442632+0000", 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442630+0000", 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442697+0000", 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6773000000000006e-05 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442707+0000", 2023-09-23T22:45:05.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5789999999999993e-06 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442714+0000", 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0979999999999996e-06 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442752+0000", 2023-09-23T22:45:05.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8105000000000002e-05 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.442757+0000", 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5249999999999996e-06 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371728+0000", 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92897071200000003 2023-09-23T22:45:05.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.371868+0000", 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000139462 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372052+0000", 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000184446 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373673+0000", 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001620511 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374175+0000", 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000502203 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374228+0000", 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2825999999999999e-05 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374318+0000", 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0082999999999998e-05 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9535 9.d 9:b4eb0ec8:::benchmark_data_smithi026_555954_object9534:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.455847+0000", 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 578.20538495200003, 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92259973200000001, 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9535 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.687 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455847+0000", 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455847+0000", 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455846+0000", 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455906+0000", 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0608e-05 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455923+0000", 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6758000000000001e-05 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455930+0000", 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7780000000000002e-06 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.455990+0000", 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0136000000000002e-05 2023-09-23T22:45:05.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456000+0000", 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8220000000000002e-06 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373303+0000", 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91730366500000005 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373384+0000", 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0984999999999998e-05 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373486+0000", 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000101313 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.692 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.375253+0000", 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001767565 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378309+0000", 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030558080000000001 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378360+0000", 2023-09-23T22:45:05.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0806000000000001e-05 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378447+0000", 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.6707999999999997e-05 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.694 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9536 9.c 9:382e37e1:::benchmark_data_smithi026_555954_object9535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.456300+0000", 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 578.20493161499996, 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92362326400000005, 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9536 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456300+0000", 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456300+0000", 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.696 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456299+0000", 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456394+0000", 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4780999999999999e-05 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456405+0000", 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0876999999999999e-05 2023-09-23T22:45:05.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456411+0000", 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7259999999999999e-06 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456453+0000", 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2178000000000002e-05 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.456466+0000", 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3247e-05 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372550+0000", 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91608398700000004 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372617+0000", 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7168000000000005e-05 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.372714+0000", 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6972000000000006e-05 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373682+0000", 2023-09-23T22:45:05.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00096728799999999996 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379826+0000", 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0061445110000000001 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379868+0000", 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1446999999999998e-05 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379924+0000", 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.5908000000000001e-05 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9540 9.d 9:bd7479be:::benchmark_data_smithi026_555954_object9539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:45:05.702 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 578.20332356200004, 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92071582799999996, 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9540 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457908+0000", 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457972+0000", 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3987999999999998e-05 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457986+0000", 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3869999999999999e-05 2023-09-23T22:45:05.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.457992+0000", 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6499999999999999e-06 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.458039+0000", 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6647000000000002e-05 2023-09-23T22:45:05.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.458044+0000", 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0610000000000003e-06 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373970+0000", 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91592621799999996 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374074+0000", 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103479 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374200+0000", 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000126317 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.375260+0000", 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010597790000000001 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378498+0000", 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032380619999999999 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378543+0000", 2023-09-23T22:45:05.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5624999999999998e-05 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378624+0000", 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0729999999999994e-05 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.710 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9544 9.d 9:b6a337e4:::benchmark_data_smithi026_555954_object9543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.459597+0000", 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 578.20163482600003, 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91919865700000003, 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9544 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.711 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459597+0000", 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459597+0000", 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459596+0000", 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459657+0000", 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0798999999999998e-05 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459668+0000", 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0868e-05 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459673+0000", 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4889999999999997e-06 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459722+0000", 2023-09-23T22:45:05.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8953999999999999e-05 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.459728+0000", 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7449999999999998e-06 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374739+0000", 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.91501085900000001 2023-09-23T22:45:05.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374843+0000", 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000104354 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1", 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374936+0000", 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2578999999999996e-05 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.375383+0000", 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00044722000000000002 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378674+0000", 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032907280000000001 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378716+0000", 2023-09-23T22:45:05.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2168e-05 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.378796+0000", 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9506999999999996e-05 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.718 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:9551 9.c 9:30ade42e:::benchmark_data_smithi026_555954_object9550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:27.465719+0000", 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 578.19551331499997, 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.914369392, 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 9551 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465719+0000", 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465719+0000", 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465718+0000", 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465753+0000", 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5376999999999997e-05 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465780+0000", 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7169e-05 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465785+0000", 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5059999999999997e-06 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465841+0000", 2023-09-23T22:45:05.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6444999999999999e-05 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:27.465846+0000", 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9490000000000004e-06 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373010+0000", 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.90716359499999999 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373077+0000", 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7021999999999996e-05 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0", 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.373205+0000", 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012822300000000001 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.374744+0000", 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015386899999999999 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.379977+0000", 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052328569999999996 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.380030+0000", 2023-09-23T22:45:05.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3511000000000003e-05 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:28.380088+0000", 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7877000000000002e-05 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.726 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104683 9.d 9:b37e8cbe:::benchmark_data_smithi026_555954_object9954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.265880+0000", 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 471.39535176999999, 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.041256663, 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104683 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265880+0000", 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265880+0000", 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265879+0000", 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265884+0000", 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3159999999999996e-06 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265896+0000", 2023-09-23T22:45:05.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2296000000000001e-05 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265901+0000", 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2499999999999997e-06 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.265962+0000", 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0973000000000001e-05 2023-09-23T22:45:05.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266094+0000", 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000131686 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,4", 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.266227+0000", 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013251100000000001 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.269476+0000", 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0032489149999999998 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.269598+0000", 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012262800000000001 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.297704+0000", 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.028105502000000001 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.297712+0000", 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3180000000000007e-06 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307040+0000", 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0093283340000001 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307137+0000", 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6446999999999993e-05 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104689 9.d 9:b5dd9fdc:::benchmark_data_smithi026_555954_object9960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.299991+0000", 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 471.361240651, 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.01128229, 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104689 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.735 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.299991+0000", 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.299991+0000", 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.299989+0000", 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.299999+0000", 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0040999999999999e-05 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.300014+0000", 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4395e-05 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.300022+0000", 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3820000000000004e-06 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.300898+0000", 2023-09-23T22:45:05.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000875894 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.300908+0000", 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0465000000000001e-05 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307652+0000", 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.006743991 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307757+0000", 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010459900000000001 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.307872+0000", 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115252 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.311035+0000", 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031630389999999999 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.311127+0000", 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1836000000000006e-05 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.311182+0000", 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4568000000000003e-05 2023-09-23T22:45:05.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.311273+0000", 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1748000000000003e-05 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.742 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104691 9.3 9.d6892893 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.306137+0000", 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 471.35509527300002, 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97170118400000005, 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104691 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.743 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306137+0000", 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306137+0000", 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306135+0000", 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306142+0000", 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1369999999999996e-06 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306156+0000", 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4102e-05 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306163+0000", 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8770000000000004e-06 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306207+0000", 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3531e-05 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.306214+0000", 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5469999999999999e-06 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.277827+0000", 2023-09-23T22:45:05.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97161278600000001 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.277838+0000", 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0689e-05 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104697 9.3 9.c29b12d3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.352031+0000", 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 471.309200476, 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92583439899999997, 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104697 2023-09-23T22:45:05.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352031+0000", 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352031+0000", 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352030+0000", 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352037+0000", 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.472e-06 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352050+0000", 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3534e-05 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352056+0000", 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4689999999999996e-06 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352104+0000", 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8214999999999998e-05 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.352112+0000", 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3950000000000001e-06 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.277858+0000", 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.92574585099999995 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.277866+0000", 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7500000000000003e-06 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112809 9.1 9.c9f688c1 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:38:54.412199+0000", 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 371.24903289700001, 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97345260199999994, 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112809 2023-09-23T22:45:05.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412199+0000", 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412199+0000", 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412197+0000", 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412204+0000", 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3919999999999997e-06 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412220+0000", 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.594e-05 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412227+0000", 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4089999999999997e-06 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412275+0000", 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7166999999999998e-05 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.412282+0000", 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2490000000000004e-06 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.385645+0000", 2023-09-23T22:45:05.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97336309399999998 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.385652+0000", 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6599999999999998e-06 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112816 9.1 9.8fe24c21 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:38:54.592383+0000", 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 371.06884868399999, 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79328532900000004, 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112816 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.761 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592383+0000", 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592383+0000", 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592382+0000", 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592387+0000", 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3170000000000001e-06 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.763 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592400+0000", 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3156000000000001e-05 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592422+0000", 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1756999999999999e-05 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592463+0000", 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1282000000000002e-05 2023-09-23T22:45:05.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.592474+0000", 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0373e-05 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.385665+0000", 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79319085899999997 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.385669+0000", 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9090000000000002e-06 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112831 9.a 9:5995d63d:::benchmark_data_smithi026_555954_object18102:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:38:54.851121+0000", 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 370.81011087299999, 2023-09-23T22:45:05.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.58216017499999995, 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112831 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851121+0000", 2023-09-23T22:45:05.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851121+0000", 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851120+0000", 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851127+0000", 2023-09-23T22:45:05.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0439999999999998e-06 2023-09-23T22:45:05.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851139+0000", 2023-09-23T22:45:05.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2616000000000001e-05 2023-09-23T22:45:05.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851145+0000", 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1850000000000003e-06 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851191+0000", 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6767999999999997e-05 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:54.851199+0000", 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2350000000000002e-06 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.416893+0000", 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.56569482400000004 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.417034+0000", 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000140464 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.417161+0000", 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000126771 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.424475+0000", 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073143879999999998 2023-09-23T22:45:05.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.425894+0000", 2023-09-23T22:45:05.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014191729999999999 2023-09-23T22:45:05.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433157+0000", 2023-09-23T22:45:05.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0072629180000000002 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433195+0000", 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8127000000000003e-05 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433281+0000", 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5987000000000002e-05 2023-09-23T22:45:05.774 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112839 9.a 9:5f6027c3:::benchmark_data_smithi026_555954_object18110:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e513)", 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:38:55.082571+0000", 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 370.57866053700002, 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.35082294800000002, 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.775 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112839 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082571+0000", 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.776 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082571+0000", 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082570+0000", 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082577+0000", 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6730000000000003e-06 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082591+0000", 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4158999999999999e-05 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082596+0000", 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.288e-06 2023-09-23T22:45:05.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082644+0000", 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7542e-05 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.082652+0000", 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.4190000000000002e-06 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.419978+0000", 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.33732560499999997 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.420067+0000", 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9258999999999994e-05 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,3", 2023-09-23T22:45:05.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.420139+0000", 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2515000000000006e-05 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.425926+0000", 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0057866640000000004 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.427909+0000", 2023-09-23T22:45:05.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019834150000000001 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433305+0000", 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0053956860000000002 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433336+0000", 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0824999999999997e-05 2023-09-23T22:45:05.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:38:55.433394+0000", 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.8346e-05 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:119011 9.3 9:c70114b4:::benchmark_data_smithi026_555954_object24282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:26.604881+0000", 2023-09-23T22:45:05.783 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 339.05635080899998, 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.77412598399999999, 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119011 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604881+0000", 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604881+0000", 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604880+0000", 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604886+0000", 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2570000000000004e-06 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604900+0000", 2023-09-23T22:45:05.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4287999999999999e-05 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604906+0000", 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4009999999999996e-06 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604968+0000", 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2269999999999998e-05 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.604978+0000", 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7210000000000006e-06 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.364445+0000", 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.75946789699999995 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.364580+0000", 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000134905 2023-09-23T22:45:05.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.364883+0000", 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000302928 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.367536+0000", 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002652298 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.789 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.368262+0000", 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00072598599999999997 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.378827+0000", 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010565827 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.378888+0000", 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0173999999999997e-05 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.379007+0000", 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000119449 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:119015 9.1 9:83bc4cab:::benchmark_data_smithi026_555954_object24286:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:45:05.791 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:26.628581+0000", 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 339.03265120399999, 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.73202734300000005, 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119015 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628581+0000", 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628581+0000", 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628579+0000", 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628585+0000", 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.524e-06 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628596+0000", 2023-09-23T22:45:05.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1279e-05 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628601+0000", 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.87e-06 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628649+0000", 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7799000000000001e-05 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.628656+0000", 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6000000000000003e-06 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.356235+0000", 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.72757985199999997 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.356371+0000", 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013551000000000001 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.356908+0000", 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00053668599999999998 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360435+0000", 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0035278319999999998 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360477+0000", 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2073999999999997e-05 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360526+0000", 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8738e-05 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360608+0000", 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1797000000000003e-05 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:119016 9.3 9:c7a55f5e:::benchmark_data_smithi026_555954_object24287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:26.635619+0000", 2023-09-23T22:45:05.799 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 339.02561323499998, 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.743613733, 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119016 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635619+0000", 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635619+0000", 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635618+0000", 2023-09-23T22:45:05.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635623+0000", 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1569999999999999e-06 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635636+0000", 2023-09-23T22:45:05.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2728000000000001e-05 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635641+0000", 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0939999999999998e-06 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635685+0000", 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4427e-05 2023-09-23T22:45:05.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.635692+0000", 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4860000000000002e-06 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.366319+0000", 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.730627684 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.366456+0000", 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00013700499999999999 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,5", 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.366620+0000", 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016379 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.367810+0000", 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011903129999999999 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.368409+0000", 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00059821199999999998 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.379061+0000", 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010652017999999999 2023-09-23T22:45:05.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.379118+0000", 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7512000000000001e-05 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.379232+0000", 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011414299999999999 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.807 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:119017 9.1 9:86eab180:::benchmark_data_smithi026_555954_object24288:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e525)", 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:26.642933+0000", 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 339.01829901799999, 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.71978275999999997, 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:05.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 119017 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642933+0000", 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642933+0000", 2023-09-23T22:45:05.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642932+0000", 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642937+0000", 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8400000000000002e-06 2023-09-23T22:45:05.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642956+0000", 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9249000000000002e-05 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642962+0000", 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.4979999999999997e-06 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.642997+0000", 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5170999999999999e-05 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.643003+0000", 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6939999999999998e-06 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:05.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.358600+0000", 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.715596071 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.358742+0000", 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014219300000000001 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 4", 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.359002+0000", 2023-09-23T22:45:05.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00026045499999999999 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.360621+0000", 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0016183829999999999 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.362562+0000", 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001941702 2023-09-23T22:45:05.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.362623+0000", 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1169999999999999e-05 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:27.362716+0000", 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.2162000000000003e-05 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.815 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.816 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.816 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:05.816 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:05.816 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:45:06.027 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T22:45:06.133 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T22:45:06.374 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T22:45:06.410 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:06.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:06 smithi070 ceph-mon[192652]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:06 smithi070 ceph-mon[192652]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:06 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1893428473' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T22:45:06.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:06 smithi070 ceph-mon[192652]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:06 smithi070 ceph-mon[192652]: osdmap e643: 8 total, 8 up, 5 in 2023-09-23T22:45:06.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:06 smithi070 ceph-mon[192652]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[223014]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[223014]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1893428473' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T22:45:06.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[223014]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[223014]: osdmap e643: 8 total, 8 up, 5 in 2023-09-23T22:45:06.668 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[223014]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[215775]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[215775]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1893428473' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T22:45:06.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[215775]: failed to encode map e643 with expected crc 2023-09-23T22:45:06.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[215775]: osdmap e643: 8 total, 8 up, 5 in 2023-09-23T22:45:06.669 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:06 smithi026 ceph-mon[215775]: failed to encode map e643 with expected crc 2023-09-23T22:45:07.587 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:07.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[215775]: pgmap v1651: 103 pgs: 103 active+clean; 8.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 321 op/s; 92 B/s, 0 objects/s recovering 2023-09-23T22:45:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[215775]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[215775]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[215775]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[215775]: osdmap e644: 8 total, 8 up, 5 in 2023-09-23T22:45:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[215775]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[215775]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[223014]: pgmap v1651: 103 pgs: 103 active+clean; 8.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 321 op/s; 92 B/s, 0 objects/s recovering 2023-09-23T22:45:07.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[223014]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[223014]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[223014]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[223014]: osdmap e644: 8 total, 8 up, 5 in 2023-09-23T22:45:07.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[223014]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:07 smithi026 ceph-mon[223014]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:07 smithi070 ceph-mon[192652]: pgmap v1651: 103 pgs: 103 active+clean; 8.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 1.2 MiB/s rd, 1.8 MiB/s wr, 321 op/s; 92 B/s, 0 objects/s recovering 2023-09-23T22:45:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:07 smithi070 ceph-mon[192652]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:07 smithi070 ceph-mon[192652]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:07 smithi070 ceph-mon[192652]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:07 smithi070 ceph-mon[192652]: osdmap e644: 8 total, 8 up, 5 in 2023-09-23T22:45:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:07 smithi070 ceph-mon[192652]: failed to encode map e644 with expected crc 2023-09-23T22:45:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:07 smithi070 ceph-mon[192652]: failed to encode map e644 with expected crc 2023-09-23T22:45:08.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[223014]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[223014]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[223014]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[223014]: osdmap e645: 8 total, 8 up, 5 in 2023-09-23T22:45:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[223014]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[223014]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[223014]: osdmap e646: 8 total, 8 up, 5 in 2023-09-23T22:45:08.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[215775]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[215775]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[215775]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[215775]: osdmap e645: 8 total, 8 up, 5 in 2023-09-23T22:45:08.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[215775]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[215775]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:08 smithi026 ceph-mon[215775]: osdmap e646: 8 total, 8 up, 5 in 2023-09-23T22:45:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:08 smithi070 ceph-mon[192652]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:08 smithi070 ceph-mon[192652]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:08 smithi070 ceph-mon[192652]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:08 smithi070 ceph-mon[192652]: osdmap e645: 8 total, 8 up, 5 in 2023-09-23T22:45:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:08 smithi070 ceph-mon[192652]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:08 smithi070 ceph-mon[192652]: failed to encode map e645 with expected crc 2023-09-23T22:45:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:08 smithi070 ceph-mon[192652]: osdmap e646: 8 total, 8 up, 5 in 2023-09-23T22:45:09.431 INFO:teuthology.orchestra.run.smithi026.stderr:noscrub is unset 2023-09-23T22:45:09.448 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:45:09.449 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:45:09.449 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:45:09.449 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:45:09.462 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: pgmap v1654: 103 pgs: 16 peering, 87 active+clean; 8.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 101 KiB/s rd, 6.9 KiB/s wr, 134 op/s 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2035020447' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:09.463 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:09.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:09.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:09.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:45:09.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[215775]: osdmap e647: 8 total, 8 up, 5 in 2023-09-23T22:45:09.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: pgmap v1654: 103 pgs: 16 peering, 87 active+clean; 8.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 101 KiB/s rd, 6.9 KiB/s wr, 134 op/s 2023-09-23T22:45:09.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.464 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2035020447' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:45:09.465 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:09 smithi026 ceph-mon[223014]: osdmap e647: 8 total, 8 up, 5 in 2023-09-23T22:45:09.733 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:45:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:45:09] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.43.0" 2023-09-23T22:45:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: pgmap v1654: 103 pgs: 16 peering, 87 active+clean; 8.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 101 KiB/s rd, 6.9 KiB/s wr, 134 op/s 2023-09-23T22:45:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: failed to encode map e646 with expected crc 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2035020447' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T22:45:09.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:09 smithi070 ceph-mon[192652]: osdmap e647: 8 total, 8 up, 5 in 2023-09-23T22:45:10.448 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[223014]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.549 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T22:45:10.583 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T22:45:10.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[215775]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[215775]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[215775]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[215775]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[215775]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:10.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[215775]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[223014]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[223014]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[223014]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[223014]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:10.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:10 smithi026 ceph-mon[223014]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:10 smithi070 ceph-mon[192652]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:10 smithi070 ceph-mon[192652]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:10 smithi070 ceph-mon[192652]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:10 smithi070 ceph-mon[192652]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:10 smithi070 ceph-mon[192652]: failed to encode map e647 with expected crc 2023-09-23T22:45:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:10 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:10 smithi070 ceph-mon[192652]: failed to encode map e647 with expected crc 2023-09-23T22:45:11.053 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:11.112 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:11.133 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T22:45:11.134 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T22:45:11.134 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T22:45:11.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:11 smithi026 ceph-mon[223014]: pgmap v1657: 103 pgs: 6 active+undersized+remapped, 16 peering, 81 active+clean; 8.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 1.0 MiB/s, 2 keys/s, 17 objects/s recovering 2023-09-23T22:45:11.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:11 smithi026 ceph-mon[215775]: pgmap v1657: 103 pgs: 6 active+undersized+remapped, 16 peering, 81 active+clean; 8.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 1.0 MiB/s, 2 keys/s, 17 objects/s recovering 2023-09-23T22:45:11.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:11 smithi070 ceph-mon[192652]: pgmap v1657: 103 pgs: 6 active+undersized+remapped, 16 peering, 81 active+clean; 8.6 MiB data, 1.7 GiB used, 356 GiB / 358 GiB avail; 1.0 MiB/s, 2 keys/s, 17 objects/s recovering 2023-09-23T22:45:12.392 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:13.298 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:45:13.299 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:45:13.299 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:45:13.299 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:45:13.299 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:45:13.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:13 smithi026 ceph-mon[223014]: pgmap v1658: 103 pgs: 6 active+undersized+remapped, 97 active+clean; 8.6 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 816 KiB/s, 1 keys/s, 21 objects/s recovering 2023-09-23T22:45:13.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:13 smithi026 ceph-mon[215775]: pgmap v1658: 103 pgs: 6 active+undersized+remapped, 97 active+clean; 8.6 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 816 KiB/s, 1 keys/s, 21 objects/s recovering 2023-09-23T22:45:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:13 smithi070 ceph-mon[192652]: pgmap v1658: 103 pgs: 6 active+undersized+remapped, 97 active+clean; 8.6 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 816 KiB/s, 1 keys/s, 21 objects/s recovering 2023-09-23T22:45:14.311 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T22:45:14.464 INFO:teuthology.orchestra.run.smithi026.stderr:nodeep-scrub is unset 2023-09-23T22:45:14.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:14 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4207959824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:14.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:14 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:14 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4207959824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:14.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:14 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:14.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:14 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4207959824' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:14.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:14 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T22:45:15.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[215775]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 688 KiB/s, 1 keys/s, 18 objects/s recovering 2023-09-23T22:45:15.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[215775]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:45:15.623 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[215775]: Cluster is now healthy 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[215775]: failed to encode map e648 with expected crc 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[215775]: failed to encode map e648 with expected crc 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[215775]: osdmap e648: 8 total, 8 up, 5 in 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[215775]: failed to encode map e648 with expected crc 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[223014]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 688 KiB/s, 1 keys/s, 18 objects/s recovering 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[223014]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[223014]: Cluster is now healthy 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[223014]: failed to encode map e648 with expected crc 2023-09-23T22:45:15.624 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[223014]: failed to encode map e648 with expected crc 2023-09-23T22:45:15.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:15.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[223014]: osdmap e648: 8 total, 8 up, 5 in 2023-09-23T22:45:15.625 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:15 smithi026 ceph-mon[223014]: failed to encode map e648 with expected crc 2023-09-23T22:45:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:15 smithi070 ceph-mon[192652]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 688 KiB/s, 1 keys/s, 18 objects/s recovering 2023-09-23T22:45:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:15 smithi070 ceph-mon[192652]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T22:45:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:15 smithi070 ceph-mon[192652]: Cluster is now healthy 2023-09-23T22:45:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:15 smithi070 ceph-mon[192652]: failed to encode map e648 with expected crc 2023-09-23T22:45:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:15 smithi070 ceph-mon[192652]: failed to encode map e648 with expected crc 2023-09-23T22:45:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:15 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T22:45:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:15 smithi070 ceph-mon[192652]: osdmap e648: 8 total, 8 up, 5 in 2023-09-23T22:45:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:15 smithi070 ceph-mon[192652]: failed to encode map e648 with expected crc 2023-09-23T22:45:15.902 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:16.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:16 smithi026 conmon[215751]: 2023-09-23T22:45:16.211+0000 7f4580f04700 -1 mon.a@0(leader).osd e648 definitely_dead 0 2023-09-23T22:45:16.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:16 smithi026 ceph-mon[215775]: failed to encode map e648 with expected crc 2023-09-23T22:45:16.464 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1610927706' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T22:45:16.471 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.4. 2023-09-23T22:45:16.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:16 smithi026 ceph-mon[223014]: failed to encode map e648 with expected crc 2023-09-23T22:45:16.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1610927706' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T22:45:16.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:16 smithi070 ceph-mon[192652]: failed to encode map e648 with expected crc 2023-09-23T22:45:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1610927706' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T22:45:17.566 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T22:45:17.567 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T22:45:17.567 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T22:45:17.567 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T22:45:17.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[223014]: pgmap v1661: 103 pgs: 103 active+clean; 8.6 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 36 KiB/s, 11 objects/s recovering 2023-09-23T22:45:17.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[223014]: failed to encode map e649 with expected crc 2023-09-23T22:45:17.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[223014]: failed to encode map e649 with expected crc 2023-09-23T22:45:17.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[223014]: failed to encode map e649 with expected crc 2023-09-23T22:45:17.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1610927706' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T22:45:17.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[223014]: osdmap e649: 8 total, 8 up, 6 in 2023-09-23T22:45:17.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[215775]: pgmap v1661: 103 pgs: 103 active+clean; 8.6 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 36 KiB/s, 11 objects/s recovering 2023-09-23T22:45:17.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[215775]: failed to encode map e649 with expected crc 2023-09-23T22:45:17.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[215775]: failed to encode map e649 with expected crc 2023-09-23T22:45:17.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[215775]: failed to encode map e649 with expected crc 2023-09-23T22:45:17.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1610927706' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T22:45:17.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:17 smithi026 ceph-mon[215775]: osdmap e649: 8 total, 8 up, 6 in 2023-09-23T22:45:17.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:17 smithi070 ceph-mon[192652]: pgmap v1661: 103 pgs: 103 active+clean; 8.6 MiB data, 2.2 GiB used, 445 GiB / 447 GiB avail; 36 KiB/s, 11 objects/s recovering 2023-09-23T22:45:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:17 smithi070 ceph-mon[192652]: failed to encode map e649 with expected crc 2023-09-23T22:45:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:17 smithi070 ceph-mon[192652]: failed to encode map e649 with expected crc 2023-09-23T22:45:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:17 smithi070 ceph-mon[192652]: failed to encode map e649 with expected crc 2023-09-23T22:45:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:17 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1610927706' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T22:45:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:17 smithi070 ceph-mon[192652]: osdmap e649: 8 total, 8 up, 6 in 2023-09-23T22:45:18.002 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:18.012 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:45:18.012 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:45:18.012 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:45:18.012 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:45:18.012 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.012 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:104686 9.2 9:4bc296ca:::benchmark_data_smithi026_555954_object9957:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:45:18.012 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:45:18.012 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 483.71478509999997, 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.032939702, 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 104686 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.013 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292570+0000", 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.014 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292569+0000", 2023-09-23T22:45:18.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:18.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.015 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.015 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.015 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292573+0000", 2023-09-23T22:45:18.015 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.6500000000000002e-06 2023-09-23T22:45:18.015 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292581+0000", 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8080000000000005e-06 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292589+0000", 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4209999999999995e-06 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.016 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292656+0000", 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7243999999999995e-05 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:14.292666+0000", 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3850000000000008e-06 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.017 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298265+0000", 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.005598754 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298394+0000", 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012897599999999999 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.018 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 3,7", 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.298532+0000", 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001381 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.308571+0000", 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010039792000000001 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.019 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.308872+0000", 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00030035000000000001 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325328+0000", 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016456529000000001 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.020 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325391+0000", 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2385999999999995e-05 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:37:15.325510+0000", 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000118841 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.021 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23700 10.4 10:2b626a3e:::benchmark_data_smithi026_640738_object23699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.405861+0000", 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 193.60149438400001, 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3690022009999998, 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.022 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23700 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405861+0000", 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.023 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405861+0000", 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405859+0000", 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.024 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405882+0000", 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2935999999999999e-05 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405893+0000", 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1070000000000001e-05 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.025 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405899+0000", 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8880000000000002e-06 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.405942+0000", 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.2895000000000002e-05 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.026 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.406020+0000", 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7571999999999995e-05 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.541149+0000", 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.135129573 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.027 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.547471+0000", 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0063218190000000002 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.570649+0000", 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023177474999999999 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.028 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.570663+0000", 2023-09-23T22:45:18.029 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4408e-05 2023-09-23T22:45:18.029 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.029 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.029 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.029 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.539860+0000", 2023-09-23T22:45:18.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96919705199999995 2023-09-23T22:45:18.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.030 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.030 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.030 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.539868+0000", 2023-09-23T22:45:18.030 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0099999999999995e-06 2023-09-23T22:45:18.030 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.031 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.747326+0000", 2023-09-23T22:45:18.031 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.207457716 2023-09-23T22:45:18.031 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.031 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.031 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.749304+0000", 2023-09-23T22:45:18.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001978133 2023-09-23T22:45:18.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.032 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.032 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.032 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774797+0000", 2023-09-23T22:45:18.032 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025492642999999999 2023-09-23T22:45:18.032 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774863+0000", 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6105000000000005e-05 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.033 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23703 10.9 10:9dc130d6:::benchmark_data_smithi026_640738_object23702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.544380+0000", 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 193.46297515699999, 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2154338739999999, 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23703 2023-09-23T22:45:18.034 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544380+0000", 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544380+0000", 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.035 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544379+0000", 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544402+0000", 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3332999999999998e-05 2023-09-23T22:45:18.036 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544413+0000", 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0912e-05 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544420+0000", 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6690000000000006e-06 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.037 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544864+0000", 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00044322699999999998 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544998+0000", 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134339 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.038 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545158+0000", 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159946 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.550111+0000", 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0049527060000000003 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.039 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.550128+0000", 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7799e-05 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.536473+0000", 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.986344318 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.040 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.536488+0000", 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5150999999999999e-05 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746698+0000", 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2102103909999999 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.041 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.748810+0000", 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021113019999999998 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.759720+0000", 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010910840999999999 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.042 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.759814+0000", 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.3207000000000004e-05 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23704 10.4 10:2f348807:::benchmark_data_smithi026_640738_object23703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.544429+0000", 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 193.46292552700001, 2023-09-23T22:45:18.043 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.230336844, 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23704 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.044 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544429+0000", 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544429+0000", 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.045 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544429+0000", 2023-09-23T22:45:18.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:18.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.046 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544450+0000", 2023-09-23T22:45:18.046 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0857e-05 2023-09-23T22:45:18.046 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.046 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.046 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544457+0000", 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6619999999999999e-06 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544462+0000", 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.631e-06 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.047 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544508+0000", 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6322000000000001e-05 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544667+0000", 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015808599999999999 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.048 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544859+0000", 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000192194 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.547477+0000", 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0026186289999999999 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.574150+0000", 2023-09-23T22:45:18.049 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.026672927999999999 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.574163+0000", 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.296e-05 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.539887+0000", 2023-09-23T22:45:18.050 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96572371099999998 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.539892+0000", 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9710000000000003e-06 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.751454+0000", 2023-09-23T22:45:18.051 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.211562539 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.752273+0000", 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00081840400000000001 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774687+0000", 2023-09-23T22:45:18.052 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022414204 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774766+0000", 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9196999999999994e-05 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.053 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23705 10.3 10:c03ba639:::benchmark_data_smithi026_640738_object23704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.555379+0000", 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 193.451975842, 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2153192160000001, 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.054 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23705 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555379+0000", 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.055 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555379+0000", 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555378+0000", 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.056 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555404+0000", 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5799000000000002e-05 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555415+0000", 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1182999999999999e-05 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555426+0000", 2023-09-23T22:45:18.057 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0934000000000001e-05 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555779+0000", 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035356400000000001 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555933+0000", 2023-09-23T22:45:18.058 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015408899999999999 2023-09-23T22:45:18.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.059 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.059 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.556090+0000", 2023-09-23T22:45:18.059 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015706300000000001 2023-09-23T22:45:18.059 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.059 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.562662+0000", 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065715909999999999 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579088+0000", 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016426211999999999 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.060 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579100+0000", 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1781e-05 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.540506+0000", 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96140631600000004 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.061 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.540519+0000", 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3169999999999999e-05 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.750405+0000", 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.209885898 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.062 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.751359+0000", 2023-09-23T22:45:18.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00095339299999999997 2023-09-23T22:45:18.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.063 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.063 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770625+0000", 2023-09-23T22:45:18.063 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019265971 2023-09-23T22:45:18.063 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.063 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.064 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.064 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770698+0000", 2023-09-23T22:45:18.064 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3579999999999997e-05 2023-09-23T22:45:18.064 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.064 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.064 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.064 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.064 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.065 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23706 10.5 10:ae9e38a0:::benchmark_data_smithi026_640738_object23705:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:18.065 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.555441+0000", 2023-09-23T22:45:18.065 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 193.45191419099999, 2023-09-23T22:45:18.065 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2154646439999999, 2023-09-23T22:45:18.065 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.066 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:18.066 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.066 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.066 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.066 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23706 2023-09-23T22:45:18.066 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.066 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.066 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.066 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555441+0000", 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555441+0000", 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.067 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555440+0000", 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555462+0000", 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1421e-05 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.068 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555470+0000", 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9990000000000008e-06 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555484+0000", 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3971000000000001e-05 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.069 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.580064+0000", 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.024580547000000001 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.580073+0000", 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.6330000000000002e-06 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.070 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.543265+0000", 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96319191999999998 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.543280+0000", 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5228000000000001e-05 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.071 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.752233+0000", 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2089533160000001 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.752376+0000", 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000142898 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.752561+0000", 2023-09-23T22:45:18.072 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018452200000000001 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.768542+0000", 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015980836000000002 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770097+0000", 2023-09-23T22:45:18.073 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015551639999999999 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770775+0000", 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00067841400000000001 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770816+0000", 2023-09-23T22:45:18.074 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0444000000000001e-05 2023-09-23T22:45:18.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.075 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.075 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.075 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770905+0000", 2023-09-23T22:45:18.075 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9864000000000006e-05 2023-09-23T22:45:18.075 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.075 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.075 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.075 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.077 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.077 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23708 10.7 10:e7b56d34:::benchmark_data_smithi026_640738_object23707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:18.077 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.555490+0000", 2023-09-23T22:45:18.077 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 193.45186467400001, 2023-09-23T22:45:18.077 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2026997179999999, 2023-09-23T22:45:18.077 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.077 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.077 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23708 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555490+0000", 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.078 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555490+0000", 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555490+0000", 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.079 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555513+0000", 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3119e-05 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555520+0000", 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1709999999999996e-06 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.080 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.555526+0000", 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0580000000000003e-06 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.561054+0000", 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055277349999999998 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.081 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.561223+0000", 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016923200000000001 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0", 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.561390+0000", 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016651599999999999 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.082 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.562883+0000", 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001493692 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579044+0000", 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016160858 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.083 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579058+0000", 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3764999999999999e-05 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.516181+0000", 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.93712273700000004 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.084 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.516196+0000", 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5617e-05 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.745713+0000", 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.229516936 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.085 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.748581+0000", 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028682149999999999 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.758101+0000", 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0095193750000000001 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.086 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.758190+0000", 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9208000000000004e-05 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23709 10.d 10:b530215a:::benchmark_data_smithi026_640738_object23708:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.571919+0000", 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 193.435436401, 2023-09-23T22:45:18.087 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1996814210000002, 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23709 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.088 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.571919+0000", 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.571919+0000", 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.089 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.571917+0000", 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.571946+0000", 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9686e-05 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.090 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.572715+0000", 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00076831999999999996 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.572735+0000", 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0168000000000001e-05 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.091 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579332+0000", 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0065975699999999997 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.579343+0000", 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0319999999999999e-05 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.092 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.543315+0000", 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96397255000000004 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.543328+0000", 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2292e-05 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.750428+0000", 2023-09-23T22:45:18.093 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2071007579999999 2023-09-23T22:45:18.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.750600+0000", 2023-09-23T22:45:18.094 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000171657 2023-09-23T22:45:18.094 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.094 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.094 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.094 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.750815+0000", 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00021523700000000001 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.753636+0000", 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0028208420000000001 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770038+0000", 2023-09-23T22:45:18.095 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.016402067999999999 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770362+0000", 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000323476 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.770429+0000", 2023-09-23T22:45:18.096 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7350999999999999e-05 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.771600+0000", 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011710710000000001 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.097 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:57477 10.3 10:c67dc132:::benchmark_data_smithi026_640738_object10081:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:55.452561+0000", 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 142.55479444900001, 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1631035160000001, 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 57477 2023-09-23T22:45:18.098 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.452561+0000", 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.452561+0000", 2023-09-23T22:45:18.099 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.452560+0000", 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.452564+0000", 2023-09-23T22:45:18.100 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.9169999999999999e-06 2023-09-23T22:45:18.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.452573+0000", 2023-09-23T22:45:18.101 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5759999999999995e-06 2023-09-23T22:45:18.101 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.101 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.101 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.101 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.452582+0000", 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8810000000000002e-06 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.456652+0000", 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0040697290000000002 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.102 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.456776+0000", 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012350100000000001 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.456901+0000", 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000125802 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.467511+0000", 2023-09-23T22:45:18.103 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010609291999999999 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.510673+0000", 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.043162725999999998 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.510680+0000", 2023-09-23T22:45:18.104 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2920000000000001e-06 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.615567+0000", 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.104887438 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.615664+0000", 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6971000000000004e-05 2023-09-23T22:45:18.105 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:57479 10.5 10:a9ff3fb3:::benchmark_data_smithi026_640738_object10083:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:55.456748+0000", 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 142.55060703500001, 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.15953803, 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.106 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.107 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.107 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.107 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.107 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 57479 2023-09-23T22:45:18.107 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.107 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.107 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.107 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.107 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.456748+0000", 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.456748+0000", 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.456747+0000", 2023-09-23T22:45:18.108 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.456753+0000", 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9190000000000002e-06 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.456766+0000", 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.218e-05 2023-09-23T22:45:18.109 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.456774+0000", 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5129999999999996e-06 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.459796+0000", 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030221800000000002 2023-09-23T22:45:18.110 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.459939+0000", 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014270999999999999 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.460291+0000", 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035236300000000001 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.111 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.468303+0000", 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0080115220000000001 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.502444+0000", 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.034140668999999998 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.112 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.536023+0000", 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033579599000000002 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.536036+0000", 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2998e-05 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.113 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.616242+0000", 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.080205503 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.616286+0000", 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4246000000000003e-05 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.114 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:57484 10.5 10:aa565620:::benchmark_data_smithi026_640738_object10088:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:55.458173+0000", 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 142.54918171400001, 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1581784479999999, 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.115 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 57484 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.458173+0000", 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.116 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.458173+0000", 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.458172+0000", 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.117 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.458178+0000", 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8139999999999997e-06 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.458193+0000", 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4728e-05 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.118 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.458201+0000", 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7340000000000008e-06 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.466424+0000", 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0082234700000000001 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.119 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.466647+0000", 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000222482 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.466766+0000", 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011951900000000001 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.120 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.479014+0000", 2023-09-23T22:45:18.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012248241999999999 2023-09-23T22:45:18.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.121 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.121 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.502545+0000", 2023-09-23T22:45:18.121 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.023531106 2023-09-23T22:45:18.121 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.121 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.536072+0000", 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.033526682000000002 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.536082+0000", 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0084e-05 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.122 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.616313+0000", 2023-09-23T22:45:18.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.080230491 2023-09-23T22:45:18.123 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.123 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.123 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.123 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.616352+0000", 2023-09-23T22:45:18.123 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.8999999999999999e-05 2023-09-23T22:45:18.123 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.124 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.124 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.124 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.124 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.124 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:57489 10.3 10:c693e99d:::benchmark_data_smithi026_640738_object10093:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T22:45:18.124 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:55.465438+0000", 2023-09-23T22:45:18.124 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 142.54191688200001, 2023-09-23T22:45:18.124 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.150326108, 2023-09-23T22:45:18.124 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 57489 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.125 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465438+0000", 2023-09-23T22:45:18.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.126 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.126 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465438+0000", 2023-09-23T22:45:18.126 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.126 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.126 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465437+0000", 2023-09-23T22:45:18.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:18.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.127 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.127 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.127 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465444+0000", 2023-09-23T22:45:18.127 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5440000000000001e-06 2023-09-23T22:45:18.127 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465455+0000", 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0689e-05 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465463+0000", 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3420000000000008e-06 2023-09-23T22:45:18.128 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465659+0000", 2023-09-23T22:45:18.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000195935 2023-09-23T22:45:18.129 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.129 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.129 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.129 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465796+0000", 2023-09-23T22:45:18.129 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013664600000000001 2023-09-23T22:45:18.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465915+0000", 2023-09-23T22:45:18.130 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011901299999999999 2023-09-23T22:45:18.130 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.130 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.130 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.130 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.502561+0000", 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.036646389000000001 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.510694+0000", 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0081328879999999996 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.131 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.510699+0000", 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4859999999999999e-06 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.615695+0000", 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.104996058 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.132 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.615764+0000", 2023-09-23T22:45:18.133 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.9338000000000006e-05 2023-09-23T22:45:18.133 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.133 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.133 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.133 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.133 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.133 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:57490 10.4 10:28c8c68d:::benchmark_data_smithi026_640738_object10094:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T22:45:18.133 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:55.465824+0000", 2023-09-23T22:45:18.133 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 142.541530997, 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1503247640000001, 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 57490 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.134 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465824+0000", 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465824+0000", 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.135 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465823+0000", 2023-09-23T22:45:18.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:18.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.136 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.136 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.136 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465829+0000", 2023-09-23T22:45:18.136 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9839999999999998e-06 2023-09-23T22:45:18.136 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465841+0000", 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1581e-05 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465849+0000", 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9580000000000007e-06 2023-09-23T22:45:18.137 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.465898+0000", 2023-09-23T22:45:18.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9869e-05 2023-09-23T22:45:18.138 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.138 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.138 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.138 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.466018+0000", 2023-09-23T22:45:18.138 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011937799999999999 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1", 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.466134+0000", 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000116018 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.478578+0000", 2023-09-23T22:45:18.139 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012443707 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.524397+0000", 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.045819907 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:55.524410+0000", 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2809000000000001e-05 2023-09-23T22:45:18.140 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.616068+0000", 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0916575530000001 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:56.616149+0000", 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.0920000000000005e-05 2023-09-23T22:45:18.141 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:64647 10.9 10:94633b6c:::benchmark_data_smithi026_640738_object17251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:10.608953+0000", 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 127.39840221, 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.245839702, 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.142 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.143 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.143 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.143 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.143 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 64647 2023-09-23T22:45:18.143 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.143 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.143 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.143 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.143 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.608953+0000", 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.608953+0000", 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.144 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.608951+0000", 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.608958+0000", 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5950000000000004e-06 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.145 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.608970+0000", 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1882000000000001e-05 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.608977+0000", 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5889999999999997e-06 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.146 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.613237+0000", 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042599070000000003 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.613760+0000", 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052256100000000005 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.147 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.626939+0000", 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013178787000000001 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.627044+0000", 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000105132 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1,6", 2023-09-23T22:45:18.148 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.627192+0000", 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014866799999999999 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.629002+0000", 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018094109999999999 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.149 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.638335+0000", 2023-09-23T22:45:18.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093335540000000008 2023-09-23T22:45:18.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.150 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.638413+0000", 2023-09-23T22:45:18.150 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7713999999999995e-05 2023-09-23T22:45:18.150 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.150 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.150 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.716922+0000", 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.078508515000000001 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.716936+0000", 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4606e-05 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.151 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.854720+0000", 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1377839329999999 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.854792+0000", 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2368999999999998e-05 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.152 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:64651 10.9 10:9ecb984e:::benchmark_data_smithi026_640738_object17255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:10.613873+0000", 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 127.393481624, 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2398489960000001, 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.153 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.154 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.154 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 64651 2023-09-23T22:45:18.154 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.154 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.154 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.155 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.613873+0000", 2023-09-23T22:45:18.155 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.155 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.155 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.155 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.613873+0000", 2023-09-23T22:45:18.156 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.156 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.156 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.156 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.156 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.613872+0000", 2023-09-23T22:45:18.157 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:18.157 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.157 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.157 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.157 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.613879+0000", 2023-09-23T22:45:18.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7660000000000001e-06 2023-09-23T22:45:18.158 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.158 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.158 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.158 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.613891+0000", 2023-09-23T22:45:18.158 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2439999999999999e-05 2023-09-23T22:45:18.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.159 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.159 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.159 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.613899+0000", 2023-09-23T22:45:18.159 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5789999999999998e-06 2023-09-23T22:45:18.159 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.160 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.160 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.613988+0000", 2023-09-23T22:45:18.160 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8984e-05 2023-09-23T22:45:18.160 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.160 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.161 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.614352+0000", 2023-09-23T22:45:18.161 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000363934 2023-09-23T22:45:18.161 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.161 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.161 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.162 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.624910+0000", 2023-09-23T22:45:18.162 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010558497999999999 2023-09-23T22:45:18.162 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.162 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.162 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625020+0000", 2023-09-23T22:45:18.163 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000109474 2023-09-23T22:45:18.163 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.163 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.163 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1,6", 2023-09-23T22:45:18.163 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625184+0000", 2023-09-23T22:45:18.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00016471899999999999 2023-09-23T22:45:18.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.164 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.628956+0000", 2023-09-23T22:45:18.164 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0037717570000000001 2023-09-23T22:45:18.164 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.164 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.164 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.638328+0000", 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093715949999999999 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.638375+0000", 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7144000000000001e-05 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.165 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.713909+0000", 2023-09-23T22:45:18.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.075534335999999994 2023-09-23T22:45:18.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.166 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.166 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.166 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.713925+0000", 2023-09-23T22:45:18.166 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5934e-05 2023-09-23T22:45:18.166 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.167 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.167 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.167 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.853641+0000", 2023-09-23T22:45:18.167 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.139715805 2023-09-23T22:45:18.167 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.168 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.168 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.168 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.853722+0000", 2023-09-23T22:45:18.168 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1384000000000005e-05 2023-09-23T22:45:18.168 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.168 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.168 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.169 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.169 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.169 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:64658 10.d 10:b73f4b3f:::benchmark_data_smithi026_640738_object17262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-23T22:45:18.169 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:10.616979+0000", 2023-09-23T22:45:18.169 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 127.390375918, 2023-09-23T22:45:18.169 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2383522810000001, 2023-09-23T22:45:18.170 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.170 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.170 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.170 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.170 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.170 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 64658 2023-09-23T22:45:18.170 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.170 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.170 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.616979+0000", 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.616979+0000", 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.171 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.616979+0000", 2023-09-23T22:45:18.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:18.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.172 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.172 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.616982+0000", 2023-09-23T22:45:18.172 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0340000000000001e-06 2023-09-23T22:45:18.172 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.172 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.616989+0000", 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.2729999999999997e-06 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.616994+0000", 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8949999999999997e-06 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.173 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617044+0000", 2023-09-23T22:45:18.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0494000000000001e-05 2023-09-23T22:45:18.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.174 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.174 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.174 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617433+0000", 2023-09-23T22:45:18.174 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038906299999999998 2023-09-23T22:45:18.174 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.632966+0000", 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015532255 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.633066+0000", 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010059 2023-09-23T22:45:18.175 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1,6", 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.633214+0000", 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014830400000000001 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634824+0000", 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001609419 2023-09-23T22:45:18.176 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635921+0000", 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001096767 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635966+0000", 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5049000000000002e-05 2023-09-23T22:45:18.177 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.729339+0000", 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.093373742999999995 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.729346+0000", 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.618e-06 2023-09-23T22:45:18.178 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.855256+0000", 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1259096239999999 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.855331+0000", 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5667000000000006e-05 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.179 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.180 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.180 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.180 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.180 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:64659 10.d 10:b394f17e:::benchmark_data_smithi026_640738_object17263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-23T22:45:18.180 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:10.617009+0000", 2023-09-23T22:45:18.180 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 127.39034642599999, 2023-09-23T22:45:18.180 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2366256529999999, 2023-09-23T22:45:18.180 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.180 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.181 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.181 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.181 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.181 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 64659 2023-09-23T22:45:18.181 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.181 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.181 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.181 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617009+0000", 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617009+0000", 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.182 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617008+0000", 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617011+0000", 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1470000000000001e-06 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.183 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617017+0000", 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9359999999999996e-06 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617022+0000", 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4839999999999998e-06 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.184 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617459+0000", 2023-09-23T22:45:18.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00043751599999999999 2023-09-23T22:45:18.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.185 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.185 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.185 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.617807+0000", 2023-09-23T22:45:18.185 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00034779500000000003 2023-09-23T22:45:18.185 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.630912+0000", 2023-09-23T22:45:18.186 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013104696000000001 2023-09-23T22:45:18.186 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.186 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.186 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.186 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.631011+0000", 2023-09-23T22:45:18.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9192999999999997e-05 2023-09-23T22:45:18.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1,6", 2023-09-23T22:45:18.187 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.631157+0000", 2023-09-23T22:45:18.187 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014611 2023-09-23T22:45:18.187 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.187 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.187 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.632937+0000", 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001779557 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634829+0000", 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018926489999999999 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.188 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635907+0000", 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001078018 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.729300+0000", 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.093392836000000007 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.189 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.729313+0000", 2023-09-23T22:45:18.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2554e-05 2023-09-23T22:45:18.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.190 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.190 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.853539+0000", 2023-09-23T22:45:18.190 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1242262519999999 2023-09-23T22:45:18.190 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.190 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.853634+0000", 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5361999999999999e-05 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:64662 10.8 10:130a8170:::benchmark_data_smithi026_640738_object17266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-23T22:45:18.191 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:10.625411+0000", 2023-09-23T22:45:18.192 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 127.381944339, 2023-09-23T22:45:18.192 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.217893959, 2023-09-23T22:45:18.192 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.192 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:18.192 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.192 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.192 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.192 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 64662 2023-09-23T22:45:18.192 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.193 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625411+0000", 2023-09-23T22:45:18.193 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.193 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.193 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.193 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.193 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625411+0000", 2023-09-23T22:45:18.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.194 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625409+0000", 2023-09-23T22:45:18.194 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:18.194 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.194 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.194 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625417+0000", 2023-09-23T22:45:18.195 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3000000000000004e-06 2023-09-23T22:45:18.195 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.195 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.195 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.195 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625430+0000", 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3484e-05 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625438+0000", 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9279999999999993e-06 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.196 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625490+0000", 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1990000000000002e-05 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625612+0000", 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000122002 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1,6", 2023-09-23T22:45:18.197 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.625743+0000", 2023-09-23T22:45:18.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013083699999999999 2023-09-23T22:45:18.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.198 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.628927+0000", 2023-09-23T22:45:18.198 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0031843729999999999 2023-09-23T22:45:18.198 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.198 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.198 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.629118+0000", 2023-09-23T22:45:18.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00019042199999999999 2023-09-23T22:45:18.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.199 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.199 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634682+0000", 2023-09-23T22:45:18.199 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0055643209999999997 2023-09-23T22:45:18.199 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.199 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.200 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.200 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.722082+0000", 2023-09-23T22:45:18.200 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.087400256999999995 2023-09-23T22:45:18.200 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.200 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.722097+0000", 2023-09-23T22:45:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4578e-05 2023-09-23T22:45:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.843233+0000", 2023-09-23T22:45:18.201 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.121136382 2023-09-23T22:45:18.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.202 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.202 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.202 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.843305+0000", 2023-09-23T22:45:18.202 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1421000000000006e-05 2023-09-23T22:45:18.202 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.202 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.202 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.202 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:64667 10.e 10:7db66264:::benchmark_data_smithi026_640738_object17271:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:10.635107+0000", 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 127.372247727, 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2445813539999999, 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.203 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 64667 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635107+0000", 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.204 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635107+0000", 2023-09-23T22:45:18.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.205 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.205 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635106+0000", 2023-09-23T22:45:18.205 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:18.205 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.205 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635110+0000", 2023-09-23T22:45:18.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5750000000000002e-06 2023-09-23T22:45:18.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.206 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.206 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.206 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635120+0000", 2023-09-23T22:45:18.206 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5359999999999998e-06 2023-09-23T22:45:18.206 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635126+0000", 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0689999999999998e-06 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635176+0000", 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0615000000000002e-05 2023-09-23T22:45:18.207 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635629+0000", 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045293800000000001 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.730785+0000", 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.095155905999999998 2023-09-23T22:45:18.208 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.730798+0000", 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3030000000000001e-05 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.829786+0000", 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0989876759999999 2023-09-23T22:45:18.209 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.829931+0000", 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014452399999999999 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1,5,6", 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.830079+0000", 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000148632 2023-09-23T22:45:18.210 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.836043+0000", 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059635469999999996 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.838010+0000", 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019671929999999999 2023-09-23T22:45:18.211 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.838172+0000", 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000162573 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.841760+0000", 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035878360000000001 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.212 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.879099+0000", 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.037339085000000001 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.879156+0000", 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6592999999999998e-05 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.214 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.214 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.879689+0000", 2023-09-23T22:45:18.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00053262800000000005 2023-09-23T22:45:18.214 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.214 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.214 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.214 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.214 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.214 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:64669 10.d 10:b8a8f3dc:::benchmark_data_smithi026_640738_object17273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-23T22:45:18.215 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:10.635138+0000", 2023-09-23T22:45:18.215 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 127.372216724, 2023-09-23T22:45:18.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2298151390000001, 2023-09-23T22:45:18.215 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:18.215 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:18.215 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:18.215 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:18.215 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:18.215 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 64669 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635138+0000", 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635138+0000", 2023-09-23T22:45:18.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635138+0000", 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635141+0000", 2023-09-23T22:45:18.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8430000000000001e-06 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635147+0000", 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4649999999999999e-06 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635153+0000", 2023-09-23T22:45:18.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7520000000000002e-06 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635349+0000", 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0001961 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for degraded object", 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.635845+0000", 2023-09-23T22:45:18.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00049642800000000004 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.729369+0000", 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.093523357000000001 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.729375+0000", 2023-09-23T22:45:18.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4300000000000003e-06 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.855613+0000", 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.126237462 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.855733+0000", 2023-09-23T22:45:18.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012026799999999999 2023-09-23T22:45:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,1,5,6", 2023-09-23T22:45:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.855881+0000", 2023-09-23T22:45:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014816800000000001 2023-09-23T22:45:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.222 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.858050+0000", 2023-09-23T22:45:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021690839999999999 2023-09-23T22:45:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.858121+0000", 2023-09-23T22:45:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.0753999999999994e-05 2023-09-23T22:45:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.859366+0000", 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001244773 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.864333+0000", 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.004967778 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.864375+0000", 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.1776000000000001e-05 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.864436+0000", 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0483999999999999e-05 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:18.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.864953+0000", 2023-09-23T22:45:18.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000517689 2023-09-23T22:45:18.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.226 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.226 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:18.226 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:18.226 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:45:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[215775]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[215775]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[215775]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[215775]: osdmap e650: 8 total, 8 up, 6 in 2023-09-23T22:45:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[215775]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[215775]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[215775]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[215775]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[215775]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[223014]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[223014]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[223014]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[223014]: osdmap e650: 8 total, 8 up, 6 in 2023-09-23T22:45:18.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[223014]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[223014]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[223014]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[223014]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.738 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:18 smithi026 ceph-mon[223014]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:18 smithi070 ceph-mon[192652]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:18 smithi070 ceph-mon[192652]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:18 smithi070 ceph-mon[192652]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:18 smithi070 ceph-mon[192652]: osdmap e650: 8 total, 8 up, 6 in 2023-09-23T22:45:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:18 smithi070 ceph-mon[192652]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:18 smithi070 ceph-mon[192652]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:18 smithi070 ceph-mon[192652]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:18 smithi070 ceph-mon[192652]: failed to encode map e650 with expected crc 2023-09-23T22:45:18.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:18 smithi070 ceph-mon[192652]: failed to encode map e650 with expected crc 2023-09-23T22:45:19.118 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T22:45:19.474 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:19 smithi070 ceph-mon[192652]: pgmap v1664: 103 pgs: 8 peering, 95 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:45:19.474 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:19 smithi070 ceph-mon[192652]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.474 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:19 smithi070 ceph-mon[192652]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.474 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:19 smithi070 ceph-mon[192652]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.474 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:19 smithi070 ceph-mon[192652]: osdmap e651: 8 total, 8 up, 6 in 2023-09-23T22:45:19.475 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:19 smithi070 ceph-mon[192652]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.475 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:19 smithi070 ceph-mon[192652]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.475 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:19 smithi070 ceph-mon[192652]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.475 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:19 smithi070 ceph-mon[192652]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.501 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:19.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[223014]: pgmap v1664: 103 pgs: 8 peering, 95 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:45:19.612 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[223014]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[223014]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[223014]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[223014]: osdmap e651: 8 total, 8 up, 6 in 2023-09-23T22:45:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[223014]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[223014]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[223014]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.613 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[223014]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[215775]: pgmap v1664: 103 pgs: 8 peering, 95 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail 2023-09-23T22:45:19.613 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[215775]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[215775]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[215775]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[215775]: osdmap e651: 8 total, 8 up, 6 in 2023-09-23T22:45:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[215775]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[215775]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[215775]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.614 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:19 smithi026 ceph-mon[215775]: failed to encode map e651 with expected crc 2023-09-23T22:45:19.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:45:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:45:19] "GET /metrics HTTP/1.1" 200 34111 "" "Prometheus/2.43.0" 2023-09-23T22:45:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[223014]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[223014]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[223014]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[223014]: osdmap e652: 8 total, 8 up, 6 in 2023-09-23T22:45:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[223014]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[223014]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[223014]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[215775]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[215775]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[215775]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[215775]: osdmap e652: 8 total, 8 up, 6 in 2023-09-23T22:45:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[215775]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[215775]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[215775]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:20 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:20.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:20 smithi026 conmon[215751]: 2023-09-23T22:45:20.516+0000 7f4580f04700 -1 mon.a@0(leader).osd e653 definitely_dead 0 2023-09-23T22:45:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:20 smithi070 ceph-mon[192652]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:20 smithi070 ceph-mon[192652]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:20 smithi070 ceph-mon[192652]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:20 smithi070 ceph-mon[192652]: osdmap e652: 8 total, 8 up, 6 in 2023-09-23T22:45:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:20 smithi070 ceph-mon[192652]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:20 smithi070 ceph-mon[192652]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:20 smithi070 ceph-mon[192652]: failed to encode map e652 with expected crc 2023-09-23T22:45:20.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:20 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:21.399 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:45:21.399 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:45:21.399 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:45:21.399 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:45:21.505 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.3. 2023-09-23T22:45:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:21 smithi070 ceph-mon[192652]: pgmap v1667: 103 pgs: 3 remapped+peering, 14 peering, 86 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 81 KiB/s, 2 keys/s, 0 objects/s recovering 2023-09-23T22:45:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:21 smithi070 ceph-mon[192652]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:21 smithi070 ceph-mon[192652]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:21 smithi070 ceph-mon[192652]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:21 smithi070 ceph-mon[192652]: osdmap e653: 8 total, 8 up, 6 in 2023-09-23T22:45:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:21 smithi070 ceph-mon[192652]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:21 smithi070 ceph-mon[192652]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:21 smithi070 ceph-mon[192652]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:21 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1434035660' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:45:21.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[215775]: pgmap v1667: 103 pgs: 3 remapped+peering, 14 peering, 86 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 81 KiB/s, 2 keys/s, 0 objects/s recovering 2023-09-23T22:45:21.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[215775]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[215775]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[215775]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[215775]: osdmap e653: 8 total, 8 up, 6 in 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[215775]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[215775]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[215775]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1434035660' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[223014]: pgmap v1667: 103 pgs: 3 remapped+peering, 14 peering, 86 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 81 KiB/s, 2 keys/s, 0 objects/s recovering 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[223014]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[223014]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[223014]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[223014]: osdmap e653: 8 total, 8 up, 6 in 2023-09-23T22:45:21.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[223014]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[223014]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[223014]: failed to encode map e653 with expected crc 2023-09-23T22:45:21.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:21 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1434035660' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T22:45:22.087 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T22:45:22.288 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T22:45:22.288 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T22:45:22.288 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-09-23T22:45:22.288 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd in 0 2023-09-23T22:45:22.429 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:22.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[215775]: failed to encode map e654 with expected crc 2023-09-23T22:45:22.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[215775]: failed to encode map e654 with expected crc 2023-09-23T22:45:22.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[215775]: failed to encode map e654 with expected crc 2023-09-23T22:45:22.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1434035660' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T22:45:22.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[215775]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T22:45:22.515 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[215775]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T22:45:22.706 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:22.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:22 smithi070 ceph-mon[192652]: failed to encode map e654 with expected crc 2023-09-23T22:45:22.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:22 smithi070 ceph-mon[192652]: failed to encode map e654 with expected crc 2023-09-23T22:45:22.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:22 smithi070 ceph-mon[192652]: failed to encode map e654 with expected crc 2023-09-23T22:45:22.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1434035660' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T22:45:22.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:22 smithi070 ceph-mon[192652]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T22:45:22.788 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:22 smithi070 ceph-mon[192652]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T22:45:22.898 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[223014]: failed to encode map e654 with expected crc 2023-09-23T22:45:22.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[223014]: failed to encode map e654 with expected crc 2023-09-23T22:45:22.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[223014]: failed to encode map e654 with expected crc 2023-09-23T22:45:22.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1434035660' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T22:45:22.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[223014]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T22:45:22.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:22 smithi026 ceph-mon[223014]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T22:45:23.773 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:23 smithi070 ceph-mon[192652]: pgmap v1670: 103 pgs: 3 remapped+peering, 18 peering, 82 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 81 KiB/s, 2 keys/s, 8 objects/s recovering 2023-09-23T22:45:23.773 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:23 smithi070 ceph-mon[192652]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.773 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:23 smithi070 ceph-mon[192652]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.773 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:23 smithi070 ceph-mon[192652]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.774 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:23 smithi070 ceph-mon[192652]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.774 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:23 smithi070 ceph-mon[192652]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T22:45:23.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[223014]: pgmap v1670: 103 pgs: 3 remapped+peering, 18 peering, 82 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 81 KiB/s, 2 keys/s, 8 objects/s recovering 2023-09-23T22:45:23.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[223014]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.813 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[223014]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.814 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[223014]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.814 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[223014]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.814 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[223014]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T22:45:23.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[215775]: pgmap v1670: 103 pgs: 3 remapped+peering, 18 peering, 82 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 81 KiB/s, 2 keys/s, 8 objects/s recovering 2023-09-23T22:45:23.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[215775]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[215775]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[215775]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[215775]: failed to encode map e655 with expected crc 2023-09-23T22:45:23.814 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:23 smithi026 ceph-mon[215775]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T22:45:23.995 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:45:23.995 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:45:23.995 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:45:23.995 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:45:23.995 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:45:24.538 INFO:teuthology.orchestra.run.smithi026.stderr:marked in osd.0. 2023-09-23T22:45:24.719 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T22:45:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:24 smithi070 ceph-mon[192652]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:24 smithi070 ceph-mon[192652]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:24 smithi070 ceph-mon[192652]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:24 smithi070 ceph-mon[192652]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:24 smithi070 ceph-mon[192652]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:24 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3689608053' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:45:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:24 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:45:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 conmon[215751]: 2023-09-23T22:45:24.493+0000 7f4580f04700 -1 mon.a@0(leader).osd e656 definitely_dead 0 2023-09-23T22:45:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[215775]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[215775]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[215775]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[215775]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[215775]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3689608053' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:45:24.987 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:45:24.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[223014]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.987 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[223014]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[223014]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[223014]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[223014]: failed to encode map e656 with expected crc 2023-09-23T22:45:24.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:24.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:24.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3689608053' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:45:24.988 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:24 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T22:45:25.068 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:25.334 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-09-23T22:45:25.334 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T22:45:25.334 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 31 2023-09-23T22:45:25.729 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:25.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[215775]: pgmap v1673: 103 pgs: 2 remapped+peering, 13 peering, 88 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 255 B/s wr, 0 op/s; 187 B/s, 8 objects/s recovering 2023-09-23T22:45:25.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[215775]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[215775]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[215775]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[215775]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[215775]: osdmap e657: 8 total, 8 up, 8 in 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[215775]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[223014]: pgmap v1673: 103 pgs: 2 remapped+peering, 13 peering, 88 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 255 B/s wr, 0 op/s; 187 B/s, 8 objects/s recovering 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[223014]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[223014]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[223014]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T22:45:25.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[223014]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[223014]: osdmap e657: 8 total, 8 up, 8 in 2023-09-23T22:45:25.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:25 smithi026 ceph-mon[223014]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:25 smithi070 ceph-mon[192652]: pgmap v1673: 103 pgs: 2 remapped+peering, 13 peering, 88 active+clean; 8.6 MiB data, 2.3 GiB used, 534 GiB / 536 GiB avail; 255 B/s wr, 0 op/s; 187 B/s, 8 objects/s recovering 2023-09-23T22:45:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:25 smithi070 ceph-mon[192652]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:25 smithi070 ceph-mon[192652]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:25 smithi070 ceph-mon[192652]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:25 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T22:45:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:25 smithi070 ceph-mon[192652]: failed to encode map e657 with expected crc 2023-09-23T22:45:25.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:25 smithi070 ceph-mon[192652]: osdmap e657: 8 total, 8 up, 8 in 2023-09-23T22:45:25.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:25 smithi070 ceph-mon[192652]: failed to encode map e657 with expected crc 2023-09-23T22:45:26.700 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:45:26.700 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:45:26.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:45:26.700 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:45:26.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.700 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48942 9.c 9.a863b59c (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.682106+0000", 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 575.01524029799998, 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99119241000000002, 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48942 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682106+0000", 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682106+0000", 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682105+0000", 2023-09-23T22:45:26.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682183+0000", 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8666999999999997e-05 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682195+0000", 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2023e-05 2023-09-23T22:45:26.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682203+0000", 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9549999999999992e-06 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691884+0000", 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096805209999999992 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691901+0000", 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7312e-05 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673292+0000", 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98139073399999999 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673299+0000", 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7959999999999998e-06 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48944 9.c 9.f6505a7c (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.682215+0000", 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 575.01513171700003, 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.99109994300000004, 2023-09-23T22:45:26.706 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48944 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682215+0000", 2023-09-23T22:45:26.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682215+0000", 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682214+0000", 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682280+0000", 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5757000000000001e-05 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682288+0000", 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.2889999999999998e-06 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682296+0000", 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2710000000000003e-06 2023-09-23T22:45:26.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691924+0000", 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0096284600000000001 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.691930+0000", 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.122e-06 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673310+0000", 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.981380107 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.673315+0000", 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5009999999999998e-06 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:48956 9.e 9.222599de (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T22:45:26.711 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:35:51.682302+0000", 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 575.01504506799995, 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98747526900000004, 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 48956 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682302+0000", 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682302+0000", 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682301+0000", 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682355+0000", 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3656999999999998e-05 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682362+0000", 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2960000000000001e-06 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.682367+0000", 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7790000000000003e-06 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.693185+0000", 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.010818533 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:51.693190+0000", 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9110000000000001e-06 2023-09-23T22:45:26.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.669765+0000", 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97657507700000001 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:35:52.669777+0000", 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1501999999999999e-05 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75839 9.e 9:747fb0cf:::benchmark_data_smithi026_555954_object75838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.254321+0000", 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 543.44302553700004, 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.009841719, 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.717 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75839 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254321+0000", 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254321+0000", 2023-09-23T22:45:26.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254319+0000", 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254433+0000", 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000113632 2023-09-23T22:45:26.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254450+0000", 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6667000000000002e-05 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254456+0000", 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7610000000000002e-06 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254520+0000", 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4104999999999997e-05 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.254529+0000", 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8799999999999997e-06 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.239678+0000", 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98514882599999998 2023-09-23T22:45:26.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.239842+0000", 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00016353099999999999 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.240035+0000", 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019364800000000001 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.249930+0000", 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.009894524 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264057+0000", 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014127293000000001 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264105+0000", 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7617999999999997e-05 2023-09-23T22:45:26.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264138+0000", 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3302000000000001e-05 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264163+0000", 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4776000000000001e-05 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.724 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75851 9.e 9:72e98536:::benchmark_data_smithi026_555954_object75850:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.266486+0000", 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 543.43086047400004, 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.997817069, 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.725 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75851 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266486+0000", 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266486+0000", 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266485+0000", 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266560+0000", 2023-09-23T22:45:26.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.4641999999999994e-05 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266591+0000", 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1872999999999998e-05 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266600+0000", 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0960000000000007e-06 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266665+0000", 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4324000000000003e-05 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.266675+0000", 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0036e-05 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.241436+0000", 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97476125800000002 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.241566+0000", 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012998400000000001 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.241731+0000", 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000164981 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.249937+0000", 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0082056939999999995 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264197+0000", 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014260324 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264251+0000", 2023-09-23T22:45:26.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3449000000000003e-05 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264281+0000", 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0531000000000003e-05 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264303+0000", 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2221999999999998e-05 2023-09-23T22:45:26.732 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:75870 9.e 9:7ffb9527:::benchmark_data_smithi026_555954_object75869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e474)", 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:36:23.291109+0000", 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 543.40623729699996, 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97326064000000001, 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.733 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 75870 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291109+0000", 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291109+0000", 2023-09-23T22:45:26.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291108+0000", 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291155+0000", 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6782e-05 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291172+0000", 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7436e-05 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291180+0000", 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.994e-06 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291269+0000", 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8403e-05 2023-09-23T22:45:26.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:23.291288+0000", 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9711000000000002e-05 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.243282+0000", 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.951993172 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.243406+0000", 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000124828 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.243551+0000", 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014428499999999999 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.253042+0000", 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0094910329999999994 2023-09-23T22:45:26.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264225+0000", 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.011183383 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264322+0000", 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6733999999999999e-05 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264349+0000", 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.6846999999999999e-05 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:36:24.264370+0000", 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1325999999999999e-05 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104674 9.e 9:7c43a090:::benchmark_data_smithi026_555954_object9945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e487)", 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 492.44442841699998, 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0412800680000001, 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104674 2023-09-23T22:45:26.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252918+0000", 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252921+0000", 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6509999999999999e-06 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252938+0000", 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6416999999999999e-05 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.252943+0000", 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8679999999999998e-06 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.254941+0000", 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001998395 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.255082+0000", 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014090000000000001 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 1,7", 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.255225+0000", 2023-09-23T22:45:26.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014290599999999999 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.260880+0000", 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0056549929999999997 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.261192+0000", 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00031205499999999999 2023-09-23T22:45:26.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.280742+0000", 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019549997 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.280749+0000", 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2590000000000003e-06 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.294129+0000", 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0133797609999999 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.294198+0000", 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9536999999999995e-05 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104670 9.4 9:28c1ceab:::benchmark_data_smithi026_555954_object9941:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.293142+0000", 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 492.40420447499997, 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0530680600000002, 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104670 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.749 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293142+0000", 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293142+0000", 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.750 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293141+0000", 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293146+0000", 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6229999999999996e-06 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293155+0000", 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4839999999999993e-06 2023-09-23T22:45:26.751 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293163+0000", 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.8669999999999994e-06 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293213+0000", 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9536999999999997e-05 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.752 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293222+0000", 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.1740000000000006e-06 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.261387+0000", 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9681656429999999 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.265588+0000", 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042009059999999999 2023-09-23T22:45:26.753 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.342211+0000", 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.076622814999999997 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.346210+0000", 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039992270000000002 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.754 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104676 9.4 9:2b72389e:::benchmark_data_smithi026_555954_object9947:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 492.40416965499998, 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0644994749999999, 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104676 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.755 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.756 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293177+0000", 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293180+0000", 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.898e-06 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293186+0000", 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5080000000000002e-06 2023-09-23T22:45:26.757 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293191+0000", 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7829999999999998e-06 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293244+0000", 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2927999999999999e-05 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.758 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293249+0000", 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1440000000000002e-06 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.265969+0000", 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9727195919999998 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.759 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.270138+0000", 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041699170000000004 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.277809+0000", 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0076701460000000001 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.310996+0000", 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.033187649999999999 2023-09-23T22:45:26.760 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.347606+0000", 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036609292000000002 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.349036+0000", 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014300789999999999 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.761 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.351072+0000", 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0020369070000000001 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.357677+0000", 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066040279999999996 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.762 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104680 9.4 9:208db86f:::benchmark_data_smithi026_555954_object9951:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e488)", 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:14.293206+0000", 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 492.40414115900001, 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0756980610000002, 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104680 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.763 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293206+0000", 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293206+0000", 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.764 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293205+0000", 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293208+0000", 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5409999999999999e-06 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.765 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293214+0000", 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2210000000000002e-06 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293218+0000", 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0640000000000004e-06 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293268+0000", 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9852999999999999e-05 2023-09-23T22:45:26.766 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:14.293273+0000", 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8940000000000001e-06 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.287851+0000", 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9945780060000002 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.767 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.293072+0000", 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052214150000000001 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.300397+0000", 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073247130000000001 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.768 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.317253+0000", 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.016856401 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.358723+0000", 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.041469682000000001 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.359839+0000", 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011156460000000001 2023-09-23T22:45:26.769 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.361542+0000", 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001703148 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.368904+0000", 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0073618570000000003 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.770 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(9.c 489/478 [PushOp(9:31531dd3:::benchmark_data_smithi026_555954_object9982:head, version: 460'627, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:31531dd3:::benchmark_data_smithi026_555954_object9982:head@460'627, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.334517+0000", 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 491.36282981099998, 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6523379540000001, 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.771 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334517+0000", 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334517+0000", 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334516+0000", 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.772 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334620+0000", 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010427799999999999 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334637+0000", 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6889999999999999e-05 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.334646+0000", 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9209999999999998e-06 2023-09-23T22:45:26.773 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.984277+0000", 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6496310489999999 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.984284+0000", 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.8950000000000001e-06 2023-09-23T22:45:26.774 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.986855+0000", 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.002571041 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.775 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104715 9.4 9:22773e84:::benchmark_data_smithi026_555954_object9986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.341862+0000", 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 491.35548493200002, 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2123820830000001, 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104715 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.776 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341862+0000", 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341862+0000", 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.777 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341859+0000", 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341870+0000", 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0808999999999999e-05 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341889+0000", 2023-09-23T22:45:26.778 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.8604e-05 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.341900+0000", 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1754e-05 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.306973+0000", 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.965072514 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.779 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.310651+0000", 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036783609999999998 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.383378+0000", 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.072726329000000006 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.387721+0000", 2023-09-23T22:45:26.780 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0043429649999999998 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.502428+0000", 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.114707641 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.506100+0000", 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0036718440000000001 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.781 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.512882+0000", 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067814260000000001 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.543660+0000", 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030778920000000001 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.782 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.544694+0000", 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001033574 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.545296+0000", 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00060233699999999999 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.547534+0000", 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022377349999999998 2023-09-23T22:45:26.783 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.554244+0000", 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0067097160000000001 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "MOSDPGPush(9.c 489/478 [PushOp(9:3a91d401:::benchmark_data_smithi026_555954_object9993:head, version: 460'628, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:3a91d401:::benchmark_data_smithi026_555954_object9993:head@460'628, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.362751+0000", 2023-09-23T22:45:26.784 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 491.334596143, 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6258491149999998, 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "started", 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362751+0000", 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.785 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362751+0000", 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362750+0000", 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362865+0000", 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000115596 2023-09-23T22:45:26.786 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362882+0000", 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7062e-05 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.362897+0000", 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4198e-05 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.986936+0000", 2023-09-23T22:45:26.787 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6240388779999999 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.986943+0000", 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.7989999999999994e-06 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.988600+0000", 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001656364 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.788 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104723 9.4 9:24a0630c:::benchmark_data_smithi026_555954_object9994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e489)", 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:15.365986+0000", 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 491.33136021299998, 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2005458440000001, 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.789 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104723 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365986+0000", 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365986+0000", 2023-09-23T22:45:26.790 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365985+0000", 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.365992+0000", 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6150000000000002e-06 2023-09-23T22:45:26.791 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.366002+0000", 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0249e-05 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:15.366007+0000", 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1039999999999998e-06 2023-09-23T22:45:26.792 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.393428+0000", 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.027421414 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.398310+0000", 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0048813049999999998 2023-09-23T22:45:26.793 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.425313+0000", 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.027003367 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.429163+0000", 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038503410000000002 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.794 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.527354+0000", 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098190762000000001 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.530473+0000", 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0031186880000000001 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.795 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.537672+0000", 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0071995219999999999 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.555295+0000", 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017622819000000001 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.556659+0000", 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013637510000000001 2023-09-23T22:45:26.796 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.557146+0000", 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00048692499999999998 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.559282+0000", 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021364320000000002 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.797 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.566532+0000", 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00725007 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:104756 9.4 9:258d171c:::benchmark_data_smithi026_555954_object10027:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:37:16.626866+0000", 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 490.07048078999998, 2023-09-23T22:45:26.798 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.026819157, 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 104756 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626866+0000", 2023-09-23T22:45:26.799 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626866+0000", 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626865+0000", 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.800 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626872+0000", 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.2779999999999996e-06 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626886+0000", 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4470999999999999e-05 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:16.626893+0000", 2023-09-23T22:45:26.801 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.5139999999999998e-06 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.398840+0000", 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.77194742100000002 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.402776+0000", 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039360929999999999 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.802 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.442574+0000", 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.039797499 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.446766+0000", 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0041918859999999997 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.803 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.575609+0000", 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.12884293299999999 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.578674+0000", 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00306552 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,3", 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.586425+0000", 2023-09-23T22:45:26.804 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0077511760000000002 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.616679+0000", 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030253802 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.617743+0000", 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010636319999999999 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.805 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.644498+0000", 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026755226999999999 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.646717+0000", 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0022190299999999999 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.806 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:37:17.653685+0000", 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069680569999999997 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:112987 9.c 9:3a416566:::benchmark_data_smithi026_555954_object18258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:05.354955+0000", 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 381.34239203499999, 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4743666209999999, 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.807 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 112987 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354955+0000", 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.808 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354955+0000", 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354953+0000", 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354958+0000", 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.69e-06 2023-09-23T22:45:26.809 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354968+0000", 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4560000000000006e-06 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.354973+0000", 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2769999999999996e-06 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.810 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.355025+0000", 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2425e-05 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:05.355032+0000", 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.1999999999999999e-06 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.714595+0000", 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3595632559999999 2023-09-23T22:45:26.811 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.717359+0000", 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027637450000000002 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.738228+0000", 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020869011 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:45:26.812 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.741256+0000", 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0030285960000000002 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.777617+0000", 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.036360874000000001 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.781007+0000", 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003390452 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.813 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.786955+0000", 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0059470399999999998 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.819024+0000", 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.032069822999999997 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.814 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.820025+0000", 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010011379999999999 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.820409+0000", 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038331999999999998 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.815 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.822395+0000", 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0019860360000000001 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:06.829321+0000", 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0069264519999999996 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.816 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:113617 9.c 9:3bc4b131:::benchmark_data_smithi026_555954_object18888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:08.817731+0000", 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 377.87961542900001, 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.98456004699999999, 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 113617 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.817 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817731+0000", 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817731+0000", 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.818 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817730+0000", 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817740+0000", 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0345e-05 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.819 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817754+0000", 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3906e-05 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.817762+0000", 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.072e-06 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.878893+0000", 2023-09-23T22:45:26.820 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.061130914000000001 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.882737+0000", 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0038442099999999998 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.933676+0000", 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.050939367999999999 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.821 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:08.936972+0000", 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.003295769 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.740736+0000", 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.80376360499999999 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.822 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.743337+0000", 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026008390000000002 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.748613+0000", 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0052764719999999999 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.775493+0000", 2023-09-23T22:45:26.823 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026879872999999999 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.792909+0000", 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017415588999999999 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.794028+0000", 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00111911 2023-09-23T22:45:26.824 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.795686+0000", 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001658723 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.802291+0000", 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0066049189999999999 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.825 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:113663 9.c 9:352b5b63:::benchmark_data_smithi026_555954_object18934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e519)", 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:09.125940+0000", 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 377.57140622399999, 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0012365830000001, 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.826 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 113663 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125940+0000", 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125940+0000", 2023-09-23T22:45:26.827 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125939+0000", 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125956+0000", 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.73e-05 2023-09-23T22:45:26.828 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125970+0000", 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3583e-05 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.125976+0000", 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2389999999999999e-06 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.829 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.224933+0000", 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.098956855999999996 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.229210+0000", 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042773409999999996 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.288240+0000", 2023-09-23T22:45:26.830 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.059029617 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:09.291651+0000", 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0034113630000000002 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.085535+0000", 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.79388345299999996 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.831 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.088218+0000", 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0026835880000000002 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.093377+0000", 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0051590380000000003 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.832 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.118971+0000", 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025593576999999999 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.119343+0000", 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00037188599999999998 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.833 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.120218+0000", 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00087530799999999999 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.121636+0000", 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014175069999999999 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.834 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:10.127177+0000", 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0055414499999999998 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:117481 9.c 9:3122b071:::benchmark_data_smithi026_555954_object22752:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:22.273591+0000", 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 364.423755293, 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.062280133, 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.835 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 117481 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273591+0000", 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.836 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273591+0000", 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273590+0000", 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.837 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273596+0000", 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.0970000000000002e-06 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273609+0000", 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3054000000000001e-05 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.273615+0000", 2023-09-23T22:45:26.838 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.2859999999999996e-06 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.293079+0000", 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019464043 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.294561+0000", 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001482209 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.839 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.821444+0000", 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.52688237299999996 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:22.824388+0000", 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0029445890000000001 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.840 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.300187+0000", 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.47579873099999997 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.302383+0000", 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021957130000000002 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.306849+0000", 2023-09-23T22:45:26.841 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0044659249999999999 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.328972+0000", 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022122882999999999 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.329730+0000", 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00075875699999999999 2023-09-23T22:45:26.842 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.329972+0000", 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024183599999999999 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.331151+0000", 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001178988 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.843 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:23.335872+0000", 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0047203760000000001 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.36252.0:118575 9.c 9:36238a6d:::benchmark_data_smithi026_555954_object23846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:39:24.992467+0000", 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 361.70487966799999, 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0767739080000001, 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:26.844 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.36252", 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 118575 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992467+0000", 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.845 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992467+0000", 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992466+0000", 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:26.846 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992470+0000", 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.783e-06 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992478+0000", 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.5189999999999992e-06 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:24.992483+0000", 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1320000000000004e-06 2023-09-23T22:45:26.847 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:25.004684+0000", 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012201900999999999 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for missing object", 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:25.006194+0000", 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015095289999999999 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.848 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:25.486627+0000", 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.48043316600000002 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for rw locks", 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:25.489033+0000", 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0024054979999999998 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.849 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.034240+0000", 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.54520759500000004 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.036416+0000", 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0021752960000000002 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.040515+0000", 2023-09-23T22:45:26.850 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0040993469999999997 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.061630+0000", 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021115373 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.062311+0000", 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00068060899999999999 2023-09-23T22:45:26.851 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.063050+0000", 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00073889099999999998 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.064177+0000", 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011268160000000001 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:26.852 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:26.853 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:39:26.069241+0000", 2023-09-23T22:45:26.853 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0050643290000000002 2023-09-23T22:45:26.853 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.853 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.853 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.853 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:26.853 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:26.853 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:45:26.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: osdmap e658: 8 total, 8 up, 8 in 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:26 smithi070 ceph-mon[192652]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: osdmap e658: 8 total, 8 up, 8 in 2023-09-23T22:45:26.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.887 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[215775]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: osdmap e658: 8 total, 8 up, 8 in 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.888 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:26.889 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:26 smithi026 ceph-mon[223014]: failed to encode map e658 with expected crc 2023-09-23T22:45:27.360 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T22:45:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:27 smithi070 ceph-mon[192652]: pgmap v1676: 103 pgs: 2 active+undersized+remapped, 2 remapped+peering, 16 peering, 83 active+clean; 8.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 255 B/s wr, 0 op/s; 49 B/s, 4 objects/s recovering 2023-09-23T22:45:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:27 smithi070 ceph-mon[192652]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:27 smithi070 ceph-mon[192652]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:27 smithi070 ceph-mon[192652]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:27 smithi070 ceph-mon[192652]: osdmap e659: 8 total, 8 up, 8 in 2023-09-23T22:45:27.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:27 smithi070 ceph-mon[192652]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.742 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:27.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[215775]: pgmap v1676: 103 pgs: 2 active+undersized+remapped, 2 remapped+peering, 16 peering, 83 active+clean; 8.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 255 B/s wr, 0 op/s; 49 B/s, 4 objects/s recovering 2023-09-23T22:45:27.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[215775]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.946 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[215775]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[215775]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[215775]: osdmap e659: 8 total, 8 up, 8 in 2023-09-23T22:45:27.947 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[215775]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[223014]: pgmap v1676: 103 pgs: 2 active+undersized+remapped, 2 remapped+peering, 16 peering, 83 active+clean; 8.6 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 255 B/s wr, 0 op/s; 49 B/s, 4 objects/s recovering 2023-09-23T22:45:27.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[223014]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[223014]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[223014]: failed to encode map e659 with expected crc 2023-09-23T22:45:27.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[223014]: osdmap e659: 8 total, 8 up, 8 in 2023-09-23T22:45:27.947 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:27 smithi026 ceph-mon[223014]: failed to encode map e659 with expected crc 2023-09-23T22:45:28.542 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[223014]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[223014]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[223014]: osdmap e660: 8 total, 8 up, 8 in 2023-09-23T22:45:28.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[223014]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[223014]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[223014]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[223014]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2264954807' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T22:45:28.543 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T22:45:28.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[215775]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[215775]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[215775]: osdmap e660: 8 total, 8 up, 8 in 2023-09-23T22:45:28.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[215775]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[215775]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[215775]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[215775]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2264954807' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T22:45:28.544 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:28 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T22:45:28.552 INFO:teuthology.orchestra.run.smithi026.stderr:set pool 1 pgp_num to 31 2023-09-23T22:45:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:28 smithi070 ceph-mon[192652]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:28 smithi070 ceph-mon[192652]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:28 smithi070 ceph-mon[192652]: osdmap e660: 8 total, 8 up, 8 in 2023-09-23T22:45:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:28 smithi070 ceph-mon[192652]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:28 smithi070 ceph-mon[192652]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:28 smithi070 ceph-mon[192652]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:28 smithi070 ceph-mon[192652]: failed to encode map e660 with expected crc 2023-09-23T22:45:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:28 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2264954807' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T22:45:28.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:28 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T22:45:29.571 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T22:45:29.724 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:45:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:45:29] "GET /metrics HTTP/1.1" 200 34115 "" "Prometheus/2.43.0" 2023-09-23T22:45:29.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: pgmap v1679: 103 pgs: 2 active+undersized+remapped, 5 remapped+peering, 14 peering, 82 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T22:45:29.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: 1.e scrub starts 2023-09-23T22:45:29.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: 1.e scrub ok 2023-09-23T22:45:29.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.725 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T22:45:29.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: osdmap e661: 8 total, 8 up, 8 in 2023-09-23T22:45:29.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.726 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[215775]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.726 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: pgmap v1679: 103 pgs: 2 active+undersized+remapped, 5 remapped+peering, 14 peering, 82 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T22:45:29.726 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: 1.e scrub starts 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: 1.e scrub ok 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: osdmap e661: 8 total, 8 up, 8 in 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.727 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:29 smithi026 ceph-mon[223014]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: pgmap v1679: 103 pgs: 2 active+undersized+remapped, 5 remapped+peering, 14 peering, 82 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 0 op/s; 0 B/s, 1 objects/s recovering 2023-09-23T22:45:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: 1.e scrub starts 2023-09-23T22:45:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: 1.e scrub ok 2023-09-23T22:45:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T22:45:29.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: osdmap e661: 8 total, 8 up, 8 in 2023-09-23T22:45:29.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:29 smithi070 ceph-mon[192652]: failed to encode map e661 with expected crc 2023-09-23T22:45:29.956 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:29.959 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:45:29.960 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:45:29.960 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:45:29.960 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:45:30.789 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T22:45:30.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:30 smithi026 ceph-mon[215775]: pgmap v1681: 103 pgs: 3 remapped+peering, 12 peering, 88 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 9 objects/s recovering 2023-09-23T22:45:30.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:30 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:30.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:30 smithi026 ceph-mon[223014]: pgmap v1681: 103 pgs: 3 remapped+peering, 12 peering, 88 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 9 objects/s recovering 2023-09-23T22:45:30.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:30 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:30 smithi070 ceph-mon[192652]: pgmap v1681: 103 pgs: 3 remapped+peering, 12 peering, 88 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 9 objects/s recovering 2023-09-23T22:45:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:30 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:31.643 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:32.079 INFO:teuthology.orchestra.run.smithi026.stdout:mon.a: {} 2023-09-23T22:45:32.090 INFO:teuthology.orchestra.run.smithi026.stdout:mon.b: {} 2023-09-23T22:45:32.101 INFO:teuthology.orchestra.run.smithi026.stdout:mon.c: {} 2023-09-23T22:45:32.102 INFO:teuthology.orchestra.run.smithi026.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T22:45:32.102 INFO:teuthology.orchestra.run.smithi026.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T22:45:32.102 INFO:teuthology.orchestra.run.smithi026.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T22:45:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:32 smithi070 ceph-mon[192652]: pgmap v1682: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 53 KiB/s, 17 objects/s recovering 2023-09-23T22:45:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:32 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3612357387' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:45:33.235 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:32 smithi026 ceph-mon[215775]: pgmap v1682: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 53 KiB/s, 17 objects/s recovering 2023-09-23T22:45:33.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:32 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3612357387' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:45:33.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:32 smithi026 ceph-mon[223014]: pgmap v1682: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 53 KiB/s, 17 objects/s recovering 2023-09-23T22:45:33.236 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:32 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3612357387' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T22:45:33.272 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T22:45:33.833 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:33.873 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:45:33.873 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:45:33.873 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:45:33.873 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:45:33.874 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:45:34.702 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T22:45:35.491 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:35.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:35 smithi026 ceph-mon[215775]: pgmap v1683: 103 pgs: 103 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 610 KiB/s, 15 objects/s recovering 2023-09-23T22:45:35.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:35 smithi026 ceph-mon[223014]: pgmap v1683: 103 pgs: 103 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 610 KiB/s, 15 objects/s recovering 2023-09-23T22:45:35.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:35 smithi070 ceph-mon[192652]: pgmap v1683: 103 pgs: 103 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 610 KiB/s, 15 objects/s recovering 2023-09-23T22:45:37.213 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:45:37.213 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:45:37.213 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:45:37.213 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:45:37.213 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112969 9.3 9:caad47cb:::benchmark_data_smithi026_555954_object18240:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 391.881440623, 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6657709080000001, 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.214 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112969 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328980+0000", 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.215 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328979+0000", 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328983+0000", 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.693e-06 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.216 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328991+0000", 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2220000000000002e-06 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.328999+0000", 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4870000000000002e-06 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.329055+0000", 2023-09-23T22:45:37.217 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.6026999999999997e-05 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.329062+0000", 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.161e-06 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.976223+0000", 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6471613999999999 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.218 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.977503+0000", 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012796629999999999 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.979185+0000", 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001682149 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.219 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.984402+0000", 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0052169599999999997 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.991082+0000", 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0066800590000000003 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.992088+0000", 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001005964 2023-09-23T22:45:37.220 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.992639+0000", 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00055137800000000005 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.994750+0000", 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021113310000000001 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.221 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112990 9.3 9:c43d78c1:::benchmark_data_smithi026_555954_object18261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 391.85135397800002, 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.640044252, 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.222 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112990 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359066+0000", 2023-09-23T22:45:37.223 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359065+0000", 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359070+0000", 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9409999999999998e-06 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.224 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359083+0000", 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2697999999999999e-05 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359089+0000", 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7200000000000003e-06 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.225 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359144+0000", 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5954e-05 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.359157+0000", 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2853e-05 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.984626+0000", 2023-09-23T22:45:37.226 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.625468431 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.986258+0000", 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016318319999999999 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.988082+0000", 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018244540000000001 2023-09-23T22:45:37.227 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.995233+0000", 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071506429999999999 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.995368+0000", 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000135067 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.228 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.996086+0000", 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00071818499999999998 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.996659+0000", 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00057308399999999996 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.229 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.999110+0000", 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024514530000000001 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112992 9.3 9:cb6055b3:::benchmark_data_smithi026_555954_object18263:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 391.84989587000001, 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6580400470000001, 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.230 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112992 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.231 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360524+0000", 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.232 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360528+0000", 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4259999999999996e-06 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360540+0000", 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1824999999999999e-05 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360545+0000", 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5289999999999999e-06 2023-09-23T22:45:37.233 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360597+0000", 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1746000000000002e-05 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.360609+0000", 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1953000000000001e-05 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.234 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:06.999299+0000", 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.638690135 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.000875+0000", 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0015752990000000001 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.235 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.002719+0000", 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0018446700000000001 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015384+0000", 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.012664969999999999 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015497+0000", 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011238 2023-09-23T22:45:37.236 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.015923+0000", 2023-09-23T22:45:37.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00042655200000000002 2023-09-23T22:45:37.237 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.237 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.237 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.237 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.016447+0000", 2023-09-23T22:45:37.237 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052434900000000002 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.018564+0000", 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0021169520000000001 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.238 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:112993 9.3 9:cd412e89:::benchmark_data_smithi026_555954_object18264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e518)", 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 391.84644853600003, 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.658274966, 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 112993 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.239 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363972+0000", 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363971+0000", 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:37.240 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363975+0000", 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0169999999999998e-06 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363986+0000", 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0569e-05 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.241 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.242 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.363991+0000", 2023-09-23T22:45:37.242 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.0599999999999998e-06 2023-09-23T22:45:37.242 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.242 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.242 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364039+0000", 2023-09-23T22:45:37.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8767999999999998e-05 2023-09-23T22:45:37.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.243 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.243 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:05.364049+0000", 2023-09-23T22:45:37.243 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.8279999999999998e-06 2023-09-23T22:45:37.243 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.243 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.008138+0000", 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6440885409999999 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.009772+0000", 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0016337540000000001 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 5,7", 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.011820+0000", 2023-09-23T22:45:37.244 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020481570000000001 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.018990+0000", 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0071701450000000002 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.019117+0000", 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012743300000000001 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.245 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.019436+0000", 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000318239 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.020027+0000", 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00059157899999999995 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:39:07.022247+0000", 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0022195629999999999 2023-09-23T22:45:37.246 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188104 9.e 9:7689f13c:::benchmark_data_smithi026_555954_object93375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 286.79675420900003, 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.09525067, 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.247 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188104 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.248 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413666+0000", 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413668+0000", 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6299999999999998e-06 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.249 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413674+0000", 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.241e-06 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.413680+0000", 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1270000000000002e-06 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419176+0000", 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0054967100000000001 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.250 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419186+0000", 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.2289999999999992e-06 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.502956+0000", 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0837702039999999 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.503056+0000", 2023-09-23T22:45:37.251 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000100003 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.503139+0000", 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.2886999999999994e-05 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505036+0000", 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001897756 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.252 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506288+0000", 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001251816 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508853+0000", 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0025648149999999998 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508885+0000", 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2224999999999997e-05 2023-09-23T22:45:37.253 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508917+0000", 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1417000000000001e-05 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188105 9.e 9:72ff4f7f:::benchmark_data_smithi026_555954_object93376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e543)", 2023-09-23T22:45:37.254 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.415823+0000", 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 286.79459718599998, 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094443748, 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188105 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.255 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415823+0000", 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415823+0000", 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415822+0000", 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:37.256 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415828+0000", 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9649999999999996e-06 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415843+0000", 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4371e-05 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.257 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.415851+0000", 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4130000000000006e-06 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419209+0000", 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.003357942 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.419219+0000", 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0190999999999999e-05 2023-09-23T22:45:37.258 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505077+0000", 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0858583820000001 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505177+0000", 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9227000000000004e-05 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.259 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.505256+0000", 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8937000000000007e-05 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508765+0000", 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0035097739999999998 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508778+0000", 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2464e-05 2023-09-23T22:45:37.260 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510218+0000", 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001440059 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510245+0000", 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6982e-05 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.261 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510267+0000", 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1889e-05 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188109 9.e 9:78387bd9:::benchmark_data_smithi026_555954_object93380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.421149+0000", 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 286.78927143700002, 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.094876424, 2023-09-23T22:45:37.262 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188109 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421149+0000", 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.263 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421149+0000", 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421148+0000", 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.264 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421154+0000", 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7089999999999996e-06 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421165+0000", 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0801e-05 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.421173+0000", 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.9389999999999997e-06 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.265 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.422829+0000", 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001656299 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.422839+0000", 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.7890000000000006e-06 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.266 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506315+0000", 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0834760299999999 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506388+0000", 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3413999999999999e-05 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.506451+0000", 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.2315000000000002e-05 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.267 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508814+0000", 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0023629089999999998 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508933+0000", 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000119478 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.515963+0000", 2023-09-23T22:45:37.268 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0070293259999999998 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.515994+0000", 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1168999999999999e-05 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516025+0000", 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1522000000000001e-05 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.269 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188140 9.e 9:78d2bdae:::benchmark_data_smithi026_555954_object93411:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.458307+0000", 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 286.75211349400001, 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.057861621, 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.270 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188140 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458307+0000", 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458307+0000", 2023-09-23T22:45:37.271 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458306+0000", 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458310+0000", 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.208e-06 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.272 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458315+0000", 2023-09-23T22:45:37.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9190000000000004e-06 2023-09-23T22:45:37.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.273 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.273 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.273 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.458320+0000", 2023-09-23T22:45:37.273 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.989e-06 2023-09-23T22:45:37.273 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471337+0000", 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013016099 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471347+0000", 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0324e-05 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.274 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.507497+0000", 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0361505470000001 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.507566+0000", 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8753999999999999e-05 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.507632+0000", 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5599999999999995e-05 2023-09-23T22:45:37.275 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508942+0000", 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001310077 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510149+0000", 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0012073489999999999 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516108+0000", 2023-09-23T22:45:37.276 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0059589150000000004 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516143+0000", 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.5042000000000001e-05 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516168+0000", 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5201000000000001e-05 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.277 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188143 9.e 9:7424b661:::benchmark_data_smithi026_555954_object93414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.463773+0000", 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 286.746646817, 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0524969799999999, 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.278 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188143 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463773+0000", 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463773+0000", 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.279 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463772+0000", 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463779+0000", 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.5320000000000003e-06 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463791+0000", 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2319e-05 2023-09-23T22:45:37.280 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463803+0000", 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2082000000000001e-05 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471367+0000", 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075638820000000001 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.281 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471373+0000", 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0279999999999997e-06 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.508966+0000", 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0375934039999999 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.509038+0000", 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1544999999999993e-05 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.282 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.509111+0000", 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3699e-05 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.511865+0000", 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.002753148 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.283 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.513751+0000", 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001886307 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516216+0000", 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0024647459999999999 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516246+0000", 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0256999999999998e-05 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.284 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516270+0000", 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4433000000000001e-05 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.36252.0:188144 9.e 9:73d3d7cc:::benchmark_data_smithi026_555954_object93415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:40:50.463813+0000", 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 286.74660724699999, 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.052639544, 2023-09-23T22:45:37.285 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.36252", 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1420838413", 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 188144 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463813+0000", 2023-09-23T22:45:37.286 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463813+0000", 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463812+0000", 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.287 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463816+0000", 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2349999999999999e-06 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463823+0000", 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.294e-06 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.463828+0000", 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.5160000000000002e-06 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.288 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471392+0000", 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0075633810000000001 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:50.471398+0000", 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.7570000000000001e-06 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510314+0000", 2023-09-23T22:45:37.289 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.038916022 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510382+0000", 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8048999999999998e-05 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.510445+0000", 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3137000000000002e-05 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.290 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.511918+0000", 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001472714 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516042+0000", 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0041248190000000001 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516374+0000", 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000331746 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.291 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516419+0000", 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4904999999999998e-05 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:40:51.516453+0000", 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3528000000000002e-05 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.292 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23685 10.c 10:3e82cace:::benchmark_data_smithi026_640738_object23684:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.400587+0000", 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.809833287, 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3549325689999998, 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23685 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.293 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400587+0000", 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400587+0000", 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.294 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400585+0000", 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400611+0000", 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6152999999999998e-05 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400621+0000", 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0336999999999999e-05 2023-09-23T22:45:37.295 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400629+0000", 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6490000000000008e-06 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400693+0000", 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4178999999999997e-05 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.400843+0000", 2023-09-23T22:45:37.296 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000149672 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.401003+0000", 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000159662 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402788+0000", 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0017856790000000001 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.297 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.543824+0000", 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.14103579799999999 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566302+0000", 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022477809000000001 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566314+0000", 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1917e-05 2023-09-23T22:45:37.298 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.523693+0000", 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.95737967599999996 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.523720+0000", 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.6058e-05 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.755413+0000", 2023-09-23T22:45:37.299 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.23169375 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.755519+0000", 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106198 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23692 10.c 10:3970b700:::benchmark_data_smithi026_640738_object23691:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.402440+0000", 2023-09-23T22:45:37.300 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.80797994, 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3583861430000002, 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23692 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402440+0000", 2023-09-23T22:45:37.301 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402440+0000", 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402439+0000", 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.302 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402469+0000", 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.0212000000000001e-05 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402482+0000", 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2894999999999999e-05 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402490+0000", 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3620000000000006e-06 2023-09-23T22:45:37.303 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402821+0000", 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033087500000000001 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402963+0000", 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014139999999999999 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.304 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403105+0000", 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014212299999999999 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.541814+0000", 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13870915 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.543831+0000", 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020176510000000001 2023-09-23T22:45:37.305 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.569547+0000", 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.025715472999999999 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.569559+0000", 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2438e-05 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.306 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.523756+0000", 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.954196453 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.523766+0000", 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0662999999999999e-05 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.760701+0000", 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.236934706 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.307 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.760826+0000", 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012522 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23693 10.2 10:4c04af01:::benchmark_data_smithi026_640738_object23692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.402504+0000", 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.80791626300001, 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3437135749999998, 2023-09-23T22:45:37.308 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23693 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402504+0000", 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.309 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402504+0000", 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402503+0000", 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402528+0000", 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4802000000000001e-05 2023-09-23T22:45:37.310 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402536+0000", 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1480000000000006e-06 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402542+0000", 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8950000000000003e-06 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.311 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403527+0000", 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00098492299999999996 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403633+0000", 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000106319 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0", 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403772+0000", 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013865399999999999 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.312 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.542749+0000", 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.13897710899999999 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564906+0000", 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022157204999999999 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564918+0000", 2023-09-23T22:45:37.313 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.146e-05 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.515244+0000", 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.950325906 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.515259+0000", 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4796e-05 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.314 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746118+0000", 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.23085961 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746218+0000", 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.9322999999999998e-05 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.315 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23696 10.0 10:07385719:::benchmark_data_smithi026_640738_object23695:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.402548+0000", 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.80787243899999, 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3550447210000001, 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23696 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.316 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402548+0000", 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402548+0000", 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.317 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402547+0000", 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402573+0000", 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.5388000000000002e-05 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402580+0000", 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.4699999999999996e-06 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.318 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402586+0000", 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.699e-06 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403782+0000", 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011965070000000001 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.403910+0000", 2023-09-23T22:45:37.319 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012775800000000001 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,7", 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.404059+0000", 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014904799999999999 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.541946+0000", 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.137886549 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.320 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.541955+0000", 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.7730000000000005e-06 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564122+0000", 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.022166994999999998 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564139+0000", 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7014999999999999e-05 2023-09-23T22:45:37.321 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529858+0000", 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96571985500000002 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529874+0000", 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5566999999999999e-05 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.322 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757503+0000", 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.227628712 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757592+0000", 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.9835999999999998e-05 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.323 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23698 10.2 10:402b652c:::benchmark_data_smithi026_640738_object23697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.402600+0000", 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.80782060000001, 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3437404329999998, 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23698 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.324 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402600+0000", 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402600+0000", 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.325 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402599+0000", 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402624+0000", 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4955000000000001e-05 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402632+0000", 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.3650000000000004e-06 2023-09-23T22:45:37.326 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.402638+0000", 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9370000000000001e-06 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.404701+0000", 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0020623759999999999 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.327 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.404809+0000", 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108125 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0", 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.404929+0000", 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000120272 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.544366+0000", 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.139437006 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.328 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.565658+0000", 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.021292285000000001 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.565670+0000", 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1100999999999999e-05 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.515294+0000", 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.94962474299999999 2023-09-23T22:45:37.329 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.515305+0000", 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0346999999999999e-05 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746259+0000", 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.230953959 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.330 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.746340+0000", 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1451000000000003e-05 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23701 10.0 10:0aff6243:::benchmark_data_smithi026_640738_object23700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.545466+0000", 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.66495389100001, 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2285612509999999, 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.331 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23701 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545466+0000", 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545466+0000", 2023-09-23T22:45:37.332 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545465+0000", 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545488+0000", 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2617999999999999e-05 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.333 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545499+0000", 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0773e-05 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545506+0000", 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.6660000000000008e-06 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564930+0000", 2023-09-23T22:45:37.334 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019424004000000002 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.564941+0000", 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0362e-05 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529900+0000", 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96495990399999998 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.335 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529909+0000", 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.4880000000000007e-06 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757621+0000", 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2277121580000001 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757762+0000", 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000141116 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.336 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757929+0000", 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000166899 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.768586+0000", 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010657217 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.769695+0000", 2023-09-23T22:45:37.337 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0011086620000000001 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.772781+0000", 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0030858219999999998 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.772848+0000", 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6799999999999997e-05 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.338 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774028+0000", 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001179951 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23702 10.0 10:081d1a08:::benchmark_data_smithi026_640738_object23701:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.545512+0000", 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.66490788300001, 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2295653080000002, 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.339 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23702 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545512+0000", 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.340 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545512+0000", 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545512+0000", 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545536+0000", 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4450000000000001e-05 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.341 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545544+0000", 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.5020000000000003e-06 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.545550+0000", 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1589999999999998e-06 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.565069+0000", 2023-09-23T22:45:37.342 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.019519334999999999 2023-09-23T22:45:37.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.565079+0000", 2023-09-23T22:45:37.343 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5519999999999993e-06 2023-09-23T22:45:37.343 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.343 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.343 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.343 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529932+0000", 2023-09-23T22:45:37.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96485308000000003 2023-09-23T22:45:37.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.344 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.344 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.344 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529940+0000", 2023-09-23T22:45:37.344 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.7149999999999998e-06 2023-09-23T22:45:37.344 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.758947+0000", 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2290078449999999 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.759087+0000", 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139126 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.345 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:45:37.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.759266+0000", 2023-09-23T22:45:37.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00017959900000000001 2023-09-23T22:45:37.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.346 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.346 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.346 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.768594+0000", 2023-09-23T22:45:37.346 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0093276999999999995 2023-09-23T22:45:37.346 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.769787+0000", 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001193103 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774081+0000", 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0042944230000000003 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.347 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.774150+0000", 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.8270999999999997e-05 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.775078+0000", 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00092792700000000005 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.348 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23707 10.0 10:0b861545:::benchmark_data_smithi026_640738_object23706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.566617+0000", 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.643803507, 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2095116400000001, 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23707 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.349 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566617+0000", 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566617+0000", 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.350 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566615+0000", 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566644+0000", 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.8364000000000002e-05 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566654+0000", 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0806e-05 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.351 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566664+0000", 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.6160000000000008e-06 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566717+0000", 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.3134000000000003e-05 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.566728+0000", 2023-09-23T22:45:37.352 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0526000000000001e-05 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529962+0000", 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.96323421600000003 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.529970+0000", 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5949999999999999e-06 2023-09-23T22:45:37.353 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.354 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.354 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.354 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.760896+0000", 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2309250709999999 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.761014+0000", 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011832600000000001 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.761123+0000", 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00010950099999999999 2023-09-23T22:45:37.355 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.768601+0000", 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0074772730000000004 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.769827+0000", 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001226013 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.356 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.775193+0000", 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0053661619999999998 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.775241+0000", 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8355e-05 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.776128+0000", 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00088716000000000001 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.357 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:23711 10.1 10:83d91f22:::benchmark_data_smithi026_640738_object23710:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:42:04.592432+0000", 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 212.617988293, 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.1646562760000001, 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 23711 2023-09-23T22:45:37.358 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592432+0000", 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592432+0000", 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.359 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592430+0000", 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592455+0000", 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.4997000000000001e-05 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592472+0000", 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6687000000000001e-05 2023-09-23T22:45:37.360 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592481+0000", 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5650000000000007e-06 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592542+0000", 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0534e-05 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.361 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:04.592557+0000", 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4868999999999999e-05 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.508097+0000", 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.91554035700000003 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:05.508110+0000", 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.322e-05 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.362 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.725178+0000", 2023-09-23T22:45:37.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2170675339999999 2023-09-23T22:45:37.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.363 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.363 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.363 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.725318+0000", 2023-09-23T22:45:37.363 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000139777 2023-09-23T22:45:37.363 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2", 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.725470+0000", 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000151924 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.736347+0000", 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.010876798999999999 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.364 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.743719+0000", 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0073726110000000003 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.756953+0000", 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.013234364 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "commit_sent", 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757003+0000", 2023-09-23T22:45:37.365 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9212000000000002e-05 2023-09-23T22:45:37.366 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.366 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.366 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.366 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:42:06.757088+0000", 2023-09-23T22:45:37.366 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.5508999999999997e-05 2023-09-23T22:45:37.366 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.366 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.367 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.367 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.367 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.367 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "osd_op(client.46634.0:64668 10.2 10:41157639:::benchmark_data_smithi026_640738_object17272:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e596)", 2023-09-23T22:45:37.367 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:43:10.634899+0000", 2023-09-23T22:45:37.367 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 146.57552132999999, 2023-09-23T22:45:37.367 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2374666329999999, 2023-09-23T22:45:37.367 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: "client_info": { 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: "client": "client.46634", 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: "tid": 64668 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634899+0000", 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.368 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634899+0000", 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634897+0000", 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:10.634905+0000", 2023-09-23T22:45:37.369 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.8790000000000003e-06 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087141+0000", 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.45223591200000002 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087153+0000", 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2293000000000001e-05 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.370 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087215+0000", 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1610000000000001e-05 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087373+0000", 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000158455 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for subops from 0,2,6", 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.087521+0000", 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014800100000000001 2023-09-23T22:45:37.371 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "op_commit", 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.089367+0000", 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00184593 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.131005+0000", 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.041637697000000001 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for peered", 2023-09-23T22:45:37.372 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.131017+0000", 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.181e-05 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.856437+0000", 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.72542065 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "waiting for missing object", 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.856569+0000", 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013176400000000001 2023-09-23T22:45:37.373 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.872254+0000", 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.015684757000000001 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:43:11.872366+0000", 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011165799999999999 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.374 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.375 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:37.375 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:37.375 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:45:37.435 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T22:45:37.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:37 smithi026 ceph-mon[215775]: pgmap v1684: 103 pgs: 103 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 536 KiB/s, 13 objects/s recovering 2023-09-23T22:45:37.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:37 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2564147273' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:37.485 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:37 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:37.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:37 smithi026 ceph-mon[223014]: pgmap v1684: 103 pgs: 103 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 536 KiB/s, 13 objects/s recovering 2023-09-23T22:45:37.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:37 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2564147273' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:37.486 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:37 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:37.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:37 smithi070 ceph-mon[192652]: pgmap v1684: 103 pgs: 103 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 536 KiB/s, 13 objects/s recovering 2023-09-23T22:45:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:37 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2564147273' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:37 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:38.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T22:45:38.510 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T22:45:38.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[215775]: failed to encode map e662 with expected crc 2023-09-23T22:45:38.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[215775]: failed to encode map e662 with expected crc 2023-09-23T22:45:38.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[215775]: failed to encode map e662 with expected crc 2023-09-23T22:45:38.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:45:38.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[215775]: osdmap e662: 8 total, 8 up, 8 in 2023-09-23T22:45:38.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[223014]: failed to encode map e662 with expected crc 2023-09-23T22:45:38.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[223014]: failed to encode map e662 with expected crc 2023-09-23T22:45:38.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[223014]: failed to encode map e662 with expected crc 2023-09-23T22:45:38.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:45:38.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:38 smithi026 ceph-mon[223014]: osdmap e662: 8 total, 8 up, 8 in 2023-09-23T22:45:38.752 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:38 smithi070 ceph-mon[192652]: failed to encode map e662 with expected crc 2023-09-23T22:45:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:38 smithi070 ceph-mon[192652]: failed to encode map e662 with expected crc 2023-09-23T22:45:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:38 smithi070 ceph-mon[192652]: failed to encode map e662 with expected crc 2023-09-23T22:45:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:38 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:45:38.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:38 smithi070 ceph-mon[192652]: osdmap e662: 8 total, 8 up, 8 in 2023-09-23T22:45:39.162 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:39.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:45:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:45:39] "GET /metrics HTTP/1.1" 200 34132 "" "Prometheus/2.43.0" 2023-09-23T22:45:39.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:39 smithi026 ceph-mon[215775]: pgmap v1686: 103 pgs: 103 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 478 KiB/s, 7 objects/s recovering 2023-09-23T22:45:39.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:39.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:39 smithi026 ceph-mon[223014]: pgmap v1686: 103 pgs: 103 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 478 KiB/s, 7 objects/s recovering 2023-09-23T22:45:39.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:39 smithi070 ceph-mon[192652]: pgmap v1686: 103 pgs: 103 active+clean; 8.6 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 478 KiB/s, 7 objects/s recovering 2023-09-23T22:45:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:41.123 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:45:41.123 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:45:41.123 INFO:teuthology.orchestra.run.smithi026.stdout: "num_ops": 0 2023-09-23T22:45:41.123 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: pgmap v1687: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 410 KiB/s, 6 objects/s recovering 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]: dispatch 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 7]}]: dispatch 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5, 3, 0]}]: dispatch 2023-09-23T22:45:41.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:40 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2023-09-23T22:45:41.157 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: pgmap v1687: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 410 KiB/s, 6 objects/s recovering 2023-09-23T22:45:41.157 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:41.157 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.157 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.157 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]: dispatch 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 7]}]: dispatch 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5, 3, 0]}]: dispatch 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: pgmap v1687: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 410 KiB/s, 6 objects/s recovering 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.158 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]: dispatch 2023-09-23T22:45:41.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2023-09-23T22:45:41.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 7]}]: dispatch 2023-09-23T22:45:41.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]: dispatch 2023-09-23T22:45:41.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5, 3, 0]}]: dispatch 2023-09-23T22:45:41.159 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:40 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]: dispatch 2023-09-23T22:45:41.830 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T22:45:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: failed to encode map e663 with expected crc 2023-09-23T22:45:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: failed to encode map e663 with expected crc 2023-09-23T22:45:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: failed to encode map e663 with expected crc 2023-09-23T22:45:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T22:45:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T22:45:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]': finished 2023-09-23T22:45:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]': finished 2023-09-23T22:45:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-09-23T22:45:42.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 7]}]': finished 2023-09-23T22:45:42.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]': finished 2023-09-23T22:45:42.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5, 3, 0]}]': finished 2023-09-23T22:45:42.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2023-09-23T22:45:42.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: osdmap e663: 8 total, 8 up, 8 in 2023-09-23T22:45:42.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3479131120' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:42.125 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:41 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: failed to encode map e663 with expected crc 2023-09-23T22:45:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: failed to encode map e663 with expected crc 2023-09-23T22:45:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: failed to encode map e663 with expected crc 2023-09-23T22:45:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T22:45:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T22:45:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]': finished 2023-09-23T22:45:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]': finished 2023-09-23T22:45:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-09-23T22:45:42.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 7]}]': finished 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]': finished 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5, 3, 0]}]': finished 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: osdmap e663: 8 total, 8 up, 8 in 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3479131120' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: failed to encode map e663 with expected crc 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: failed to encode map e663 with expected crc 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: failed to encode map e663 with expected crc 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.6", "id": [1, 5]}]': finished 2023-09-23T22:45:42.237 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 0]}]': finished 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 5]}]': finished 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.19", "id": [3, 7]}]': finished 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]': finished 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 5, 3, 0]}]': finished 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1e", "id": [3, 2]}]': finished 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: osdmap e663: 8 total, 8 up, 8 in 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3479131120' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:42.238 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:41 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T22:45:42.430 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T22:45:42.602 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T22:45:42.814 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:43.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[223014]: pgmap v1689: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:43.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[223014]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[223014]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[223014]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:45:43.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[223014]: osdmap e664: 8 total, 8 up, 8 in 2023-09-23T22:45:43.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[223014]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.085 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[223014]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[215775]: pgmap v1689: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:43.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[215775]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[215775]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[215775]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:45:43.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[215775]: osdmap e664: 8 total, 8 up, 8 in 2023-09-23T22:45:43.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[215775]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.086 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:42 smithi026 ceph-mon[215775]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:42 smithi070 ceph-mon[192652]: pgmap v1689: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:42 smithi070 ceph-mon[192652]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:42 smithi070 ceph-mon[192652]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:42 smithi070 ceph-mon[192652]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:42 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T22:45:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:42 smithi070 ceph-mon[192652]: osdmap e664: 8 total, 8 up, 8 in 2023-09-23T22:45:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:42 smithi070 ceph-mon[192652]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:42 smithi070 ceph-mon[192652]: failed to encode map e664 with expected crc 2023-09-23T22:45:43.320 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:44.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[215775]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[215775]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[215775]: osdmap e665: 8 total, 8 up, 8 in 2023-09-23T22:45:44.096 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[215775]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.097 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[215775]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[223014]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[223014]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[223014]: osdmap e665: 8 total, 8 up, 8 in 2023-09-23T22:45:44.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[223014]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.097 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:43 smithi026 ceph-mon[223014]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:43 smithi070 ceph-mon[192652]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:43 smithi070 ceph-mon[192652]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:43 smithi070 ceph-mon[192652]: osdmap e665: 8 total, 8 up, 8 in 2023-09-23T22:45:44.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:43 smithi070 ceph-mon[192652]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:43 smithi070 ceph-mon[192652]: failed to encode map e665 with expected crc 2023-09-23T22:45:44.868 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:45:44.868 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [], 2023-09-23T22:45:44.868 INFO:teuthology.orchestra.run.smithi026.stdout: "complaint_time": 30, 2023-09-23T22:45:44.868 INFO:teuthology.orchestra.run.smithi026.stdout: "num_blocked_ops": 0 2023-09-23T22:45:44.868 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:45:45.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:45 smithi026 ceph-mon[223014]: pgmap v1692: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:45.514 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:45 smithi026 ceph-mon[215775]: pgmap v1692: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:45.750 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T22:45:45.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:45 smithi070 ceph-mon[192652]: pgmap v1692: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:46.231 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:46.439 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T22:45:46.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:46 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2860002241' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:45:46.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:46 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:45:46.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:46 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2860002241' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:45:46.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:46 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:45:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:46 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2860002241' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:45:46.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:46 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:45:47.293 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T22:45:47.527 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[215775]: pgmap v1693: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:47.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[215775]: failed to encode map e666 with expected crc 2023-09-23T22:45:47.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[215775]: failed to encode map e666 with expected crc 2023-09-23T22:45:47.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:45:47.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[215775]: osdmap e666: 8 total, 8 up, 8 in 2023-09-23T22:45:47.528 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[215775]: failed to encode map e666 with expected crc 2023-09-23T22:45:47.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[223014]: pgmap v1693: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:47.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[223014]: failed to encode map e666 with expected crc 2023-09-23T22:45:47.528 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[223014]: failed to encode map e666 with expected crc 2023-09-23T22:45:47.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:45:47.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[223014]: osdmap e666: 8 total, 8 up, 8 in 2023-09-23T22:45:47.529 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:47 smithi026 ceph-mon[223014]: failed to encode map e666 with expected crc 2023-09-23T22:45:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:47 smithi070 ceph-mon[192652]: pgmap v1693: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:47 smithi070 ceph-mon[192652]: failed to encode map e666 with expected crc 2023-09-23T22:45:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:47 smithi070 ceph-mon[192652]: failed to encode map e666 with expected crc 2023-09-23T22:45:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:47 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:45:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:47 smithi070 ceph-mon[192652]: osdmap e666: 8 total, 8 up, 8 in 2023-09-23T22:45:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:47 smithi070 ceph-mon[192652]: failed to encode map e666 with expected crc 2023-09-23T22:45:48.180 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout:{ 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout: "size": 20, 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 600, 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout: "ops": [ 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(3.7 655/654 [PushOp(3:e0b41d06:::609.00000000:head, version: 67'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:e0b41d06:::609.00000000:head@67'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:22.519894+0000", 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 26.002424940000001, 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00037484699999999998, 2023-09-23T22:45:48.525 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.519894+0000", 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.519894+0000", 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.526 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.519894+0000", 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.519914+0000", 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0007999999999999e-05 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.527 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.519927+0000", 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2832e-05 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.519946+0000", 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9298000000000001e-05 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.520019+0000", 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.3059000000000006e-05 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.528 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.520034+0000", 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4739999999999999e-05 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.520269+0000", 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023524700000000001 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.529 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(3.10 655/654 [PushOp(3:0d82a743:::600.00000000:head, version: 67'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:0d82a743:::600.00000000:head@67'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:22.523505+0000", 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 25.998813937000001, 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00038301600000000002, 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.523505+0000", 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.530 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.523505+0000", 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.523504+0000", 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.531 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.523527+0000", 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.3640999999999999e-05 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.523560+0000", 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.2144999999999998e-05 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.523573+0000", 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3382e-05 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.532 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.523664+0000", 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0931000000000001e-05 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.523679+0000", 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4955e-05 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.523888+0000", 2023-09-23T22:45:48.533 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00020934099999999999 2023-09-23T22:45:48.534 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.534 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.534 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.534 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.534 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.534 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(3.10 655/654 [PushOp(3:09401a81:::601.00000000:head, version: 67'4, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:09401a81:::601.00000000:head@67'4, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:48.534 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:22.524814+0000", 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 25.997505572000001, 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00023030500000000001, 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.524814+0000", 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.535 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.524814+0000", 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.524813+0000", 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.524828+0000", 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5755000000000001e-05 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.536 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.524847+0000", 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.8700000000000001e-05 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.524861+0000", 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4479e-05 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.524926+0000", 2023-09-23T22:45:48.537 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4560000000000005e-05 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.524935+0000", 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0899999999999994e-06 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.525044+0000", 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000108755 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.538 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(3.10 655/654 [PushOp(3:0bd6d154:::602.00000000:head, version: 67'6, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:0bd6d154:::602.00000000:head@67'6, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:22.525830+0000", 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 25.996488794000001, 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029042599999999999, 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.525830+0000", 2023-09-23T22:45:48.539 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.525830+0000", 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.525830+0000", 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.540 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.525842+0000", 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2829e-05 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.525860+0000", 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7146000000000002e-05 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.525880+0000", 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9958000000000001e-05 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.541 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.525960+0000", 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.03e-05 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.525973+0000", 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.306e-05 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526121+0000", 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00014782900000000001 2023-09-23T22:45:48.542 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(3.15 655/654 [PushOp(3:a93a17c2:::604.00000000:head, version: 67'2, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:a93a17c2:::604.00000000:head@67'2, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:22.526716+0000", 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 25.995602697999999, 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000356786, 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.543 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526716+0000", 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526716+0000", 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526716+0000", 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.544 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526727+0000", 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0773e-05 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526740+0000", 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2875e-05 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526748+0000", 2023-09-23T22:45:48.545 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.8570000000000001e-06 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526800+0000", 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1471000000000002e-05 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526807+0000", 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 7.1470000000000004e-06 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.546 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.527073+0000", 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00026619000000000002 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(3.10 655/654 [PushOp(3:0d89b25e:::603.00000000:head, version: 67'8, data_included: [], data_size: 0, omap_header_size: 274, omap_entries_size: 0, attrset_size: 4, recovery_info: ObjectRecoveryInfo(3:0d89b25e:::603.00000000:head@67'8, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:22.526943+0000", 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 25.995376414999999, 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000239671, 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.547 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526943+0000", 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526943+0000", 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.548 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526942+0000", 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526954+0000", 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1772e-05 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526970+0000", 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5845000000000002e-05 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.549 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.526983+0000", 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.296e-05 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.527049+0000", 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.6611000000000003e-05 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.527063+0000", 2023-09-23T22:45:48.550 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3692e-05 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.527182+0000", 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00011932800000000001 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(3.15 655/654 [PushOp(3:aa448500:::500.00000000:head, version: 67'3, data_included: [0~90], data_size: 90, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:aa448500:::500.00000000:head@67'3, size: 90, copy_subset: [0~90], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:90, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:22.528327+0000", 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 25.993991828999999, 2023-09-23T22:45:48.551 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00052607399999999996, 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.528327+0000", 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.528327+0000", 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.552 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.528327+0000", 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.528340+0000", 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2984e-05 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.553 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.528357+0000", 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7421000000000001e-05 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.528373+0000", 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5560999999999999e-05 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.528436+0000", 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.3533000000000003e-05 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.554 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.528448+0000", 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.2011000000000001e-05 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.528853+0000", 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00040501300000000002 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.555 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(3.b 655/654 [PushOp(3:d0630e4c:::mds0_sessionmap:head, version: 67'1, data_included: [], data_size: 0, omap_header_size: 14, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(3:d0630e4c:::mds0_sessionmap:head@67'1, size: 0, copy_subset: [], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:0, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:22.531502+0000", 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 25.990817491000001, 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00046453199999999998, 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.531502+0000", 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.556 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.531502+0000", 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.531501+0000", 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.531521+0000", 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9803999999999999e-05 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.557 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.531541+0000", 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.9848e-05 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.531557+0000", 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.6209e-05 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.531617+0000", 2023-09-23T22:45:48.558 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.0004999999999999e-05 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.531631+0000", 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.451e-05 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:22.531966+0000", 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00033469800000000003 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.559 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPush(1.c 656/655 [PushOp(1:30306672:devicehealth::main.db.0000000000000000:head, version: 89'161, data_included: [0~327680], data_size: 327680, omap_header_size: 0, omap_entries_size: 0, attrset_size: 10, recovery_info: ObjectRecoveryInfo(1:30306672:devicehealth::main.db.0000000000000000:head@89'161, size: 327680, copy_subset: [0~327680], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:327680, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:23.747941+0000", 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.774377821000002, 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.0019197789999999999, 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.747941+0000", 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.560 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.747941+0000", 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.747941+0000", 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.561 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.749290+0000", 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.001348997 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.749307+0000", 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7146000000000002e-05 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.749327+0000", 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.0392999999999999e-05 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.562 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.749388+0000", 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1148999999999993e-05 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.749403+0000", 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.4686e-05 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.749861+0000", 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00045792899999999999 2023-09-23T22:45:48.563 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGRecoveryDelete(2.0 e656,655 2:1c4216a6:::rbd_data.3a08970e7b7e.0000000000000018:head,71'3)", 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:23.751356+0000", 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.770962780000001, 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012652099999999999, 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.564 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.751356+0000", 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.751356+0000", 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.751356+0000", 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.565 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.751360+0000", 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6199999999999998e-06 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.751372+0000", 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1901e-05 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.566 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.751382+0000", 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.815e-06 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.751431+0000", 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9268000000000003e-05 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.751437+0000", 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9800000000000003e-06 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.567 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.751483+0000", 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5504999999999999e-05 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGRecoveryDelete(2.0 e656,655 2:1ffbff89:::rbd_object_map.b86131379f28.0000000000000014:head,632'179)", 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:23.761703+0000", 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.760616466999998, 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000134771, 2023-09-23T22:45:48.568 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.761703+0000", 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.761703+0000", 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.569 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.761702+0000", 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.761706+0000", 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.98e-06 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.761728+0000", 2023-09-23T22:45:48.570 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.2198999999999999e-05 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.761734+0000", 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.215e-06 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.761780+0000", 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6094999999999998e-05 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.571 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.761790+0000", 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0523000000000001e-05 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.761837+0000", 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.7182000000000003e-05 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.572 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGRecoveryDelete(2.1 e656,655 2:935fe252:::rbd_data.609240dd5ed5.00000000000000f4:head,70'5)", 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:23.766321+0000", 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.755998506000001, 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012659300000000001, 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.766321+0000", 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.573 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.766321+0000", 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.766320+0000", 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.766325+0000", 2023-09-23T22:45:48.574 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4599999999999996e-06 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.766335+0000", 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0900000000000001e-05 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.766341+0000", 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.8209999999999998e-06 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.575 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.766385+0000", 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3829999999999999e-05 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.766396+0000", 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0479000000000001e-05 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.766447+0000", 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.1644999999999998e-05 2023-09-23T22:45:48.576 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGRecoveryDelete(2.1 e656,655 2:8496e2e6:::rbd_object_map.90e4cc1a1b1c.0000000000000011:head,626'114)", 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:23.775522+0000", 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.746796703000001, 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012554100000000001, 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.577 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.775522+0000", 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.775522+0000", 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.775522+0000", 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:48.578 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.775527+0000", 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.13e-06 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.775539+0000", 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.1992e-05 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.579 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.775548+0000", 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.0420000000000008e-06 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.775588+0000", 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0243000000000001e-05 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.775594+0000", 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.536e-06 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.580 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.775648+0000", 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4336999999999998e-05 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGRecoveryDelete(2.1 e656,655 2:8d736321:::rbd_object_map.90c68b6a7311:head,628'127)", 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:23.776338+0000", 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.745981391000001, 2023-09-23T22:45:48.581 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012426700000000001, 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.776338+0000", 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.776338+0000", 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.582 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.776337+0000", 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.999999 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.776342+0000", 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6500000000000004e-06 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.583 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.776352+0000", 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0373e-05 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.776357+0000", 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.4929999999999998e-06 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.776402+0000", 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.5052999999999997e-05 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.584 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.776413+0000", 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0501e-05 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.776462+0000", 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9030999999999999e-05 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.585 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGRecoveryDelete(2.1 e656,655 2:970d7d0d:::rbd_header.912c8f657f87:head,628'147)", 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:23.777088+0000", 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.745231509, 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00013318299999999999, 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.777088+0000", 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.586 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.777088+0000", 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.777087+0000", 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.777091+0000", 2023-09-23T22:45:48.587 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.0949999999999998e-06 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.777101+0000", 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0002e-05 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.777117+0000", 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.5668e-05 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.588 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.777163+0000", 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6638999999999998e-05 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.777174+0000", 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.0688e-05 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.777221+0000", 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.6647999999999998e-05 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.589 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGRecoveryDelete(2.1 e656,655 2:978ab7ea:::rbd_id.testimg:head,636'157)", 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:23.778848+0000", 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 24.743471437, 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00012831, 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.778848+0000", 2023-09-23T22:45:48.590 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.778848+0000", 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.778847+0000", 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.591 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.778852+0000", 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.8119999999999998e-06 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.778866+0000", 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.3814e-05 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.778872+0000", 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1480000000000003e-06 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.592 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.778916+0000", 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.4567000000000002e-05 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.778926+0000", 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 9.5719999999999992e-06 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:23.778976+0000", 2023-09-23T22:45:48.593 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.9969000000000002e-05 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPushReply(3.b 658/657 [PushReplyOp(3:d0630e4c:::mds0_sessionmap:head)])", 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:25.570756+0000", 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.951563085, 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018452299999999999, 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.594 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.570756+0000", 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.570756+0000", 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.570756+0000", 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.595 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.570759+0000", 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.3079999999999999e-06 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.570766+0000", 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.7220000000000002e-06 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.596 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.570774+0000", 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 8.1810000000000001e-06 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.570811+0000", 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.7073000000000002e-05 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.570941+0000", 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000129498 2023-09-23T22:45:48.597 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPushReply(3.7 658/657 [PushReplyOp(3:e0b41d06:::609.00000000:head)])", 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:25.572820+0000", 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 22.949498939000001, 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.000231237, 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.598 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.572820+0000", 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.572820+0000", 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.572820+0000", 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:48.599 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.572823+0000", 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.1470000000000001e-06 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.572830+0000", 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.4219999999999997e-06 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.600 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.572836+0000", 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 6.1120000000000001e-06 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.572870+0000", 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4218999999999999e-05 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:25.573051+0000", 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00018157700000000001 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.601 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDPGPushReply(1.c 659/658 [PushReplyOp(1:30306672:devicehealth::main.db.0000000000000000:head)])", 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:27.240603+0000", 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 21.281715707, 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00035182699999999998, 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "reached pg", 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.602 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:27.240603+0000", 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:27.240603+0000", 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:27.240603+0000", 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967296 2023-09-23T22:45:48.603 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:27.240606+0000", 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9220000000000001e-06 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:27.240611+0000", 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.792e-06 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.604 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:27.240617+0000", 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.9270000000000002e-06 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:27.240660+0000", 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4.3350000000000003e-05 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:27.240955+0000", 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00029501899999999999 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.605 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: "description": "MOSDScrubReserve(1.e RELEASE e660)", 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: "initiated_at": "2023-09-23T22:45:28.346074+0000", 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: "age": 20.176245579, 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0.00015620200000000001, 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: "type_data": { 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: "flag_point": "started", 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: "events": [ 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "initiated", 2023-09-23T22:45:48.606 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:28.346074+0000", 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "throttled", 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:28.346074+0000", 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 0 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "header_read", 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:28.346072+0000", 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.607 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "all_read", 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:28.346084+0000", 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.184e-05 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "dispatched", 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:28.346113+0000", 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 2.9776000000000001e-05 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "queued_for_pg", 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:28.346126+0000", 2023-09-23T22:45:48.608 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.22e-05 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "reached_pg", 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:28.346178+0000", 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 5.2262999999999998e-05 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "started", 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:28.346195+0000", 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 1.7516e-05 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: }, 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: { 2023-09-23T22:45:48.609 INFO:teuthology.orchestra.run.smithi026.stdout: "event": "done", 2023-09-23T22:45:48.610 INFO:teuthology.orchestra.run.smithi026.stdout: "time": "2023-09-23T22:45:28.346230+0000", 2023-09-23T22:45:48.610 INFO:teuthology.orchestra.run.smithi026.stdout: "duration": 3.4390000000000001e-05 2023-09-23T22:45:48.610 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.610 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.610 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.610 INFO:teuthology.orchestra.run.smithi026.stdout: } 2023-09-23T22:45:48.610 INFO:teuthology.orchestra.run.smithi026.stdout: ] 2023-09-23T22:45:48.610 INFO:teuthology.orchestra.run.smithi026.stdout:} 2023-09-23T22:45:48.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:48 smithi026 ceph-mon[223014]: failed to encode map e666 with expected crc 2023-09-23T22:45:48.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:48 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:45:48.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:48 smithi026 ceph-mon[215775]: failed to encode map e666 with expected crc 2023-09-23T22:45:48.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:48 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:45:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:48 smithi070 ceph-mon[192652]: failed to encode map e666 with expected crc 2023-09-23T22:45:48.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:48 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:45:49.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:49 smithi026 ceph-mon[215775]: pgmap v1695: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:49.736 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:45:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:45:49] "GET /metrics HTTP/1.1" 200 34132 "" "Prometheus/2.43.0" 2023-09-23T22:45:49.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:49 smithi026 ceph-mon[223014]: pgmap v1695: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:49.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:49 smithi070 ceph-mon[192652]: pgmap v1695: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:49.957 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T22:45:50.382 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:51.448 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T22:45:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:51 smithi070 ceph-mon[192652]: pgmap v1696: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:51 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2129174290' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:45:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:51 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:51.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:51 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:51.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:51 smithi026 ceph-mon[223014]: pgmap v1696: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:51.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:51 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2129174290' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:45:51.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:51 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:51.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:51 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:51.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:51 smithi026 ceph-mon[215775]: pgmap v1696: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:51.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:51 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2129174290' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T22:45:51.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:51 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:51.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:51 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:52.603 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:45:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:45:52.603 INFO:teuthology.orchestra.run.smithi070.stdout: "num_ops": 0 2023-09-23T22:45:52.603 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:45:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[223014]: failed to encode map e667 with expected crc 2023-09-23T22:45:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[223014]: failed to encode map e667 with expected crc 2023-09-23T22:45:52.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2129174290' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:45:52.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[223014]: osdmap e667: 8 total, 8 up, 8 in 2023-09-23T22:45:52.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[223014]: failed to encode map e667 with expected crc 2023-09-23T22:45:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[215775]: failed to encode map e667 with expected crc 2023-09-23T22:45:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[215775]: failed to encode map e667 with expected crc 2023-09-23T22:45:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2129174290' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:45:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[215775]: osdmap e667: 8 total, 8 up, 8 in 2023-09-23T22:45:52.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:52 smithi026 ceph-mon[215775]: failed to encode map e667 with expected crc 2023-09-23T22:45:52.769 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T22:45:52.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:52 smithi070 ceph-mon[192652]: failed to encode map e667 with expected crc 2023-09-23T22:45:52.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:52 smithi070 ceph-mon[192652]: failed to encode map e667 with expected crc 2023-09-23T22:45:52.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:52 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2129174290' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T22:45:52.806 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:52 smithi070 ceph-mon[192652]: osdmap e667: 8 total, 8 up, 8 in 2023-09-23T22:45:52.807 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:52 smithi070 ceph-mon[192652]: failed to encode map e667 with expected crc 2023-09-23T22:45:53.152 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:53.280 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T22:45:53.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:53 smithi070 ceph-mon[192652]: pgmap v1698: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:45:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:45:53.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:53.631 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:53.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[215775]: pgmap v1698: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:53.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:53.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:53.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:45:53.734 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:45:53.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:53.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[223014]: pgmap v1698: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:53.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:53.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:53.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:45:53.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:45:53.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:45:54.511 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:54.512 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:45:55.099 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:45:55.100 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [], 2023-09-23T22:45:55.100 INFO:teuthology.orchestra.run.smithi070.stdout: "complaint_time": 30, 2023-09-23T22:45:55.100 INFO:teuthology.orchestra.run.smithi070.stdout: "num_blocked_ops": 0 2023-09-23T22:45:55.100 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:45:55.465 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T22:45:55.623 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:55 smithi070 ceph-mon[192652]: pgmap v1699: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:55.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2030733224' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:45:55.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:55 smithi026 ceph-mon[215775]: pgmap v1699: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:55.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2030733224' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:45:55.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:55 smithi026 ceph-mon[223014]: pgmap v1699: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:55.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2030733224' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:45:55.762 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T22:45:56.103 INFO:teuthology.orchestra.run.smithi070.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T22:45:56.171 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T22:45:56.539 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:56 smithi070 ceph-mon[192652]: failed to encode map e668 with expected crc 2023-09-23T22:45:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:56 smithi070 ceph-mon[192652]: failed to encode map e668 with expected crc 2023-09-23T22:45:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:56 smithi070 ceph-mon[192652]: failed to encode map e668 with expected crc 2023-09-23T22:45:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2030733224' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:45:56.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:56 smithi070 ceph-mon[192652]: osdmap e668: 8 total, 8 up, 8 in 2023-09-23T22:45:56.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[223014]: failed to encode map e668 with expected crc 2023-09-23T22:45:56.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[223014]: failed to encode map e668 with expected crc 2023-09-23T22:45:56.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[223014]: failed to encode map e668 with expected crc 2023-09-23T22:45:56.734 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2030733224' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:45:56.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[223014]: osdmap e668: 8 total, 8 up, 8 in 2023-09-23T22:45:56.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[215775]: failed to encode map e668 with expected crc 2023-09-23T22:45:56.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[215775]: failed to encode map e668 with expected crc 2023-09-23T22:45:56.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[215775]: failed to encode map e668 with expected crc 2023-09-23T22:45:56.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2030733224' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:45:56.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:56 smithi026 ceph-mon[215775]: osdmap e668: 8 total, 8 up, 8 in 2023-09-23T22:45:57.612 INFO:teuthology.orchestra.run.smithi070.stdout:{ 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "size": 20, 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 600, 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "ops": [ 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:57492 10.f 10.9da7365f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e592)", 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:42:55.503047+0000", 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 182.10657842000001, 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.97292683499999999, 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:57.613 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 57492 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:55.503047+0000", 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.614 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:55.503047+0000", 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:55.503046+0000", 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.615 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:55.503056+0000", 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8919999999999995e-06 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:55.503065+0000", 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.9590000000000001e-06 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:55.503072+0000", 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5120000000000002e-06 2023-09-23T22:45:57.616 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:55.526013+0000", 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022940947 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:55.526027+0000", 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4235e-05 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.617 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:56.475962+0000", 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.94993516899999997 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:42:56.475974+0000", 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1301000000000001e-05 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.618 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:79649 10.a 10:5b516bea:::benchmark_data_smithi026_640738_object32253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:33.449500+0000", 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 144.16012497099999, 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019500182000000001, 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 79649 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.619 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.449500+0000", 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.449500+0000", 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.620 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.449499+0000", 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.449503+0000", 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.3519999999999998e-06 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.449513+0000", 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.7799999999999995e-06 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.621 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.449517+0000", 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0330000000000002e-06 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.449568+0000", 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1418999999999997e-05 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.622 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.449693+0000", 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000125251 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.449829+0000", 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000136079 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.450781+0000", 2023-09-23T22:45:57.623 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00095151800000000005 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.451167+0000", 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038599199999999999 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.468933+0000", 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017766110000000002 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.624 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.468973+0000", 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9626000000000003e-05 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.469000+0000", 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7855e-05 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.625 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:80045 10.a 10:5b6431e1:::benchmark_data_smithi026_640738_object32649:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:33.888286+0000", 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 143.72133927100001, 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19497397999999999, 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 80045 2023-09-23T22:45:57.626 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.888286+0000", 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.888286+0000", 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.627 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.888285+0000", 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.888290+0000", 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0270000000000003e-06 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.888302+0000", 2023-09-23T22:45:57.628 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1861999999999999e-05 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.888306+0000", 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5800000000000002e-06 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.888348+0000", 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.1227999999999999e-05 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.629 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.888452+0000", 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00010383299999999999 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.888535+0000", 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3108000000000005e-05 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.630 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.889603+0000", 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001068804 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:33.891423+0000", 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0018196359999999999 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:34.083153+0000", 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.19172963600000001 2023-09-23T22:45:57.631 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:34.083209+0000", 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6490999999999998e-05 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:34.083260+0000", 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.0800999999999997e-05 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.632 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:82279 10.a 10:5cc94f88:::benchmark_data_smithi026_640738_object34883:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:36.742740+0000", 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 140.86688514599999, 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.066084026000000004, 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 82279 2023-09-23T22:45:57.633 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.742740+0000", 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.742740+0000", 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.634 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.742739+0000", 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.742745+0000", 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.6279999999999999e-06 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.635 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.742761+0000", 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5891000000000001e-05 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.742767+0000", 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3810000000000004e-06 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.742814+0000", 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7027999999999998e-05 2023-09-23T22:45:57.636 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.742965+0000", 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000150901 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.743108+0000", 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014228299999999999 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.637 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.744227+0000", 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011195319999999999 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.744616+0000", 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00038853599999999998 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.808718+0000", 2023-09-23T22:45:57.638 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.064101933999999999 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.808774+0000", 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.6808000000000002e-05 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:36.808824+0000", 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9734000000000002e-05 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.639 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:83276 10.a 10:59f72546:::benchmark_data_smithi026_640738_object35880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:37.965883+0000", 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 139.64374217700001, 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.022108913000000001, 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.640 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 83276 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.965883+0000", 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.965883+0000", 2023-09-23T22:45:57.641 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.965881+0000", 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999981 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.965888+0000", 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.7619999999999998e-06 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.642 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.965903+0000", 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.4857e-05 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.965909+0000", 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3820000000000001e-06 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.965967+0000", 2023-09-23T22:45:57.643 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.7620000000000001e-05 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.966111+0000", 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014375100000000001 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.966241+0000", 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000130557 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.644 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.967386+0000", 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00114437 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.967961+0000", 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00057580499999999998 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.645 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.987895+0000", 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019933519 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.987943+0000", 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8554000000000003e-05 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:37.987992+0000", 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.8619999999999999e-05 2023-09-23T22:45:57.646 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:83399 10.a 10:5ddcdfed:::benchmark_data_smithi026_640738_object36003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:38.155041+0000", 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 139.45458449399999, 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.025487656000000001, 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.647 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.648 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.648 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 83399 2023-09-23T22:45:57.648 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.648 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.648 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.648 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.648 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155041+0000", 2023-09-23T22:45:57.648 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155041+0000", 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155031+0000", 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999905 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.649 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155050+0000", 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9113000000000001e-05 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155082+0000", 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1804999999999998e-05 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155096+0000", 2023-09-23T22:45:57.650 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.3577000000000001e-05 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155213+0000", 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000117656 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155490+0000", 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00027703100000000001 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.651 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155613+0000", 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122485 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.158456+0000", 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028434630000000001 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.158467+0000", 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0929e-05 2023-09-23T22:45:57.652 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.180429+0000", 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021961415000000001 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.180482+0000", 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.3698e-05 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.653 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.180528+0000", 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6156999999999999e-05 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:83405 10.a 10:5e49cbbd:::benchmark_data_smithi026_640738_object36009:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:38.155115+0000", 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 139.454510056, 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.030293163000000001, 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.654 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 83405 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155115+0000", 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.655 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155115+0000", 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155114+0000", 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155118+0000", 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9369999999999997e-06 2023-09-23T22:45:57.656 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155128+0000", 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.9969999999999993e-06 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.155135+0000", 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.3160000000000002e-06 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.657 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.156618+0000", 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014828280000000001 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.156732+0000", 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114345 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.156854+0000", 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000122067 2023-09-23T22:45:57.658 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.158471+0000", 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001617014 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.158667+0000", 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00019630200000000001 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.659 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.185339+0000", 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026671619000000001 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.185371+0000", 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.1902000000000001e-05 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.185408+0000", 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7580000000000003e-05 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.660 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:83411 10.a 10:55484c36:::benchmark_data_smithi026_640738_object36015:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:38.156180+0000", 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 139.45344551299999, 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.029384117000000001, 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.661 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 83411 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.156180+0000", 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.156180+0000", 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.662 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.156179+0000", 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.156184+0000", 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9570000000000001e-06 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.156199+0000", 2023-09-23T22:45:57.663 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5274e-05 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.156206+0000", 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.4999999999999996e-06 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.157537+0000", 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0013315880000000001 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.664 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.157617+0000", 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0098000000000005e-05 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.157691+0000", 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3610999999999997e-05 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.158510+0000", 2023-09-23T22:45:57.665 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00081922400000000002 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.158755+0000", 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024455700000000001 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.185504+0000", 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.026748946999999999 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.666 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.185539+0000", 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5108999999999999e-05 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.185564+0000", 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.51e-05 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.667 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:83812 10.a 10:5e5b70e9:::benchmark_data_smithi026_640738_object36416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:38.803816+0000", 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.805809092, 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021198460999999998, 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 83812 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.668 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.803816+0000", 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.803816+0000", 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.803815+0000", 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.669 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.803819+0000", 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.585e-06 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.803827+0000", 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.7350000000000002e-06 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.670 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.803831+0000", 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.5810000000000001e-06 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.804594+0000", 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00076258000000000001 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.804708+0000", 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114926 2023-09-23T22:45:57.671 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.804804+0000", 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.5089999999999999e-05 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.805844+0000", 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010403579999999999 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.672 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.806371+0000", 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00052672299999999999 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.824904+0000", 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018533239 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.824975+0000", 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.1130000000000005e-05 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.673 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.825015+0000", 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9609e-05 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:83837 10.a 10:5e1eafb1:::benchmark_data_smithi026_640738_object36441:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:38.848062+0000", 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.76156350599999, 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019919942, 2023-09-23T22:45:57.674 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 83837 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848062+0000", 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.675 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848062+0000", 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848061+0000", 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848065+0000", 2023-09-23T22:45:57.676 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8829999999999999e-06 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848075+0000", 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0030000000000001e-05 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848081+0000", 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.9499999999999998e-06 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.677 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.849611+0000", 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0015300229999999999 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.849725+0000", 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000114215 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.678 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.849801+0000", 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.5569999999999996e-05 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.854006+0000", 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0042056250000000002 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.855026+0000", 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001019289 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.679 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.867916+0000", 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012890603 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.867956+0000", 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.9889999999999999e-05 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.867982+0000", 2023-09-23T22:45:57.680 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5610999999999999e-05 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:83839 10.a 10:5e8954d7:::benchmark_data_smithi026_640738_object36443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:38.848131+0000", 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.76149421299999, 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019939611999999999, 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.681 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 83839 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848131+0000", 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.682 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848131+0000", 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848131+0000", 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848133+0000", 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0020000000000001e-06 2023-09-23T22:45:57.683 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848137+0000", 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8380000000000004e-06 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848139+0000", 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.3580000000000001e-06 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.684 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.850931+0000", 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0027916070000000002 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.851040+0000", 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000109107 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.851157+0000", 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00011747300000000001 2023-09-23T22:45:57.685 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.854009+0000", 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0028519679999999999 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.855049+0000", 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001039562 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.686 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.868021+0000", 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.012972711 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.868049+0000", 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7540999999999999e-05 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.868071+0000", 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.1716999999999999e-05 2023-09-23T22:45:57.687 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:83841 10.a 10:538129c4:::benchmark_data_smithi026_640738_object36445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:38.848143+0000", 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.76148211399999, 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019997561000000001, 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.688 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 83841 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848143+0000", 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848143+0000", 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.689 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848143+0000", 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848145+0000", 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.6619999999999999e-06 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.690 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848148+0000", 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0000000000000001e-06 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.848150+0000", 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.418e-06 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.691 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.852108+0000", 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0039581140000000004 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.852188+0000", 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.9936000000000002e-05 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.852261+0000", 2023-09-23T22:45:57.693 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.3003999999999994e-05 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.854012+0000", 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017510500000000001 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.855088+0000", 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001075924 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.694 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.868095+0000", 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.013007240999999999 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.868121+0000", 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.5477e-05 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:38.868141+0000", 2023-09-23T22:45:57.695 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0057000000000001e-05 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:84046 10.a 10:52738353:::benchmark_data_smithi026_640738_object36650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:39.112937+0000", 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.49668785899999, 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021359240000000002, 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.696 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84046 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.112937+0000", 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.112937+0000", 2023-09-23T22:45:57.697 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.112936+0000", 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.112941+0000", 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6090000000000003e-06 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.698 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.112960+0000", 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.9032999999999999e-05 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.112965+0000", 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.5630000000000002e-06 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.113034+0000", 2023-09-23T22:45:57.699 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 6.9465e-05 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.113131+0000", 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.6848000000000005e-05 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.113215+0000", 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.3523999999999995e-05 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.700 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.114214+0000", 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00099909200000000004 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.114633+0000", 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00041963999999999999 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.134232+0000", 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019598925999999999 2023-09-23T22:45:57.701 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.134268+0000", 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6186000000000002e-05 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.134297+0000", 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8294999999999999e-05 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.702 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:84073 10.a 10:55bc6227:::benchmark_data_smithi026_640738_object36677:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:39.149482+0000", 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 138.46014278300001, 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018241035999999999, 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 84073 2023-09-23T22:45:57.703 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.149482+0000", 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.149482+0000", 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.704 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.149482+0000", 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.149484+0000", 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.815e-06 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.149488+0000", 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6559999999999998e-06 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.705 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.149490+0000", 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.4080000000000001e-06 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.150877+0000", 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001387157 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.706 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.150998+0000", 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012094700000000001 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.151068+0000", 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 7.0261999999999999e-05 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.151930+0000", 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00086137300000000002 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.707 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.152540+0000", 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00061010999999999997 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.167634+0000", 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.01509455 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.167681+0000", 2023-09-23T22:45:57.708 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.6288999999999997e-05 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:39.167724+0000", 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2750999999999998e-05 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:86758 10.a 10:53d59d95:::benchmark_data_smithi026_640738_object39362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T22:45:57.709 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:42.924524+0000", 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 134.68510085200001, 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.021667369999999998, 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 86758 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.710 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.924524+0000", 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.924524+0000", 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.924523+0000", 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999986 2023-09-23T22:45:57.711 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.924528+0000", 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.4449999999999998e-06 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.924538+0000", 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.0105000000000001e-05 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.712 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.924542+0000", 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.2740000000000001e-06 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.924593+0000", 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.1360000000000003e-05 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.924734+0000", 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000140359 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.713 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.924874+0000", 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00014065599999999999 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.925995+0000", 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0011201640000000001 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.926366+0000", 2023-09-23T22:45:57.714 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00037186199999999998 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.946115+0000", 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019748371000000001 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.946151+0000", 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6491e-05 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.715 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:42.946192+0000", 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.0519000000000003e-05 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:86907 10.a 10:59304254:::benchmark_data_smithi026_640738_object39511:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:43.132836+0000", 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 134.476789564, 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.023685548000000001, 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.716 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 86907 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.132836+0000", 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.717 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.132836+0000", 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.132835+0000", 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.132839+0000", 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7000000000000002e-06 2023-09-23T22:45:57.718 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.132847+0000", 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.738e-06 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.132851+0000", 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.7179999999999998e-06 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.719 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.132879+0000", 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7438000000000001e-05 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.132977+0000", 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.8240999999999997e-05 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.133058+0000", 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.1571999999999998e-05 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.720 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.134127+0000", 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0010689199999999999 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.135457+0000", 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001329427 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.156450+0000", 2023-09-23T22:45:57.721 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.020993537999999999 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.156494+0000", 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3634999999999998e-05 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.156521+0000", 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.7368e-05 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.722 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:87128 10.a 10:5ec9fa52:::benchmark_data_smithi026_640738_object39732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:43.443353+0000", 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 134.16627232299999, 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.018503405000000001, 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.723 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 87128 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443353+0000", 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443353+0000", 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.724 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443350+0000", 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999976 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443362+0000", 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1997000000000001e-05 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443393+0000", 2023-09-23T22:45:57.725 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.0216e-05 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443405+0000", 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.2459e-05 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443496+0000", 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.0878000000000006e-05 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.726 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443635+0000", 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000139052 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443788+0000", 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00015343600000000001 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.446412+0000", 2023-09-23T22:45:57.727 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00262327 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.446794+0000", 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000382677 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.461791+0000", 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.014996323000000001 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.728 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.461827+0000", 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.6356000000000002e-05 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.461856+0000", 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9320999999999999e-05 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.729 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:87129 10.a 10:59249ac5:::benchmark_data_smithi026_640738_object39733:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:43.443439+0000", 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 134.16618657999999, 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019286464999999999, 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 87129 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.730 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443439+0000", 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443439+0000", 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443438+0000", 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.9999995 2023-09-23T22:45:57.731 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443440+0000", 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.7209999999999999e-06 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443461+0000", 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.0513000000000002e-05 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.732 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.443463+0000", 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.8329999999999998e-06 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.444907+0000", 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0014435819999999999 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.445047+0000", 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000139481 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.733 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.445168+0000", 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00012152200000000001 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.446907+0000", 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.0017388460000000001 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.447155+0000", 2023-09-23T22:45:57.734 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00024843 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.462674+0000", 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.015518389 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.462703+0000", 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.9343e-05 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.735 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.462725+0000", 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 2.2070999999999999e-05 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.46634.0:87195 10.a 10:5e47547f:::benchmark_data_smithi026_640738_object39799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:43:43.534044+0000", 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 134.07558126399999, 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.019619408000000001, 2023-09-23T22:45:57.736 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.46634", 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1937414636", 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 87195 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.534044+0000", 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.737 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.534044+0000", 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.534043+0000", 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.738 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.534047+0000", 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.3379999999999998e-06 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.534057+0000", 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 9.4620000000000002e-06 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.534061+0000", 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8079999999999998e-06 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.739 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.534096+0000", 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.553e-05 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "started", 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.534200+0000", 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000103643 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for subops from 2,5", 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.534288+0000", 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.8412000000000007e-05 2023-09-23T22:45:57.740 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "op_commit", 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.535402+0000", 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.001114028 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.536071+0000", 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.00066870899999999997 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.741 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "sub_op_commit_rec", 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.553592+0000", 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.017520844000000001 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "commit_sent", 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.553625+0000", 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.2846999999999999e-05 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:43:43.553663+0000", 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 3.8865000000000001e-05 2023-09-23T22:45:57.742 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "description": "osd_op(client.47297.0:90 2.5 2:a1c9e6f8:::journal.b888711a537:head [stat] snapc 0=[] ondisk+read+known_if_redirected+full_try+supports_pool_eio e636)", 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "initiated_at": "2023-09-23T22:44:54.630789+0000", 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "age": 62.978836141999999, 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.803975962, 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "type_data": { 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "flag_point": "reached pg", 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "client_info": { 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "client": "client.47297", 2023-09-23T22:45:57.743 INFO:teuthology.orchestra.run.smithi070.stdout: "client_addr": "172.21.15.26:0/1815957787", 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "tid": 90 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "events": [ 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "initiated", 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.630789+0000", 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "throttled", 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.630789+0000", 2023-09-23T22:45:57.744 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "header_read", 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.630788+0000", 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4294967295.999999 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "all_read", 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.630793+0000", 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 5.2700000000000004e-06 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.745 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "dispatched", 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.630805+0000", 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.1878e-05 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "queued_for_pg", 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.630810+0000", 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.7820000000000001e-06 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.630860+0000", 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 4.9886999999999999e-05 2023-09-23T22:45:57.746 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:54.630868+0000", 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 8.0299999999999994e-06 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:55.433018+0000", 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.80214988600000003 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "waiting for peered", 2023-09-23T22:45:57.747 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:55.433034+0000", 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.5903e-05 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "reached_pg", 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:56.434649+0000", 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 1.001614845 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: }, 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: { 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "event": "done", 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "time": "2023-09-23T22:44:56.434765+0000", 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: "duration": 0.000116555 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.748 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: } 2023-09-23T22:45:57.749 INFO:teuthology.orchestra.run.smithi070.stdout: ] 2023-09-23T22:45:57.749 INFO:teuthology.orchestra.run.smithi070.stdout:} 2023-09-23T22:45:57.749 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:57 smithi026 ceph-mon[223014]: pgmap v1701: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:57.750 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:57 smithi026 ceph-mon[215775]: pgmap v1701: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:57.847 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:57 smithi070 ceph-mon[192652]: pgmap v1701: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:58.477 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T22:45:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:58 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/4260907502' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:45:58.624 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:58 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:45:58.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:58 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/4260907502' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:45:58.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:58 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:45:58.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:58 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/4260907502' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:45:58.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:58 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T22:45:59.165 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T22:45:59.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[215775]: pgmap v1702: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:59.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[215775]: failed to encode map e669 with expected crc 2023-09-23T22:45:59.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[215775]: failed to encode map e669 with expected crc 2023-09-23T22:45:59.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[215775]: failed to encode map e669 with expected crc 2023-09-23T22:45:59.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:45:59.473 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[215775]: osdmap e669: 8 total, 8 up, 8 in 2023-09-23T22:45:59.537 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:45:59.735 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:45:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:45:59] "GET /metrics HTTP/1.1" 200 34107 "" "Prometheus/2.43.0" 2023-09-23T22:45:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[223014]: pgmap v1702: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[223014]: failed to encode map e669 with expected crc 2023-09-23T22:45:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[223014]: failed to encode map e669 with expected crc 2023-09-23T22:45:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[223014]: failed to encode map e669 with expected crc 2023-09-23T22:45:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:45:59.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:45:59 smithi026 ceph-mon[223014]: osdmap e669: 8 total, 8 up, 8 in 2023-09-23T22:45:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:59 smithi070 ceph-mon[192652]: pgmap v1702: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:45:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:59 smithi070 ceph-mon[192652]: failed to encode map e669 with expected crc 2023-09-23T22:45:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:59 smithi070 ceph-mon[192652]: failed to encode map e669 with expected crc 2023-09-23T22:45:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:59 smithi070 ceph-mon[192652]: failed to encode map e669 with expected crc 2023-09-23T22:45:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:59 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T22:45:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:45:59 smithi070 ceph-mon[192652]: osdmap e669: 8 total, 8 up, 8 in 2023-09-23T22:46:00.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:00 smithi026 ceph-mon[215775]: failed to encode map e670 with expected crc 2023-09-23T22:46:00.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:00 smithi026 ceph-mon[215775]: failed to encode map e670 with expected crc 2023-09-23T22:46:00.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:00 smithi026 ceph-mon[215775]: failed to encode map e670 with expected crc 2023-09-23T22:46:00.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:00 smithi026 ceph-mon[215775]: osdmap e670: 8 total, 8 up, 8 in 2023-09-23T22:46:00.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:00 smithi026 ceph-mon[223014]: failed to encode map e670 with expected crc 2023-09-23T22:46:00.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:00 smithi026 ceph-mon[223014]: failed to encode map e670 with expected crc 2023-09-23T22:46:00.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:00 smithi026 ceph-mon[223014]: failed to encode map e670 with expected crc 2023-09-23T22:46:00.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:00 smithi026 ceph-mon[223014]: osdmap e670: 8 total, 8 up, 8 in 2023-09-23T22:46:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:00 smithi070 ceph-mon[192652]: failed to encode map e670 with expected crc 2023-09-23T22:46:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:00 smithi070 ceph-mon[192652]: failed to encode map e670 with expected crc 2023-09-23T22:46:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:00 smithi070 ceph-mon[192652]: failed to encode map e670 with expected crc 2023-09-23T22:46:00.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:00 smithi070 ceph-mon[192652]: osdmap e670: 8 total, 8 up, 8 in 2023-09-23T22:46:01.493 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T22:46:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:01 smithi070 ceph-mon[192652]: pgmap v1705: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/670716786' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:46:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:01 smithi026 ceph-mon[223014]: pgmap v1705: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/670716786' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:46:01.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:01 smithi026 ceph-mon[215775]: pgmap v1705: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:01.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/670716786' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:46:02.173 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T22:46:02.546 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:02.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[215775]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[215775]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/670716786' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:46:02.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[215775]: osdmap e671: 8 total, 8 up, 8 in 2023-09-23T22:46:02.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[215775]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[215775]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.764 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[223014]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[223014]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/670716786' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:46:02.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[223014]: osdmap e671: 8 total, 8 up, 8 in 2023-09-23T22:46:02.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[223014]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:02 smithi026 ceph-mon[223014]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:02 smithi070 ceph-mon[192652]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:02 smithi070 ceph-mon[192652]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/670716786' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:46:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:02 smithi070 ceph-mon[192652]: osdmap e671: 8 total, 8 up, 8 in 2023-09-23T22:46:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:02 smithi070 ceph-mon[192652]: failed to encode map e671 with expected crc 2023-09-23T22:46:02.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:02 smithi070 ceph-mon[192652]: failed to encode map e671 with expected crc 2023-09-23T22:46:03.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[215775]: pgmap v1707: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:03.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[215775]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[215775]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.835 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[215775]: osdmap e672: 8 total, 8 up, 8 in 2023-09-23T22:46:03.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[215775]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.836 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[215775]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[223014]: pgmap v1707: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:03.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[223014]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[223014]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[223014]: osdmap e672: 8 total, 8 up, 8 in 2023-09-23T22:46:03.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[223014]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.836 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:03 smithi026 ceph-mon[223014]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:03 smithi070 ceph-mon[192652]: pgmap v1707: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:03 smithi070 ceph-mon[192652]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:03 smithi070 ceph-mon[192652]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:03 smithi070 ceph-mon[192652]: osdmap e672: 8 total, 8 up, 8 in 2023-09-23T22:46:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:03 smithi070 ceph-mon[192652]: failed to encode map e672 with expected crc 2023-09-23T22:46:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:03 smithi070 ceph-mon[192652]: failed to encode map e672 with expected crc 2023-09-23T22:46:04.504 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T22:46:04.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:04 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1103034061' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:46:04.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:04 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1103034061' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:46:04.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:04 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1103034061' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T22:46:05.170 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T22:46:05.543 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:05.752 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[223014]: pgmap v1709: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:05.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[223014]: failed to encode map e673 with expected crc 2023-09-23T22:46:05.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[223014]: failed to encode map e673 with expected crc 2023-09-23T22:46:05.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1103034061' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:46:05.753 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[223014]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T22:46:05.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[215775]: pgmap v1709: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:05.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[215775]: failed to encode map e673 with expected crc 2023-09-23T22:46:05.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[215775]: failed to encode map e673 with expected crc 2023-09-23T22:46:05.753 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1103034061' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:46:05.754 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:05 smithi026 ceph-mon[215775]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T22:46:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:05 smithi070 ceph-mon[192652]: pgmap v1709: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:05 smithi070 ceph-mon[192652]: failed to encode map e673 with expected crc 2023-09-23T22:46:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:05 smithi070 ceph-mon[192652]: failed to encode map e673 with expected crc 2023-09-23T22:46:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:05 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1103034061' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T22:46:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:05 smithi070 ceph-mon[192652]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T22:46:07.509 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T22:46:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:07 smithi070 ceph-mon[192652]: pgmap v1711: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:07 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3957755642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:46:07.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:07 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:46:07.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:07 smithi026 ceph-mon[215775]: pgmap v1711: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:07.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:07 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3957755642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:46:07.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:07 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:46:07.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:07 smithi026 ceph-mon[223014]: pgmap v1711: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:07.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:07 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3957755642' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:46:07.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:07 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:46:08.199 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T22:46:08.569 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:08.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:08 smithi026 ceph-mon[215775]: failed to encode map e674 with expected crc 2023-09-23T22:46:08.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:08 smithi026 ceph-mon[215775]: failed to encode map e674 with expected crc 2023-09-23T22:46:08.764 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:08 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:46:08.765 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:08 smithi026 ceph-mon[215775]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T22:46:08.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:08 smithi026 ceph-mon[223014]: failed to encode map e674 with expected crc 2023-09-23T22:46:08.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:08 smithi026 ceph-mon[223014]: failed to encode map e674 with expected crc 2023-09-23T22:46:08.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:08 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:46:08.765 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:08 smithi026 ceph-mon[223014]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T22:46:08.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:08 smithi070 ceph-mon[192652]: failed to encode map e674 with expected crc 2023-09-23T22:46:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:08 smithi070 ceph-mon[192652]: failed to encode map e674 with expected crc 2023-09-23T22:46:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:08 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:46:08.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:08 smithi070 ceph-mon[192652]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T22:46:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:09 smithi070 ceph-mon[192652]: pgmap v1713: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-09-23T22:46:09.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:09 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:09.949 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:46:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:46:09] "GET /metrics HTTP/1.1" 200 34097 "" "Prometheus/2.43.0" 2023-09-23T22:46:09.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:09 smithi026 ceph-mon[215775]: pgmap v1713: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-09-23T22:46:09.949 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:09 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:09.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:09 smithi026 ceph-mon[223014]: pgmap v1713: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-09-23T22:46:09.950 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:09 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:10.523 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T22:46:10.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:10 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2592975593' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:46:10.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:10 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2592975593' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:46:10.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:10 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2592975593' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T22:46:11.232 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T22:46:11.606 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:11.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[215775]: pgmap v1714: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 75 B/s, 0 objects/s recovering 2023-09-23T22:46:11.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[215775]: failed to encode map e675 with expected crc 2023-09-23T22:46:11.798 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[215775]: failed to encode map e675 with expected crc 2023-09-23T22:46:11.799 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[215775]: failed to encode map e675 with expected crc 2023-09-23T22:46:11.801 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2592975593' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:46:11.802 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[215775]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T22:46:11.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[223014]: pgmap v1714: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 75 B/s, 0 objects/s recovering 2023-09-23T22:46:11.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[223014]: failed to encode map e675 with expected crc 2023-09-23T22:46:11.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[223014]: failed to encode map e675 with expected crc 2023-09-23T22:46:11.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[223014]: failed to encode map e675 with expected crc 2023-09-23T22:46:11.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2592975593' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:46:11.802 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:11 smithi026 ceph-mon[223014]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T22:46:11.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:11 smithi070 ceph-mon[192652]: pgmap v1714: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 75 B/s, 0 objects/s recovering 2023-09-23T22:46:11.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:11 smithi070 ceph-mon[192652]: failed to encode map e675 with expected crc 2023-09-23T22:46:11.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:11 smithi070 ceph-mon[192652]: failed to encode map e675 with expected crc 2023-09-23T22:46:11.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:11 smithi070 ceph-mon[192652]: failed to encode map e675 with expected crc 2023-09-23T22:46:11.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:11 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2592975593' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T22:46:11.875 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:11 smithi070 ceph-mon[192652]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T22:46:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:12 smithi070 ceph-mon[192652]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:12 smithi070 ceph-mon[192652]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:12 smithi070 ceph-mon[192652]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:12 smithi070 ceph-mon[192652]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T22:46:12.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:12 smithi070 ceph-mon[192652]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.966 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[215775]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[215775]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[215775]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[215775]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T22:46:12.967 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[215775]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.967 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[223014]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[223014]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[223014]: failed to encode map e676 with expected crc 2023-09-23T22:46:12.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[223014]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T22:46:12.968 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:12 smithi026 ceph-mon[223014]: failed to encode map e676 with expected crc 2023-09-23T22:46:13.537 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T22:46:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:13 smithi070 ceph-mon[192652]: pgmap v1717: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-09-23T22:46:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:13 smithi070 ceph-mon[192652]: failed to encode map e676 with expected crc 2023-09-23T22:46:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:13 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2866330596' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:13.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:13 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:13.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:13 smithi026 ceph-mon[215775]: pgmap v1717: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-09-23T22:46:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:13 smithi026 ceph-mon[215775]: failed to encode map e676 with expected crc 2023-09-23T22:46:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:13 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2866330596' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:13.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:13 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:13 smithi026 ceph-mon[223014]: pgmap v1717: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 92 B/s, 0 objects/s recovering 2023-09-23T22:46:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:13 smithi026 ceph-mon[223014]: failed to encode map e676 with expected crc 2023-09-23T22:46:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:13 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2866330596' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:13.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:13 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:14.198 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T22:46:14.562 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:14.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:14 smithi026 ceph-mon[215775]: failed to encode map e677 with expected crc 2023-09-23T22:46:14.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:14 smithi026 ceph-mon[215775]: failed to encode map e677 with expected crc 2023-09-23T22:46:14.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:14 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:46:14.806 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:14 smithi026 ceph-mon[215775]: osdmap e677: 8 total, 8 up, 8 in 2023-09-23T22:46:14.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:14 smithi026 ceph-mon[223014]: failed to encode map e677 with expected crc 2023-09-23T22:46:14.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:14 smithi026 ceph-mon[223014]: failed to encode map e677 with expected crc 2023-09-23T22:46:14.806 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:14 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:46:14.807 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:14 smithi026 ceph-mon[223014]: osdmap e677: 8 total, 8 up, 8 in 2023-09-23T22:46:14.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:14 smithi070 ceph-mon[192652]: failed to encode map e677 with expected crc 2023-09-23T22:46:14.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:14 smithi070 ceph-mon[192652]: failed to encode map e677 with expected crc 2023-09-23T22:46:14.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:14 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:46:14.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:14 smithi070 ceph-mon[192652]: osdmap e677: 8 total, 8 up, 8 in 2023-09-23T22:46:15.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:15 smithi070 ceph-mon[192652]: pgmap v1719: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T22:46:15.899 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:15 smithi026 ceph-mon[223014]: pgmap v1719: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T22:46:15.900 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:15 smithi026 ceph-mon[215775]: pgmap v1719: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T22:46:16.554 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T22:46:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:16 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1110951136' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:16.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:16 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:16 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1110951136' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:16.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:16 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:16 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1110951136' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:16.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:16 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T22:46:17.247 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T22:46:17.613 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:17.814 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[223014]: pgmap v1720: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T22:46:17.814 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[223014]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.815 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[223014]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.815 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:46:17.815 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[223014]: osdmap e678: 8 total, 8 up, 8 in 2023-09-23T22:46:17.815 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[223014]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.815 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[223014]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[215775]: pgmap v1720: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T22:46:17.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[215775]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[215775]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:46:17.816 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[215775]: osdmap e678: 8 total, 8 up, 8 in 2023-09-23T22:46:17.816 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[215775]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.816 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:17 smithi026 ceph-mon[215775]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:17 smithi070 ceph-mon[192652]: pgmap v1720: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T22:46:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:17 smithi070 ceph-mon[192652]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:17 smithi070 ceph-mon[192652]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:17 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T22:46:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:17 smithi070 ceph-mon[192652]: osdmap e678: 8 total, 8 up, 8 in 2023-09-23T22:46:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:17 smithi070 ceph-mon[192652]: failed to encode map e678 with expected crc 2023-09-23T22:46:17.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:17 smithi070 ceph-mon[192652]: failed to encode map e678 with expected crc 2023-09-23T22:46:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:18 smithi070 ceph-mon[192652]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:18 smithi070 ceph-mon[192652]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:18 smithi070 ceph-mon[192652]: osdmap e679: 8 total, 8 up, 8 in 2023-09-23T22:46:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:18 smithi070 ceph-mon[192652]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:18 smithi070 ceph-mon[192652]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:18 smithi070 ceph-mon[192652]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:18 smithi070 ceph-mon[192652]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[215775]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[215775]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[215775]: osdmap e679: 8 total, 8 up, 8 in 2023-09-23T22:46:18.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[215775]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[215775]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[215775]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.924 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[215775]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[223014]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[223014]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[223014]: osdmap e679: 8 total, 8 up, 8 in 2023-09-23T22:46:18.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[223014]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[223014]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[223014]: failed to encode map e679 with expected crc 2023-09-23T22:46:18.925 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:18 smithi026 ceph-mon[223014]: failed to encode map e679 with expected crc 2023-09-23T22:46:19.561 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T22:46:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:19 smithi070 ceph-mon[192652]: pgmap v1723: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T22:46:19.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:19 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3352170794' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:46:19.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:19 smithi026 ceph-mon[223014]: pgmap v1723: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T22:46:19.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:19 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3352170794' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:46:19.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:19 smithi026 ceph-mon[215775]: pgmap v1723: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 788 B/s, 0 objects/s recovering 2023-09-23T22:46:19.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:19 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3352170794' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:46:19.986 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:46:19 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:46:19] "GET /metrics HTTP/1.1" 200 34097 "" "Prometheus/2.43.0" 2023-09-23T22:46:20.256 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T22:46:20.564 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:20 smithi026 ceph-mon[223014]: failed to encode map e680 with expected crc 2023-09-23T22:46:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:20 smithi026 ceph-mon[223014]: failed to encode map e680 with expected crc 2023-09-23T22:46:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:20 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3352170794' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:46:20.565 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:20 smithi026 ceph-mon[223014]: osdmap e680: 8 total, 8 up, 8 in 2023-09-23T22:46:20.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:20 smithi026 ceph-mon[215775]: failed to encode map e680 with expected crc 2023-09-23T22:46:20.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:20 smithi026 ceph-mon[215775]: failed to encode map e680 with expected crc 2023-09-23T22:46:20.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:20 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3352170794' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:46:20.565 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:20 smithi026 ceph-mon[215775]: osdmap e680: 8 total, 8 up, 8 in 2023-09-23T22:46:20.635 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:20 smithi070 ceph-mon[192652]: failed to encode map e680 with expected crc 2023-09-23T22:46:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:20 smithi070 ceph-mon[192652]: failed to encode map e680 with expected crc 2023-09-23T22:46:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:20 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3352170794' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:46:20.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:20 smithi070 ceph-mon[192652]: osdmap e680: 8 total, 8 up, 8 in 2023-09-23T22:46:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:21 smithi070 ceph-mon[192652]: pgmap v1725: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:21 smithi070 ceph-mon[192652]: failed to encode map e680 with expected crc 2023-09-23T22:46:21.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:21 smithi070 ceph-mon[192652]: failed to encode map e680 with expected crc 2023-09-23T22:46:21.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:21 smithi026 ceph-mon[223014]: pgmap v1725: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:21.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:21 smithi026 ceph-mon[223014]: failed to encode map e680 with expected crc 2023-09-23T22:46:21.915 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:21 smithi026 ceph-mon[223014]: failed to encode map e680 with expected crc 2023-09-23T22:46:21.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:21 smithi026 ceph-mon[215775]: pgmap v1725: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:21.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:21 smithi026 ceph-mon[215775]: failed to encode map e680 with expected crc 2023-09-23T22:46:21.916 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:21 smithi026 ceph-mon[215775]: failed to encode map e680 with expected crc 2023-09-23T22:46:22.572 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T22:46:22.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:22 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2545057648' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:46:22.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:22 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2545057648' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:46:22.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:22 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2545057648' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T22:46:23.295 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T22:46:23.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[223014]: pgmap v1726: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 682 KiB/s, 4 objects/s recovering 2023-09-23T22:46:23.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[223014]: failed to encode map e681 with expected crc 2023-09-23T22:46:23.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[223014]: failed to encode map e681 with expected crc 2023-09-23T22:46:23.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[223014]: failed to encode map e681 with expected crc 2023-09-23T22:46:23.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2545057648' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:46:23.575 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[223014]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T22:46:23.663 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:23.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[215775]: pgmap v1726: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 682 KiB/s, 4 objects/s recovering 2023-09-23T22:46:23.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[215775]: failed to encode map e681 with expected crc 2023-09-23T22:46:23.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[215775]: failed to encode map e681 with expected crc 2023-09-23T22:46:23.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[215775]: failed to encode map e681 with expected crc 2023-09-23T22:46:23.863 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2545057648' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:46:23.864 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:23 smithi026 ceph-mon[215775]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T22:46:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:23 smithi070 ceph-mon[192652]: pgmap v1726: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 682 KiB/s, 4 objects/s recovering 2023-09-23T22:46:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:23 smithi070 ceph-mon[192652]: failed to encode map e681 with expected crc 2023-09-23T22:46:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:23 smithi070 ceph-mon[192652]: failed to encode map e681 with expected crc 2023-09-23T22:46:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:23 smithi070 ceph-mon[192652]: failed to encode map e681 with expected crc 2023-09-23T22:46:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:23 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2545057648' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T22:46:23.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:23 smithi070 ceph-mon[192652]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T22:46:24.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:24 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:24.973 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:24 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:24.974 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:24 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:25.589 INFO:teuthology.orchestra.run.smithi026.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T22:46:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:25 smithi070 ceph-mon[192652]: pgmap v1728: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 657 KiB/s, 4 objects/s recovering 2023-09-23T22:46:25.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:25 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1555903377' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:46:25.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:25 smithi026 ceph-mon[215775]: pgmap v1728: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 657 KiB/s, 4 objects/s recovering 2023-09-23T22:46:25.985 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:25 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1555903377' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:46:25.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:25 smithi026 ceph-mon[223014]: pgmap v1728: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 657 KiB/s, 4 objects/s recovering 2023-09-23T22:46:25.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:25 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1555903377' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:46:26.230 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T22:46:26.597 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:26.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[215775]: failed to encode map e682 with expected crc 2023-09-23T22:46:26.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[215775]: failed to encode map e682 with expected crc 2023-09-23T22:46:26.844 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[215775]: failed to encode map e682 with expected crc 2023-09-23T22:46:26.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/1555903377' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:46:26.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[215775]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T22:46:26.845 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[215775]: pgmap v1730: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 657 KiB/s, 4 objects/s recovering 2023-09-23T22:46:26.845 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[223014]: failed to encode map e682 with expected crc 2023-09-23T22:46:26.845 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[223014]: failed to encode map e682 with expected crc 2023-09-23T22:46:26.845 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[223014]: failed to encode map e682 with expected crc 2023-09-23T22:46:26.845 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/1555903377' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:46:26.845 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[223014]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T22:46:26.845 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:26 smithi026 ceph-mon[223014]: pgmap v1730: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 657 KiB/s, 4 objects/s recovering 2023-09-23T22:46:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:26 smithi070 ceph-mon[192652]: failed to encode map e682 with expected crc 2023-09-23T22:46:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:26 smithi070 ceph-mon[192652]: failed to encode map e682 with expected crc 2023-09-23T22:46:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:26 smithi070 ceph-mon[192652]: failed to encode map e682 with expected crc 2023-09-23T22:46:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:26 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/1555903377' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:46:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:26 smithi070 ceph-mon[192652]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T22:46:26.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:26 smithi070 ceph-mon[192652]: pgmap v1730: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 657 KiB/s, 4 objects/s recovering 2023-09-23T22:46:28.852 INFO:teuthology.orchestra.run.smithi026.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T22:46:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:28 smithi070 ceph-mon[192652]: pgmap v1731: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 512 KiB/s, 3 objects/s recovering 2023-09-23T22:46:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:28 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2227516664' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:46:29.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:28 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:46:29.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:28 smithi026 ceph-mon[223014]: pgmap v1731: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 512 KiB/s, 3 objects/s recovering 2023-09-23T22:46:29.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:28 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2227516664' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:46:29.235 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:28 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:46:29.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:28 smithi026 ceph-mon[215775]: pgmap v1731: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 512 KiB/s, 3 objects/s recovering 2023-09-23T22:46:29.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:28 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2227516664' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:46:29.236 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:28 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T22:46:29.529 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T22:46:29.529 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T22:46:29.530 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T22:46:29.530 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T22:46:29.530 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T22:46:29.530 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:46:29.850 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:46:29 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:46:29] "GET /metrics HTTP/1.1" 200 34093 "" "Prometheus/2.43.0" 2023-09-23T22:46:29.904 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:30.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:29 smithi026 ceph-mon[215775]: failed to encode map e683 with expected crc 2023-09-23T22:46:30.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:29 smithi026 ceph-mon[215775]: failed to encode map e683 with expected crc 2023-09-23T22:46:30.111 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:29 smithi026 ceph-mon[215775]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:46:30.112 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:29 smithi026 ceph-mon[215775]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T22:46:30.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:29 smithi026 ceph-mon[223014]: failed to encode map e683 with expected crc 2023-09-23T22:46:30.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:29 smithi026 ceph-mon[223014]: failed to encode map e683 with expected crc 2023-09-23T22:46:30.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:29 smithi026 ceph-mon[223014]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:46:30.112 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:29 smithi026 ceph-mon[223014]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T22:46:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:29 smithi070 ceph-mon[192652]: failed to encode map e683 with expected crc 2023-09-23T22:46:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:29 smithi070 ceph-mon[192652]: failed to encode map e683 with expected crc 2023-09-23T22:46:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:29 smithi070 ceph-mon[192652]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T22:46:30.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:29 smithi070 ceph-mon[192652]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T22:46:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:30 smithi070 ceph-mon[192652]: pgmap v1733: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:30 smithi070 ceph-mon[192652]: failed to encode map e683 with expected crc 2023-09-23T22:46:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:30 smithi070 ceph-mon[192652]: failed to encode map e683 with expected crc 2023-09-23T22:46:31.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:30 smithi070 ceph-mon[192652]: failed to encode map e683 with expected crc 2023-09-23T22:46:31.232 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:30 smithi026 ceph-mon[215775]: pgmap v1733: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:31.232 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:30 smithi026 ceph-mon[215775]: failed to encode map e683 with expected crc 2023-09-23T22:46:31.232 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:30 smithi026 ceph-mon[215775]: failed to encode map e683 with expected crc 2023-09-23T22:46:31.232 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:30 smithi026 ceph-mon[215775]: failed to encode map e683 with expected crc 2023-09-23T22:46:31.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:30 smithi026 ceph-mon[223014]: pgmap v1733: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:31.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:30 smithi026 ceph-mon[223014]: failed to encode map e683 with expected crc 2023-09-23T22:46:31.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:30 smithi026 ceph-mon[223014]: failed to encode map e683 with expected crc 2023-09-23T22:46:31.232 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:30 smithi026 ceph-mon[223014]: failed to encode map e683 with expected crc 2023-09-23T22:46:31.626 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:46:31.626 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":683,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:46:28.839877+0000","last_up_change":"2023-09-23T22:42:55.450523+0000","last_in_change":"2023-09-23T22:45:24.515300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"661","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"557","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:04:04.880309+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"636","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":636,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:04:53.313440+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:04:53.801900+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":671,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6803","nonce":507489654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6805","nonce":507489654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6809","nonce":507489654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6807","nonce":507489654}]},"public_addr":"172.21.15.26:6803/507489654","cluster_addr":"172.21.15.26:6805/507489654","heartbeat_back_addr":"172.21.15.26:6809/507489654","heartbeat_front_addr":"172.21.15.26:6807/507489654","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":98,"last_clean_end":487,"up_from":495,"up_thru":678,"down_at":488,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":1220464431},{"type":"v1","addr":"172.21.15.26:6811","nonce":1220464431}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":1220464431},{"type":"v1","addr":"172.21.15.26:6813","nonce":1220464431}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":1220464431},{"type":"v1","addr":"172.21.15.26:6817","nonce":1220464431}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":1220464431},{"type":"v1","addr":"172.21.15.26:6815","nonce":1220464431}]},"public_addr":"172.21.15.26:6811/1220464431","cluster_addr":"172.21.15.26:6813/1220464431","heartbeat_back_addr":"172.21.15.26:6817/1220464431","heartbeat_front_addr":"172.21.15.26:6815/1220464431","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":103,"last_clean_end":333,"up_from":349,"up_thru":671,"down_at":334,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":160909150},{"type":"v1","addr":"172.21.15.26:6819","nonce":160909150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":160909150},{"type":"v1","addr":"172.21.15.26:6821","nonce":160909150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":160909150},{"type":"v1","addr":"172.21.15.26:6825","nonce":160909150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":160909150},{"type":"v1","addr":"172.21.15.26:6823","nonce":160909150}]},"public_addr":"172.21.15.26:6819/160909150","cluster_addr":"172.21.15.26:6821/160909150","heartbeat_back_addr":"172.21.15.26:6825/160909150","heartbeat_front_addr":"172.21.15.26:6823/160909150","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":108,"last_clean_end":516,"up_from":584,"up_thru":663,"down_at":517,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":2763378290},{"type":"v1","addr":"172.21.15.26:6827","nonce":2763378290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":2763378290},{"type":"v1","addr":"172.21.15.26:6829","nonce":2763378290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":2763378290},{"type":"v1","addr":"172.21.15.26:6833","nonce":2763378290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":2763378290},{"type":"v1","addr":"172.21.15.26:6831","nonce":2763378290}]},"public_addr":"172.21.15.26:6827/2763378290","cluster_addr":"172.21.15.26:6829/2763378290","heartbeat_back_addr":"172.21.15.26:6833/2763378290","heartbeat_front_addr":"172.21.15.26:6831/2763378290","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":458,"up_from":465,"up_thru":675,"down_at":459,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":3366415869},{"type":"v1","addr":"172.21.15.70:6801","nonce":3366415869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":3366415869},{"type":"v1","addr":"172.21.15.70:6803","nonce":3366415869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":3366415869},{"type":"v1","addr":"172.21.15.70:6807","nonce":3366415869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":3366415869},{"type":"v1","addr":"172.21.15.70:6805","nonce":3366415869}]},"public_addr":"172.21.15.70:6801/3366415869","cluster_addr":"172.21.15.70:6803/3366415869","heartbeat_back_addr":"172.21.15.70:6807/3366415869","heartbeat_front_addr":"172.21.15.70:6805/3366415869","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":46,"last_clean_end":512,"up_from":592,"up_thru":663,"down_at":513,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":1249954970},{"type":"v1","addr":"172.21.15.70:6817","nonce":1249954970}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":1249954970},{"type":"v1","addr":"172.21.15.70:6819","nonce":1249954970}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":1249954970},{"type":"v1","addr":"172.21.15.70:6823","nonce":1249954970}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":1249954970},{"type":"v1","addr":"172.21.15.70:6821","nonce":1249954970}]},"public_addr":"172.21.15.70:6817/1249954970","cluster_addr":"172.21.15.70:6819/1249954970","heartbeat_back_addr":"172.21.15.70:6823/1249954970","heartbeat_front_addr":"172.21.15.70:6821/1249954970","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":669,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":487},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":333},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":516},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":458},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":512},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.e","mappings":[{"from":1,"to":0}]},{"pgid":"4.10","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]},{"pgid":"4.19","mappings":[{"from":3,"to":7}]},{"pgid":"4.1a","mappings":[{"from":3,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":5},{"from":3,"to":0}]},{"pgid":"4.1e","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.26:0/575035623":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/472809172":"2023-09-24T22:08:38.898443+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3419561437":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/771693589":"2023-09-24T22:07:17.644537+0000","172.21.15.70:6835/1647959066":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/2583870620":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/1692711239":"2023-09-24T22:07:17.644537+0000","172.21.15.70:0/202130526":"2023-09-24T22:07:55.974675+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3618866901":"2023-09-24T22:07:17.644537+0000","172.21.15.70:0/3676980533":"2023-09-24T22:07:55.974675+0000","172.21.15.70:0/1888936544":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/574101533":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/4237902526":"2023-09-24T22:08:38.898443+0000","172.21.15.70:6834/1647959066":"2023-09-24T22:07:55.974675+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.70:0/562032249":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/1211771677":"2023-09-24T22:08:38.898443+0000","172.21.15.26:6801/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3996051568":"2023-09-24T22:07:17.644537+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/1887156644":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/577898280":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6800/2951021001":"2023-09-24T22:08:38.898443+0000","172.21.15.70:0/3435183798":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/3350176691":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6800/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3418308803":"2023-09-24T22:07:17.644537+0000","172.21.15.26:6801/3418308803":"2023-09-24T22:07:17.644537+0000","172.21.15.70:0/1453808871":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/2951021001":"2023-09-24T22:08:38.898443+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:46:31.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:31 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2557313736' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:46:31.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:31 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2557313736' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:46:32.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:31 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2557313736' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:46:32.396 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T22:46:32.397 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T22:46:32.767 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:33.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:32 smithi070 ceph-mon[192652]: pgmap v1734: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:33.171 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:32 smithi026 ceph-mon[215775]: pgmap v1734: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:33.171 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:32 smithi026 ceph-mon[223014]: pgmap v1734: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:34.467 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:46:34.467 INFO:teuthology.orchestra.run.smithi026.stdout:{"epoch":683,"fsid":"2a3049e2-5a5c-11ee-8db4-212e2dc638e7","created":"2023-09-23T21:58:20.282373+0000","modified":"2023-09-23T22:46:28.839877+0000","last_up_change":"2023-09-23T22:42:55.450523+0000","last_in_change":"2023-09-23T22:45:24.515300+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":27,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T22:01:25.159293+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"661","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"557","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T22:04:04.880309+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"636","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":636,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T22:04:53.313440+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T22:04:53.801900+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"fdf0a195-9329-458c-b103-38db9a4408a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":671,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6802","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6803","nonce":507489654}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6804","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6805","nonce":507489654}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6808","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6809","nonce":507489654}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6806","nonce":507489654},{"type":"v1","addr":"172.21.15.26:6807","nonce":507489654}]},"public_addr":"172.21.15.26:6803/507489654","cluster_addr":"172.21.15.26:6805/507489654","heartbeat_back_addr":"172.21.15.26:6809/507489654","heartbeat_front_addr":"172.21.15.26:6807/507489654","state":["exists","up"]},{"osd":1,"uuid":"9c0507d8-f63e-47b2-841d-394196258d6b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":98,"last_clean_end":487,"up_from":495,"up_thru":678,"down_at":488,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6810","nonce":1220464431},{"type":"v1","addr":"172.21.15.26:6811","nonce":1220464431}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6812","nonce":1220464431},{"type":"v1","addr":"172.21.15.26:6813","nonce":1220464431}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6816","nonce":1220464431},{"type":"v1","addr":"172.21.15.26:6817","nonce":1220464431}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6814","nonce":1220464431},{"type":"v1","addr":"172.21.15.26:6815","nonce":1220464431}]},"public_addr":"172.21.15.26:6811/1220464431","cluster_addr":"172.21.15.26:6813/1220464431","heartbeat_back_addr":"172.21.15.26:6817/1220464431","heartbeat_front_addr":"172.21.15.26:6815/1220464431","state":["exists","up"]},{"osd":2,"uuid":"4b241a88-8860-4616-8af5-383823ce290f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":103,"last_clean_end":333,"up_from":349,"up_thru":671,"down_at":334,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6818","nonce":160909150},{"type":"v1","addr":"172.21.15.26:6819","nonce":160909150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6820","nonce":160909150},{"type":"v1","addr":"172.21.15.26:6821","nonce":160909150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6824","nonce":160909150},{"type":"v1","addr":"172.21.15.26:6825","nonce":160909150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6822","nonce":160909150},{"type":"v1","addr":"172.21.15.26:6823","nonce":160909150}]},"public_addr":"172.21.15.26:6819/160909150","cluster_addr":"172.21.15.26:6821/160909150","heartbeat_back_addr":"172.21.15.26:6825/160909150","heartbeat_front_addr":"172.21.15.26:6823/160909150","state":["exists","up"]},{"osd":3,"uuid":"8f0b5ef0-55d0-45ff-8e3a-a35b3f9220ea","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":108,"last_clean_end":516,"up_from":584,"up_thru":663,"down_at":517,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6826","nonce":2763378290},{"type":"v1","addr":"172.21.15.26:6827","nonce":2763378290}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6828","nonce":2763378290},{"type":"v1","addr":"172.21.15.26:6829","nonce":2763378290}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6832","nonce":2763378290},{"type":"v1","addr":"172.21.15.26:6833","nonce":2763378290}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.26:6830","nonce":2763378290},{"type":"v1","addr":"172.21.15.26:6831","nonce":2763378290}]},"public_addr":"172.21.15.26:6827/2763378290","cluster_addr":"172.21.15.26:6829/2763378290","heartbeat_back_addr":"172.21.15.26:6833/2763378290","heartbeat_front_addr":"172.21.15.26:6831/2763378290","state":["exists","up"]},{"osd":4,"uuid":"6d4a4d17-075d-4538-9109-7a1dde1b422d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":30,"last_clean_end":458,"up_from":465,"up_thru":675,"down_at":459,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6800","nonce":3366415869},{"type":"v1","addr":"172.21.15.70:6801","nonce":3366415869}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6802","nonce":3366415869},{"type":"v1","addr":"172.21.15.70:6803","nonce":3366415869}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6806","nonce":3366415869},{"type":"v1","addr":"172.21.15.70:6807","nonce":3366415869}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6804","nonce":3366415869},{"type":"v1","addr":"172.21.15.70:6805","nonce":3366415869}]},"public_addr":"172.21.15.70:6801/3366415869","cluster_addr":"172.21.15.70:6803/3366415869","heartbeat_back_addr":"172.21.15.70:6807/3366415869","heartbeat_front_addr":"172.21.15.70:6805/3366415869","state":["exists","up"]},{"osd":5,"uuid":"b25ee343-9def-42dc-8b45-a3c85bf14805","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":663,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6808","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6809","nonce":4236773994}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6810","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6811","nonce":4236773994}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6814","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6815","nonce":4236773994}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6812","nonce":4236773994},{"type":"v1","addr":"172.21.15.70:6813","nonce":4236773994}]},"public_addr":"172.21.15.70:6809/4236773994","cluster_addr":"172.21.15.70:6811/4236773994","heartbeat_back_addr":"172.21.15.70:6815/4236773994","heartbeat_front_addr":"172.21.15.70:6813/4236773994","state":["exists","up"]},{"osd":6,"uuid":"f7ecfbb9-ea1a-480b-b31e-114e2d7b1721","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":46,"last_clean_end":512,"up_from":592,"up_thru":663,"down_at":513,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6816","nonce":1249954970},{"type":"v1","addr":"172.21.15.70:6817","nonce":1249954970}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6818","nonce":1249954970},{"type":"v1","addr":"172.21.15.70:6819","nonce":1249954970}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6822","nonce":1249954970},{"type":"v1","addr":"172.21.15.70:6823","nonce":1249954970}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6820","nonce":1249954970},{"type":"v1","addr":"172.21.15.70:6821","nonce":1249954970}]},"public_addr":"172.21.15.70:6817/1249954970","cluster_addr":"172.21.15.70:6819/1249954970","heartbeat_back_addr":"172.21.15.70:6823/1249954970","heartbeat_front_addr":"172.21.15.70:6821/1249954970","state":["exists","up"]},{"osd":7,"uuid":"e9057258-2f16-4805-a17d-b672eacc4cb0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":54,"up_thru":669,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6824","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6825","nonce":265778624}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6826","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6827","nonce":265778624}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6830","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6831","nonce":265778624}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.70:6828","nonce":265778624},{"type":"v1","addr":"172.21.15.70:6829","nonce":265778624}]},"public_addr":"172.21.15.70:6825/265778624","cluster_addr":"172.21.15.70:6827/265778624","heartbeat_back_addr":"172.21.15.70:6831/265778624","heartbeat_front_addr":"172.21.15.70:6829/265778624","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:35.403018+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:00:58.677375+0000","dead_epoch":487},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:22.065752+0000","dead_epoch":333},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:01:46.284975+0000","dead_epoch":516},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:07.830487+0000","dead_epoch":458},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:31.223690+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:02:53.039604+0000","dead_epoch":512},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T22:03:15.927526+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.6","mappings":[{"from":1,"to":5}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.e","mappings":[{"from":1,"to":0}]},{"pgid":"4.10","mappings":[{"from":1,"to":5}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]},{"pgid":"4.19","mappings":[{"from":3,"to":7}]},{"pgid":"4.1a","mappings":[{"from":3,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":5},{"from":3,"to":0}]},{"pgid":"4.1e","mappings":[{"from":3,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.70:0/1453808871":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/575035623":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/771693589":"2023-09-24T22:07:17.644537+0000","172.21.15.70:6835/1647959066":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/2583870620":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/1692711239":"2023-09-24T22:07:17.644537+0000","172.21.15.26:0/3568300881":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3618866901":"2023-09-24T22:07:17.644537+0000","172.21.15.70:0/3676980533":"2023-09-24T22:07:55.974675+0000","172.21.15.70:0/1888936544":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/574101533":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/4237902526":"2023-09-24T22:08:38.898443+0000","172.21.15.26:6800/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.70:6834/1647959066":"2023-09-24T22:07:55.974675+0000","172.21.15.26:6800/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3419561437":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/250294092":"2023-09-24T21:58:49.560739+0000","172.21.15.70:0/202130526":"2023-09-24T22:07:55.974675+0000","172.21.15.26:6801/3973672550":"2023-09-24T21:58:49.560739+0000","172.21.15.26:0/3643922551":"2023-09-24T21:59:16.628901+0000","172.21.15.70:0/562032249":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/1211771677":"2023-09-24T22:08:38.898443+0000","172.21.15.26:6801/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3996051568":"2023-09-24T22:07:17.644537+0000","172.21.15.26:0/3678729612":"2023-09-24T21:59:16.628901+0000","172.21.15.26:0/1887156644":"2023-09-24T22:03:52.071180+0000","172.21.15.26:0/3021908149":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/2951021001":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/577898280":"2023-09-24T22:03:52.071180+0000","172.21.15.70:0/3435183798":"2023-09-24T22:07:55.974675+0000","172.21.15.26:0/3350176691":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6800/3849079523":"2023-09-24T22:03:52.071180+0000","172.21.15.26:6801/985150005":"2023-09-24T21:59:16.628901+0000","172.21.15.26:6800/3418308803":"2023-09-24T22:07:17.644537+0000","172.21.15.26:6801/3418308803":"2023-09-24T22:07:17.644537+0000","172.21.15.26:0/2705141594":"2023-09-24T21:58:49.560739+0000","172.21.15.26:6801/2951021001":"2023-09-24T22:08:38.898443+0000","172.21.15.26:0/472809172":"2023-09-24T22:08:38.898443+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T22:46:35.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T22:46:35.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T22:46:35.180 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T22:46:35.180 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T22:46:35.180 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T22:46:35.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T22:46:35.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T22:46:35.181 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T22:46:35.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:35 smithi026 ceph-mon[215775]: pgmap v1735: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:35.482 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:35 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/942344231' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:46:35.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:35 smithi026 ceph-mon[223014]: pgmap v1735: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:35.482 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:35 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/942344231' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:46:35.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:35 smithi070 ceph-mon[192652]: pgmap v1735: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:35.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:35 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/942344231' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T22:46:36.188 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:36.489 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:37.283 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:37.284 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:37.284 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:37.284 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:37.646 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:37.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:37 smithi026 ceph-mon[215775]: pgmap v1736: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:37.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:37 smithi026 ceph-mon[223014]: pgmap v1736: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:37.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:37 smithi070 ceph-mon[192652]: pgmap v1736: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:39 smithi026 ceph-mon[223014]: pgmap v1737: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:39.611 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:39 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:39 smithi026 ceph-mon[215775]: pgmap v1737: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:39.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:39 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:39.756 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:39 smithi070 ceph-mon[192652]: pgmap v1737: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:39.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:39 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:39.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:46:39 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:46:39] "GET /metrics HTTP/1.1" 200 34079 "" "Prometheus/2.43.0" 2023-09-23T22:46:41.179 INFO:teuthology.orchestra.run.smithi026.stdout:399431958945 2023-09-23T22:46:41.179 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T22:46:41.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:41 smithi026 ceph-mon[223014]: pgmap v1738: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:41.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:41 smithi026 ceph-mon[215775]: pgmap v1738: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:41.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:41 smithi070 ceph-mon[192652]: pgmap v1738: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:43.488 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:43 smithi026 ceph-mon[223014]: pgmap v1739: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:43.737 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:43 smithi026 ceph-mon[215775]: pgmap v1739: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:43.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:43 smithi070 ceph-mon[192652]: pgmap v1739: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:44.176 INFO:teuthology.orchestra.run.smithi026.stdout:1498943586509 2023-09-23T22:46:44.176 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T22:46:45.681 INFO:teuthology.orchestra.run.smithi026.stdout:2542620639279 2023-09-23T22:46:45.682 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T22:46:45.812 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:45 smithi026 ceph-mon[215775]: pgmap v1740: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:45.812 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:45 smithi026 ceph-mon[223014]: pgmap v1740: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:45.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:45 smithi070 ceph-mon[192652]: pgmap v1740: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:46.311 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:46.336 INFO:teuthology.orchestra.run.smithi026.stdout:231928234508 2023-09-23T22:46:46.337 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T22:46:47.238 INFO:teuthology.orchestra.run.smithi026.stdout:1997159792772 2023-09-23T22:46:47.238 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T22:46:47.519 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:47 smithi026 ceph-mon[223014]: pgmap v1741: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:47.519 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:47 smithi026 ceph-mon[215775]: pgmap v1741: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:47.610 INFO:teuthology.orchestra.run.smithi026.stdout:2508260900916 2023-09-23T22:46:47.610 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T22:46:47.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:47 smithi070 ceph-mon[192652]: pgmap v1741: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:48.379 INFO:teuthology.orchestra.run.smithi026.stdout:163208757781 2023-09-23T22:46:48.380 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T22:46:49.461 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:49.513 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:49.612 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:49.761 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:46:49 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:46:49] "GET /metrics HTTP/1.1" 200 34079 "" "Prometheus/2.43.0" 2023-09-23T22:46:49.761 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:49 smithi026 ceph-mon[215775]: pgmap v1742: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:49.761 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:49 smithi026 ceph-mon[223014]: pgmap v1742: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:49.806 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:49.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:49 smithi070 ceph-mon[192652]: pgmap v1742: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:50.300 INFO:teuthology.orchestra.run.smithi026.stdout:399431958947 2023-09-23T22:46:50.526 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:50.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:50 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2134289912' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:46:50.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:50 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2134289912' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:46:50.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:50 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2134289912' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T22:46:51.735 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:51 smithi026 ceph-mon[223014]: pgmap v1743: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:51.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:51 smithi026 ceph-mon[215775]: pgmap v1743: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:51.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:51 smithi070 ceph-mon[192652]: pgmap v1743: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:53.648 INFO:teuthology.orchestra.run.smithi026.stdout:2126008811631 2023-09-23T22:46:53.648 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T22:46:53.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:53 smithi026 ceph-mon[215775]: pgmap v1744: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:53.676 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:53 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:46:53.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:53 smithi026 ceph-mon[223014]: pgmap v1744: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:53.676 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:53 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:46:53.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:53 smithi070 ceph-mon[192652]: pgmap v1744: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:53.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:53 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T22:46:53.910 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:54.124 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958945 got 399431958947 for osd.0 2023-09-23T22:46:54.125 DEBUG:teuthology.parallel:result is None 2023-09-23T22:46:54.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:54 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:54.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:54 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:54.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:54 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T22:46:54.904 INFO:teuthology.orchestra.run.smithi026.stdout:1498943586511 2023-09-23T22:46:55.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:55 smithi026 ceph-mon[215775]: pgmap v1745: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:55.815 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:55 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3637557576' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:46:55.815 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:55 smithi026 ceph-mon[223014]: pgmap v1745: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:55.816 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:55 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3637557576' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:46:55.816 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1498943586509 got 1498943586511 for osd.2 2023-09-23T22:46:55.816 DEBUG:teuthology.parallel:result is None 2023-09-23T22:46:55.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:55 smithi070 ceph-mon[192652]: pgmap v1745: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:55.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:55 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3637557576' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T22:46:55.951 INFO:teuthology.orchestra.run.smithi026.stdout:1997159792774 2023-09-23T22:46:56.198 INFO:teuthology.orchestra.run.smithi026.stdout:231928234510 2023-09-23T22:46:56.595 INFO:teuthology.orchestra.run.smithi026.stdout:2508260900919 2023-09-23T22:46:56.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2855547676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:46:56.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:56 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/2165181010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:46:56.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:56 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:46:56.621 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:56 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:46:56.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2855547676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:46:56.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:56 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/2165181010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:46:56.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:56 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:46:56.622 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:56 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:46:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2855547676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T22:46:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:56 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/2165181010' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T22:46:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:56 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:46:56.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:56 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:46:57.233 INFO:teuthology.orchestra.run.smithi026.stdout:2542620639282 2023-09-23T22:46:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:57 smithi070 ceph-mon[192652]: pgmap v1746: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/33182208' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:46:57.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:57 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/357756627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:46:57.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:57 smithi026 ceph-mon[223014]: pgmap v1746: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/33182208' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:46:57.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:57 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/357756627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:46:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:57 smithi026 ceph-mon[215775]: pgmap v1746: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/33182208' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T22:46:57.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:57 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/357756627' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T22:46:58.147 INFO:tasks.cephadm.ceph_manager.ceph:need seq 231928234508 got 231928234510 for osd.7 2023-09-23T22:46:58.147 DEBUG:teuthology.parallel:result is None 2023-09-23T22:46:58.194 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1997159792772 got 1997159792774 for osd.4 2023-09-23T22:46:58.195 DEBUG:teuthology.parallel:result is None 2023-09-23T22:46:58.303 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2508260900916 got 2508260900919 for osd.3 2023-09-23T22:46:58.303 DEBUG:teuthology.parallel:result is None 2023-09-23T22:46:58.843 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2542620639279 got 2542620639282 for osd.6 2023-09-23T22:46:58.843 DEBUG:teuthology.parallel:result is None 2023-09-23T22:46:59.149 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:46:59.682 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:46:59 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:46:59] "GET /metrics HTTP/1.1" 200 34082 "" "Prometheus/2.43.0" 2023-09-23T22:46:59.683 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:46:59 smithi026 ceph-mon[215775]: pgmap v1747: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:59.683 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:46:59 smithi026 ceph-mon[223014]: pgmap v1747: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:46:59.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:46:59 smithi070 ceph-mon[192652]: pgmap v1747: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:00.903 INFO:teuthology.orchestra.run.smithi026.stdout:163208757784 2023-09-23T22:47:01.561 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757781 got 163208757784 for osd.5 2023-09-23T22:47:01.561 DEBUG:teuthology.parallel:result is None 2023-09-23T22:47:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:01 smithi070 ceph-mon[192652]: pgmap v1748: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:01.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:01 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/387116743' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:47:01.985 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:01 smithi026 ceph-mon[223014]: pgmap v1748: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:01.986 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:01 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/387116743' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:47:01.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:01 smithi026 ceph-mon[215775]: pgmap v1748: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:01.986 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:01 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/387116743' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T22:47:02.054 INFO:teuthology.orchestra.run.smithi026.stdout:2126008811633 2023-09-23T22:47:02.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:02 smithi026 ceph-mon[215775]: from='client.? 172.21.15.26:0/3045740823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:47:02.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:02 smithi026 ceph-mon[223014]: from='client.? 172.21.15.26:0/3045740823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:47:02.861 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2126008811631 got 2126008811633 for osd.1 2023-09-23T22:47:02.861 DEBUG:teuthology.parallel:result is None 2023-09-23T22:47:02.861 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T22:47:02.862 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:47:02.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:02 smithi070 ceph-mon[192652]: from='client.? 172.21.15.26:0/3045740823' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T22:47:03.345 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:03.773 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:03 smithi026 ceph-mon[215775]: pgmap v1749: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:03.773 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:03 smithi026 ceph-mon[223014]: pgmap v1749: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:03.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:03 smithi070 ceph-mon[192652]: pgmap v1749: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:05.385 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:47:05.387 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:47:05.735 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[215775]: pgmap v1750: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:47:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:47:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:47:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:47:05.736 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[215775]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:47:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[223014]: pgmap v1750: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:05.736 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:47:05.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:47:05.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:47:05.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:47:05.737 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:05 smithi026 ceph-mon[223014]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:47:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:05 smithi070 ceph-mon[192652]: pgmap v1750: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:05 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:47:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:05 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:47:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:05 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T22:47:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:05 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T22:47:05.874 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:05 smithi070 ceph-mon[192652]: from='mgr.25772 172.21.15.26:0/810369918' entity='mgr.y' 2023-09-23T22:47:06.013 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1750,"stamp":"2023-09-23T22:47:03.796484+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11971,"num_read_kb":25396,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":762,"num_bytes_recovered":26400651,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3617564,"kb_used_data":90652,"kb_used_omap":66,"kb_used_meta":3526781,"kb_avail":746343652,"statfs":{"total":767960285184,"available":764255899648,"internally_reserved":0,"allocated":92827648,"data_stored":71262317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67826,"internal_metadata":3611424526},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003196"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":1185,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587835+0000","last_change":"2023-09-23T22:45:09.435050+0000","last_active":"2023-09-23T22:46:25.587835+0000","last_peered":"2023-09-23T22:46:25.587835+0000","last_clean":"2023-09-23T22:46:25.587835+0000","last_became_active":"2023-09-23T22:45:09.434512+0000","last_became_peered":"2023-09-23T22:45:09.434512+0000","last_unstale":"2023-09-23T22:46:25.587835+0000","last_undegraded":"2023-09-23T22:46:25.587835+0000","last_fullsized":"2023-09-23T22:46:25.587835+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.209739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.209739+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:43.842917+0000","scrub_duration":0.0016266950000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":43,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700962+0000","last_change":"2023-09-23T22:45:17.506717+0000","last_active":"2023-09-23T22:46:26.700962+0000","last_peered":"2023-09-23T22:46:26.700962+0000","last_clean":"2023-09-23T22:46:26.700962+0000","last_became_active":"2023-09-23T22:45:17.506273+0000","last_became_peered":"2023-09-23T22:45:17.506273+0000","last_unstale":"2023-09-23T22:46:26.700962+0000","last_undegraded":"2023-09-23T22:46:26.700962+0000","last_fullsized":"2023-09-23T22:46:26.700962+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:35.732850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834129+0000","last_change":"2023-09-23T22:45:22.504651+0000","last_active":"2023-09-23T22:46:25.834129+0000","last_peered":"2023-09-23T22:46:25.834129+0000","last_clean":"2023-09-23T22:46:25.834129+0000","last_became_active":"2023-09-23T22:45:22.504357+0000","last_became_peered":"2023-09-23T22:45:22.504357+0000","last_unstale":"2023-09-23T22:46:25.834129+0000","last_undegraded":"2023-09-23T22:46:25.834129+0000","last_fullsized":"2023-09-23T22:46:25.834129+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:25.702961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":954,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588621+0000","last_change":"2023-09-23T22:45:28.553711+0000","last_active":"2023-09-23T22:46:25.588621+0000","last_peered":"2023-09-23T22:46:25.588621+0000","last_clean":"2023-09-23T22:46:25.588621+0000","last_became_active":"2023-09-23T22:45:28.553291+0000","last_became_peered":"2023-09-23T22:45:28.553291+0000","last_unstale":"2023-09-23T22:46:25.588621+0000","last_undegraded":"2023-09-23T22:46:25.588621+0000","last_fullsized":"2023-09-23T22:46:25.588621+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:31.111983+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:31.111983+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:58.638288+0000","scrub_duration":0.001731486,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":483,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.508950+0000","last_change":"2023-09-23T22:45:22.505474+0000","last_active":"2023-09-23T22:46:26.508950+0000","last_peered":"2023-09-23T22:46:26.508950+0000","last_clean":"2023-09-23T22:46:26.508950+0000","last_became_active":"2023-09-23T22:45:22.505139+0000","last_became_peered":"2023-09-23T22:45:22.505139+0000","last_unstale":"2023-09-23T22:46:26.508950+0000","last_undegraded":"2023-09-23T22:46:26.508950+0000","last_fullsized":"2023-09-23T22:46:26.508950+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.186813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":44,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700748+0000","last_change":"2023-09-23T22:45:42.916524+0000","last_active":"2023-09-23T22:46:26.700748+0000","last_peered":"2023-09-23T22:46:26.700748+0000","last_clean":"2023-09-23T22:46:26.700748+0000","last_became_active":"2023-09-23T22:45:42.916176+0000","last_became_peered":"2023-09-23T22:45:42.916176+0000","last_unstale":"2023-09-23T22:46:26.700748+0000","last_undegraded":"2023-09-23T22:46:26.700748+0000","last_fullsized":"2023-09-23T22:46:26.700748+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:44.151573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":913,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700985+0000","last_change":"2023-09-23T22:45:20.493782+0000","last_active":"2023-09-23T22:46:26.700985+0000","last_peered":"2023-09-23T22:46:26.700985+0000","last_clean":"2023-09-23T22:46:26.700985+0000","last_became_active":"2023-09-23T22:45:20.493321+0000","last_became_peered":"2023-09-23T22:45:20.493321+0000","last_unstale":"2023-09-23T22:46:26.700985+0000","last_undegraded":"2023-09-23T22:46:26.700985+0000","last_fullsized":"2023-09-23T22:46:26.700985+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:31.625786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:31.625786+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:14.192194+0000","scrub_duration":0.0014301240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":48,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833741+0000","last_change":"2023-09-23T22:45:25.550834+0000","last_active":"2023-09-23T22:46:25.833741+0000","last_peered":"2023-09-23T22:46:25.833741+0000","last_clean":"2023-09-23T22:46:25.833741+0000","last_became_active":"2023-09-23T22:45:25.550529+0000","last_became_peered":"2023-09-23T22:45:25.550529+0000","last_unstale":"2023-09-23T22:46:25.833741+0000","last_undegraded":"2023-09-23T22:46:25.833741+0000","last_fullsized":"2023-09-23T22:46:25.833741+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:20.657338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":298,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589178+0000","last_change":"2023-09-23T22:42:56.509390+0000","last_active":"2023-09-23T22:46:25.589178+0000","last_peered":"2023-09-23T22:46:25.589178+0000","last_clean":"2023-09-23T22:46:25.589178+0000","last_became_active":"2023-09-23T22:42:56.509066+0000","last_became_peered":"2023-09-23T22:42:56.509066+0000","last_unstale":"2023-09-23T22:46:25.589178+0000","last_undegraded":"2023-09-23T22:46:25.589178+0000","last_fullsized":"2023-09-23T22:46:25.589178+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:36.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1010,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589315+0000","last_change":"2023-09-23T22:45:59.494924+0000","last_active":"2023-09-23T22:46:25.589315+0000","last_peered":"2023-09-23T22:46:25.589315+0000","last_clean":"2023-09-23T22:46:25.589315+0000","last_became_active":"2023-09-23T22:45:59.494310+0000","last_became_peered":"2023-09-23T22:45:59.494310+0000","last_unstale":"2023-09-23T22:46:25.589315+0000","last_undegraded":"2023-09-23T22:46:25.589315+0000","last_fullsized":"2023-09-23T22:46:25.589315+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:57.273632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:57.273632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:07.883227+0000","scrub_duration":0.018058204000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":110,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700996+0000","last_change":"2023-09-23T22:45:42.841665+0000","last_active":"2023-09-23T22:46:26.700996+0000","last_peered":"2023-09-23T22:46:26.700996+0000","last_clean":"2023-09-23T22:46:26.700996+0000","last_became_active":"2023-09-23T22:45:42.841311+0000","last_became_peered":"2023-09-23T22:45:42.841311+0000","last_unstale":"2023-09-23T22:46:26.700996+0000","last_undegraded":"2023-09-23T22:46:26.700996+0000","last_fullsized":"2023-09-23T22:46:26.700996+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:51.118344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":48,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833673+0000","last_change":"2023-09-23T22:45:41.831660+0000","last_active":"2023-09-23T22:46:25.833673+0000","last_peered":"2023-09-23T22:46:25.833673+0000","last_clean":"2023-09-23T22:46:25.833673+0000","last_became_active":"2023-09-23T22:45:41.831242+0000","last_became_peered":"2023-09-23T22:45:41.831242+0000","last_unstale":"2023-09-23T22:46:25.833673+0000","last_undegraded":"2023-09-23T22:46:25.833673+0000","last_fullsized":"2023-09-23T22:46:25.833673+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:48.814841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":868,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:28.038076+0000","last_change":"2023-09-23T22:45:20.494097+0000","last_active":"2023-09-23T22:46:28.038076+0000","last_peered":"2023-09-23T22:46:28.038076+0000","last_clean":"2023-09-23T22:46:28.038076+0000","last_became_active":"2023-09-23T22:45:20.493775+0000","last_became_peered":"2023-09-23T22:45:20.493775+0000","last_unstale":"2023-09-23T22:46:28.038076+0000","last_undegraded":"2023-09-23T22:46:28.038076+0000","last_fullsized":"2023-09-23T22:46:28.038076+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:02.056099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:02.056099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:28.358390+0000","scrub_duration":0.012145022,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":104,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588208+0000","last_change":"2023-09-23T22:45:06.452276+0000","last_active":"2023-09-23T22:46:25.588208+0000","last_peered":"2023-09-23T22:46:25.588208+0000","last_clean":"2023-09-23T22:46:25.588208+0000","last_became_active":"2023-09-23T22:45:06.451936+0000","last_became_peered":"2023-09-23T22:45:06.451936+0000","last_unstale":"2023-09-23T22:46:25.588208+0000","last_undegraded":"2023-09-23T22:46:25.588208+0000","last_fullsized":"2023-09-23T22:46:25.588208+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:25.418898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833872+0000","last_change":"2023-09-23T22:45:22.511553+0000","last_active":"2023-09-23T22:46:25.833872+0000","last_peered":"2023-09-23T22:46:25.833872+0000","last_clean":"2023-09-23T22:46:25.833872+0000","last_became_active":"2023-09-23T22:45:22.511245+0000","last_became_peered":"2023-09-23T22:45:22.511245+0000","last_unstale":"2023-09-23T22:46:25.833872+0000","last_undegraded":"2023-09-23T22:46:25.833872+0000","last_fullsized":"2023-09-23T22:46:25.833872+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:04.426439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":999,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588185+0000","last_change":"2023-09-23T22:45:28.556099+0000","last_active":"2023-09-23T22:46:25.588185+0000","last_peered":"2023-09-23T22:46:25.588185+0000","last_clean":"2023-09-23T22:46:25.588185+0000","last_became_active":"2023-09-23T22:45:28.555729+0000","last_became_peered":"2023-09-23T22:45:28.555729+0000","last_unstale":"2023-09-23T22:46:25.588185+0000","last_undegraded":"2023-09-23T22:46:25.588185+0000","last_fullsized":"2023-09-23T22:46:25.588185+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:29.054342+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:29.054342+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:19.435489+0000","scrub_duration":0.001695039,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":109,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589401+0000","last_change":"2023-09-23T22:45:59.487631+0000","last_active":"2023-09-23T22:46:25.589401+0000","last_peered":"2023-09-23T22:46:25.589401+0000","last_clean":"2023-09-23T22:46:25.589401+0000","last_became_active":"2023-09-23T22:45:59.487286+0000","last_became_peered":"2023-09-23T22:45:59.487286+0000","last_unstale":"2023-09-23T22:46:25.589401+0000","last_undegraded":"2023-09-23T22:46:25.589401+0000","last_fullsized":"2023-09-23T22:46:25.589401+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:45.210248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":336,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T22:46:29.935702+0000","last_change":"2023-09-23T22:42:56.567330+0000","last_active":"2023-09-23T22:46:29.935702+0000","last_peered":"2023-09-23T22:46:29.935702+0000","last_clean":"2023-09-23T22:46:29.935702+0000","last_became_active":"2023-09-23T22:42:56.567016+0000","last_became_peered":"2023-09-23T22:42:56.567016+0000","last_unstale":"2023-09-23T22:46:29.935702+0000","last_undegraded":"2023-09-23T22:46:29.935702+0000","last_fullsized":"2023-09-23T22:46:29.935702+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:00.456335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":938,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T22:46:29.934436+0000","last_change":"2023-09-23T22:45:28.550354+0000","last_active":"2023-09-23T22:46:29.934436+0000","last_peered":"2023-09-23T22:46:29.934436+0000","last_clean":"2023-09-23T22:46:29.934436+0000","last_became_active":"2023-09-23T22:45:28.549956+0000","last_became_peered":"2023-09-23T22:45:28.549956+0000","last_unstale":"2023-09-23T22:46:29.934436+0000","last_undegraded":"2023-09-23T22:46:29.934436+0000","last_fullsized":"2023-09-23T22:46:29.934436+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:30.597926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:30.597926+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:41.683726+0000","scrub_duration":0.001666528,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":681,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701150+0000","last_change":"2023-09-23T22:46:11.531687+0000","last_active":"2023-09-23T22:46:26.701150+0000","last_peered":"2023-09-23T22:46:26.701150+0000","last_clean":"2023-09-23T22:46:26.701150+0000","last_became_active":"2023-09-23T22:46:11.531313+0000","last_became_peered":"2023-09-23T22:46:11.531313+0000","last_unstale":"2023-09-23T22:46:26.701150+0000","last_undegraded":"2023-09-23T22:46:26.701150+0000","last_fullsized":"2023-09-23T22:46:26.701150+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:07.100563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":47340,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":43,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701053+0000","last_change":"2023-09-23T22:45:17.508332+0000","last_active":"2023-09-23T22:46:26.701053+0000","last_peered":"2023-09-23T22:46:26.701053+0000","last_clean":"2023-09-23T22:46:26.701053+0000","last_became_active":"2023-09-23T22:45:17.507811+0000","last_became_peered":"2023-09-23T22:45:17.507811+0000","last_unstale":"2023-09-23T22:46:26.701053+0000","last_undegraded":"2023-09-23T22:46:26.701053+0000","last_fullsized":"2023-09-23T22:46:26.701053+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:04.334173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":1152,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700437+0000","last_change":"2023-09-23T22:45:42.841085+0000","last_active":"2023-09-23T22:46:26.700437+0000","last_peered":"2023-09-23T22:46:26.700437+0000","last_clean":"2023-09-23T22:46:26.700437+0000","last_became_active":"2023-09-23T22:45:42.840750+0000","last_became_peered":"2023-09-23T22:45:42.840750+0000","last_unstale":"2023-09-23T22:46:26.700437+0000","last_undegraded":"2023-09-23T22:46:26.700437+0000","last_fullsized":"2023-09-23T22:46:26.700437+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:19.077318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:19.077318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.621150+0000","scrub_duration":0.0080980949999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":357,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587986+0000","last_change":"2023-09-23T22:45:59.490356+0000","last_active":"2023-09-23T22:46:25.587986+0000","last_peered":"2023-09-23T22:46:25.587986+0000","last_clean":"2023-09-23T22:46:25.587986+0000","last_became_active":"2023-09-23T22:45:59.490049+0000","last_became_peered":"2023-09-23T22:45:59.490049+0000","last_unstale":"2023-09-23T22:46:25.587986+0000","last_undegraded":"2023-09-23T22:46:25.587986+0000","last_fullsized":"2023-09-23T22:46:25.587986+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:35.738748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834258+0000","last_change":"2023-09-23T22:45:22.512222+0000","last_active":"2023-09-23T22:46:25.834258+0000","last_peered":"2023-09-23T22:46:25.834258+0000","last_clean":"2023-09-23T22:46:25.834258+0000","last_became_active":"2023-09-23T22:45:22.511892+0000","last_became_peered":"2023-09-23T22:45:22.511892+0000","last_unstale":"2023-09-23T22:46:25.834258+0000","last_undegraded":"2023-09-23T22:46:25.834258+0000","last_fullsized":"2023-09-23T22:46:25.834258+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:50.029752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":999,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587670+0000","last_change":"2023-09-23T22:45:09.437073+0000","last_active":"2023-09-23T22:46:25.587670+0000","last_peered":"2023-09-23T22:46:25.587670+0000","last_clean":"2023-09-23T22:46:25.587670+0000","last_became_active":"2023-09-23T22:45:09.436611+0000","last_became_peered":"2023-09-23T22:45:09.436611+0000","last_unstale":"2023-09-23T22:46:25.587670+0000","last_undegraded":"2023-09-23T22:46:25.587670+0000","last_fullsized":"2023-09-23T22:46:25.587670+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.556720+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:09.783596+0000","scrub_duration":0.0022349409999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":44,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700578+0000","last_change":"2023-09-23T22:45:42.835144+0000","last_active":"2023-09-23T22:46:26.700578+0000","last_peered":"2023-09-23T22:46:26.700578+0000","last_clean":"2023-09-23T22:46:26.700578+0000","last_became_active":"2023-09-23T22:45:42.834794+0000","last_became_peered":"2023-09-23T22:45:42.834794+0000","last_unstale":"2023-09-23T22:46:26.700578+0000","last_undegraded":"2023-09-23T22:46:26.700578+0000","last_fullsized":"2023-09-23T22:46:26.700578+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:37.053170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":114,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588363+0000","last_change":"2023-09-23T22:45:22.504560+0000","last_active":"2023-09-23T22:46:25.588363+0000","last_peered":"2023-09-23T22:46:25.588363+0000","last_clean":"2023-09-23T22:46:25.588363+0000","last_became_active":"2023-09-23T22:45:22.503994+0000","last_became_peered":"2023-09-23T22:45:22.503994+0000","last_unstale":"2023-09-23T22:46:25.588363+0000","last_undegraded":"2023-09-23T22:46:25.588363+0000","last_fullsized":"2023-09-23T22:46:25.588363+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:22.098112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":738,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509311+0000","last_change":"2023-09-23T22:45:25.553371+0000","last_active":"2023-09-23T22:46:26.509311+0000","last_peered":"2023-09-23T22:46:26.509311+0000","last_clean":"2023-09-23T22:46:26.509311+0000","last_became_active":"2023-09-23T22:45:25.552907+0000","last_became_peered":"2023-09-23T22:45:25.552907+0000","last_unstale":"2023-09-23T22:46:26.509311+0000","last_undegraded":"2023-09-23T22:46:26.509311+0000","last_fullsized":"2023-09-23T22:46:26.509311+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:57.922489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:57.922489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:05:44.724877+0000","scrub_duration":0.001544351,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":514,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588996+0000","last_change":"2023-09-23T22:45:59.490918+0000","last_active":"2023-09-23T22:46:25.588996+0000","last_peered":"2023-09-23T22:46:25.588996+0000","last_clean":"2023-09-23T22:46:25.588996+0000","last_became_active":"2023-09-23T22:45:59.490573+0000","last_became_peered":"2023-09-23T22:45:59.490573+0000","last_unstale":"2023-09-23T22:46:25.588996+0000","last_undegraded":"2023-09-23T22:46:25.588996+0000","last_fullsized":"2023-09-23T22:46:25.588996+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.807697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":44,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700778+0000","last_change":"2023-09-23T22:45:42.836299+0000","last_active":"2023-09-23T22:46:26.700778+0000","last_peered":"2023-09-23T22:46:26.700778+0000","last_clean":"2023-09-23T22:46:26.700778+0000","last_became_active":"2023-09-23T22:45:42.836003+0000","last_became_peered":"2023-09-23T22:45:42.836003+0000","last_unstale":"2023-09-23T22:46:26.700778+0000","last_undegraded":"2023-09-23T22:46:26.700778+0000","last_fullsized":"2023-09-23T22:46:26.700778+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:15.194210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":1018,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588822+0000","last_change":"2023-09-23T22:44:05.892047+0000","last_active":"2023-09-23T22:46:25.588822+0000","last_peered":"2023-09-23T22:46:25.588822+0000","last_clean":"2023-09-23T22:46:25.588822+0000","last_became_active":"2023-09-23T22:44:05.891675+0000","last_became_peered":"2023-09-23T22:44:05.891675+0000","last_unstale":"2023-09-23T22:46:25.588822+0000","last_undegraded":"2023-09-23T22:46:25.588822+0000","last_fullsized":"2023-09-23T22:46:25.588822+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:26.313042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:26.313042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:06.388598+0000","scrub_duration":0.0017143830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":382,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587404+0000","last_change":"2023-09-23T22:45:25.565473+0000","last_active":"2023-09-23T22:46:25.587404+0000","last_peered":"2023-09-23T22:46:25.587404+0000","last_clean":"2023-09-23T22:46:25.587404+0000","last_became_active":"2023-09-23T22:45:25.546116+0000","last_became_peered":"2023-09-23T22:45:25.546116+0000","last_unstale":"2023-09-23T22:46:25.587404+0000","last_undegraded":"2023-09-23T22:46:25.587404+0000","last_fullsized":"2023-09-23T22:46:25.587404+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:54.957330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":48,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834324+0000","last_change":"2023-09-23T22:45:41.837664+0000","last_active":"2023-09-23T22:46:25.834324+0000","last_peered":"2023-09-23T22:46:25.834324+0000","last_clean":"2023-09-23T22:46:25.834324+0000","last_became_active":"2023-09-23T22:45:41.837302+0000","last_became_peered":"2023-09-23T22:45:41.837302+0000","last_unstale":"2023-09-23T22:46:25.834324+0000","last_undegraded":"2023-09-23T22:46:25.834324+0000","last_fullsized":"2023-09-23T22:46:25.834324+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:56.197841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":1062,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589263+0000","last_change":"2023-09-23T22:45:20.494718+0000","last_active":"2023-09-23T22:46:25.589263+0000","last_peered":"2023-09-23T22:46:25.589263+0000","last_clean":"2023-09-23T22:46:25.589263+0000","last_became_active":"2023-09-23T22:45:20.494224+0000","last_became_peered":"2023-09-23T22:45:20.494224+0000","last_unstale":"2023-09-23T22:46:25.589263+0000","last_undegraded":"2023-09-23T22:46:25.589263+0000","last_fullsized":"2023-09-23T22:46:25.589263+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:34.147486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:34.147486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:41.573558+0000","scrub_duration":0.0018320190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":210,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588137+0000","last_change":"2023-09-23T22:45:59.493695+0000","last_active":"2023-09-23T22:46:25.588137+0000","last_peered":"2023-09-23T22:46:25.588137+0000","last_clean":"2023-09-23T22:46:25.588137+0000","last_became_active":"2023-09-23T22:45:59.493124+0000","last_became_peered":"2023-09-23T22:45:59.493124+0000","last_unstale":"2023-09-23T22:46:25.588137+0000","last_undegraded":"2023-09-23T22:46:25.588137+0000","last_fullsized":"2023-09-23T22:46:25.588137+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:38.627199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":104,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587827+0000","last_change":"2023-09-23T22:45:06.458827+0000","last_active":"2023-09-23T22:46:25.587827+0000","last_peered":"2023-09-23T22:46:25.587827+0000","last_clean":"2023-09-23T22:46:25.587827+0000","last_became_active":"2023-09-23T22:45:06.458533+0000","last_became_peered":"2023-09-23T22:45:06.458533+0000","last_unstale":"2023-09-23T22:46:25.587827+0000","last_undegraded":"2023-09-23T22:46:25.587827+0000","last_fullsized":"2023-09-23T22:46:25.587827+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:02.882529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":816,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834206+0000","last_change":"2023-09-23T22:45:28.556001+0000","last_active":"2023-09-23T22:46:25.834206+0000","last_peered":"2023-09-23T22:46:25.834206+0000","last_clean":"2023-09-23T22:46:25.834206+0000","last_became_active":"2023-09-23T22:45:28.555527+0000","last_became_peered":"2023-09-23T22:45:28.555527+0000","last_unstale":"2023-09-23T22:46:25.834206+0000","last_undegraded":"2023-09-23T22:46:25.834206+0000","last_fullsized":"2023-09-23T22:46:25.834206+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:04.048606+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:04.048606+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:33.382480+0000","scrub_duration":0.0064015249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":588,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588538+0000","last_change":"2023-09-23T22:45:59.489789+0000","last_active":"2023-09-23T22:46:25.588538+0000","last_peered":"2023-09-23T22:46:25.588538+0000","last_clean":"2023-09-23T22:46:25.588538+0000","last_became_active":"2023-09-23T22:45:59.489485+0000","last_became_peered":"2023-09-23T22:45:59.489485+0000","last_unstale":"2023-09-23T22:46:25.588538+0000","last_undegraded":"2023-09-23T22:46:25.588538+0000","last_fullsized":"2023-09-23T22:46:25.588538+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:15.906128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":706,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700723+0000","last_change":"2023-09-23T22:46:11.933058+0000","last_active":"2023-09-23T22:46:26.700723+0000","last_peered":"2023-09-23T22:46:26.700723+0000","last_clean":"2023-09-23T22:46:26.700723+0000","last_became_active":"2023-09-23T22:46:11.932695+0000","last_became_peered":"2023-09-23T22:46:11.932695+0000","last_unstale":"2023-09-23T22:46:26.700723+0000","last_undegraded":"2023-09-23T22:46:26.700723+0000","last_fullsized":"2023-09-23T22:46:26.700723+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:36.351064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":997,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589081+0000","last_change":"2023-09-23T22:45:20.549256+0000","last_active":"2023-09-23T22:46:25.589081+0000","last_peered":"2023-09-23T22:46:25.589081+0000","last_clean":"2023-09-23T22:46:25.589081+0000","last_became_active":"2023-09-23T22:45:20.547976+0000","last_became_peered":"2023-09-23T22:45:20.547976+0000","last_unstale":"2023-09-23T22:46:25.589081+0000","last_undegraded":"2023-09-23T22:46:25.589081+0000","last_fullsized":"2023-09-23T22:46:25.589081+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:36.397150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:36.397150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:24.867382+0000","scrub_duration":0.001754996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":86,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589450+0000","last_change":"2023-09-23T22:45:59.494780+0000","last_active":"2023-09-23T22:46:25.589450+0000","last_peered":"2023-09-23T22:46:25.589450+0000","last_clean":"2023-09-23T22:46:25.589450+0000","last_became_active":"2023-09-23T22:45:59.494281+0000","last_became_peered":"2023-09-23T22:45:59.494281+0000","last_unstale":"2023-09-23T22:46:25.589450+0000","last_undegraded":"2023-09-23T22:46:25.589450+0000","last_fullsized":"2023-09-23T22:46:25.589450+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:54.063310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":326,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589148+0000","last_change":"2023-09-23T22:45:06.451816+0000","last_active":"2023-09-23T22:46:25.589148+0000","last_peered":"2023-09-23T22:46:25.589148+0000","last_clean":"2023-09-23T22:46:25.589148+0000","last_became_active":"2023-09-23T22:45:06.451507+0000","last_became_peered":"2023-09-23T22:45:06.451507+0000","last_unstale":"2023-09-23T22:46:25.589148+0000","last_undegraded":"2023-09-23T22:46:25.589148+0000","last_fullsized":"2023-09-23T22:46:25.589148+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:05.149871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":864,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701438+0000","last_change":"2023-09-23T22:45:28.345915+0000","last_active":"2023-09-23T22:46:26.701438+0000","last_peered":"2023-09-23T22:46:26.701438+0000","last_clean":"2023-09-23T22:46:26.701438+0000","last_became_active":"2023-09-23T22:45:25.547347+0000","last_became_peered":"2023-09-23T22:45:25.547347+0000","last_unstale":"2023-09-23T22:46:26.701438+0000","last_undegraded":"2023-09-23T22:46:26.701438+0000","last_fullsized":"2023-09-23T22:46:26.701438+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:45:28.345788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:45:28.345788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:07.959240+0000","scrub_duration":0.0015759280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":104,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588785+0000","last_change":"2023-09-23T22:45:22.505328+0000","last_active":"2023-09-23T22:46:25.588785+0000","last_peered":"2023-09-23T22:46:25.588785+0000","last_clean":"2023-09-23T22:46:25.588785+0000","last_became_active":"2023-09-23T22:45:22.505022+0000","last_became_peered":"2023-09-23T22:45:22.505022+0000","last_unstale":"2023-09-23T22:46:25.588785+0000","last_undegraded":"2023-09-23T22:46:25.588785+0000","last_fullsized":"2023-09-23T22:46:25.588785+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:37.601573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":512,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700822+0000","last_change":"2023-09-23T22:45:42.841970+0000","last_active":"2023-09-23T22:46:26.700822+0000","last_peered":"2023-09-23T22:46:26.700822+0000","last_clean":"2023-09-23T22:46:26.700822+0000","last_became_active":"2023-09-23T22:45:42.840926+0000","last_became_peered":"2023-09-23T22:45:42.840926+0000","last_unstale":"2023-09-23T22:46:26.700822+0000","last_undegraded":"2023-09-23T22:46:26.700822+0000","last_fullsized":"2023-09-23T22:46:26.700822+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:52.257621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1025,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588827+0000","last_change":"2023-09-23T22:45:59.491149+0000","last_active":"2023-09-23T22:46:25.588827+0000","last_peered":"2023-09-23T22:46:25.588827+0000","last_clean":"2023-09-23T22:46:25.588827+0000","last_became_active":"2023-09-23T22:45:59.490803+0000","last_became_peered":"2023-09-23T22:45:59.490803+0000","last_unstale":"2023-09-23T22:46:25.588827+0000","last_undegraded":"2023-09-23T22:46:25.588827+0000","last_fullsized":"2023-09-23T22:46:25.588827+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:58.211060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:58.211060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.147929+0000","scrub_duration":0.0035013240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":56,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833894+0000","last_change":"2023-09-23T22:45:25.571211+0000","last_active":"2023-09-23T22:46:25.833894+0000","last_peered":"2023-09-23T22:46:25.833894+0000","last_clean":"2023-09-23T22:46:25.833894+0000","last_became_active":"2023-09-23T22:45:25.548340+0000","last_became_peered":"2023-09-23T22:45:25.548340+0000","last_unstale":"2023-09-23T22:46:25.833894+0000","last_undegraded":"2023-09-23T22:46:25.833894+0000","last_fullsized":"2023-09-23T22:46:25.833894+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:10.421715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":522,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701142+0000","last_change":"2023-09-23T22:46:11.531961+0000","last_active":"2023-09-23T22:46:26.701142+0000","last_peered":"2023-09-23T22:46:26.701142+0000","last_clean":"2023-09-23T22:46:26.701142+0000","last_became_active":"2023-09-23T22:46:11.531618+0000","last_became_peered":"2023-09-23T22:46:11.531618+0000","last_unstale":"2023-09-23T22:46:26.701142+0000","last_undegraded":"2023-09-23T22:46:26.701142+0000","last_fullsized":"2023-09-23T22:46:26.701142+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:00.040458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":1161,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587917+0000","last_change":"2023-09-23T22:45:25.555386+0000","last_active":"2023-09-23T22:46:25.587917+0000","last_peered":"2023-09-23T22:46:25.587917+0000","last_clean":"2023-09-23T22:46:25.587917+0000","last_became_active":"2023-09-23T22:45:25.555134+0000","last_became_peered":"2023-09-23T22:45:25.555134+0000","last_unstale":"2023-09-23T22:46:25.587917+0000","last_undegraded":"2023-09-23T22:46:25.587917+0000","last_fullsized":"2023-09-23T22:46:25.587917+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.183183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.183183+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:31.061518+0000","scrub_duration":0.0015656730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":323,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589319+0000","last_change":"2023-09-23T22:45:17.502689+0000","last_active":"2023-09-23T22:46:25.589319+0000","last_peered":"2023-09-23T22:46:25.589319+0000","last_clean":"2023-09-23T22:46:25.589319+0000","last_became_active":"2023-09-23T22:45:17.502401+0000","last_became_peered":"2023-09-23T22:45:17.502401+0000","last_unstale":"2023-09-23T22:46:25.589319+0000","last_undegraded":"2023-09-23T22:46:25.589319+0000","last_fullsized":"2023-09-23T22:46:25.589319+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:00.890452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":342,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701432+0000","last_change":"2023-09-23T22:46:11.936276+0000","last_active":"2023-09-23T22:46:26.701432+0000","last_peered":"2023-09-23T22:46:26.701432+0000","last_clean":"2023-09-23T22:46:26.701432+0000","last_became_active":"2023-09-23T22:46:11.935887+0000","last_became_peered":"2023-09-23T22:46:11.935887+0000","last_unstale":"2023-09-23T22:46:26.701432+0000","last_undegraded":"2023-09-23T22:46:26.701432+0000","last_fullsized":"2023-09-23T22:46:26.701432+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:47.592106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":658,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509400+0000","last_change":"2023-09-23T22:46:17.563136+0000","last_active":"2023-09-23T22:46:26.509400+0000","last_peered":"2023-09-23T22:46:26.509400+0000","last_clean":"2023-09-23T22:46:26.509400+0000","last_became_active":"2023-09-23T22:46:17.562788+0000","last_became_peered":"2023-09-23T22:46:17.562788+0000","last_unstale":"2023-09-23T22:46:26.509400+0000","last_undegraded":"2023-09-23T22:46:26.509400+0000","last_fullsized":"2023-09-23T22:46:26.509400+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:32.087146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:32.087146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:28.127046+0000","scrub_duration":0.001693546,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":43,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700734+0000","last_change":"2023-09-23T22:45:17.496669+0000","last_active":"2023-09-23T22:46:26.700734+0000","last_peered":"2023-09-23T22:46:26.700734+0000","last_clean":"2023-09-23T22:46:26.700734+0000","last_became_active":"2023-09-23T22:45:17.495007+0000","last_became_peered":"2023-09-23T22:45:17.495007+0000","last_unstale":"2023-09-23T22:46:26.700734+0000","last_undegraded":"2023-09-23T22:46:26.700734+0000","last_fullsized":"2023-09-23T22:46:26.700734+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:43.132602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":625,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701400+0000","last_change":"2023-09-23T22:46:11.934154+0000","last_active":"2023-09-23T22:46:26.701400+0000","last_peered":"2023-09-23T22:46:26.701400+0000","last_clean":"2023-09-23T22:46:26.701400+0000","last_became_active":"2023-09-23T22:46:11.933777+0000","last_became_peered":"2023-09-23T22:46:11.933777+0000","last_unstale":"2023-09-23T22:46:26.701400+0000","last_undegraded":"2023-09-23T22:46:26.701400+0000","last_fullsized":"2023-09-23T22:46:26.701400+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:36.797336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":1090,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588992+0000","last_change":"2023-09-23T22:43:57.157649+0000","last_active":"2023-09-23T22:46:25.588992+0000","last_peered":"2023-09-23T22:46:25.588992+0000","last_clean":"2023-09-23T22:46:25.588992+0000","last_became_active":"2023-09-23T22:42:56.530915+0000","last_became_peered":"2023-09-23T22:42:56.530915+0000","last_unstale":"2023-09-23T22:46:25.588992+0000","last_undegraded":"2023-09-23T22:46:25.588992+0000","last_fullsized":"2023-09-23T22:46:25.588992+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_clean_scrub_stamp":"2023-09-23T22:43:57.157560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:20.159228+0000","scrub_duration":0.001566287,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833758+0000","last_change":"2023-09-23T22:45:22.511359+0000","last_active":"2023-09-23T22:46:25.833758+0000","last_peered":"2023-09-23T22:46:25.833758+0000","last_clean":"2023-09-23T22:46:25.833758+0000","last_became_active":"2023-09-23T22:45:22.510056+0000","last_became_peered":"2023-09-23T22:45:22.510056+0000","last_unstale":"2023-09-23T22:46:25.833758+0000","last_undegraded":"2023-09-23T22:46:25.833758+0000","last_fullsized":"2023-09-23T22:46:25.833758+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:34.850604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":534,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T22:46:29.934970+0000","last_change":"2023-09-23T22:45:22.506360+0000","last_active":"2023-09-23T22:46:29.934970+0000","last_peered":"2023-09-23T22:46:29.934970+0000","last_clean":"2023-09-23T22:46:29.934970+0000","last_became_active":"2023-09-23T22:45:22.505618+0000","last_became_peered":"2023-09-23T22:45:22.505618+0000","last_unstale":"2023-09-23T22:46:29.934970+0000","last_undegraded":"2023-09-23T22:46:29.934970+0000","last_fullsized":"2023-09-23T22:46:29.934970+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:21.115523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":1038,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587653+0000","last_change":"2023-09-23T22:45:25.546635+0000","last_active":"2023-09-23T22:46:25.587653+0000","last_peered":"2023-09-23T22:46:25.587653+0000","last_clean":"2023-09-23T22:46:25.587653+0000","last_became_active":"2023-09-23T22:45:25.545649+0000","last_became_peered":"2023-09-23T22:45:25.545649+0000","last_unstale":"2023-09-23T22:46:25.587653+0000","last_undegraded":"2023-09-23T22:46:25.587653+0000","last_fullsized":"2023-09-23T22:46:25.587653+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:03.570110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:03.570110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:43.773804+0000","scrub_duration":0.001987071,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"640'271","reported_seq":1450,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509324+0000","last_change":"2023-09-23T22:46:17.557802+0000","last_active":"2023-09-23T22:46:26.509324+0000","last_peered":"2023-09-23T22:46:26.509324+0000","last_clean":"2023-09-23T22:46:26.509324+0000","last_became_active":"2023-09-23T22:46:17.557196+0000","last_became_peered":"2023-09-23T22:46:17.557196+0000","last_unstale":"2023-09-23T22:46:26.509324+0000","last_undegraded":"2023-09-23T22:46:26.509324+0000","last_fullsized":"2023-09-23T22:46:26.509324+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":271,"log_dups_size":0,"ondisk_log_size":271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:52.972098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073668799999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1192,"num_read_kb":5960,"num_write":287,"num_write_kb":9768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":104,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588207+0000","last_change":"2023-09-23T22:45:22.504383+0000","last_active":"2023-09-23T22:46:25.588207+0000","last_peered":"2023-09-23T22:46:25.588207+0000","last_clean":"2023-09-23T22:46:25.588207+0000","last_became_active":"2023-09-23T22:45:22.503573+0000","last_became_peered":"2023-09-23T22:45:22.503573+0000","last_unstale":"2023-09-23T22:46:25.588207+0000","last_undegraded":"2023-09-23T22:46:25.588207+0000","last_fullsized":"2023-09-23T22:46:25.588207+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:22.322562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":450,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509157+0000","last_change":"2023-09-23T22:45:17.503914+0000","last_active":"2023-09-23T22:46:26.509157+0000","last_peered":"2023-09-23T22:46:26.509157+0000","last_clean":"2023-09-23T22:46:26.509157+0000","last_became_active":"2023-09-23T22:45:17.503111+0000","last_became_peered":"2023-09-23T22:45:17.503111+0000","last_unstale":"2023-09-23T22:46:26.509157+0000","last_undegraded":"2023-09-23T22:46:26.509157+0000","last_fullsized":"2023-09-23T22:46:26.509157+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:52.519749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":1154,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588300+0000","last_change":"2023-09-23T22:45:28.552025+0000","last_active":"2023-09-23T22:46:25.588300+0000","last_peered":"2023-09-23T22:46:25.588300+0000","last_clean":"2023-09-23T22:46:25.588300+0000","last_became_active":"2023-09-23T22:45:28.551589+0000","last_became_peered":"2023-09-23T22:45:28.551589+0000","last_unstale":"2023-09-23T22:46:25.588300+0000","last_undegraded":"2023-09-23T22:46:25.588300+0000","last_fullsized":"2023-09-23T22:46:25.588300+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:16.067396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:16.067396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.205560+0000","scrub_duration":0.0072300120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"640'221","reported_seq":2324,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589083+0000","last_change":"2023-09-23T22:44:54.429423+0000","last_active":"2023-09-23T22:46:25.589083+0000","last_peered":"2023-09-23T22:46:25.589083+0000","last_clean":"2023-09-23T22:46:25.589083+0000","last_became_active":"2023-09-23T22:42:56.548134+0000","last_became_peered":"2023-09-23T22:42:56.548134+0000","last_unstale":"2023-09-23T22:46:25.589083+0000","last_undegraded":"2023-09-23T22:46:25.589083+0000","last_fullsized":"2023-09-23T22:46:25.589083+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":221,"log_dups_size":0,"ondisk_log_size":221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:36.511009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0028955169999999998,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2322,"num_read_kb":4116,"num_write":184,"num_write_kb":6241,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":584,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701015+0000","last_change":"2023-09-23T22:46:11.934797+0000","last_active":"2023-09-23T22:46:26.701015+0000","last_peered":"2023-09-23T22:46:26.701015+0000","last_clean":"2023-09-23T22:46:26.701015+0000","last_became_active":"2023-09-23T22:46:11.934474+0000","last_became_peered":"2023-09-23T22:46:11.934474+0000","last_unstale":"2023-09-23T22:46:26.701015+0000","last_undegraded":"2023-09-23T22:46:26.701015+0000","last_fullsized":"2023-09-23T22:46:26.701015+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:53.890160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":319,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.508978+0000","last_change":"2023-09-23T22:46:17.905264+0000","last_active":"2023-09-23T22:46:26.508978+0000","last_peered":"2023-09-23T22:46:26.508978+0000","last_clean":"2023-09-23T22:46:26.508978+0000","last_became_active":"2023-09-23T22:46:17.904890+0000","last_became_peered":"2023-09-23T22:46:17.904890+0000","last_unstale":"2023-09-23T22:46:26.508978+0000","last_undegraded":"2023-09-23T22:46:26.508978+0000","last_fullsized":"2023-09-23T22:46:26.508978+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:29.607630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":1043,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587423+0000","last_change":"2023-09-23T22:45:59.493261+0000","last_active":"2023-09-23T22:46:25.587423+0000","last_peered":"2023-09-23T22:46:25.587423+0000","last_clean":"2023-09-23T22:46:25.587423+0000","last_became_active":"2023-09-23T22:45:59.492759+0000","last_became_peered":"2023-09-23T22:45:59.492759+0000","last_unstale":"2023-09-23T22:46:25.587423+0000","last_undegraded":"2023-09-23T22:46:25.587423+0000","last_fullsized":"2023-09-23T22:46:25.587423+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:07.291196+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:07.291196+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:23:52.331068+0000","scrub_duration":0.0016895390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"640'185","reported_seq":925,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.611939+0000","last_change":"2023-09-23T22:45:28.548846+0000","last_active":"2023-09-23T22:46:26.611939+0000","last_peered":"2023-09-23T22:46:26.611939+0000","last_clean":"2023-09-23T22:46:26.611939+0000","last_became_active":"2023-09-23T22:45:28.548123+0000","last_became_peered":"2023-09-23T22:45:28.548123+0000","last_unstale":"2023-09-23T22:46:26.611939+0000","last_undegraded":"2023-09-23T22:46:26.611939+0000","last_fullsized":"2023-09-23T22:46:26.611939+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":185,"log_dups_size":0,"ondisk_log_size":185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:52.606762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048572900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":3296,"num_write":166,"num_write_kb":2410,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":255,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.508812+0000","last_change":"2023-09-23T22:46:17.905222+0000","last_active":"2023-09-23T22:46:26.508812+0000","last_peered":"2023-09-23T22:46:26.508812+0000","last_clean":"2023-09-23T22:46:26.508812+0000","last_became_active":"2023-09-23T22:46:17.904872+0000","last_became_peered":"2023-09-23T22:46:17.904872+0000","last_unstale":"2023-09-23T22:46:26.508812+0000","last_undegraded":"2023-09-23T22:46:26.508812+0000","last_fullsized":"2023-09-23T22:46:26.508812+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:44.143193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":394,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.510342+0000","last_change":"2023-09-23T22:45:25.552027+0000","last_active":"2023-09-23T22:46:26.510342+0000","last_peered":"2023-09-23T22:46:26.510342+0000","last_clean":"2023-09-23T22:46:26.510342+0000","last_became_active":"2023-09-23T22:45:25.551697+0000","last_became_peered":"2023-09-23T22:45:25.551697+0000","last_unstale":"2023-09-23T22:46:26.510342+0000","last_undegraded":"2023-09-23T22:46:26.510342+0000","last_fullsized":"2023-09-23T22:46:26.510342+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:06:41.170247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"640'420","reported_seq":2264,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588763+0000","last_change":"2023-09-23T22:45:09.438706+0000","last_active":"2023-09-23T22:46:25.588763+0000","last_peered":"2023-09-23T22:46:25.588763+0000","last_clean":"2023-09-23T22:46:25.588763+0000","last_became_active":"2023-09-23T22:45:09.437974+0000","last_became_peered":"2023-09-23T22:45:09.437974+0000","last_unstale":"2023-09-23T22:46:25.588763+0000","last_undegraded":"2023-09-23T22:46:25.588763+0000","last_fullsized":"2023-09-23T22:46:25.588763+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":420,"log_dups_size":0,"ondisk_log_size":420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:22.520495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020164129999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1916,"num_read_kb":3106,"num_write":395,"num_write_kb":2468,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":873,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588099+0000","last_change":"2023-09-23T22:45:59.493669+0000","last_active":"2023-09-23T22:46:25.588099+0000","last_peered":"2023-09-23T22:46:25.588099+0000","last_clean":"2023-09-23T22:46:25.588099+0000","last_became_active":"2023-09-23T22:45:59.492788+0000","last_became_peered":"2023-09-23T22:45:59.492788+0000","last_unstale":"2023-09-23T22:46:25.588099+0000","last_undegraded":"2023-09-23T22:46:25.588099+0000","last_fullsized":"2023-09-23T22:46:25.588099+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.574795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.574795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:20.792256+0000","scrub_duration":0.0023357170000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833818+0000","last_change":"2023-09-23T22:45:22.509218+0000","last_active":"2023-09-23T22:46:25.833818+0000","last_peered":"2023-09-23T22:46:25.833818+0000","last_clean":"2023-09-23T22:46:25.833818+0000","last_became_active":"2023-09-23T22:45:22.508907+0000","last_became_peered":"2023-09-23T22:45:22.508907+0000","last_unstale":"2023-09-23T22:46:25.833818+0000","last_undegraded":"2023-09-23T22:46:25.833818+0000","last_fullsized":"2023-09-23T22:46:25.833818+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:34.923418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":348,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588731+0000","last_change":"2023-09-23T22:45:17.503488+0000","last_active":"2023-09-23T22:46:25.588731+0000","last_peered":"2023-09-23T22:46:25.588731+0000","last_clean":"2023-09-23T22:46:25.588731+0000","last_became_active":"2023-09-23T22:45:17.503018+0000","last_became_peered":"2023-09-23T22:45:17.503018+0000","last_unstale":"2023-09-23T22:46:25.588731+0000","last_undegraded":"2023-09-23T22:46:25.588731+0000","last_fullsized":"2023-09-23T22:46:25.588731+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:54.032873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":637,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509495+0000","last_change":"2023-09-23T22:46:17.558912+0000","last_active":"2023-09-23T22:46:26.509495+0000","last_peered":"2023-09-23T22:46:26.509495+0000","last_clean":"2023-09-23T22:46:26.509495+0000","last_became_active":"2023-09-23T22:46:17.558673+0000","last_became_peered":"2023-09-23T22:46:17.558673+0000","last_unstale":"2023-09-23T22:46:26.509495+0000","last_undegraded":"2023-09-23T22:46:26.509495+0000","last_fullsized":"2023-09-23T22:46:26.509495+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:38.451955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:38.451955+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:16.777533+0000","scrub_duration":0.0018627959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"640'215","reported_seq":1636,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588256+0000","last_change":"2023-09-23T22:45:59.493329+0000","last_active":"2023-09-23T22:46:25.588256+0000","last_peered":"2023-09-23T22:46:25.588256+0000","last_clean":"2023-09-23T22:46:25.588256+0000","last_became_active":"2023-09-23T22:45:59.492566+0000","last_became_peered":"2023-09-23T22:45:59.492566+0000","last_unstale":"2023-09-23T22:46:25.588256+0000","last_undegraded":"2023-09-23T22:46:25.588256+0000","last_fullsized":"2023-09-23T22:46:25.588256+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":215,"log_dups_size":0,"ondisk_log_size":215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:15:18.478665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018983450000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1446,"num_read_kb":3631,"num_write":157,"num_write_kb":2284,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":59,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700627+0000","last_change":"2023-09-23T22:45:42.842740+0000","last_active":"2023-09-23T22:46:26.700627+0000","last_peered":"2023-09-23T22:46:26.700627+0000","last_clean":"2023-09-23T22:46:26.700627+0000","last_became_active":"2023-09-23T22:45:42.842420+0000","last_became_peered":"2023-09-23T22:45:42.842420+0000","last_unstale":"2023-09-23T22:46:26.700627+0000","last_undegraded":"2023-09-23T22:46:26.700627+0000","last_fullsized":"2023-09-23T22:46:26.700627+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:11.531744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":530,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700869+0000","last_change":"2023-09-23T22:45:42.916672+0000","last_active":"2023-09-23T22:46:26.700869+0000","last_peered":"2023-09-23T22:46:26.700869+0000","last_clean":"2023-09-23T22:46:26.700869+0000","last_became_active":"2023-09-23T22:45:42.916180+0000","last_became_peered":"2023-09-23T22:45:42.916180+0000","last_unstale":"2023-09-23T22:46:26.700869+0000","last_undegraded":"2023-09-23T22:46:26.700869+0000","last_fullsized":"2023-09-23T22:46:26.700869+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:58.883770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":1080,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587639+0000","last_change":"2023-09-23T22:45:28.557202+0000","last_active":"2023-09-23T22:46:25.587639+0000","last_peered":"2023-09-23T22:46:25.587639+0000","last_clean":"2023-09-23T22:46:25.587639+0000","last_became_active":"2023-09-23T22:45:28.556683+0000","last_became_peered":"2023-09-23T22:45:28.556683+0000","last_unstale":"2023-09-23T22:46:25.587639+0000","last_undegraded":"2023-09-23T22:46:25.587639+0000","last_fullsized":"2023-09-23T22:46:25.587639+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_clean_scrub_stamp":"2023-09-23T22:44:02.567345+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:30.226289+0000","scrub_duration":0.0019136540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"640'181","reported_seq":601,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589202+0000","last_change":"2023-09-23T22:45:59.494626+0000","last_active":"2023-09-23T22:46:25.589202+0000","last_peered":"2023-09-23T22:46:25.589202+0000","last_clean":"2023-09-23T22:46:25.589202+0000","last_became_active":"2023-09-23T22:45:59.492597+0000","last_became_peered":"2023-09-23T22:45:59.492597+0000","last_unstale":"2023-09-23T22:46:25.589202+0000","last_undegraded":"2023-09-23T22:46:25.589202+0000","last_fullsized":"2023-09-23T22:46:25.589202+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:37.964127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050434000000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1303,"num_read_kb":1560,"num_write":136,"num_write_kb":302,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":257,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.508867+0000","last_change":"2023-09-23T22:46:17.905607+0000","last_active":"2023-09-23T22:46:26.508867+0000","last_peered":"2023-09-23T22:46:26.508867+0000","last_clean":"2023-09-23T22:46:26.508867+0000","last_became_active":"2023-09-23T22:46:17.904956+0000","last_became_peered":"2023-09-23T22:46:17.904956+0000","last_unstale":"2023-09-23T22:46:26.508867+0000","last_undegraded":"2023-09-23T22:46:26.508867+0000","last_fullsized":"2023-09-23T22:46:26.508867+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:43.177950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":120,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700831+0000","last_change":"2023-09-23T22:45:42.842394+0000","last_active":"2023-09-23T22:46:26.700831+0000","last_peered":"2023-09-23T22:46:26.700831+0000","last_clean":"2023-09-23T22:46:26.700831+0000","last_became_active":"2023-09-23T22:45:42.841979+0000","last_became_peered":"2023-09-23T22:45:42.841979+0000","last_unstale":"2023-09-23T22:46:26.700831+0000","last_undegraded":"2023-09-23T22:46:26.700831+0000","last_fullsized":"2023-09-23T22:46:26.700831+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:43.176130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":1037,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701443+0000","last_change":"2023-09-23T22:45:20.546576+0000","last_active":"2023-09-23T22:46:26.701443+0000","last_peered":"2023-09-23T22:46:26.701443+0000","last_clean":"2023-09-23T22:46:26.701443+0000","last_became_active":"2023-09-23T22:45:20.546222+0000","last_became_peered":"2023-09-23T22:45:20.546222+0000","last_unstale":"2023-09-23T22:46:26.701443+0000","last_undegraded":"2023-09-23T22:46:26.701443+0000","last_fullsized":"2023-09-23T22:46:26.701443+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:27.290379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:27.290379+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:53.972924+0000","scrub_duration":0.0016473779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":56,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834253+0000","last_change":"2023-09-23T22:45:25.573308+0000","last_active":"2023-09-23T22:46:25.834253+0000","last_peered":"2023-09-23T22:46:25.834253+0000","last_clean":"2023-09-23T22:46:25.834253+0000","last_became_active":"2023-09-23T22:45:25.551602+0000","last_became_peered":"2023-09-23T22:45:25.551602+0000","last_unstale":"2023-09-23T22:46:25.834253+0000","last_undegraded":"2023-09-23T22:46:25.834253+0000","last_fullsized":"2023-09-23T22:46:25.834253+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:30.986848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"640'291","reported_seq":1790,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509444+0000","last_change":"2023-09-23T22:45:20.547402+0000","last_active":"2023-09-23T22:46:26.509444+0000","last_peered":"2023-09-23T22:46:26.509444+0000","last_clean":"2023-09-23T22:46:26.509444+0000","last_became_active":"2023-09-23T22:45:20.547014+0000","last_became_peered":"2023-09-23T22:45:20.547014+0000","last_unstale":"2023-09-23T22:46:26.509444+0000","last_undegraded":"2023-09-23T22:46:26.509444+0000","last_fullsized":"2023-09-23T22:46:26.509444+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":291,"log_dups_size":0,"ondisk_log_size":291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:23.216759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059489999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1986,"num_read_kb":2085,"num_write":204,"num_write_kb":320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":48,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833650+0000","last_change":"2023-09-23T22:45:25.548381+0000","last_active":"2023-09-23T22:46:25.833650+0000","last_peered":"2023-09-23T22:46:25.833650+0000","last_clean":"2023-09-23T22:46:25.833650+0000","last_became_active":"2023-09-23T22:45:25.547994+0000","last_became_peered":"2023-09-23T22:45:25.547994+0000","last_unstale":"2023-09-23T22:46:25.833650+0000","last_undegraded":"2023-09-23T22:46:25.833650+0000","last_fullsized":"2023-09-23T22:46:25.833650+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:16.603990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":872,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834078+0000","last_change":"2023-09-23T22:45:25.550997+0000","last_active":"2023-09-23T22:46:25.834078+0000","last_peered":"2023-09-23T22:46:25.834078+0000","last_clean":"2023-09-23T22:46:25.834078+0000","last_became_active":"2023-09-23T22:45:25.550468+0000","last_became_peered":"2023-09-23T22:45:25.550468+0000","last_unstale":"2023-09-23T22:46:25.834078+0000","last_undegraded":"2023-09-23T22:46:25.834078+0000","last_fullsized":"2023-09-23T22:46:25.834078+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:03.084589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:03.084589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:27.030089+0000","scrub_duration":0.013840414000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"640'166","reported_seq":741,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589267+0000","last_change":"2023-09-23T22:45:09.436414+0000","last_active":"2023-09-23T22:46:25.589267+0000","last_peered":"2023-09-23T22:46:25.589267+0000","last_clean":"2023-09-23T22:46:25.589267+0000","last_became_active":"2023-09-23T22:45:09.435610+0000","last_became_peered":"2023-09-23T22:45:09.435610+0000","last_unstale":"2023-09-23T22:46:25.589267+0000","last_undegraded":"2023-09-23T22:46:25.589267+0000","last_fullsized":"2023-09-23T22:46:25.589267+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":166,"log_dups_size":0,"ondisk_log_size":166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:00.370983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044240799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":819,"num_read_kb":1642,"num_write":161,"num_write_kb":1597,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":520,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700879+0000","last_change":"2023-09-23T22:45:42.840850+0000","last_active":"2023-09-23T22:46:26.700879+0000","last_peered":"2023-09-23T22:46:26.700879+0000","last_clean":"2023-09-23T22:46:26.700879+0000","last_became_active":"2023-09-23T22:45:42.840300+0000","last_became_peered":"2023-09-23T22:45:42.840300+0000","last_unstale":"2023-09-23T22:46:26.700879+0000","last_undegraded":"2023-09-23T22:46:26.700879+0000","last_fullsized":"2023-09-23T22:46:26.700879+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:28.519370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":92,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700617+0000","last_change":"2023-09-23T22:46:11.935394+0000","last_active":"2023-09-23T22:46:26.700617+0000","last_peered":"2023-09-23T22:46:26.700617+0000","last_clean":"2023-09-23T22:46:26.700617+0000","last_became_active":"2023-09-23T22:46:11.935105+0000","last_became_peered":"2023-09-23T22:46:11.935105+0000","last_unstale":"2023-09-23T22:46:26.700617+0000","last_undegraded":"2023-09-23T22:46:26.700617+0000","last_fullsized":"2023-09-23T22:46:26.700617+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:39.722745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":1063,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833929+0000","last_change":"2023-09-23T22:45:28.549683+0000","last_active":"2023-09-23T22:46:25.833929+0000","last_peered":"2023-09-23T22:46:25.833929+0000","last_clean":"2023-09-23T22:46:25.833929+0000","last_became_active":"2023-09-23T22:45:28.549245+0000","last_became_peered":"2023-09-23T22:45:28.549245+0000","last_unstale":"2023-09-23T22:46:25.833929+0000","last_undegraded":"2023-09-23T22:46:25.833929+0000","last_fullsized":"2023-09-23T22:46:25.833929+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:39.417809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:39.417809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:31.897644+0000","scrub_duration":0.0016817830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":36,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700920+0000","last_change":"2023-09-23T22:45:25.549660+0000","last_active":"2023-09-23T22:46:26.700920+0000","last_peered":"2023-09-23T22:46:26.700920+0000","last_clean":"2023-09-23T22:46:26.700920+0000","last_became_active":"2023-09-23T22:45:25.549284+0000","last_became_peered":"2023-09-23T22:45:25.549284+0000","last_unstale":"2023-09-23T22:46:26.700920+0000","last_undegraded":"2023-09-23T22:46:26.700920+0000","last_fullsized":"2023-09-23T22:46:26.700920+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:04.359582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.611665+0000","last_change":"2023-09-23T22:46:02.574561+0000","last_active":"2023-09-23T22:46:26.611665+0000","last_peered":"2023-09-23T22:46:26.611665+0000","last_clean":"2023-09-23T22:46:26.611665+0000","last_became_active":"2023-09-23T22:46:02.574207+0000","last_became_peered":"2023-09-23T22:46:02.574207+0000","last_unstale":"2023-09-23T22:46:26.611665+0000","last_undegraded":"2023-09-23T22:46:26.611665+0000","last_fullsized":"2023-09-23T22:46:26.611665+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:18.613134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":1062,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834157+0000","last_change":"2023-09-23T22:45:25.550225+0000","last_active":"2023-09-23T22:46:25.834157+0000","last_peered":"2023-09-23T22:46:25.834157+0000","last_clean":"2023-09-23T22:46:25.834157+0000","last_became_active":"2023-09-23T22:45:25.549867+0000","last_became_peered":"2023-09-23T22:45:25.549867+0000","last_unstale":"2023-09-23T22:46:25.834157+0000","last_undegraded":"2023-09-23T22:46:25.834157+0000","last_fullsized":"2023-09-23T22:46:25.834157+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:41.386557+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:41.386557+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:52.547517+0000","scrub_duration":0.0019422809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":748,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588889+0000","last_change":"2023-09-23T22:45:17.530896+0000","last_active":"2023-09-23T22:46:25.588889+0000","last_peered":"2023-09-23T22:46:25.588889+0000","last_clean":"2023-09-23T22:46:25.588889+0000","last_became_active":"2023-09-23T22:45:17.501647+0000","last_became_peered":"2023-09-23T22:45:17.501647+0000","last_unstale":"2023-09-23T22:46:25.588889+0000","last_undegraded":"2023-09-23T22:46:25.588889+0000","last_fullsized":"2023-09-23T22:46:25.588889+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:44.145773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":709,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701145+0000","last_change":"2023-09-23T22:46:11.935068+0000","last_active":"2023-09-23T22:46:26.701145+0000","last_peered":"2023-09-23T22:46:26.701145+0000","last_clean":"2023-09-23T22:46:26.701145+0000","last_became_active":"2023-09-23T22:46:11.934724+0000","last_became_peered":"2023-09-23T22:46:11.934724+0000","last_unstale":"2023-09-23T22:46:26.701145+0000","last_undegraded":"2023-09-23T22:46:26.701145+0000","last_fullsized":"2023-09-23T22:46:26.701145+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:23.099835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":763,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588459+0000","last_change":"2023-09-23T22:45:17.527461+0000","last_active":"2023-09-23T22:46:25.588459+0000","last_peered":"2023-09-23T22:46:25.588459+0000","last_clean":"2023-09-23T22:46:25.588459+0000","last_became_active":"2023-09-23T22:45:17.501280+0000","last_became_peered":"2023-09-23T22:45:17.501280+0000","last_unstale":"2023-09-23T22:46:25.588459+0000","last_undegraded":"2023-09-23T22:46:25.588459+0000","last_fullsized":"2023-09-23T22:46:25.588459+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:42.501570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":162,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700854+0000","last_change":"2023-09-23T22:46:11.936343+0000","last_active":"2023-09-23T22:46:26.700854+0000","last_peered":"2023-09-23T22:46:26.700854+0000","last_clean":"2023-09-23T22:46:26.700854+0000","last_became_active":"2023-09-23T22:46:11.935390+0000","last_became_peered":"2023-09-23T22:46:11.935390+0000","last_unstale":"2023-09-23T22:46:26.700854+0000","last_undegraded":"2023-09-23T22:46:26.700854+0000","last_fullsized":"2023-09-23T22:46:26.700854+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:19:41.846656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":782,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587550+0000","last_change":"2023-09-23T22:45:59.489049+0000","last_active":"2023-09-23T22:46:25.587550+0000","last_peered":"2023-09-23T22:46:25.587550+0000","last_clean":"2023-09-23T22:46:25.587550+0000","last_became_active":"2023-09-23T22:45:59.488402+0000","last_became_peered":"2023-09-23T22:45:59.488402+0000","last_unstale":"2023-09-23T22:46:25.587550+0000","last_undegraded":"2023-09-23T22:46:25.587550+0000","last_fullsized":"2023-09-23T22:46:25.587550+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:56.024308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:56.024308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.046514+0000","scrub_duration":0.0045490360000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834216+0000","last_change":"2023-09-23T22:45:22.510829+0000","last_active":"2023-09-23T22:46:25.834216+0000","last_peered":"2023-09-23T22:46:25.834216+0000","last_clean":"2023-09-23T22:46:25.834216+0000","last_became_active":"2023-09-23T22:45:22.510400+0000","last_became_peered":"2023-09-23T22:45:22.510400+0000","last_unstale":"2023-09-23T22:46:25.834216+0000","last_undegraded":"2023-09-23T22:46:25.834216+0000","last_fullsized":"2023-09-23T22:46:25.834216+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:30.648352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":290,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588728+0000","last_change":"2023-09-23T22:45:59.493580+0000","last_active":"2023-09-23T22:46:25.588728+0000","last_peered":"2023-09-23T22:46:25.588728+0000","last_clean":"2023-09-23T22:46:25.588728+0000","last_became_active":"2023-09-23T22:45:59.492686+0000","last_became_peered":"2023-09-23T22:45:59.492686+0000","last_unstale":"2023-09-23T22:46:25.588728+0000","last_undegraded":"2023-09-23T22:46:25.588728+0000","last_fullsized":"2023-09-23T22:46:25.588728+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:26.630282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":1072,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T22:46:29.934860+0000","last_change":"2023-09-23T22:45:25.544754+0000","last_active":"2023-09-23T22:46:29.934860+0000","last_peered":"2023-09-23T22:46:29.934860+0000","last_clean":"2023-09-23T22:46:29.934860+0000","last_became_active":"2023-09-23T22:45:25.544366+0000","last_became_peered":"2023-09-23T22:45:25.544366+0000","last_unstale":"2023-09-23T22:46:29.934860+0000","last_undegraded":"2023-09-23T22:46:29.934860+0000","last_fullsized":"2023-09-23T22:46:29.934860+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:53.343498+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:53.343498+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:01.725748+0000","scrub_duration":0.0041572029999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":169,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700336+0000","last_change":"2023-09-23T22:46:02.508263+0000","last_active":"2023-09-23T22:46:26.700336+0000","last_peered":"2023-09-23T22:46:26.700336+0000","last_clean":"2023-09-23T22:46:26.700336+0000","last_became_active":"2023-09-23T22:46:02.504295+0000","last_became_peered":"2023-09-23T22:46:02.504295+0000","last_unstale":"2023-09-23T22:46:26.700336+0000","last_undegraded":"2023-09-23T22:46:26.700336+0000","last_fullsized":"2023-09-23T22:46:26.700336+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:36.779405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":43,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700611+0000","last_change":"2023-09-23T22:45:17.507532+0000","last_active":"2023-09-23T22:46:26.700611+0000","last_peered":"2023-09-23T22:46:26.700611+0000","last_clean":"2023-09-23T22:46:26.700611+0000","last_became_active":"2023-09-23T22:45:17.507180+0000","last_became_peered":"2023-09-23T22:45:17.507180+0000","last_unstale":"2023-09-23T22:46:26.700611+0000","last_undegraded":"2023-09-23T22:46:26.700611+0000","last_fullsized":"2023-09-23T22:46:26.700611+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:05.877635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":175,"num_bytes_recovered":53996,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62789,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11971,"num_read_kb":25396,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":583,"num_bytes_recovered":25165919,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234512,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":541464,"kb_used_data":11664,"kb_used_omap":0,"kb_used_meta":529792,"kb_avail":93203688,"statfs":{"total":95995035648,"available":95440576512,"internally_reserved":0,"allocated":11943936,"data_stored":9915759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":542507008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.98599999999999999,"15min":2.0419999999999998},"min":{"1min":0.37,"5min":0.23699999999999999,"15min":0.19700000000000001},"max":{"1min":0.72599999999999998,"5min":25.308,"15min":124.34699999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.63,"5min":1.022,"15min":2.1219999999999999},"min":{"1min":0.40200000000000002,"5min":0.26400000000000001,"15min":0.23000000000000001},"max":{"1min":0.88500000000000001,"5min":27.265000000000001,"15min":124.73099999999999},"last":0.47799999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:46:50 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.0589999999999999,"15min":0.93400000000000005},"min":{"1min":0.40600000000000003,"5min":0.251,"15min":0.223},"max":{"1min":0.87,"5min":20.045999999999999,"15min":42.329999999999998},"last":0.502},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.95899999999999996,"15min":0.89500000000000002},"min":{"1min":0.46400000000000002,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":0.80300000000000005,"5min":19.920000000000002,"15min":42.703000000000003},"last":0.432}]},{"osd":2,"last update":"Sat Sep 23 22:46:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.4039999999999999,"15min":1.702},"min":{"1min":0.41499999999999998,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":0.84399999999999997,"5min":29.189,"15min":70.183999999999997},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.5580000000000001,"15min":1.889},"min":{"1min":0.443,"5min":0.30399999999999999,"15min":0.28899999999999998},"max":{"1min":0.81200000000000006,"5min":31.373000000000001,"15min":70.140000000000001},"last":0.49199999999999999}]},{"osd":3,"last update":"Sat Sep 23 22:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.85699999999999998,"15min":1.1240000000000001},"min":{"1min":0.55100000000000005,"5min":0.316,"15min":0.316},"max":{"1min":0.84899999999999998,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.65100000000000002,"5min":1.0469999999999999,"15min":1.45},"min":{"1min":0.499,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.82999999999999996,"5min":7.1429999999999998,"15min":7.1429999999999998},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.45500000000000002,"15min":0.58599999999999997},"min":{"1min":0.28499999999999998,"5min":0.27400000000000002,"15min":0.24099999999999999},"max":{"1min":0.68200000000000005,"5min":0.89500000000000002,"15min":2.8090000000000002},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.45600000000000002,"15min":1.234},"min":{"1min":0.35499999999999998,"5min":0.25700000000000001,"15min":0.23899999999999999},"max":{"1min":0.80200000000000005,"5min":0.871,"15min":48.960000000000001},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 22:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.51000000000000001,"15min":1.7290000000000001},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.28100000000000003},"max":{"1min":1.006,"5min":1.591,"15min":125.083},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.53100000000000003,"15min":1.6379999999999999},"min":{"1min":0.39800000000000002,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":1.04,"5min":1.413,"15min":122.351},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.70699999999999996,"15min":0.61499999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":12.616,"5min":12.616,"15min":12.616},"last":0.37},{"interface":"front","average":{"1min":1.218,"5min":0.67000000000000004,"15min":0.56200000000000006},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":12.632999999999999,"5min":12.632999999999999,"15min":12.632999999999999},"last":0.46700000000000003}]}]},{"osd":6,"up_from":592,"seq":2542620639283,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":136336,"kb_used_data":11720,"kb_used_omap":8,"kb_used_meta":124599,"kb_avail":93608816,"statfs":{"total":95995035648,"available":95855427584,"internally_reserved":0,"allocated":12001280,"data_stored":9980947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":127590242},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.6059999999999999,"15min":3.532},"min":{"1min":0.32100000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.82499999999999996,"5min":47.795000000000002,"15min":47.795000000000002},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":2.1970000000000001,"15min":2.9350000000000001},"min":{"1min":0.39800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.81299999999999994,"5min":47.682000000000002,"15min":47.682000000000002},"last":0.46000000000000002}]},{"osd":1,"last update":"Sat Sep 23 22:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":4.6950000000000003,"15min":6.5579999999999998},"min":{"1min":0.35999999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.71399999999999997,"5min":115.848,"15min":115.848},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.60499999999999998,"5min":4.8529999999999998,"15min":6.7690000000000001},"min":{"1min":0.45000000000000001,"5min":0.251,"15min":0.251},"max":{"1min":0.745,"5min":116.517,"15min":116.517},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.90400000000000003,"15min":1.0569999999999999},"min":{"1min":0.373,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.79000000000000004,"5min":3.6379999999999999,"15min":3.6379999999999999},"last":0.437},{"interface":"front","average":{"1min":0.57999999999999996,"5min":1.8069999999999999,"15min":2.3599999999999999},"min":{"1min":0.38300000000000001,"5min":0.254,"15min":0.254},"max":{"1min":0.72199999999999998,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.375}]},{"osd":3,"last update":"Sat Sep 23 22:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.81299999999999994,"15min":0.91100000000000003},"min":{"1min":0.38800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.746,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.624,"5min":0.95199999999999996,"15min":1.117},"min":{"1min":0.39500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.78300000000000003,"5min":5.6260000000000003,"15min":5.6260000000000003},"last":0.505}]},{"osd":4,"last update":"Sat Sep 23 22:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51500000000000001,"15min":0.51900000000000002},"min":{"1min":0.34100000000000003,"5min":0.311,"15min":0.311},"max":{"1min":0.72399999999999998,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.51600000000000001,"15min":0.495},"min":{"1min":0.38600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.79200000000000004,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.48699999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.51500000000000001,"15min":0.497},"min":{"1min":0.308,"5min":0.255,"15min":0.255},"max":{"1min":0.81499999999999995,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.318},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.56399999999999995,"15min":0.57499999999999996},"min":{"1min":0.32400000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.79300000000000004,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:45:57 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.60399999999999998,"15min":0.59599999999999997},"min":{"1min":0.42499999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.97299999999999998,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.629,"5min":0.58899999999999997,"15min":0.58599999999999997},"min":{"1min":0.46000000000000002,"5min":0.373,"15min":0.373},"max":{"1min":0.81999999999999995,"5min":1.339,"15min":1.339},"last":0.47899999999999998}]}]},{"osd":1,"up_from":495,"seq":2126008811634,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":463084,"kb_used_data":16932,"kb_used_omap":12,"kb_used_meta":446131,"kb_avail":93282068,"statfs":{"total":95995035648,"available":95520837632,"internally_reserved":0,"allocated":17338368,"data_stored":14010502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":456838830},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.1299999999999999,"15min":1.119},"min":{"1min":0.21099999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.6139999999999999,"5min":12.361000000000001,"15min":64.507000000000005},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":1.3420000000000001,"15min":1.282},"min":{"1min":0.27000000000000002,"5min":0.23899999999999999,"15min":0.20399999999999999},"max":{"1min":3.4049999999999998,"5min":22.393000000000001,"15min":64.352000000000004},"last":0.375}]},{"osd":2,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":3.6110000000000002,"15min":1.696},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.192},"max":{"1min":3.7450000000000001,"5min":124.447,"15min":124.447},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":3.0859999999999999,"15min":1.583},"min":{"1min":0.246,"5min":0.246,"15min":0.218},"max":{"1min":1.2569999999999999,"5min":124.03,"15min":124.03},"last":0.44}]},{"osd":3,"last update":"Sat Sep 23 22:46:35 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":1.1100000000000001,"15min":1.5660000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.91400000000000003,"5min":11.800000000000001,"15min":11.800000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.98499999999999999,"15min":1.4630000000000001},"min":{"1min":0.31,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.46,"5min":6.4530000000000003,"15min":6.4530000000000003},"last":0.56999999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.046,"15min":0.76500000000000001},"min":{"1min":0.44600000000000001,"5min":0.34899999999999998,"15min":0.26400000000000001},"max":{"1min":3.6429999999999998,"5min":9.4410000000000007,"15min":9.4410000000000007},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.90900000000000003,"15min":0.93100000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.28499999999999998},"max":{"1min":3.4340000000000002,"5min":5.524,"15min":5.6740000000000004},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.96699999999999997,"15min":0.91500000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.98699999999999999,"5min":11.734,"15min":11.734},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.74299999999999999,"5min":1.069,"15min":0.79900000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":3.5550000000000002,"5min":9.4169999999999998,"15min":9.4169999999999998},"last":0.63400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.1060000000000001,"15min":1.526},"min":{"1min":0.44800000000000001,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":3.585,"5min":9.3759999999999994,"15min":9.3759999999999994},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.75,"15min":0.83399999999999996},"min":{"1min":0.44,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.3620000000000001,"5min":3.3620000000000001,"15min":3.3620000000000001},"last":0.69899999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.022,"15min":0.96599999999999997},"min":{"1min":0.42999999999999999,"5min":0.40600000000000003,"15min":0.30199999999999999},"max":{"1min":0.95999999999999996,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.622},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.1020000000000001,"15min":0.84699999999999998},"min":{"1min":0.45200000000000001,"5min":0.41599999999999998,"15min":0.32800000000000001},"max":{"1min":3.5150000000000001,"5min":9.4819999999999993,"15min":9.4819999999999993},"last":0.67900000000000005}]}]},{"osd":0,"up_from":93,"seq":399431958950,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1134932,"kb_used_data":13196,"kb_used_omap":8,"kb_used_meta":1121719,"kb_avail":92610220,"statfs":{"total":95995035648,"available":94832865280,"internally_reserved":0,"allocated":13512704,"data_stored":10145658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":1148641130},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":6.1689999999999996,"15min":2.6789999999999998},"min":{"1min":0.222,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.55500000000000005,"5min":195.91999999999999,"15min":195.91999999999999},"last":0.307},{"interface":"front","average":{"1min":0.41899999999999998,"5min":6.6050000000000004,"15min":2.673},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.252},"max":{"1min":0.80200000000000005,"5min":195.90799999999999,"15min":195.90799999999999},"last":0.38100000000000001}]},{"osd":2,"last update":"Sat Sep 23 22:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":4.569,"15min":3.2639999999999998},"min":{"1min":0.27100000000000002,"5min":0.216,"15min":0.216},"max":{"1min":0.53900000000000003,"5min":196.16499999999999,"15min":196.16499999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":4.7389999999999999,"15min":3.1360000000000001},"min":{"1min":0.245,"5min":0.216,"15min":0.216},"max":{"1min":0.77900000000000003,"5min":191.999,"15min":191.999},"last":0.29099999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:46:38 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.595,"15min":2.6469999999999998},"min":{"1min":0.33500000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.751,"5min":9.4149999999999991,"15min":9.4149999999999991},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":1.0660000000000001,"15min":1.6100000000000001},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61299999999999999,"5min":4.4080000000000004,"15min":4.4080000000000004},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.96899999999999997,"15min":1.167},"min":{"1min":0.34899999999999998,"5min":0.32200000000000001,"15min":0.26000000000000001},"max":{"1min":0.85999999999999999,"5min":4.2709999999999999,"15min":9.968},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.98999999999999999,"15min":1.202},"min":{"1min":0.40899999999999997,"5min":0.377,"15min":0.33300000000000002},"max":{"1min":0.92800000000000005,"5min":4.298,"15min":9.9949999999999992},"last":0.72799999999999998}]},{"osd":5,"last update":"Sat Sep 23 22:46:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.72799999999999998,"15min":2.0299999999999998},"min":{"1min":0.38600000000000001,"5min":0.31,"15min":0.28699999999999998},"max":{"1min":1.0009999999999999,"5min":2.9220000000000002,"15min":126.872},"last":0.83099999999999996},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.99299999999999999,"15min":1.4630000000000001},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.27000000000000002},"max":{"1min":0.83799999999999997,"5min":9.3819999999999997,"15min":126.521},"last":0.623}]},{"osd":6,"last update":"Sat Sep 23 22:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":1.671,"15min":2.6970000000000001},"min":{"1min":0.504,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95499999999999996,"5min":26.431000000000001,"15min":26.431000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.8480000000000001,"15min":3.024},"min":{"1min":0.42999999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.89300000000000002,"5min":26.387,"15min":26.387},"last":0.79900000000000004}]},{"osd":7,"last update":"Sat Sep 23 22:46:38 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":1.012,"15min":1.266},"min":{"1min":0.45200000000000001,"5min":0.32400000000000001,"15min":0.316},"max":{"1min":0.89000000000000001,"5min":4.2199999999999998,"15min":10.377000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.78400000000000003,"15min":1.3720000000000001},"min":{"1min":0.48099999999999998,"5min":0.35599999999999998,"15min":0.35499999999999998},"max":{"1min":0.91000000000000003,"5min":2.9649999999999999,"15min":9.0879999999999992},"last":0.85799999999999998}]}]},{"osd":2,"up_from":349,"seq":1498943586513,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":518172,"kb_used_data":8980,"kb_used_omap":15,"kb_used_meta":509168,"kb_avail":93226980,"statfs":{"total":95995035648,"available":95464427520,"internally_reserved":0,"allocated":9195520,"data_stored":5879586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":521388790},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":1.7529999999999999,"15min":2.3260000000000001},"min":{"1min":0.216,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":0.46000000000000002,"5min":61.712000000000003,"15min":86.620000000000005},"last":0.502},{"interface":"front","average":{"1min":0.34999999999999998,"5min":2.2040000000000002,"15min":2.6040000000000001},"min":{"1min":0.247,"5min":0.223,"15min":0.215},"max":{"1min":0.45300000000000001,"5min":61.177999999999997,"15min":86.355000000000004},"last":0.252}]},{"osd":1,"last update":"Sat Sep 23 22:46:58 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.865,"15min":1.329},"min":{"1min":0.23100000000000001,"5min":0.20399999999999999,"15min":0.19500000000000001},"max":{"1min":1.034,"5min":47.018000000000001,"15min":70.042000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.41999999999999998,"5min":3.9540000000000002,"15min":2.25},"min":{"1min":0.27500000000000002,"5min":0.23599999999999999,"15min":0.184},"max":{"1min":0.84099999999999997,"5min":127.327,"15min":127.327},"last":0.314}]},{"osd":3,"last update":"Sat Sep 23 22:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":2.0190000000000001,"15min":3.2839999999999998},"min":{"1min":0.308,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.52300000000000002,"5min":14.561,"15min":14.561},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.36799999999999999,"5min":1.0620000000000001,"15min":1.6180000000000001},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.434,"5min":4.7720000000000002,"15min":4.7720000000000002},"last":0.33800000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.1779999999999999,"15min":1.6799999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.23799999999999999},"max":{"1min":0.89300000000000002,"5min":8.8710000000000004,"15min":45.576999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.96199999999999997,"15min":1.2230000000000001},"min":{"1min":0.42799999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.91300000000000003,"5min":5.157,"15min":45.420999999999999},"last":0.436}]},{"osd":5,"last update":"Sat Sep 23 22:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.155,"15min":3.2490000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.27300000000000002},"max":{"1min":0.88500000000000001,"5min":8.8119999999999994,"15min":163.56399999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":2.246,"15min":3.3730000000000002},"min":{"1min":0.38500000000000001,"5min":0.30099999999999999,"15min":0.28599999999999998},"max":{"1min":0.83999999999999997,"5min":126.81100000000001,"15min":163.541},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.5980000000000001,"15min":2.0409999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.90100000000000002,"5min":7.351,"15min":7.351},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.129,"15min":1.3600000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.86199999999999999,"5min":3.895,"15min":3.895},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.3220000000000001,"15min":1.8520000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.26400000000000001},"max":{"1min":0.82699999999999996,"5min":7.3979999999999997,"15min":108.113},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.1990000000000001,"15min":2.1749999999999998},"min":{"1min":0.438,"5min":0.432,"15min":0.30599999999999999},"max":{"1min":0.81000000000000005,"5min":8.8480000000000008,"15min":108.093},"last":0.46600000000000003}]}]},{"osd":3,"up_from":584,"seq":2508260900920,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":133304,"kb_used_data":9200,"kb_used_omap":9,"kb_used_meta":124086,"kb_avail":93611848,"statfs":{"total":95995035648,"available":95858532352,"internally_reserved":0,"allocated":9420800,"data_stored":5959070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10195,"internal_metadata":127064109},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":1.0529999999999999,"15min":1.6459999999999999},"min":{"1min":0.25700000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.55100000000000005,"5min":5.2939999999999996,"15min":5.2939999999999996},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.874,"15min":1.202},"min":{"1min":0.26900000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.54300000000000004,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":0.42899999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":9.8450000000000006,"15min":17.018999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.64700000000000002,"5min":303.267,"15min":303.267},"last":0.26500000000000001},{"interface":"front","average":{"1min":0.39000000000000001,"5min":9.6419999999999995,"15min":16.539999999999999},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":303.09699999999998,"15min":303.09699999999998},"last":0.34899999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":13.375,"15min":25.969999999999999},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.54200000000000004,"5min":381.87299999999999,"15min":381.87299999999999},"last":0.38},{"interface":"front","average":{"1min":0.41199999999999998,"5min":13.739000000000001,"15min":26.841000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.55300000000000005,"5min":382.73500000000001,"15min":382.73500000000001},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.0580000000000001,"15min":1.488},"min":{"1min":0.36699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.753,"5min":5.0460000000000003,"15min":5.0460000000000003},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.0009999999999999,"15min":1.359},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.871,"5min":4.5410000000000004,"15min":4.5410000000000004},"last":0.439}]},{"osd":5,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.86599999999999999,"15min":1.1240000000000001},"min":{"1min":0.45300000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.83899999999999997,"5min":3.762,"15min":3.762},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.093,"15min":1.524},"min":{"1min":0.499,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.77000000000000002,"5min":5.1079999999999997,"15min":5.1079999999999997},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.80400000000000005,"15min":0.90900000000000003},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.81499999999999995,"5min":3.7839999999999998,"15min":3.7839999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.133,"15min":1.377},"min":{"1min":0.42199999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.96199999999999997,"5min":5.1529999999999996,"15min":5.1529999999999996},"last":0.48499999999999999}]},{"osd":7,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.159,"15min":1.613},"min":{"1min":0.46700000000000003,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.002,"5min":5.1929999999999996,"15min":5.1929999999999996},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.048,"15min":1.4079999999999999},"min":{"1min":0.47799999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.91200000000000003,"5min":4.5800000000000001,"15min":4.5800000000000001},"last":0.50800000000000001}]}]},{"osd":4,"up_from":465,"seq":1997159792776,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":123348,"kb_used_data":7308,"kb_used_omap":12,"kb_used_meta":116019,"kb_avail":93621804,"statfs":{"total":95995035648,"available":95868727296,"internally_reserved":0,"allocated":7483392,"data_stored":5459020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":118804081},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.79400000000000004,"15min":0.98799999999999999},"min":{"1min":0.35499999999999998,"5min":0.23100000000000001,"15min":0.21199999999999999},"max":{"1min":1.0309999999999999,"5min":8.7289999999999992,"15min":8.7289999999999992},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.69799999999999995,"15min":0.90800000000000003},"min":{"1min":0.38600000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.83299999999999996,"5min":3.5990000000000002,"15min":9.1110000000000007},"last":0.48999999999999999}]},{"osd":1,"last update":"Sat Sep 23 22:46:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.89400000000000002,"15min":0.82399999999999995},"min":{"1min":0.38500000000000001,"5min":0.246,"15min":0.23100000000000001},"max":{"1min":0.69299999999999995,"5min":8.3539999999999992,"15min":38.142000000000003},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.82299999999999995,"15min":0.92500000000000004},"min":{"1min":0.38400000000000001,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":0.78600000000000003,"5min":5.6040000000000001,"15min":38.167000000000002},"last":0.46300000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.0699999999999998,"15min":1.611},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.247},"max":{"1min":1.0980000000000001,"5min":104.13,"15min":104.13},"last":0.624},{"interface":"front","average":{"1min":0.624,"5min":2.4729999999999999,"15min":1.9950000000000001},"min":{"1min":0.42999999999999999,"5min":0.30599999999999999,"15min":0.29999999999999999},"max":{"1min":0.89900000000000002,"5min":109.82899999999999,"15min":109.82899999999999},"last":0.58299999999999996}]},{"osd":3,"last update":"Sat Sep 23 22:46:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.72799999999999998,"15min":0.76200000000000001},"min":{"1min":0.43099999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.91600000000000004,"5min":2.8879999999999999,"15min":2.8879999999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.85499999999999998,"15min":1.0569999999999999},"min":{"1min":0.45900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.88,"5min":4.2770000000000001,"15min":4.2770000000000001},"last":0.68300000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.395,"15min":1.262},"min":{"1min":0.309,"5min":0.224,"15min":0.224},"max":{"1min":0.84599999999999997,"5min":80.213999999999999,"15min":80.213999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.3979999999999999,"15min":1.244},"min":{"1min":0.32500000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.79300000000000004,"5min":80.244,"15min":80.244},"last":0.55400000000000005}]},{"osd":6,"last update":"Sat Sep 23 22:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.52100000000000002,"15min":0.46700000000000003},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.76700000000000002,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.54800000000000004,"15min":0.55000000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.91400000000000003,"5min":2.1469999999999998,"15min":2.1469999999999998},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.51200000000000001,"15min":1.0900000000000001},"min":{"1min":0.378,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":0.81200000000000006,"5min":0.96099999999999997,"15min":8.1950000000000003},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.62,"5min":0.52500000000000002,"15min":1.341},"min":{"1min":0.44500000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.81299999999999994,"5min":0.87,"15min":8.6140000000000008},"last":0.69599999999999995}]}]},{"osd":5,"up_from":38,"seq":163208757785,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":566924,"kb_used_data":11652,"kb_used_omap":0,"kb_used_meta":555264,"kb_avail":93178228,"statfs":{"total":95995035648,"available":95414505472,"internally_reserved":0,"allocated":11931648,"data_stored":9911775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":568590336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:48 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":1.0600000000000001,"15min":3.4089999999999998},"min":{"1min":0.28100000000000003,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.624,"5min":26.925000000000001,"15min":145.46600000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.071,"15min":3.4489999999999998},"min":{"1min":0.33200000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.75700000000000001,"5min":30.609000000000002,"15min":144.429},"last":0.54200000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:46:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":1.6970000000000001,"15min":1.075},"min":{"1min":0.315,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.71299999999999997,"5min":35.643999999999998,"15min":35.643999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.52800000000000002,"5min":1.7090000000000001,"15min":1.008},"min":{"1min":0.33500000000000002,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":0.71599999999999997,"5min":35.316000000000003,"15min":35.316000000000003},"last":0.47699999999999998}]},{"osd":2,"last update":"Sat Sep 23 22:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":2.2250000000000001,"15min":2.2469999999999999},"min":{"1min":0.34399999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.72499999999999998,"5min":77.813999999999993,"15min":77.813999999999993},"last":0.378},{"interface":"front","average":{"1min":0.54700000000000004,"5min":1.9850000000000001,"15min":2.1019999999999999},"min":{"1min":0.41699999999999998,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":0.75900000000000001,"5min":76.265000000000001,"15min":76.265000000000001},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:46:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.996,"15min":1.353},"min":{"1min":0.39600000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.78000000000000003,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.98599999999999999,"15min":1.3180000000000001},"min":{"1min":0.441,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.73799999999999999,"5min":7.8959999999999999,"15min":7.8959999999999999},"last":0.65500000000000003}]},{"osd":4,"last update":"Sat Sep 23 22:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.442,"15min":1.958},"min":{"1min":0.29999999999999999,"5min":0.24399999999999999,"15min":0.23999999999999999},"max":{"1min":0.75800000000000001,"5min":1.476,"15min":61.173999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.53200000000000003,"15min":2.0329999999999999},"min":{"1min":0.39000000000000001,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":0.80300000000000005,"5min":1.5409999999999999,"15min":61.203000000000003},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Sep 23 22:46:03 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.47799999999999998,"15min":0.44700000000000001},"min":{"1min":0.33600000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.746,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.52300000000000002,"15min":0.48199999999999998},"min":{"1min":0.36599999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.874,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.59099999999999997}]},{"osd":7,"last update":"Sat Sep 23 22:46:48 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.54900000000000004,"15min":1.3160000000000001},"min":{"1min":0.34699999999999998,"5min":0.255,"15min":0.249},"max":{"1min":0.81899999999999995,"5min":1.4550000000000001,"15min":59.101999999999997},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.56999999999999995,"15min":1.496},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.27600000000000002},"max":{"1min":0.80100000000000005,"5min":1.6499999999999999,"15min":59.151000000000003},"last":0.60599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8798,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:47:06.020 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T22:47:06.386 INFO:teuthology.orchestra.run.smithi026.stderr:Inferring config /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T22:47:06.782 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:06 smithi026 ceph-mon[223014]: from='client.37989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:47:06.783 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:06 smithi026 ceph-mon[215775]: from='client.37989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:47:06.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:06 smithi070 ceph-mon[192652]: from='client.37989 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T22:47:07.732 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:07 smithi026 ceph-mon[215775]: pgmap v1751: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:07.732 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:07 smithi026 ceph-mon[223014]: pgmap v1751: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:07.873 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:07 smithi070 ceph-mon[192652]: pgmap v1751: 103 pgs: 103 active+clean; 8.6 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail 2023-09-23T22:47:08.162 INFO:teuthology.orchestra.run.smithi026.stdout: 2023-09-23T22:47:08.164 INFO:teuthology.orchestra.run.smithi026.stderr:dumped all 2023-09-23T22:47:08.821 INFO:teuthology.orchestra.run.smithi026.stdout:{"pg_ready":true,"pg_map":{"version":1752,"stamp":"2023-09-23T22:47:07.797850+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11971,"num_read_kb":25396,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":762,"num_bytes_recovered":26400651,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3617564,"kb_used_data":90652,"kb_used_omap":66,"kb_used_meta":3526781,"kb_avail":746343652,"statfs":{"total":767960285184,"available":764255899648,"internally_reserved":0,"allocated":92827648,"data_stored":71262317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":67826,"internal_metadata":3611424526},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003535"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":1185,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587835+0000","last_change":"2023-09-23T22:45:09.435050+0000","last_active":"2023-09-23T22:46:25.587835+0000","last_peered":"2023-09-23T22:46:25.587835+0000","last_clean":"2023-09-23T22:46:25.587835+0000","last_became_active":"2023-09-23T22:45:09.434512+0000","last_became_peered":"2023-09-23T22:45:09.434512+0000","last_unstale":"2023-09-23T22:46:25.587835+0000","last_undegraded":"2023-09-23T22:46:25.587835+0000","last_fullsized":"2023-09-23T22:46:25.587835+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.209739+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.209739+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:43.842917+0000","scrub_duration":0.0016266950000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":43,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700962+0000","last_change":"2023-09-23T22:45:17.506717+0000","last_active":"2023-09-23T22:46:26.700962+0000","last_peered":"2023-09-23T22:46:26.700962+0000","last_clean":"2023-09-23T22:46:26.700962+0000","last_became_active":"2023-09-23T22:45:17.506273+0000","last_became_peered":"2023-09-23T22:45:17.506273+0000","last_unstale":"2023-09-23T22:46:26.700962+0000","last_undegraded":"2023-09-23T22:46:26.700962+0000","last_fullsized":"2023-09-23T22:46:26.700962+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:35.732850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834129+0000","last_change":"2023-09-23T22:45:22.504651+0000","last_active":"2023-09-23T22:46:25.834129+0000","last_peered":"2023-09-23T22:46:25.834129+0000","last_clean":"2023-09-23T22:46:25.834129+0000","last_became_active":"2023-09-23T22:45:22.504357+0000","last_became_peered":"2023-09-23T22:45:22.504357+0000","last_unstale":"2023-09-23T22:46:25.834129+0000","last_undegraded":"2023-09-23T22:46:25.834129+0000","last_fullsized":"2023-09-23T22:46:25.834129+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:57:25.702961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":954,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588621+0000","last_change":"2023-09-23T22:45:28.553711+0000","last_active":"2023-09-23T22:46:25.588621+0000","last_peered":"2023-09-23T22:46:25.588621+0000","last_clean":"2023-09-23T22:46:25.588621+0000","last_became_active":"2023-09-23T22:45:28.553291+0000","last_became_peered":"2023-09-23T22:45:28.553291+0000","last_unstale":"2023-09-23T22:46:25.588621+0000","last_undegraded":"2023-09-23T22:46:25.588621+0000","last_fullsized":"2023-09-23T22:46:25.588621+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:31.111983+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:31.111983+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:58.638288+0000","scrub_duration":0.001731486,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":483,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.508950+0000","last_change":"2023-09-23T22:45:22.505474+0000","last_active":"2023-09-23T22:46:26.508950+0000","last_peered":"2023-09-23T22:46:26.508950+0000","last_clean":"2023-09-23T22:46:26.508950+0000","last_became_active":"2023-09-23T22:45:22.505139+0000","last_became_peered":"2023-09-23T22:45:22.505139+0000","last_unstale":"2023-09-23T22:46:26.508950+0000","last_undegraded":"2023-09-23T22:46:26.508950+0000","last_fullsized":"2023-09-23T22:46:26.508950+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:24.186813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":44,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700748+0000","last_change":"2023-09-23T22:45:42.916524+0000","last_active":"2023-09-23T22:46:26.700748+0000","last_peered":"2023-09-23T22:46:26.700748+0000","last_clean":"2023-09-23T22:46:26.700748+0000","last_became_active":"2023-09-23T22:45:42.916176+0000","last_became_peered":"2023-09-23T22:45:42.916176+0000","last_unstale":"2023-09-23T22:46:26.700748+0000","last_undegraded":"2023-09-23T22:46:26.700748+0000","last_fullsized":"2023-09-23T22:46:26.700748+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:44.151573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":913,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700985+0000","last_change":"2023-09-23T22:45:20.493782+0000","last_active":"2023-09-23T22:46:26.700985+0000","last_peered":"2023-09-23T22:46:26.700985+0000","last_clean":"2023-09-23T22:46:26.700985+0000","last_became_active":"2023-09-23T22:45:20.493321+0000","last_became_peered":"2023-09-23T22:45:20.493321+0000","last_unstale":"2023-09-23T22:46:26.700985+0000","last_undegraded":"2023-09-23T22:46:26.700985+0000","last_fullsized":"2023-09-23T22:46:26.700985+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:31.625786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:31.625786+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:14.192194+0000","scrub_duration":0.0014301240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":48,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833741+0000","last_change":"2023-09-23T22:45:25.550834+0000","last_active":"2023-09-23T22:46:25.833741+0000","last_peered":"2023-09-23T22:46:25.833741+0000","last_clean":"2023-09-23T22:46:25.833741+0000","last_became_active":"2023-09-23T22:45:25.550529+0000","last_became_peered":"2023-09-23T22:45:25.550529+0000","last_unstale":"2023-09-23T22:46:25.833741+0000","last_undegraded":"2023-09-23T22:46:25.833741+0000","last_fullsized":"2023-09-23T22:46:25.833741+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:20.657338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":298,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589178+0000","last_change":"2023-09-23T22:42:56.509390+0000","last_active":"2023-09-23T22:46:25.589178+0000","last_peered":"2023-09-23T22:46:25.589178+0000","last_clean":"2023-09-23T22:46:25.589178+0000","last_became_active":"2023-09-23T22:42:56.509066+0000","last_became_peered":"2023-09-23T22:42:56.509066+0000","last_unstale":"2023-09-23T22:46:25.589178+0000","last_undegraded":"2023-09-23T22:46:25.589178+0000","last_fullsized":"2023-09-23T22:46:25.589178+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:36.790258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1010,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589315+0000","last_change":"2023-09-23T22:45:59.494924+0000","last_active":"2023-09-23T22:46:25.589315+0000","last_peered":"2023-09-23T22:46:25.589315+0000","last_clean":"2023-09-23T22:46:25.589315+0000","last_became_active":"2023-09-23T22:45:59.494310+0000","last_became_peered":"2023-09-23T22:45:59.494310+0000","last_unstale":"2023-09-23T22:46:25.589315+0000","last_undegraded":"2023-09-23T22:46:25.589315+0000","last_fullsized":"2023-09-23T22:46:25.589315+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:57.273632+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:57.273632+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:52:07.883227+0000","scrub_duration":0.018058204000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":110,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700996+0000","last_change":"2023-09-23T22:45:42.841665+0000","last_active":"2023-09-23T22:46:26.700996+0000","last_peered":"2023-09-23T22:46:26.700996+0000","last_clean":"2023-09-23T22:46:26.700996+0000","last_became_active":"2023-09-23T22:45:42.841311+0000","last_became_peered":"2023-09-23T22:45:42.841311+0000","last_unstale":"2023-09-23T22:46:26.700996+0000","last_undegraded":"2023-09-23T22:46:26.700996+0000","last_fullsized":"2023-09-23T22:46:26.700996+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:51.118344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":48,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833673+0000","last_change":"2023-09-23T22:45:41.831660+0000","last_active":"2023-09-23T22:46:25.833673+0000","last_peered":"2023-09-23T22:46:25.833673+0000","last_clean":"2023-09-23T22:46:25.833673+0000","last_became_active":"2023-09-23T22:45:41.831242+0000","last_became_peered":"2023-09-23T22:45:41.831242+0000","last_unstale":"2023-09-23T22:46:25.833673+0000","last_undegraded":"2023-09-23T22:46:25.833673+0000","last_fullsized":"2023-09-23T22:46:25.833673+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:48.814841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":868,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:28.038076+0000","last_change":"2023-09-23T22:45:20.494097+0000","last_active":"2023-09-23T22:46:28.038076+0000","last_peered":"2023-09-23T22:46:28.038076+0000","last_clean":"2023-09-23T22:46:28.038076+0000","last_became_active":"2023-09-23T22:45:20.493775+0000","last_became_peered":"2023-09-23T22:45:20.493775+0000","last_unstale":"2023-09-23T22:46:28.038076+0000","last_undegraded":"2023-09-23T22:46:28.038076+0000","last_fullsized":"2023-09-23T22:46:28.038076+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:02.056099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:02.056099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:28.358390+0000","scrub_duration":0.012145022,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":104,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588208+0000","last_change":"2023-09-23T22:45:06.452276+0000","last_active":"2023-09-23T22:46:25.588208+0000","last_peered":"2023-09-23T22:46:25.588208+0000","last_clean":"2023-09-23T22:46:25.588208+0000","last_became_active":"2023-09-23T22:45:06.451936+0000","last_became_peered":"2023-09-23T22:45:06.451936+0000","last_unstale":"2023-09-23T22:46:25.588208+0000","last_undegraded":"2023-09-23T22:46:25.588208+0000","last_fullsized":"2023-09-23T22:46:25.588208+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:25.418898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833872+0000","last_change":"2023-09-23T22:45:22.511553+0000","last_active":"2023-09-23T22:46:25.833872+0000","last_peered":"2023-09-23T22:46:25.833872+0000","last_clean":"2023-09-23T22:46:25.833872+0000","last_became_active":"2023-09-23T22:45:22.511245+0000","last_became_peered":"2023-09-23T22:45:22.511245+0000","last_unstale":"2023-09-23T22:46:25.833872+0000","last_undegraded":"2023-09-23T22:46:25.833872+0000","last_fullsized":"2023-09-23T22:46:25.833872+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:04.426439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":999,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588185+0000","last_change":"2023-09-23T22:45:28.556099+0000","last_active":"2023-09-23T22:46:25.588185+0000","last_peered":"2023-09-23T22:46:25.588185+0000","last_clean":"2023-09-23T22:46:25.588185+0000","last_became_active":"2023-09-23T22:45:28.555729+0000","last_became_peered":"2023-09-23T22:45:28.555729+0000","last_unstale":"2023-09-23T22:46:25.588185+0000","last_undegraded":"2023-09-23T22:46:25.588185+0000","last_fullsized":"2023-09-23T22:46:25.588185+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:29.054342+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:29.054342+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:19.435489+0000","scrub_duration":0.001695039,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":109,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589401+0000","last_change":"2023-09-23T22:45:59.487631+0000","last_active":"2023-09-23T22:46:25.589401+0000","last_peered":"2023-09-23T22:46:25.589401+0000","last_clean":"2023-09-23T22:46:25.589401+0000","last_became_active":"2023-09-23T22:45:59.487286+0000","last_became_peered":"2023-09-23T22:45:59.487286+0000","last_unstale":"2023-09-23T22:46:25.589401+0000","last_undegraded":"2023-09-23T22:46:25.589401+0000","last_fullsized":"2023-09-23T22:46:25.589401+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:45.210248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":336,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T22:46:29.935702+0000","last_change":"2023-09-23T22:42:56.567330+0000","last_active":"2023-09-23T22:46:29.935702+0000","last_peered":"2023-09-23T22:46:29.935702+0000","last_clean":"2023-09-23T22:46:29.935702+0000","last_became_active":"2023-09-23T22:42:56.567016+0000","last_became_peered":"2023-09-23T22:42:56.567016+0000","last_unstale":"2023-09-23T22:46:29.935702+0000","last_undegraded":"2023-09-23T22:46:29.935702+0000","last_fullsized":"2023-09-23T22:46:29.935702+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:00.456335+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":938,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T22:46:29.934436+0000","last_change":"2023-09-23T22:45:28.550354+0000","last_active":"2023-09-23T22:46:29.934436+0000","last_peered":"2023-09-23T22:46:29.934436+0000","last_clean":"2023-09-23T22:46:29.934436+0000","last_became_active":"2023-09-23T22:45:28.549956+0000","last_became_peered":"2023-09-23T22:45:28.549956+0000","last_unstale":"2023-09-23T22:46:29.934436+0000","last_undegraded":"2023-09-23T22:46:29.934436+0000","last_fullsized":"2023-09-23T22:46:29.934436+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:30.597926+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:30.597926+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:41.683726+0000","scrub_duration":0.001666528,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":681,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701150+0000","last_change":"2023-09-23T22:46:11.531687+0000","last_active":"2023-09-23T22:46:26.701150+0000","last_peered":"2023-09-23T22:46:26.701150+0000","last_clean":"2023-09-23T22:46:26.701150+0000","last_became_active":"2023-09-23T22:46:11.531313+0000","last_became_peered":"2023-09-23T22:46:11.531313+0000","last_unstale":"2023-09-23T22:46:26.701150+0000","last_undegraded":"2023-09-23T22:46:26.701150+0000","last_fullsized":"2023-09-23T22:46:26.701150+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:07.100563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":47340,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":43,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701053+0000","last_change":"2023-09-23T22:45:17.508332+0000","last_active":"2023-09-23T22:46:26.701053+0000","last_peered":"2023-09-23T22:46:26.701053+0000","last_clean":"2023-09-23T22:46:26.701053+0000","last_became_active":"2023-09-23T22:45:17.507811+0000","last_became_peered":"2023-09-23T22:45:17.507811+0000","last_unstale":"2023-09-23T22:46:26.701053+0000","last_undegraded":"2023-09-23T22:46:26.701053+0000","last_fullsized":"2023-09-23T22:46:26.701053+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:07:04.334173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":1152,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700437+0000","last_change":"2023-09-23T22:45:42.841085+0000","last_active":"2023-09-23T22:46:26.700437+0000","last_peered":"2023-09-23T22:46:26.700437+0000","last_clean":"2023-09-23T22:46:26.700437+0000","last_became_active":"2023-09-23T22:45:42.840750+0000","last_became_peered":"2023-09-23T22:45:42.840750+0000","last_unstale":"2023-09-23T22:46:26.700437+0000","last_undegraded":"2023-09-23T22:46:26.700437+0000","last_fullsized":"2023-09-23T22:46:26.700437+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:19.077318+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:19.077318+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.621150+0000","scrub_duration":0.0080980949999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":357,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587986+0000","last_change":"2023-09-23T22:45:59.490356+0000","last_active":"2023-09-23T22:46:25.587986+0000","last_peered":"2023-09-23T22:46:25.587986+0000","last_clean":"2023-09-23T22:46:25.587986+0000","last_became_active":"2023-09-23T22:45:59.490049+0000","last_became_peered":"2023-09-23T22:45:59.490049+0000","last_unstale":"2023-09-23T22:46:25.587986+0000","last_undegraded":"2023-09-23T22:46:25.587986+0000","last_fullsized":"2023-09-23T22:46:25.587986+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:35.738748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834258+0000","last_change":"2023-09-23T22:45:22.512222+0000","last_active":"2023-09-23T22:46:25.834258+0000","last_peered":"2023-09-23T22:46:25.834258+0000","last_clean":"2023-09-23T22:46:25.834258+0000","last_became_active":"2023-09-23T22:45:22.511892+0000","last_became_peered":"2023-09-23T22:45:22.511892+0000","last_unstale":"2023-09-23T22:46:25.834258+0000","last_undegraded":"2023-09-23T22:46:25.834258+0000","last_fullsized":"2023-09-23T22:46:25.834258+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:37:50.029752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":999,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587670+0000","last_change":"2023-09-23T22:45:09.437073+0000","last_active":"2023-09-23T22:46:25.587670+0000","last_peered":"2023-09-23T22:46:25.587670+0000","last_clean":"2023-09-23T22:46:25.587670+0000","last_became_active":"2023-09-23T22:45:09.436611+0000","last_became_peered":"2023-09-23T22:45:09.436611+0000","last_unstale":"2023-09-23T22:46:25.587670+0000","last_undegraded":"2023-09-23T22:46:25.587670+0000","last_fullsized":"2023-09-23T22:46:25.587670+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:01.556720+0000","last_clean_scrub_stamp":"2023-09-23T22:44:01.556720+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:09.783596+0000","scrub_duration":0.0022349409999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":44,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700578+0000","last_change":"2023-09-23T22:45:42.835144+0000","last_active":"2023-09-23T22:46:26.700578+0000","last_peered":"2023-09-23T22:46:26.700578+0000","last_clean":"2023-09-23T22:46:26.700578+0000","last_became_active":"2023-09-23T22:45:42.834794+0000","last_became_peered":"2023-09-23T22:45:42.834794+0000","last_unstale":"2023-09-23T22:46:26.700578+0000","last_undegraded":"2023-09-23T22:46:26.700578+0000","last_fullsized":"2023-09-23T22:46:26.700578+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:37.053170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":114,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588363+0000","last_change":"2023-09-23T22:45:22.504560+0000","last_active":"2023-09-23T22:46:25.588363+0000","last_peered":"2023-09-23T22:46:25.588363+0000","last_clean":"2023-09-23T22:46:25.588363+0000","last_became_active":"2023-09-23T22:45:22.503994+0000","last_became_peered":"2023-09-23T22:45:22.503994+0000","last_unstale":"2023-09-23T22:46:25.588363+0000","last_undegraded":"2023-09-23T22:46:25.588363+0000","last_fullsized":"2023-09-23T22:46:25.588363+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:22.098112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":738,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509311+0000","last_change":"2023-09-23T22:45:25.553371+0000","last_active":"2023-09-23T22:46:26.509311+0000","last_peered":"2023-09-23T22:46:26.509311+0000","last_clean":"2023-09-23T22:46:26.509311+0000","last_became_active":"2023-09-23T22:45:25.552907+0000","last_became_peered":"2023-09-23T22:45:25.552907+0000","last_unstale":"2023-09-23T22:46:26.509311+0000","last_undegraded":"2023-09-23T22:46:26.509311+0000","last_fullsized":"2023-09-23T22:46:26.509311+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:57.922489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:57.922489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:05:44.724877+0000","scrub_duration":0.001544351,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":514,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588996+0000","last_change":"2023-09-23T22:45:59.490918+0000","last_active":"2023-09-23T22:46:25.588996+0000","last_peered":"2023-09-23T22:46:25.588996+0000","last_clean":"2023-09-23T22:46:25.588996+0000","last_became_active":"2023-09-23T22:45:59.490573+0000","last_became_peered":"2023-09-23T22:45:59.490573+0000","last_unstale":"2023-09-23T22:46:25.588996+0000","last_undegraded":"2023-09-23T22:46:25.588996+0000","last_fullsized":"2023-09-23T22:46:25.588996+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:36.807697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":44,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700778+0000","last_change":"2023-09-23T22:45:42.836299+0000","last_active":"2023-09-23T22:46:26.700778+0000","last_peered":"2023-09-23T22:46:26.700778+0000","last_clean":"2023-09-23T22:46:26.700778+0000","last_became_active":"2023-09-23T22:45:42.836003+0000","last_became_peered":"2023-09-23T22:45:42.836003+0000","last_unstale":"2023-09-23T22:46:26.700778+0000","last_undegraded":"2023-09-23T22:46:26.700778+0000","last_fullsized":"2023-09-23T22:46:26.700778+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:15.194210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":1018,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588822+0000","last_change":"2023-09-23T22:44:05.892047+0000","last_active":"2023-09-23T22:46:25.588822+0000","last_peered":"2023-09-23T22:46:25.588822+0000","last_clean":"2023-09-23T22:46:25.588822+0000","last_became_active":"2023-09-23T22:44:05.891675+0000","last_became_peered":"2023-09-23T22:44:05.891675+0000","last_unstale":"2023-09-23T22:46:25.588822+0000","last_undegraded":"2023-09-23T22:46:25.588822+0000","last_fullsized":"2023-09-23T22:46:25.588822+0000","mapping_epoch":615,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":616,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:26.313042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:26.313042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:06.388598+0000","scrub_duration":0.0017143830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":382,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587404+0000","last_change":"2023-09-23T22:45:25.565473+0000","last_active":"2023-09-23T22:46:25.587404+0000","last_peered":"2023-09-23T22:46:25.587404+0000","last_clean":"2023-09-23T22:46:25.587404+0000","last_became_active":"2023-09-23T22:45:25.546116+0000","last_became_peered":"2023-09-23T22:45:25.546116+0000","last_unstale":"2023-09-23T22:46:25.587404+0000","last_undegraded":"2023-09-23T22:46:25.587404+0000","last_fullsized":"2023-09-23T22:46:25.587404+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:54.957330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":48,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834324+0000","last_change":"2023-09-23T22:45:41.837664+0000","last_active":"2023-09-23T22:46:25.834324+0000","last_peered":"2023-09-23T22:46:25.834324+0000","last_clean":"2023-09-23T22:46:25.834324+0000","last_became_active":"2023-09-23T22:45:41.837302+0000","last_became_peered":"2023-09-23T22:45:41.837302+0000","last_unstale":"2023-09-23T22:46:25.834324+0000","last_undegraded":"2023-09-23T22:46:25.834324+0000","last_fullsized":"2023-09-23T22:46:25.834324+0000","mapping_epoch":663,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":664,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:56.197841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":1062,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589263+0000","last_change":"2023-09-23T22:45:20.494718+0000","last_active":"2023-09-23T22:46:25.589263+0000","last_peered":"2023-09-23T22:46:25.589263+0000","last_clean":"2023-09-23T22:46:25.589263+0000","last_became_active":"2023-09-23T22:45:20.494224+0000","last_became_peered":"2023-09-23T22:45:20.494224+0000","last_unstale":"2023-09-23T22:46:25.589263+0000","last_undegraded":"2023-09-23T22:46:25.589263+0000","last_fullsized":"2023-09-23T22:46:25.589263+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:34.147486+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:34.147486+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:41.573558+0000","scrub_duration":0.0018320190000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":210,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588137+0000","last_change":"2023-09-23T22:45:59.493695+0000","last_active":"2023-09-23T22:46:25.588137+0000","last_peered":"2023-09-23T22:46:25.588137+0000","last_clean":"2023-09-23T22:46:25.588137+0000","last_became_active":"2023-09-23T22:45:59.493124+0000","last_became_peered":"2023-09-23T22:45:59.493124+0000","last_unstale":"2023-09-23T22:46:25.588137+0000","last_undegraded":"2023-09-23T22:46:25.588137+0000","last_fullsized":"2023-09-23T22:46:25.588137+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:38.627199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":154,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":104,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587827+0000","last_change":"2023-09-23T22:45:06.458827+0000","last_active":"2023-09-23T22:46:25.587827+0000","last_peered":"2023-09-23T22:46:25.587827+0000","last_clean":"2023-09-23T22:46:25.587827+0000","last_became_active":"2023-09-23T22:45:06.458533+0000","last_became_peered":"2023-09-23T22:45:06.458533+0000","last_unstale":"2023-09-23T22:46:25.587827+0000","last_undegraded":"2023-09-23T22:46:25.587827+0000","last_fullsized":"2023-09-23T22:46:25.587827+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:02.882529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":816,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834206+0000","last_change":"2023-09-23T22:45:28.556001+0000","last_active":"2023-09-23T22:46:25.834206+0000","last_peered":"2023-09-23T22:46:25.834206+0000","last_clean":"2023-09-23T22:46:25.834206+0000","last_became_active":"2023-09-23T22:45:28.555527+0000","last_became_peered":"2023-09-23T22:45:28.555527+0000","last_unstale":"2023-09-23T22:46:25.834206+0000","last_undegraded":"2023-09-23T22:46:25.834206+0000","last_fullsized":"2023-09-23T22:46:25.834206+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:04.048606+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:04.048606+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:33.382480+0000","scrub_duration":0.0064015249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":588,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588538+0000","last_change":"2023-09-23T22:45:59.489789+0000","last_active":"2023-09-23T22:46:25.588538+0000","last_peered":"2023-09-23T22:46:25.588538+0000","last_clean":"2023-09-23T22:46:25.588538+0000","last_became_active":"2023-09-23T22:45:59.489485+0000","last_became_peered":"2023-09-23T22:45:59.489485+0000","last_unstale":"2023-09-23T22:46:25.588538+0000","last_undegraded":"2023-09-23T22:46:25.588538+0000","last_fullsized":"2023-09-23T22:46:25.588538+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:15.906128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":706,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700723+0000","last_change":"2023-09-23T22:46:11.933058+0000","last_active":"2023-09-23T22:46:26.700723+0000","last_peered":"2023-09-23T22:46:26.700723+0000","last_clean":"2023-09-23T22:46:26.700723+0000","last_became_active":"2023-09-23T22:46:11.932695+0000","last_became_peered":"2023-09-23T22:46:11.932695+0000","last_unstale":"2023-09-23T22:46:26.700723+0000","last_undegraded":"2023-09-23T22:46:26.700723+0000","last_fullsized":"2023-09-23T22:46:26.700723+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:36.351064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":997,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589081+0000","last_change":"2023-09-23T22:45:20.549256+0000","last_active":"2023-09-23T22:46:25.589081+0000","last_peered":"2023-09-23T22:46:25.589081+0000","last_clean":"2023-09-23T22:46:25.589081+0000","last_became_active":"2023-09-23T22:45:20.547976+0000","last_became_peered":"2023-09-23T22:45:20.547976+0000","last_unstale":"2023-09-23T22:46:25.589081+0000","last_undegraded":"2023-09-23T22:46:25.589081+0000","last_fullsized":"2023-09-23T22:46:25.589081+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:36.397150+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:36.397150+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:24.867382+0000","scrub_duration":0.001754996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":86,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589450+0000","last_change":"2023-09-23T22:45:59.494780+0000","last_active":"2023-09-23T22:46:25.589450+0000","last_peered":"2023-09-23T22:46:25.589450+0000","last_clean":"2023-09-23T22:46:25.589450+0000","last_became_active":"2023-09-23T22:45:59.494281+0000","last_became_peered":"2023-09-23T22:45:59.494281+0000","last_unstale":"2023-09-23T22:46:25.589450+0000","last_undegraded":"2023-09-23T22:46:25.589450+0000","last_fullsized":"2023-09-23T22:46:25.589450+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:54.063310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":326,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589148+0000","last_change":"2023-09-23T22:45:06.451816+0000","last_active":"2023-09-23T22:46:25.589148+0000","last_peered":"2023-09-23T22:46:25.589148+0000","last_clean":"2023-09-23T22:46:25.589148+0000","last_became_active":"2023-09-23T22:45:06.451507+0000","last_became_peered":"2023-09-23T22:45:06.451507+0000","last_unstale":"2023-09-23T22:46:25.589148+0000","last_undegraded":"2023-09-23T22:46:25.589148+0000","last_fullsized":"2023-09-23T22:46:25.589148+0000","mapping_epoch":643,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":644,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:05.149871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":864,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701438+0000","last_change":"2023-09-23T22:45:28.345915+0000","last_active":"2023-09-23T22:46:26.701438+0000","last_peered":"2023-09-23T22:46:26.701438+0000","last_clean":"2023-09-23T22:46:26.701438+0000","last_became_active":"2023-09-23T22:45:25.547347+0000","last_became_peered":"2023-09-23T22:45:25.547347+0000","last_unstale":"2023-09-23T22:46:26.701438+0000","last_undegraded":"2023-09-23T22:46:26.701438+0000","last_fullsized":"2023-09-23T22:46:26.701438+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:45:28.345788+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:45:28.345788+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:07.959240+0000","scrub_duration":0.0015759280000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":104,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588785+0000","last_change":"2023-09-23T22:45:22.505328+0000","last_active":"2023-09-23T22:46:25.588785+0000","last_peered":"2023-09-23T22:46:25.588785+0000","last_clean":"2023-09-23T22:46:25.588785+0000","last_became_active":"2023-09-23T22:45:22.505022+0000","last_became_peered":"2023-09-23T22:45:22.505022+0000","last_unstale":"2023-09-23T22:46:25.588785+0000","last_undegraded":"2023-09-23T22:46:25.588785+0000","last_fullsized":"2023-09-23T22:46:25.588785+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:37.601573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":512,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700822+0000","last_change":"2023-09-23T22:45:42.841970+0000","last_active":"2023-09-23T22:46:26.700822+0000","last_peered":"2023-09-23T22:46:26.700822+0000","last_clean":"2023-09-23T22:46:26.700822+0000","last_became_active":"2023-09-23T22:45:42.840926+0000","last_became_peered":"2023-09-23T22:45:42.840926+0000","last_unstale":"2023-09-23T22:46:26.700822+0000","last_undegraded":"2023-09-23T22:46:26.700822+0000","last_fullsized":"2023-09-23T22:46:26.700822+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:52.257621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1025,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588827+0000","last_change":"2023-09-23T22:45:59.491149+0000","last_active":"2023-09-23T22:46:25.588827+0000","last_peered":"2023-09-23T22:46:25.588827+0000","last_clean":"2023-09-23T22:46:25.588827+0000","last_became_active":"2023-09-23T22:45:59.490803+0000","last_became_peered":"2023-09-23T22:45:59.490803+0000","last_unstale":"2023-09-23T22:46:25.588827+0000","last_undegraded":"2023-09-23T22:46:25.588827+0000","last_fullsized":"2023-09-23T22:46:25.588827+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:58.211060+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:58.211060+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.147929+0000","scrub_duration":0.0035013240000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":56,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833894+0000","last_change":"2023-09-23T22:45:25.571211+0000","last_active":"2023-09-23T22:46:25.833894+0000","last_peered":"2023-09-23T22:46:25.833894+0000","last_clean":"2023-09-23T22:46:25.833894+0000","last_became_active":"2023-09-23T22:45:25.548340+0000","last_became_peered":"2023-09-23T22:45:25.548340+0000","last_unstale":"2023-09-23T22:46:25.833894+0000","last_undegraded":"2023-09-23T22:46:25.833894+0000","last_fullsized":"2023-09-23T22:46:25.833894+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:10.421715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":522,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701142+0000","last_change":"2023-09-23T22:46:11.531961+0000","last_active":"2023-09-23T22:46:26.701142+0000","last_peered":"2023-09-23T22:46:26.701142+0000","last_clean":"2023-09-23T22:46:26.701142+0000","last_became_active":"2023-09-23T22:46:11.531618+0000","last_became_peered":"2023-09-23T22:46:11.531618+0000","last_unstale":"2023-09-23T22:46:26.701142+0000","last_undegraded":"2023-09-23T22:46:26.701142+0000","last_fullsized":"2023-09-23T22:46:26.701142+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:56:00.040458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":1161,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587917+0000","last_change":"2023-09-23T22:45:25.555386+0000","last_active":"2023-09-23T22:46:25.587917+0000","last_peered":"2023-09-23T22:46:25.587917+0000","last_clean":"2023-09-23T22:46:25.587917+0000","last_became_active":"2023-09-23T22:45:25.555134+0000","last_became_peered":"2023-09-23T22:45:25.555134+0000","last_unstale":"2023-09-23T22:46:25.587917+0000","last_undegraded":"2023-09-23T22:46:25.587917+0000","last_fullsized":"2023-09-23T22:46:25.587917+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.183183+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.183183+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:31.061518+0000","scrub_duration":0.0015656730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":323,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589319+0000","last_change":"2023-09-23T22:45:17.502689+0000","last_active":"2023-09-23T22:46:25.589319+0000","last_peered":"2023-09-23T22:46:25.589319+0000","last_clean":"2023-09-23T22:46:25.589319+0000","last_became_active":"2023-09-23T22:45:17.502401+0000","last_became_peered":"2023-09-23T22:45:17.502401+0000","last_unstale":"2023-09-23T22:46:25.589319+0000","last_undegraded":"2023-09-23T22:46:25.589319+0000","last_fullsized":"2023-09-23T22:46:25.589319+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:00.890452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":342,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701432+0000","last_change":"2023-09-23T22:46:11.936276+0000","last_active":"2023-09-23T22:46:26.701432+0000","last_peered":"2023-09-23T22:46:26.701432+0000","last_clean":"2023-09-23T22:46:26.701432+0000","last_became_active":"2023-09-23T22:46:11.935887+0000","last_became_peered":"2023-09-23T22:46:11.935887+0000","last_unstale":"2023-09-23T22:46:26.701432+0000","last_undegraded":"2023-09-23T22:46:26.701432+0000","last_fullsized":"2023-09-23T22:46:26.701432+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:47.592106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":658,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509400+0000","last_change":"2023-09-23T22:46:17.563136+0000","last_active":"2023-09-23T22:46:26.509400+0000","last_peered":"2023-09-23T22:46:26.509400+0000","last_clean":"2023-09-23T22:46:26.509400+0000","last_became_active":"2023-09-23T22:46:17.562788+0000","last_became_peered":"2023-09-23T22:46:17.562788+0000","last_unstale":"2023-09-23T22:46:26.509400+0000","last_undegraded":"2023-09-23T22:46:26.509400+0000","last_fullsized":"2023-09-23T22:46:26.509400+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":472,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:32.087146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:32.087146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:28.127046+0000","scrub_duration":0.001693546,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":43,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700734+0000","last_change":"2023-09-23T22:45:17.496669+0000","last_active":"2023-09-23T22:46:26.700734+0000","last_peered":"2023-09-23T22:46:26.700734+0000","last_clean":"2023-09-23T22:46:26.700734+0000","last_became_active":"2023-09-23T22:45:17.495007+0000","last_became_peered":"2023-09-23T22:45:17.495007+0000","last_unstale":"2023-09-23T22:46:26.700734+0000","last_undegraded":"2023-09-23T22:46:26.700734+0000","last_fullsized":"2023-09-23T22:46:26.700734+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:43.132602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":625,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701400+0000","last_change":"2023-09-23T22:46:11.934154+0000","last_active":"2023-09-23T22:46:26.701400+0000","last_peered":"2023-09-23T22:46:26.701400+0000","last_clean":"2023-09-23T22:46:26.701400+0000","last_became_active":"2023-09-23T22:46:11.933777+0000","last_became_peered":"2023-09-23T22:46:11.933777+0000","last_unstale":"2023-09-23T22:46:26.701400+0000","last_undegraded":"2023-09-23T22:46:26.701400+0000","last_fullsized":"2023-09-23T22:46:26.701400+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:08:36.797336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":1090,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588992+0000","last_change":"2023-09-23T22:43:57.157649+0000","last_active":"2023-09-23T22:46:25.588992+0000","last_peered":"2023-09-23T22:46:25.588992+0000","last_clean":"2023-09-23T22:46:25.588992+0000","last_became_active":"2023-09-23T22:42:56.530915+0000","last_became_peered":"2023-09-23T22:42:56.530915+0000","last_unstale":"2023-09-23T22:46:25.588992+0000","last_undegraded":"2023-09-23T22:46:25.588992+0000","last_fullsized":"2023-09-23T22:46:25.588992+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:43:57.157560+0000","last_clean_scrub_stamp":"2023-09-23T22:43:57.157560+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:20.159228+0000","scrub_duration":0.001566287,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833758+0000","last_change":"2023-09-23T22:45:22.511359+0000","last_active":"2023-09-23T22:46:25.833758+0000","last_peered":"2023-09-23T22:46:25.833758+0000","last_clean":"2023-09-23T22:46:25.833758+0000","last_became_active":"2023-09-23T22:45:22.510056+0000","last_became_peered":"2023-09-23T22:45:22.510056+0000","last_unstale":"2023-09-23T22:46:25.833758+0000","last_undegraded":"2023-09-23T22:46:25.833758+0000","last_fullsized":"2023-09-23T22:46:25.833758+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:34.850604+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":534,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T22:46:29.934970+0000","last_change":"2023-09-23T22:45:22.506360+0000","last_active":"2023-09-23T22:46:29.934970+0000","last_peered":"2023-09-23T22:46:29.934970+0000","last_clean":"2023-09-23T22:46:29.934970+0000","last_became_active":"2023-09-23T22:45:22.505618+0000","last_became_peered":"2023-09-23T22:45:22.505618+0000","last_unstale":"2023-09-23T22:46:29.934970+0000","last_undegraded":"2023-09-23T22:46:29.934970+0000","last_fullsized":"2023-09-23T22:46:29.934970+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:21.115523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":1038,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587653+0000","last_change":"2023-09-23T22:45:25.546635+0000","last_active":"2023-09-23T22:46:25.587653+0000","last_peered":"2023-09-23T22:46:25.587653+0000","last_clean":"2023-09-23T22:46:25.587653+0000","last_became_active":"2023-09-23T22:45:25.545649+0000","last_became_peered":"2023-09-23T22:45:25.545649+0000","last_unstale":"2023-09-23T22:46:25.587653+0000","last_undegraded":"2023-09-23T22:46:25.587653+0000","last_fullsized":"2023-09-23T22:46:25.587653+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:03.570110+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:03.570110+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:43.773804+0000","scrub_duration":0.001987071,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"640'271","reported_seq":1450,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509324+0000","last_change":"2023-09-23T22:46:17.557802+0000","last_active":"2023-09-23T22:46:26.509324+0000","last_peered":"2023-09-23T22:46:26.509324+0000","last_clean":"2023-09-23T22:46:26.509324+0000","last_became_active":"2023-09-23T22:46:17.557196+0000","last_became_peered":"2023-09-23T22:46:17.557196+0000","last_unstale":"2023-09-23T22:46:26.509324+0000","last_undegraded":"2023-09-23T22:46:26.509324+0000","last_fullsized":"2023-09-23T22:46:26.509324+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":271,"log_dups_size":0,"ondisk_log_size":271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:20:52.972098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00073668799999999999,"stat_sum":{"num_bytes":4194304,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":1192,"num_read_kb":5960,"num_write":287,"num_write_kb":9768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":104,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588207+0000","last_change":"2023-09-23T22:45:22.504383+0000","last_active":"2023-09-23T22:46:25.588207+0000","last_peered":"2023-09-23T22:46:25.588207+0000","last_clean":"2023-09-23T22:46:25.588207+0000","last_became_active":"2023-09-23T22:45:22.503573+0000","last_became_peered":"2023-09-23T22:45:22.503573+0000","last_unstale":"2023-09-23T22:46:25.588207+0000","last_undegraded":"2023-09-23T22:46:25.588207+0000","last_fullsized":"2023-09-23T22:46:25.588207+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:19:22.322562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":450,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509157+0000","last_change":"2023-09-23T22:45:17.503914+0000","last_active":"2023-09-23T22:46:26.509157+0000","last_peered":"2023-09-23T22:46:26.509157+0000","last_clean":"2023-09-23T22:46:26.509157+0000","last_became_active":"2023-09-23T22:45:17.503111+0000","last_became_peered":"2023-09-23T22:45:17.503111+0000","last_unstale":"2023-09-23T22:46:26.509157+0000","last_undegraded":"2023-09-23T22:46:26.509157+0000","last_fullsized":"2023-09-23T22:46:26.509157+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:52.519749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":1154,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588300+0000","last_change":"2023-09-23T22:45:28.552025+0000","last_active":"2023-09-23T22:46:25.588300+0000","last_peered":"2023-09-23T22:46:25.588300+0000","last_clean":"2023-09-23T22:46:25.588300+0000","last_became_active":"2023-09-23T22:45:28.551589+0000","last_became_peered":"2023-09-23T22:45:28.551589+0000","last_unstale":"2023-09-23T22:46:25.588300+0000","last_undegraded":"2023-09-23T22:46:25.588300+0000","last_fullsized":"2023-09-23T22:46:25.588300+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:40:16.067396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:40:16.067396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:47.205560+0000","scrub_duration":0.0072300120000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"640'221","reported_seq":2324,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589083+0000","last_change":"2023-09-23T22:44:54.429423+0000","last_active":"2023-09-23T22:46:25.589083+0000","last_peered":"2023-09-23T22:46:25.589083+0000","last_clean":"2023-09-23T22:46:25.589083+0000","last_became_active":"2023-09-23T22:42:56.548134+0000","last_became_peered":"2023-09-23T22:42:56.548134+0000","last_unstale":"2023-09-23T22:46:25.589083+0000","last_undegraded":"2023-09-23T22:46:25.589083+0000","last_fullsized":"2023-09-23T22:46:25.589083+0000","mapping_epoch":592,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":221,"log_dups_size":0,"ondisk_log_size":221,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:36.511009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0028955169999999998,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2322,"num_read_kb":4116,"num_write":184,"num_write_kb":6241,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":20971615,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":584,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701015+0000","last_change":"2023-09-23T22:46:11.934797+0000","last_active":"2023-09-23T22:46:26.701015+0000","last_peered":"2023-09-23T22:46:26.701015+0000","last_clean":"2023-09-23T22:46:26.701015+0000","last_became_active":"2023-09-23T22:46:11.934474+0000","last_became_peered":"2023-09-23T22:46:11.934474+0000","last_unstale":"2023-09-23T22:46:26.701015+0000","last_undegraded":"2023-09-23T22:46:26.701015+0000","last_fullsized":"2023-09-23T22:46:26.701015+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:53.890160+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":319,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.508978+0000","last_change":"2023-09-23T22:46:17.905264+0000","last_active":"2023-09-23T22:46:26.508978+0000","last_peered":"2023-09-23T22:46:26.508978+0000","last_clean":"2023-09-23T22:46:26.508978+0000","last_became_active":"2023-09-23T22:46:17.904890+0000","last_became_peered":"2023-09-23T22:46:17.904890+0000","last_unstale":"2023-09-23T22:46:26.508978+0000","last_undegraded":"2023-09-23T22:46:26.508978+0000","last_fullsized":"2023-09-23T22:46:26.508978+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:29.607630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":1043,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587423+0000","last_change":"2023-09-23T22:45:59.493261+0000","last_active":"2023-09-23T22:46:25.587423+0000","last_peered":"2023-09-23T22:46:25.587423+0000","last_clean":"2023-09-23T22:46:25.587423+0000","last_became_active":"2023-09-23T22:45:59.492759+0000","last_became_peered":"2023-09-23T22:45:59.492759+0000","last_unstale":"2023-09-23T22:46:25.587423+0000","last_undegraded":"2023-09-23T22:46:25.587423+0000","last_fullsized":"2023-09-23T22:46:25.587423+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:07.291196+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:07.291196+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:23:52.331068+0000","scrub_duration":0.0016895390000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"640'185","reported_seq":925,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.611939+0000","last_change":"2023-09-23T22:45:28.548846+0000","last_active":"2023-09-23T22:46:26.611939+0000","last_peered":"2023-09-23T22:46:26.611939+0000","last_clean":"2023-09-23T22:46:26.611939+0000","last_became_active":"2023-09-23T22:45:28.548123+0000","last_became_peered":"2023-09-23T22:45:28.548123+0000","last_unstale":"2023-09-23T22:46:26.611939+0000","last_undegraded":"2023-09-23T22:46:26.611939+0000","last_fullsized":"2023-09-23T22:46:26.611939+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":185,"log_dups_size":0,"ondisk_log_size":185,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:14:52.606762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00048572900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":987,"num_read_kb":3296,"num_write":166,"num_write_kb":2410,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":255,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.508812+0000","last_change":"2023-09-23T22:46:17.905222+0000","last_active":"2023-09-23T22:46:26.508812+0000","last_peered":"2023-09-23T22:46:26.508812+0000","last_clean":"2023-09-23T22:46:26.508812+0000","last_became_active":"2023-09-23T22:46:17.904872+0000","last_became_peered":"2023-09-23T22:46:17.904872+0000","last_unstale":"2023-09-23T22:46:26.508812+0000","last_undegraded":"2023-09-23T22:46:26.508812+0000","last_fullsized":"2023-09-23T22:46:26.508812+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:44.143193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":394,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.510342+0000","last_change":"2023-09-23T22:45:25.552027+0000","last_active":"2023-09-23T22:46:26.510342+0000","last_peered":"2023-09-23T22:46:26.510342+0000","last_clean":"2023-09-23T22:46:26.510342+0000","last_became_active":"2023-09-23T22:45:25.551697+0000","last_became_peered":"2023-09-23T22:45:25.551697+0000","last_unstale":"2023-09-23T22:46:26.510342+0000","last_undegraded":"2023-09-23T22:46:26.510342+0000","last_fullsized":"2023-09-23T22:46:26.510342+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:06:41.170247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"640'420","reported_seq":2264,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588763+0000","last_change":"2023-09-23T22:45:09.438706+0000","last_active":"2023-09-23T22:46:25.588763+0000","last_peered":"2023-09-23T22:46:25.588763+0000","last_clean":"2023-09-23T22:46:25.588763+0000","last_became_active":"2023-09-23T22:45:09.437974+0000","last_became_peered":"2023-09-23T22:45:09.437974+0000","last_unstale":"2023-09-23T22:46:25.588763+0000","last_undegraded":"2023-09-23T22:46:25.588763+0000","last_fullsized":"2023-09-23T22:46:25.588763+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":420,"log_dups_size":0,"ondisk_log_size":420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:22.520495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0020164129999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1916,"num_read_kb":3106,"num_write":395,"num_write_kb":2468,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":873,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588099+0000","last_change":"2023-09-23T22:45:59.493669+0000","last_active":"2023-09-23T22:46:25.588099+0000","last_peered":"2023-09-23T22:46:25.588099+0000","last_clean":"2023-09-23T22:46:25.588099+0000","last_became_active":"2023-09-23T22:45:59.492788+0000","last_became_peered":"2023-09-23T22:45:59.492788+0000","last_unstale":"2023-09-23T22:46:25.588099+0000","last_undegraded":"2023-09-23T22:46:25.588099+0000","last_fullsized":"2023-09-23T22:46:25.588099+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:00.574795+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:00.574795+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:39:20.792256+0000","scrub_duration":0.0023357170000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833818+0000","last_change":"2023-09-23T22:45:22.509218+0000","last_active":"2023-09-23T22:46:25.833818+0000","last_peered":"2023-09-23T22:46:25.833818+0000","last_clean":"2023-09-23T22:46:25.833818+0000","last_became_active":"2023-09-23T22:45:22.508907+0000","last_became_peered":"2023-09-23T22:45:22.508907+0000","last_unstale":"2023-09-23T22:46:25.833818+0000","last_undegraded":"2023-09-23T22:46:25.833818+0000","last_fullsized":"2023-09-23T22:46:25.833818+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:34.923418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":348,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588731+0000","last_change":"2023-09-23T22:45:17.503488+0000","last_active":"2023-09-23T22:46:25.588731+0000","last_peered":"2023-09-23T22:46:25.588731+0000","last_clean":"2023-09-23T22:46:25.588731+0000","last_became_active":"2023-09-23T22:45:17.503018+0000","last_became_peered":"2023-09-23T22:45:17.503018+0000","last_unstale":"2023-09-23T22:46:25.588731+0000","last_undegraded":"2023-09-23T22:46:25.588731+0000","last_fullsized":"2023-09-23T22:46:25.588731+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:23:54.032873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":637,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509495+0000","last_change":"2023-09-23T22:46:17.558912+0000","last_active":"2023-09-23T22:46:26.509495+0000","last_peered":"2023-09-23T22:46:26.509495+0000","last_clean":"2023-09-23T22:46:26.509495+0000","last_became_active":"2023-09-23T22:46:17.558673+0000","last_became_peered":"2023-09-23T22:46:17.558673+0000","last_unstale":"2023-09-23T22:46:26.509495+0000","last_undegraded":"2023-09-23T22:46:26.509495+0000","last_fullsized":"2023-09-23T22:46:26.509495+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:38.451955+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:38.451955+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:38:16.777533+0000","scrub_duration":0.0018627959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"640'215","reported_seq":1636,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588256+0000","last_change":"2023-09-23T22:45:59.493329+0000","last_active":"2023-09-23T22:46:25.588256+0000","last_peered":"2023-09-23T22:46:25.588256+0000","last_clean":"2023-09-23T22:46:25.588256+0000","last_became_active":"2023-09-23T22:45:59.492566+0000","last_became_peered":"2023-09-23T22:45:59.492566+0000","last_unstale":"2023-09-23T22:46:25.588256+0000","last_undegraded":"2023-09-23T22:46:25.588256+0000","last_fullsized":"2023-09-23T22:46:25.588256+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":215,"log_dups_size":0,"ondisk_log_size":215,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:15:18.478665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0018983450000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1446,"num_read_kb":3631,"num_write":157,"num_write_kb":2284,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":59,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700627+0000","last_change":"2023-09-23T22:45:42.842740+0000","last_active":"2023-09-23T22:46:26.700627+0000","last_peered":"2023-09-23T22:46:26.700627+0000","last_clean":"2023-09-23T22:46:26.700627+0000","last_became_active":"2023-09-23T22:45:42.842420+0000","last_became_peered":"2023-09-23T22:45:42.842420+0000","last_unstale":"2023-09-23T22:46:26.700627+0000","last_undegraded":"2023-09-23T22:46:26.700627+0000","last_fullsized":"2023-09-23T22:46:26.700627+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:24:11.531744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":530,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700869+0000","last_change":"2023-09-23T22:45:42.916672+0000","last_active":"2023-09-23T22:46:26.700869+0000","last_peered":"2023-09-23T22:46:26.700869+0000","last_clean":"2023-09-23T22:46:26.700869+0000","last_became_active":"2023-09-23T22:45:42.916180+0000","last_became_peered":"2023-09-23T22:45:42.916180+0000","last_unstale":"2023-09-23T22:46:26.700869+0000","last_undegraded":"2023-09-23T22:46:26.700869+0000","last_fullsized":"2023-09-23T22:46:26.700869+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:58.883770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":1080,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587639+0000","last_change":"2023-09-23T22:45:28.557202+0000","last_active":"2023-09-23T22:46:25.587639+0000","last_peered":"2023-09-23T22:46:25.587639+0000","last_clean":"2023-09-23T22:46:25.587639+0000","last_became_active":"2023-09-23T22:45:28.556683+0000","last_became_peered":"2023-09-23T22:45:28.556683+0000","last_unstale":"2023-09-23T22:46:25.587639+0000","last_undegraded":"2023-09-23T22:46:25.587639+0000","last_fullsized":"2023-09-23T22:46:25.587639+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T22:44:02.567345+0000","last_clean_scrub_stamp":"2023-09-23T22:44:02.567345+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T10:07:30.226289+0000","scrub_duration":0.0019136540000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"640'181","reported_seq":601,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589202+0000","last_change":"2023-09-23T22:45:59.494626+0000","last_active":"2023-09-23T22:46:25.589202+0000","last_peered":"2023-09-23T22:46:25.589202+0000","last_clean":"2023-09-23T22:46:25.589202+0000","last_became_active":"2023-09-23T22:45:59.492597+0000","last_became_peered":"2023-09-23T22:45:59.492597+0000","last_unstale":"2023-09-23T22:46:25.589202+0000","last_undegraded":"2023-09-23T22:46:25.589202+0000","last_fullsized":"2023-09-23T22:46:25.589202+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:37.964127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050434000000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1303,"num_read_kb":1560,"num_write":136,"num_write_kb":302,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":257,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.508867+0000","last_change":"2023-09-23T22:46:17.905607+0000","last_active":"2023-09-23T22:46:26.508867+0000","last_peered":"2023-09-23T22:46:26.508867+0000","last_clean":"2023-09-23T22:46:26.508867+0000","last_became_active":"2023-09-23T22:46:17.904956+0000","last_became_peered":"2023-09-23T22:46:17.904956+0000","last_unstale":"2023-09-23T22:46:26.508867+0000","last_undegraded":"2023-09-23T22:46:26.508867+0000","last_fullsized":"2023-09-23T22:46:26.508867+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:43.177950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":120,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700831+0000","last_change":"2023-09-23T22:45:42.842394+0000","last_active":"2023-09-23T22:46:26.700831+0000","last_peered":"2023-09-23T22:46:26.700831+0000","last_clean":"2023-09-23T22:46:26.700831+0000","last_became_active":"2023-09-23T22:45:42.841979+0000","last_became_peered":"2023-09-23T22:45:42.841979+0000","last_unstale":"2023-09-23T22:46:26.700831+0000","last_undegraded":"2023-09-23T22:46:26.700831+0000","last_fullsized":"2023-09-23T22:46:26.700831+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:43.176130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":1037,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701443+0000","last_change":"2023-09-23T22:45:20.546576+0000","last_active":"2023-09-23T22:46:26.701443+0000","last_peered":"2023-09-23T22:46:26.701443+0000","last_clean":"2023-09-23T22:46:26.701443+0000","last_became_active":"2023-09-23T22:45:20.546222+0000","last_became_peered":"2023-09-23T22:45:20.546222+0000","last_unstale":"2023-09-23T22:46:26.701443+0000","last_undegraded":"2023-09-23T22:46:26.701443+0000","last_fullsized":"2023-09-23T22:46:26.701443+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:27.290379+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:27.290379+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:36:53.972924+0000","scrub_duration":0.0016473779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":56,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834253+0000","last_change":"2023-09-23T22:45:25.573308+0000","last_active":"2023-09-23T22:46:25.834253+0000","last_peered":"2023-09-23T22:46:25.834253+0000","last_clean":"2023-09-23T22:46:25.834253+0000","last_became_active":"2023-09-23T22:45:25.551602+0000","last_became_peered":"2023-09-23T22:45:25.551602+0000","last_unstale":"2023-09-23T22:46:25.834253+0000","last_undegraded":"2023-09-23T22:46:25.834253+0000","last_fullsized":"2023-09-23T22:46:25.834253+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:30.986848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"640'291","reported_seq":1790,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.509444+0000","last_change":"2023-09-23T22:45:20.547402+0000","last_active":"2023-09-23T22:46:26.509444+0000","last_peered":"2023-09-23T22:46:26.509444+0000","last_clean":"2023-09-23T22:46:26.509444+0000","last_became_active":"2023-09-23T22:45:20.547014+0000","last_became_peered":"2023-09-23T22:45:20.547014+0000","last_unstale":"2023-09-23T22:46:26.509444+0000","last_undegraded":"2023-09-23T22:46:26.509444+0000","last_fullsized":"2023-09-23T22:46:26.509444+0000","mapping_epoch":652,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":653,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":291,"log_dups_size":0,"ondisk_log_size":291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:23.216759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059489999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1986,"num_read_kb":2085,"num_write":204,"num_write_kb":320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":48,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833650+0000","last_change":"2023-09-23T22:45:25.548381+0000","last_active":"2023-09-23T22:46:25.833650+0000","last_peered":"2023-09-23T22:46:25.833650+0000","last_clean":"2023-09-23T22:46:25.833650+0000","last_became_active":"2023-09-23T22:45:25.547994+0000","last_became_peered":"2023-09-23T22:45:25.547994+0000","last_unstale":"2023-09-23T22:46:25.833650+0000","last_undegraded":"2023-09-23T22:46:25.833650+0000","last_fullsized":"2023-09-23T22:46:25.833650+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:16.603990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":872,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834078+0000","last_change":"2023-09-23T22:45:25.550997+0000","last_active":"2023-09-23T22:46:25.834078+0000","last_peered":"2023-09-23T22:46:25.834078+0000","last_clean":"2023-09-23T22:46:25.834078+0000","last_became_active":"2023-09-23T22:45:25.550468+0000","last_became_peered":"2023-09-23T22:45:25.550468+0000","last_unstale":"2023-09-23T22:46:25.834078+0000","last_undegraded":"2023-09-23T22:46:25.834078+0000","last_fullsized":"2023-09-23T22:46:25.834078+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":164,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:42:03.084589+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:42:03.084589+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:27.030089+0000","scrub_duration":0.013840414000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"640'166","reported_seq":741,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.589267+0000","last_change":"2023-09-23T22:45:09.436414+0000","last_active":"2023-09-23T22:46:25.589267+0000","last_peered":"2023-09-23T22:46:25.589267+0000","last_clean":"2023-09-23T22:46:25.589267+0000","last_became_active":"2023-09-23T22:45:09.435610+0000","last_became_peered":"2023-09-23T22:45:09.435610+0000","last_unstale":"2023-09-23T22:46:25.589267+0000","last_undegraded":"2023-09-23T22:46:25.589267+0000","last_fullsized":"2023-09-23T22:46:25.589267+0000","mapping_epoch":646,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":647,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:05.138652+0000","last_clean_scrub_stamp":"2023-09-23T22:04:05.138652+0000","objects_scrubbed":0,"log_size":166,"log_dups_size":0,"ondisk_log_size":166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:00.370983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044240799999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":819,"num_read_kb":1642,"num_write":161,"num_write_kb":1597,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":520,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700879+0000","last_change":"2023-09-23T22:45:42.840850+0000","last_active":"2023-09-23T22:46:26.700879+0000","last_peered":"2023-09-23T22:46:26.700879+0000","last_clean":"2023-09-23T22:46:26.700879+0000","last_became_active":"2023-09-23T22:45:42.840300+0000","last_became_peered":"2023-09-23T22:45:42.840300+0000","last_unstale":"2023-09-23T22:46:26.700879+0000","last_undegraded":"2023-09-23T22:46:26.700879+0000","last_fullsized":"2023-09-23T22:46:26.700879+0000","mapping_epoch":664,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":665,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:20:28.519370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":92,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700617+0000","last_change":"2023-09-23T22:46:11.935394+0000","last_active":"2023-09-23T22:46:26.700617+0000","last_peered":"2023-09-23T22:46:26.700617+0000","last_clean":"2023-09-23T22:46:26.700617+0000","last_became_active":"2023-09-23T22:46:11.935105+0000","last_became_peered":"2023-09-23T22:46:11.935105+0000","last_unstale":"2023-09-23T22:46:26.700617+0000","last_undegraded":"2023-09-23T22:46:26.700617+0000","last_fullsized":"2023-09-23T22:46:26.700617+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:39.722745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":1063,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.833929+0000","last_change":"2023-09-23T22:45:28.549683+0000","last_active":"2023-09-23T22:46:25.833929+0000","last_peered":"2023-09-23T22:46:25.833929+0000","last_clean":"2023-09-23T22:46:25.833929+0000","last_became_active":"2023-09-23T22:45:28.549245+0000","last_became_peered":"2023-09-23T22:45:28.549245+0000","last_unstale":"2023-09-23T22:46:25.833929+0000","last_undegraded":"2023-09-23T22:46:25.833929+0000","last_fullsized":"2023-09-23T22:46:25.833929+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":661,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:39.417809+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:39.417809+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:44:31.897644+0000","scrub_duration":0.0016817830000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":36,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700920+0000","last_change":"2023-09-23T22:45:25.549660+0000","last_active":"2023-09-23T22:46:26.700920+0000","last_peered":"2023-09-23T22:46:26.700920+0000","last_clean":"2023-09-23T22:46:26.700920+0000","last_became_active":"2023-09-23T22:45:25.549284+0000","last_became_peered":"2023-09-23T22:45:25.549284+0000","last_unstale":"2023-09-23T22:46:26.700920+0000","last_undegraded":"2023-09-23T22:46:26.700920+0000","last_fullsized":"2023-09-23T22:46:26.700920+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:04.359582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":22,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.611665+0000","last_change":"2023-09-23T22:46:02.574561+0000","last_active":"2023-09-23T22:46:26.611665+0000","last_peered":"2023-09-23T22:46:26.611665+0000","last_clean":"2023-09-23T22:46:26.611665+0000","last_became_active":"2023-09-23T22:46:02.574207+0000","last_became_peered":"2023-09-23T22:46:02.574207+0000","last_unstale":"2023-09-23T22:46:26.611665+0000","last_undegraded":"2023-09-23T22:46:26.611665+0000","last_fullsized":"2023-09-23T22:46:26.611665+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:18.613134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":1062,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834157+0000","last_change":"2023-09-23T22:45:25.550225+0000","last_active":"2023-09-23T22:46:25.834157+0000","last_peered":"2023-09-23T22:46:25.834157+0000","last_clean":"2023-09-23T22:46:25.834157+0000","last_became_active":"2023-09-23T22:45:25.549867+0000","last_became_peered":"2023-09-23T22:45:25.549867+0000","last_unstale":"2023-09-23T22:46:25.834157+0000","last_undegraded":"2023-09-23T22:46:25.834157+0000","last_fullsized":"2023-09-23T22:46:25.834157+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:44:41.386557+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:44:41.386557+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:34:52.547517+0000","scrub_duration":0.0019422809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":748,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588889+0000","last_change":"2023-09-23T22:45:17.530896+0000","last_active":"2023-09-23T22:46:25.588889+0000","last_peered":"2023-09-23T22:46:25.588889+0000","last_clean":"2023-09-23T22:46:25.588889+0000","last_became_active":"2023-09-23T22:45:17.501647+0000","last_became_peered":"2023-09-23T22:45:17.501647+0000","last_unstale":"2023-09-23T22:46:25.588889+0000","last_undegraded":"2023-09-23T22:46:25.588889+0000","last_fullsized":"2023-09-23T22:46:25.588889+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:44.145773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":709,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.701145+0000","last_change":"2023-09-23T22:46:11.935068+0000","last_active":"2023-09-23T22:46:26.701145+0000","last_peered":"2023-09-23T22:46:26.701145+0000","last_clean":"2023-09-23T22:46:26.701145+0000","last_became_active":"2023-09-23T22:46:11.934724+0000","last_became_peered":"2023-09-23T22:46:11.934724+0000","last_unstale":"2023-09-23T22:46:26.701145+0000","last_undegraded":"2023-09-23T22:46:26.701145+0000","last_fullsized":"2023-09-23T22:46:26.701145+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:29:23.099835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":763,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588459+0000","last_change":"2023-09-23T22:45:17.527461+0000","last_active":"2023-09-23T22:46:25.588459+0000","last_peered":"2023-09-23T22:46:25.588459+0000","last_clean":"2023-09-23T22:46:25.588459+0000","last_became_active":"2023-09-23T22:45:17.501280+0000","last_became_peered":"2023-09-23T22:45:17.501280+0000","last_unstale":"2023-09-23T22:46:25.588459+0000","last_undegraded":"2023-09-23T22:46:25.588459+0000","last_fullsized":"2023-09-23T22:46:25.588459+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:42.501570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":162,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700854+0000","last_change":"2023-09-23T22:46:11.936343+0000","last_active":"2023-09-23T22:46:26.700854+0000","last_peered":"2023-09-23T22:46:26.700854+0000","last_clean":"2023-09-23T22:46:26.700854+0000","last_became_active":"2023-09-23T22:46:11.935390+0000","last_became_peered":"2023-09-23T22:46:11.935390+0000","last_unstale":"2023-09-23T22:46:26.700854+0000","last_undegraded":"2023-09-23T22:46:26.700854+0000","last_fullsized":"2023-09-23T22:46:26.700854+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:19:41.846656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":782,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.587550+0000","last_change":"2023-09-23T22:45:59.489049+0000","last_active":"2023-09-23T22:46:25.587550+0000","last_peered":"2023-09-23T22:46:25.587550+0000","last_clean":"2023-09-23T22:46:25.587550+0000","last_became_active":"2023-09-23T22:45:59.488402+0000","last_became_peered":"2023-09-23T22:45:59.488402+0000","last_unstale":"2023-09-23T22:46:25.587550+0000","last_undegraded":"2023-09-23T22:46:25.587550+0000","last_fullsized":"2023-09-23T22:46:25.587550+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:56.024308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:56.024308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:35:26.046514+0000","scrub_duration":0.0045490360000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":39,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.834216+0000","last_change":"2023-09-23T22:45:22.510829+0000","last_active":"2023-09-23T22:46:25.834216+0000","last_peered":"2023-09-23T22:46:25.834216+0000","last_clean":"2023-09-23T22:46:25.834216+0000","last_became_active":"2023-09-23T22:45:22.510400+0000","last_became_peered":"2023-09-23T22:45:22.510400+0000","last_unstale":"2023-09-23T22:46:25.834216+0000","last_undegraded":"2023-09-23T22:46:25.834216+0000","last_fullsized":"2023-09-23T22:46:25.834216+0000","mapping_epoch":654,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":655,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:30.648352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":290,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:25.588728+0000","last_change":"2023-09-23T22:45:59.493580+0000","last_active":"2023-09-23T22:46:25.588728+0000","last_peered":"2023-09-23T22:46:25.588728+0000","last_clean":"2023-09-23T22:46:25.588728+0000","last_became_active":"2023-09-23T22:45:59.492686+0000","last_became_peered":"2023-09-23T22:45:59.492686+0000","last_unstale":"2023-09-23T22:46:25.588728+0000","last_undegraded":"2023-09-23T22:46:25.588728+0000","last_fullsized":"2023-09-23T22:46:25.588728+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:26.630282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":1072,"reported_epoch":683,"state":"active+clean","last_fresh":"2023-09-23T22:46:29.934860+0000","last_change":"2023-09-23T22:45:25.544754+0000","last_active":"2023-09-23T22:46:29.934860+0000","last_peered":"2023-09-23T22:46:29.934860+0000","last_clean":"2023-09-23T22:46:29.934860+0000","last_became_active":"2023-09-23T22:45:25.544366+0000","last_became_peered":"2023-09-23T22:45:25.544366+0000","last_unstale":"2023-09-23T22:46:29.934860+0000","last_undegraded":"2023-09-23T22:46:29.934860+0000","last_fullsized":"2023-09-23T22:46:29.934860+0000","mapping_epoch":657,"log_start":"0'0","ondisk_log_start":"0'0","created":557,"last_epoch_clean":658,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T22:41:53.343498+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:01:25.160012+0000","last_clean_scrub_stamp":"2023-09-23T22:41:53.343498+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T09:46:01.725748+0000","scrub_duration":0.0041572029999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":169,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700336+0000","last_change":"2023-09-23T22:46:02.508263+0000","last_active":"2023-09-23T22:46:26.700336+0000","last_peered":"2023-09-23T22:46:26.700336+0000","last_clean":"2023-09-23T22:46:26.700336+0000","last_became_active":"2023-09-23T22:46:02.504295+0000","last_became_peered":"2023-09-23T22:46:02.504295+0000","last_unstale":"2023-09-23T22:46:26.700336+0000","last_undegraded":"2023-09-23T22:46:26.700336+0000","last_fullsized":"2023-09-23T22:46:26.700336+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:53.363610+0000","last_clean_scrub_stamp":"2023-09-23T22:04:53.363610+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:36.779405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":43,"reported_epoch":682,"state":"active+clean","last_fresh":"2023-09-23T22:46:26.700611+0000","last_change":"2023-09-23T22:45:17.507532+0000","last_active":"2023-09-23T22:46:26.700611+0000","last_peered":"2023-09-23T22:46:26.700611+0000","last_clean":"2023-09-23T22:46:26.700611+0000","last_became_active":"2023-09-23T22:45:17.507180+0000","last_became_peered":"2023-09-23T22:45:17.507180+0000","last_unstale":"2023-09-23T22:46:26.700611+0000","last_undegraded":"2023-09-23T22:46:26.700611+0000","last_fullsized":"2023-09-23T22:46:26.700611+0000","mapping_epoch":649,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T22:04:54.364840+0000","last_clean_scrub_stamp":"2023-09-23T22:04:54.364840+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:05.877635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":175,"num_bytes_recovered":53996,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62789,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1180736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11971,"num_read_kb":25396,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":583,"num_bytes_recovered":25165919,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5037,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":54,"seq":231928234513,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":541464,"kb_used_data":11664,"kb_used_omap":0,"kb_used_meta":529792,"kb_avail":93203688,"statfs":{"total":95995035648,"available":95440576512,"internally_reserved":0,"allocated":11943936,"data_stored":9915759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":542507008},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.98599999999999999,"15min":2.0419999999999998},"min":{"1min":0.37,"5min":0.23699999999999999,"15min":0.19700000000000001},"max":{"1min":0.72599999999999998,"5min":25.308,"15min":124.34699999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.63,"5min":1.022,"15min":2.1219999999999999},"min":{"1min":0.40200000000000002,"5min":0.26400000000000001,"15min":0.23000000000000001},"max":{"1min":0.88500000000000001,"5min":27.265000000000001,"15min":124.73099999999999},"last":0.68600000000000005}]},{"osd":1,"last update":"Sat Sep 23 22:46:50 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.0589999999999999,"15min":0.93400000000000005},"min":{"1min":0.40600000000000003,"5min":0.251,"15min":0.223},"max":{"1min":0.87,"5min":20.045999999999999,"15min":42.329999999999998},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.95899999999999996,"15min":0.89500000000000002},"min":{"1min":0.46400000000000002,"5min":0.26000000000000001,"15min":0.25},"max":{"1min":0.80300000000000005,"5min":19.920000000000002,"15min":42.703000000000003},"last":0.54100000000000004}]},{"osd":2,"last update":"Sat Sep 23 22:46:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.4039999999999999,"15min":1.702},"min":{"1min":0.41499999999999998,"5min":0.27200000000000002,"15min":0.27100000000000002},"max":{"1min":0.84399999999999997,"5min":29.189,"15min":70.183999999999997},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":1.5580000000000001,"15min":1.889},"min":{"1min":0.443,"5min":0.30399999999999999,"15min":0.28899999999999998},"max":{"1min":0.81200000000000006,"5min":31.373000000000001,"15min":70.140000000000001},"last":0.59999999999999998}]},{"osd":3,"last update":"Sat Sep 23 22:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.85699999999999998,"15min":1.1240000000000001},"min":{"1min":0.55100000000000005,"5min":0.316,"15min":0.316},"max":{"1min":0.84899999999999998,"5min":3.7610000000000001,"15min":3.7610000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.65100000000000002,"5min":1.0469999999999999,"15min":1.45},"min":{"1min":0.499,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.82999999999999996,"5min":7.1429999999999998,"15min":7.1429999999999998},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.47999999999999998,"5min":0.45500000000000002,"15min":0.58599999999999997},"min":{"1min":0.28499999999999998,"5min":0.27400000000000002,"15min":0.24099999999999999},"max":{"1min":0.68200000000000005,"5min":0.89500000000000002,"15min":2.8090000000000002},"last":0.44},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.45600000000000002,"15min":1.234},"min":{"1min":0.35499999999999998,"5min":0.25700000000000001,"15min":0.23899999999999999},"max":{"1min":0.80200000000000005,"5min":0.871,"15min":48.960000000000001},"last":0.56100000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:46:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.51000000000000001,"15min":1.7290000000000001},"min":{"1min":0.33900000000000002,"5min":0.29599999999999999,"15min":0.28100000000000003},"max":{"1min":1.006,"5min":1.591,"15min":125.083},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.53100000000000003,"15min":1.6379999999999999},"min":{"1min":0.39800000000000002,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":1.04,"5min":1.413,"15min":122.351},"last":0.50900000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:46:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.70699999999999996,"15min":0.61499999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":12.616,"5min":12.616,"15min":12.616},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.218,"5min":0.67000000000000004,"15min":0.56200000000000006},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":12.632999999999999,"5min":12.632999999999999,"15min":12.632999999999999},"last":0.63700000000000001}]}]},{"osd":6,"up_from":592,"seq":2542620639284,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":136336,"kb_used_data":11720,"kb_used_omap":8,"kb_used_meta":124599,"kb_avail":93608816,"statfs":{"total":95995035648,"available":95855427584,"internally_reserved":0,"allocated":12001280,"data_stored":9980947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":127590242},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":1.913,"15min":3.3010000000000002},"min":{"1min":0.36099999999999999,"5min":0.218,"15min":0.218},"max":{"1min":0.73899999999999999,"5min":47.795000000000002,"15min":47.795000000000002},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.502,"5min":1.637,"15min":2.7480000000000002},"min":{"1min":0.35699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.77600000000000002,"5min":47.682000000000002,"15min":47.682000000000002},"last":0.54400000000000004}]},{"osd":1,"last update":"Sat Sep 23 22:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":3.306,"15min":6.0949999999999998},"min":{"1min":0.39000000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.72999999999999998,"5min":115.848,"15min":115.848},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.502,"5min":3.4079999999999999,"15min":6.2880000000000003},"min":{"1min":0.33700000000000002,"5min":0.251,"15min":0.251},"max":{"1min":0.69399999999999995,"5min":116.517,"15min":116.517},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.78400000000000003,"15min":1.0169999999999999},"min":{"1min":0.39600000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.72399999999999998,"5min":3.6379999999999999,"15min":3.6379999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.50800000000000001,"5min":1.3819999999999999,"15min":2.218},"min":{"1min":0.375,"5min":0.254,"15min":0.254},"max":{"1min":0.67600000000000005,"5min":12.502000000000001,"15min":12.502000000000001},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.73199999999999998,"15min":0.88400000000000001},"min":{"1min":0.46899999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.70799999999999996,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.82699999999999996,"15min":1.075},"min":{"1min":0.36199999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.746,"5min":5.6260000000000003,"15min":5.6260000000000003},"last":0.49199999999999999}]},{"osd":4,"last update":"Sat Sep 23 22:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51800000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.69699999999999995,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.52500000000000002,"15min":0.498},"min":{"1min":0.33600000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.64900000000000002,"5min":1.4590000000000001,"15min":1.4590000000000001},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Sep 23 22:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.53500000000000003,"15min":0.504},"min":{"1min":0.318,"5min":0.255,"15min":0.255},"max":{"1min":0.81499999999999995,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.56699999999999995,"15min":0.57599999999999996},"min":{"1min":0.371,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.75600000000000001,"5min":1.1399999999999999,"15min":1.1399999999999999},"last":0.63500000000000001}]},{"osd":7,"last update":"Sat Sep 23 22:46:59 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.61299999999999999,"15min":0.59899999999999998},"min":{"1min":0.42499999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.87,"5min":1.5029999999999999,"15min":1.5029999999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.60599999999999998,"15min":0.59199999999999997},"min":{"1min":0.47899999999999998,"5min":0.373,"15min":0.373},"max":{"1min":0.82699999999999996,"5min":1.339,"15min":1.339},"last":0.68600000000000005}]}]},{"osd":1,"up_from":495,"seq":2126008811635,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":463084,"kb_used_data":16932,"kb_used_omap":12,"kb_used_meta":446131,"kb_avail":93282068,"statfs":{"total":95995035648,"available":95520837632,"internally_reserved":0,"allocated":17338368,"data_stored":14010502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":456838830},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.1299999999999999,"15min":1.119},"min":{"1min":0.21099999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":3.6139999999999999,"5min":12.361000000000001,"15min":64.507000000000005},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":1.3420000000000001,"15min":1.282},"min":{"1min":0.27000000000000002,"5min":0.23899999999999999,"15min":0.20399999999999999},"max":{"1min":3.4049999999999998,"5min":22.393000000000001,"15min":64.352000000000004},"last":0.375}]},{"osd":2,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":3.6110000000000002,"15min":1.696},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.192},"max":{"1min":3.7450000000000001,"5min":124.447,"15min":124.447},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":3.0859999999999999,"15min":1.583},"min":{"1min":0.246,"5min":0.246,"15min":0.218},"max":{"1min":1.2569999999999999,"5min":124.03,"15min":124.03},"last":0.44}]},{"osd":3,"last update":"Sat Sep 23 22:46:35 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":1.1100000000000001,"15min":1.5660000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.91400000000000003,"5min":11.800000000000001,"15min":11.800000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.98499999999999999,"15min":1.4630000000000001},"min":{"1min":0.31,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.46,"5min":6.4530000000000003,"15min":6.4530000000000003},"last":0.56999999999999995}]},{"osd":4,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.046,"15min":0.76500000000000001},"min":{"1min":0.44600000000000001,"5min":0.34899999999999998,"15min":0.26400000000000001},"max":{"1min":3.6429999999999998,"5min":9.4410000000000007,"15min":9.4410000000000007},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.90900000000000003,"15min":0.93100000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.28499999999999998},"max":{"1min":3.4340000000000002,"5min":5.524,"15min":5.6740000000000004},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.96699999999999997,"15min":0.91500000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.98699999999999999,"5min":11.734,"15min":11.734},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.74299999999999999,"5min":1.069,"15min":0.79900000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.31},"max":{"1min":3.5550000000000002,"5min":9.4169999999999998,"15min":9.4169999999999998},"last":0.63400000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.1060000000000001,"15min":1.526},"min":{"1min":0.44800000000000001,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":3.585,"5min":9.3759999999999994,"15min":9.3759999999999994},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.75,"15min":0.83399999999999996},"min":{"1min":0.44,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":3.3620000000000001,"5min":3.3620000000000001,"15min":3.3620000000000001},"last":0.69899999999999995}]},{"osd":7,"last update":"Sat Sep 23 22:46:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.022,"15min":0.96599999999999997},"min":{"1min":0.42999999999999999,"5min":0.40600000000000003,"15min":0.30199999999999999},"max":{"1min":0.95999999999999996,"5min":11.768000000000001,"15min":11.768000000000001},"last":0.622},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.1020000000000001,"15min":0.84699999999999998},"min":{"1min":0.45200000000000001,"5min":0.41599999999999998,"15min":0.32800000000000001},"max":{"1min":3.5150000000000001,"5min":9.4819999999999993,"15min":9.4819999999999993},"last":0.67900000000000005}]}]},{"osd":0,"up_from":93,"seq":399431958951,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1134932,"kb_used_data":13196,"kb_used_omap":8,"kb_used_meta":1121719,"kb_avail":92610220,"statfs":{"total":95995035648,"available":94832865280,"internally_reserved":0,"allocated":13512704,"data_stored":10145658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":1148641130},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 22:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.36599999999999999,"5min":6.1689999999999996,"15min":2.6789999999999998},"min":{"1min":0.222,"5min":0.182,"15min":0.16600000000000001},"max":{"1min":0.55500000000000005,"5min":195.91999999999999,"15min":195.91999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.41899999999999998,"5min":6.6050000000000004,"15min":2.673},"min":{"1min":0.29099999999999998,"5min":0.253,"15min":0.252},"max":{"1min":0.80200000000000005,"5min":195.90799999999999,"15min":195.90799999999999},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 22:46:27 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":4.569,"15min":3.2639999999999998},"min":{"1min":0.27100000000000002,"5min":0.216,"15min":0.216},"max":{"1min":0.53900000000000003,"5min":196.16499999999999,"15min":196.16499999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.40400000000000003,"5min":4.7389999999999999,"15min":3.1360000000000001},"min":{"1min":0.245,"5min":0.216,"15min":0.216},"max":{"1min":0.77900000000000003,"5min":191.999,"15min":191.999},"last":0.503}]},{"osd":3,"last update":"Sat Sep 23 22:46:38 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":1.595,"15min":2.6469999999999998},"min":{"1min":0.33500000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.751,"5min":9.4149999999999991,"15min":9.4149999999999991},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":1.0660000000000001,"15min":1.6100000000000001},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.61299999999999999,"5min":4.4080000000000004,"15min":4.4080000000000004},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Sep 23 22:46:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.96899999999999997,"15min":1.167},"min":{"1min":0.34899999999999998,"5min":0.32200000000000001,"15min":0.26000000000000001},"max":{"1min":0.85999999999999999,"5min":4.2709999999999999,"15min":9.968},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.98999999999999999,"15min":1.202},"min":{"1min":0.40899999999999997,"5min":0.377,"15min":0.33300000000000002},"max":{"1min":0.92800000000000005,"5min":4.298,"15min":9.9949999999999992},"last":0.80400000000000005}]},{"osd":5,"last update":"Sat Sep 23 22:46:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.72799999999999998,"15min":2.0299999999999998},"min":{"1min":0.38600000000000001,"5min":0.31,"15min":0.28699999999999998},"max":{"1min":1.0009999999999999,"5min":2.9220000000000002,"15min":126.872},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.99299999999999999,"15min":1.4630000000000001},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.27000000000000002},"max":{"1min":0.83799999999999997,"5min":9.3819999999999997,"15min":126.521},"last":0.79400000000000004}]},{"osd":6,"last update":"Sat Sep 23 22:47:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":1.671,"15min":2.6970000000000001},"min":{"1min":0.504,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.95499999999999996,"5min":26.431000000000001,"15min":26.431000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.68100000000000005,"5min":1.8480000000000001,"15min":3.024},"min":{"1min":0.42999999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.89300000000000002,"5min":26.387,"15min":26.387},"last":0.70299999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:46:38 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":1.012,"15min":1.266},"min":{"1min":0.45200000000000001,"5min":0.32400000000000001,"15min":0.316},"max":{"1min":0.89000000000000001,"5min":4.2199999999999998,"15min":10.377000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.78400000000000003,"15min":1.3720000000000001},"min":{"1min":0.48099999999999998,"5min":0.35599999999999998,"15min":0.35499999999999998},"max":{"1min":0.91000000000000003,"5min":2.9649999999999999,"15min":9.0879999999999992},"last":0.78000000000000003}]}]},{"osd":2,"up_from":349,"seq":1498943586513,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":518172,"kb_used_data":8980,"kb_used_omap":15,"kb_used_meta":509168,"kb_avail":93226980,"statfs":{"total":95995035648,"available":95464427520,"internally_reserved":0,"allocated":9195520,"data_stored":5879586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":15626,"internal_metadata":521388790},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":1.7529999999999999,"15min":2.3260000000000001},"min":{"1min":0.216,"5min":0.16900000000000001,"15min":0.16900000000000001},"max":{"1min":0.46000000000000002,"5min":61.712000000000003,"15min":86.620000000000005},"last":0.502},{"interface":"front","average":{"1min":0.34999999999999998,"5min":2.2040000000000002,"15min":2.6040000000000001},"min":{"1min":0.247,"5min":0.223,"15min":0.215},"max":{"1min":0.45300000000000001,"5min":61.177999999999997,"15min":86.355000000000004},"last":0.252}]},{"osd":1,"last update":"Sat Sep 23 22:46:58 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.865,"15min":1.329},"min":{"1min":0.23100000000000001,"5min":0.20399999999999999,"15min":0.19500000000000001},"max":{"1min":1.034,"5min":47.018000000000001,"15min":70.042000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.41999999999999998,"5min":3.9540000000000002,"15min":2.25},"min":{"1min":0.27500000000000002,"5min":0.23599999999999999,"15min":0.184},"max":{"1min":0.84099999999999997,"5min":127.327,"15min":127.327},"last":0.314}]},{"osd":3,"last update":"Sat Sep 23 22:46:41 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":2.0190000000000001,"15min":3.2839999999999998},"min":{"1min":0.308,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.52300000000000002,"5min":14.561,"15min":14.561},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.36799999999999999,"5min":1.0620000000000001,"15min":1.6180000000000001},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.434,"5min":4.7720000000000002,"15min":4.7720000000000002},"last":0.33800000000000002}]},{"osd":4,"last update":"Sat Sep 23 22:46:13 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":1.1779999999999999,"15min":1.6799999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.23799999999999999},"max":{"1min":0.89300000000000002,"5min":8.8710000000000004,"15min":45.576999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.96199999999999997,"15min":1.2230000000000001},"min":{"1min":0.42799999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.91300000000000003,"5min":5.157,"15min":45.420999999999999},"last":0.436}]},{"osd":5,"last update":"Sat Sep 23 22:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.155,"15min":3.2490000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.27300000000000002},"max":{"1min":0.88500000000000001,"5min":8.8119999999999994,"15min":163.56399999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":2.246,"15min":3.3730000000000002},"min":{"1min":0.38500000000000001,"5min":0.30099999999999999,"15min":0.28599999999999998},"max":{"1min":0.83999999999999997,"5min":126.81100000000001,"15min":163.541},"last":0.47599999999999998}]},{"osd":6,"last update":"Sat Sep 23 22:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":1.5980000000000001,"15min":2.0409999999999999},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.90100000000000002,"5min":7.351,"15min":7.351},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.129,"15min":1.3600000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.86199999999999999,"5min":3.895,"15min":3.895},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.3220000000000001,"15min":1.8520000000000001},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.26400000000000001},"max":{"1min":0.82699999999999996,"5min":7.3979999999999997,"15min":108.113},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.1990000000000001,"15min":2.1749999999999998},"min":{"1min":0.438,"5min":0.432,"15min":0.30599999999999999},"max":{"1min":0.81000000000000005,"5min":8.8480000000000008,"15min":108.093},"last":0.46600000000000003}]}]},{"osd":3,"up_from":584,"seq":2508260900921,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":133304,"kb_used_data":9200,"kb_used_omap":9,"kb_used_meta":124086,"kb_avail":93611848,"statfs":{"total":95995035648,"available":95858532352,"internally_reserved":0,"allocated":9420800,"data_stored":5959070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":10195,"internal_metadata":127064109},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.34599999999999997,"5min":1.0529999999999999,"15min":1.6459999999999999},"min":{"1min":0.25700000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.55100000000000005,"5min":5.2939999999999996,"15min":5.2939999999999996},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.874,"15min":1.202},"min":{"1min":0.26900000000000002,"5min":0.222,"15min":0.222},"max":{"1min":0.54300000000000004,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":0.35299999999999998}]},{"osd":1,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":9.8450000000000006,"15min":17.018999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.64700000000000002,"5min":303.267,"15min":303.267},"last":0.246},{"interface":"front","average":{"1min":0.39000000000000001,"5min":9.6419999999999995,"15min":16.539999999999999},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":0.63200000000000001,"5min":303.09699999999998,"15min":303.09699999999998},"last":0.27400000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":13.375,"15min":25.969999999999999},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.54200000000000004,"5min":381.87299999999999,"15min":381.87299999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.41199999999999998,"5min":13.739000000000001,"15min":26.841000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.55300000000000005,"5min":382.73500000000001,"15min":382.73500000000001},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.0580000000000001,"15min":1.488},"min":{"1min":0.36699999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.753,"5min":5.0460000000000003,"15min":5.0460000000000003},"last":0.501},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.0009999999999999,"15min":1.359},"min":{"1min":0.41899999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.871,"5min":4.5410000000000004,"15min":4.5410000000000004},"last":0.54400000000000004}]},{"osd":5,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.86599999999999999,"15min":1.1240000000000001},"min":{"1min":0.45300000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.83899999999999997,"5min":3.762,"15min":3.762},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.093,"15min":1.524},"min":{"1min":0.499,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.77000000000000002,"5min":5.1079999999999997,"15min":5.1079999999999997},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 23 22:47:02 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.73299999999999998,"15min":0.88500000000000001},"min":{"1min":0.46400000000000002,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.88600000000000001,"5min":3.7839999999999998,"15min":3.7839999999999998},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.95399999999999996,"15min":1.3180000000000001},"min":{"1min":0.42199999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.79600000000000004,"5min":5.1529999999999996,"15min":5.1529999999999996},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 22:46:32 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":1.159,"15min":1.613},"min":{"1min":0.46700000000000003,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.002,"5min":5.1929999999999996,"15min":5.1929999999999996},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.65000000000000002,"5min":1.048,"15min":1.4079999999999999},"min":{"1min":0.47799999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.91200000000000003,"5min":4.5800000000000001,"15min":4.5800000000000001},"last":0.52000000000000002}]}]},{"osd":4,"up_from":465,"seq":1997159792777,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":123348,"kb_used_data":7308,"kb_used_omap":12,"kb_used_meta":116019,"kb_avail":93621804,"statfs":{"total":95995035648,"available":95868727296,"internally_reserved":0,"allocated":7483392,"data_stored":5459020,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":118804081},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.79400000000000004,"15min":0.98799999999999999},"min":{"1min":0.35499999999999998,"5min":0.23100000000000001,"15min":0.21199999999999999},"max":{"1min":1.0309999999999999,"5min":8.7289999999999992,"15min":8.7289999999999992},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.69799999999999995,"15min":0.90800000000000003},"min":{"1min":0.38600000000000001,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.83299999999999996,"5min":3.5990000000000002,"15min":9.1110000000000007},"last":0.72099999999999997}]},{"osd":1,"last update":"Sat Sep 23 22:46:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.89400000000000002,"15min":0.82399999999999995},"min":{"1min":0.38500000000000001,"5min":0.246,"15min":0.23100000000000001},"max":{"1min":0.69299999999999995,"5min":8.3539999999999992,"15min":38.142000000000003},"last":0.621},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.82299999999999995,"15min":0.92500000000000004},"min":{"1min":0.38400000000000001,"5min":0.27300000000000002,"15min":0.25},"max":{"1min":0.78600000000000003,"5min":5.6040000000000001,"15min":38.167000000000002},"last":0.58399999999999996}]},{"osd":2,"last update":"Sat Sep 23 22:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":2.0699999999999998,"15min":1.611},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.247},"max":{"1min":1.0980000000000001,"5min":104.13,"15min":104.13},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.624,"5min":2.4729999999999999,"15min":1.9950000000000001},"min":{"1min":0.42999999999999999,"5min":0.30599999999999999,"15min":0.29999999999999999},"max":{"1min":0.89900000000000002,"5min":109.82899999999999,"15min":109.82899999999999},"last":0.59399999999999997}]},{"osd":3,"last update":"Sat Sep 23 22:46:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.72799999999999998,"15min":0.76200000000000001},"min":{"1min":0.43099999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.91600000000000004,"5min":2.8879999999999999,"15min":2.8879999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.85499999999999998,"15min":1.0569999999999999},"min":{"1min":0.45900000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.88,"5min":4.2770000000000001,"15min":4.2770000000000001},"last":0.60899999999999999}]},{"osd":5,"last update":"Sat Sep 23 22:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":1.395,"15min":1.262},"min":{"1min":0.309,"5min":0.224,"15min":0.224},"max":{"1min":0.84599999999999997,"5min":80.213999999999999,"15min":80.213999999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.3979999999999999,"15min":1.244},"min":{"1min":0.32500000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.79300000000000004,"5min":80.244,"15min":80.244},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 22:47:01 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.52100000000000002,"15min":0.46700000000000003},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.76700000000000002,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.54800000000000004,"15min":0.55000000000000004},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.91400000000000003,"5min":2.1469999999999998,"15min":2.1469999999999998},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Sep 23 22:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":0.51200000000000001,"15min":1.0900000000000001},"min":{"1min":0.378,"5min":0.247,"15min":0.22800000000000001},"max":{"1min":0.81200000000000006,"5min":0.96099999999999997,"15min":8.1950000000000003},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.62,"5min":0.52500000000000002,"15min":1.341},"min":{"1min":0.44500000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.81299999999999994,"5min":0.87,"15min":8.6140000000000008},"last":0.63900000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757786,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":566924,"kb_used_data":11652,"kb_used_omap":0,"kb_used_meta":555264,"kb_avail":93178228,"statfs":{"total":95995035648,"available":95414505472,"internally_reserved":0,"allocated":11931648,"data_stored":9911775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":568590336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 22:46:48 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":1.0600000000000001,"15min":3.4089999999999998},"min":{"1min":0.28100000000000003,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":0.624,"5min":26.925000000000001,"15min":145.46600000000001},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.071,"15min":3.4489999999999998},"min":{"1min":0.33200000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.75700000000000001,"5min":30.609000000000002,"15min":144.429},"last":0.68700000000000006}]},{"osd":1,"last update":"Sat Sep 23 22:46:54 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":1.6970000000000001,"15min":1.075},"min":{"1min":0.315,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.71299999999999997,"5min":35.643999999999998,"15min":35.643999999999998},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.52800000000000002,"5min":1.7090000000000001,"15min":1.008},"min":{"1min":0.33500000000000002,"5min":0.26200000000000001,"15min":0.249},"max":{"1min":0.71599999999999997,"5min":35.316000000000003,"15min":35.316000000000003},"last":0.76900000000000002}]},{"osd":2,"last update":"Sat Sep 23 22:46:19 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":2.2250000000000001,"15min":2.2469999999999999},"min":{"1min":0.34399999999999997,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.72499999999999998,"5min":77.813999999999993,"15min":77.813999999999993},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.54700000000000004,"5min":1.9850000000000001,"15min":2.1019999999999999},"min":{"1min":0.41699999999999998,"5min":0.28599999999999998,"15min":0.28399999999999997},"max":{"1min":0.75900000000000001,"5min":76.265000000000001,"15min":76.265000000000001},"last":0.63500000000000001}]},{"osd":3,"last update":"Sat Sep 23 22:46:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.996,"15min":1.353},"min":{"1min":0.39600000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.78000000000000003,"5min":6.0919999999999996,"15min":6.0919999999999996},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.98599999999999999,"15min":1.3180000000000001},"min":{"1min":0.441,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.73799999999999999,"5min":7.8959999999999999,"15min":7.8959999999999999},"last":0.80900000000000005}]},{"osd":4,"last update":"Sat Sep 23 22:46:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.442,"15min":1.958},"min":{"1min":0.29999999999999999,"5min":0.24399999999999999,"15min":0.23999999999999999},"max":{"1min":0.75800000000000001,"5min":1.476,"15min":61.173999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.53200000000000003,"15min":2.0329999999999999},"min":{"1min":0.39000000000000001,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":0.80300000000000005,"5min":1.5409999999999999,"15min":61.203000000000003},"last":0.747}]},{"osd":6,"last update":"Sat Sep 23 22:47:04 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.5,"15min":0.45400000000000001},"min":{"1min":0.378,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.72499999999999998,"5min":0.97599999999999998,"15min":0.97599999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.54800000000000004,"15min":0.48999999999999999},"min":{"1min":0.42299999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78600000000000003,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 23 22:46:48 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.54900000000000004,"15min":1.3160000000000001},"min":{"1min":0.34699999999999998,"5min":0.255,"15min":0.249},"max":{"1min":0.81899999999999995,"5min":1.4550000000000001,"15min":59.101999999999997},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.56999999999999995,"15min":1.496},"min":{"1min":0.36099999999999999,"5min":0.28499999999999998,"15min":0.27600000000000002},"max":{"1min":0.80100000000000005,"5min":1.6499999999999999,"15min":59.151000000000003},"last":0.65500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3640,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1397,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8342,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12626,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11986,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8798,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12687,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8350,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T22:47:08.827 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T22:47:08.828 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T22:47:08.899 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=6c4335fce8894e1381218277d33c3589 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T22:47:08.901 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T22:47:08.911 INFO:tasks.cephadm:Teardown begin 2023-09-23T22:47:08.912 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:47:08.962 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:47:08.999 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T22:47:08.999 DEBUG:teuthology.orchestra.run.smithi026:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T22:47:09.019 DEBUG:teuthology.orchestra.run.smithi070:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T22:47:09.058 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T22:47:09.058 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T22:47:09.058 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T22:47:09.341 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:09 smithi026 systemd[1]: Stopping Ceph mon.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:09.611 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:09 smithi026 conmon[215751]: 2023-09-23T22:47:09.339+0000 7f458670f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:09.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:09 smithi026 conmon[215751]: 2023-09-23T22:47:09.339+0000 7f458670f700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T22:47:09.612 INFO:journalctl@ceph.mon.a.smithi026.stdout:Sep 23 22:47:09 smithi026 bash[723477]: 0028efef866221db5b1dde28b65e86961eb64fd91d4ea3ceaca98040eb313454 2023-09-23T22:47:09.612 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:47:09 smithi026 conmon[202047]: [23/Sep/2023:22:47:09] ENGINE Bus STOPPING 2023-09-23T22:47:09.612 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:47:09 smithi026 conmon[202047]: [23/Sep/2023:22:47:09] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T22:47:09.613 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:47:09 smithi026 conmon[202047]: [23/Sep/2023:22:47:09] ENGINE Bus STOPPED 2023-09-23T22:47:09.613 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:47:09 smithi026 conmon[202047]: [23/Sep/2023:22:47:09] ENGINE Bus STARTING 2023-09-23T22:47:09.613 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:47:09 smithi026 conmon[202047]: [23/Sep/2023:22:47:09] ENGINE Serving on http://:::9283 2023-09-23T22:47:09.613 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:47:09 smithi026 conmon[202047]: [23/Sep/2023:22:47:09] ENGINE Bus STARTED 2023-09-23T22:47:09.985 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:47:09 smithi026 conmon[202047]: ::ffff:172.21.15.70 - - [23/Sep/2023:22:47:09] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2023-09-23T22:47:10.234 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T22:47:10.284 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:10.285 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T22:47:10.285 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T22:47:10.285 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T22:47:10.485 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:10 smithi026 systemd[1]: Stopping Ceph mon.c for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:10.847 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:10 smithi026 conmon[222990]: 2023-09-23T22:47:10.592+0000 7f88426df700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:10.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:10 smithi026 conmon[222990]: 2023-09-23T22:47:10.592+0000 7f88426df700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T22:47:10.848 INFO:journalctl@ceph.mon.c.smithi026.stdout:Sep 23 22:47:10 smithi026 bash[723790]: 25f6f9fc0ac743ddec0a6c3048cd4506081ff9dd39b8829d695df36de35336a2 2023-09-23T22:47:11.358 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T22:47:11.399 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:11.399 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T22:47:11.399 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T22:47:11.400 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T22:47:11.681 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:11 smithi070 systemd[1]: Stopping Ceph mon.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:12.123 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:11 smithi070 conmon[192628]: 2023-09-23T22:47:11.679+0000 7fea05503700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:12.124 INFO:journalctl@ceph.mon.b.smithi070.stdout:Sep 23 22:47:11 smithi070 conmon[192628]: 2023-09-23T22:47:11.679+0000 7fea05503700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T22:47:12.723 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T22:47:12.763 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:12.763 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T22:47:12.763 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T22:47:12.764 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T22:47:13.068 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:47:12 smithi026 systemd[1]: Stopping Ceph mgr.y for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:13.068 INFO:journalctl@ceph.mgr.y.smithi026.stdout:Sep 23 22:47:13 smithi026 conmon[202047]: 2023-09-23T22:47:13.033+0000 7f47d0814700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:13.892 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T22:47:13.930 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:13.931 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T22:47:13.931 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T22:47:13.931 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T22:47:14.373 INFO:journalctl@ceph.mgr.x.smithi070.stdout:Sep 23 22:47:13 smithi070 systemd[1]: Stopping Ceph mgr.x for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:15.088 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T22:47:15.125 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:15.126 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T22:47:15.126 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T22:47:15.126 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T22:47:15.421 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:47:15 smithi026 systemd[1]: Stopping Ceph osd.0 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:15.735 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:47:15 smithi026 conmon[237380]: 2023-09-23T22:47:15.420+0000 7fd2a8dac700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:15.735 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:47:15 smithi026 conmon[237380]: 2023-09-23T22:47:15.420+0000 7fd2a8dac700 -1 osd.0 683 *** Got signal Terminated *** 2023-09-23T22:47:15.736 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:47:15 smithi026 conmon[237380]: 2023-09-23T22:47:15.420+0000 7fd2a8dac700 -1 osd.0 683 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:47:21.986 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:47:21 smithi026 bash[724394]: dd9b16818fa43f894b879ea8eb90e8fca940b3f8846252894af29d0147a17fb6 2023-09-23T22:47:23.469 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T22:47:23.496 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:47:23 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T22:47:23.496 INFO:journalctl@ceph.osd.0.smithi026.stdout:Sep 23 22:47:23 smithi026 systemd[1]: Stopped Ceph osd.0 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:47:23.511 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:23.511 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T22:47:23.512 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T22:47:23.512 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T22:47:23.836 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:47:23 smithi026 systemd[1]: Stopping Ceph osd.1 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:24.235 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:47:23 smithi026 conmon[584057]: 2023-09-23T22:47:23.835+0000 7f97135aa700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:24.236 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:47:23 smithi026 conmon[584057]: 2023-09-23T22:47:23.835+0000 7f97135aa700 -1 osd.1 683 *** Got signal Terminated *** 2023-09-23T22:47:24.236 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:47:23 smithi026 conmon[584057]: 2023-09-23T22:47:23.835+0000 7f97135aa700 -1 osd.1 683 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:47:29.735 INFO:journalctl@ceph.osd.1.smithi026.stdout:Sep 23 22:47:29 smithi026 bash[724968]: 3877a14b80dfc78d9a2c26a0dbdd4a84406cd87bdcc70479348643e7c189547e 2023-09-23T22:47:31.569 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T22:47:31.613 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:31.614 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T22:47:31.614 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T22:47:31.614 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T22:47:31.952 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:47:31 smithi026 systemd[1]: Stopping Ceph osd.2 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:32.235 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:47:31 smithi026 conmon[473736]: 2023-09-23T22:47:31.950+0000 7fd60ba03700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:32.236 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:47:31 smithi026 conmon[473736]: 2023-09-23T22:47:31.950+0000 7fd60ba03700 -1 osd.2 683 *** Got signal Terminated *** 2023-09-23T22:47:32.236 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:47:31 smithi026 conmon[473736]: 2023-09-23T22:47:31.950+0000 7fd60ba03700 -1 osd.2 683 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:47:37.985 INFO:journalctl@ceph.osd.2.smithi026.stdout:Sep 23 22:47:37 smithi026 bash[725546]: 11475c0ad819687fa136cd691f2eb1319f98924d4f7f7c39233e1a595327b27a 2023-09-23T22:47:39.483 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T22:47:39.525 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:39.525 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T22:47:39.525 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T22:47:39.525 DEBUG:teuthology.orchestra.run.smithi026:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T22:47:39.735 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:47:39 smithi026 systemd[1]: Stopping Ceph osd.3 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:40.235 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:47:39 smithi026 conmon[660192]: 2023-09-23T22:47:39.840+0000 7f84db724700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:40.236 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:47:39 smithi026 conmon[660192]: 2023-09-23T22:47:39.840+0000 7f84db724700 -1 osd.3 683 *** Got signal Terminated *** 2023-09-23T22:47:40.236 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:47:39 smithi026 conmon[660192]: 2023-09-23T22:47:39.840+0000 7f84db724700 -1 osd.3 683 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:47:42.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:42 smithi070 conmon[410182]: 2023-09-23T22:47:42.583+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:42.985 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:47:42 smithi026 conmon[660192]: 2023-09-23T22:47:42.558+0000 7f84d6517700 -1 osd.3 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.538597+0000 front 2023-09-23T22:47:18.538627+0000 (oldest deadline 2023-09-23T22:47:42.038688+0000) 2023-09-23T22:47:43.865 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:43 smithi070 conmon[410182]: 2023-09-23T22:47:43.536+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:43.985 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:47:43 smithi026 conmon[660192]: 2023-09-23T22:47:43.513+0000 7f84d6517700 -1 osd.3 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.538597+0000 front 2023-09-23T22:47:18.538627+0000 (oldest deadline 2023-09-23T22:47:42.038688+0000) 2023-09-23T22:47:44.124 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:43 smithi070 conmon[341485]: 2023-09-23T22:47:43.864+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.089228+0000 front 2023-09-23T22:47:20.089110+0000 (oldest deadline 2023-09-23T22:47:43.589277+0000) 2023-09-23T22:47:44.579 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:44 smithi070 conmon[140233]: 2023-09-23T22:47:44.150+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:44.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:44 smithi070 conmon[410182]: 2023-09-23T22:47:44.578+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:44.985 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:47:44 smithi026 conmon[660192]: 2023-09-23T22:47:44.498+0000 7f84d6517700 -1 osd.3 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.538597+0000 front 2023-09-23T22:47:18.538627+0000 (oldest deadline 2023-09-23T22:47:42.038688+0000) 2023-09-23T22:47:45.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:45 smithi070 conmon[140233]: 2023-09-23T22:47:45.108+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:45.374 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:44 smithi070 conmon[341485]: 2023-09-23T22:47:44.893+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.089228+0000 front 2023-09-23T22:47:20.089110+0000 (oldest deadline 2023-09-23T22:47:43.589277+0000) 2023-09-23T22:47:45.375 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:47:45 smithi070 conmon[172396]: logger=cleanup t=2023-09-23T22:47:45.008939451Z level=info msg="Completed cleanup jobs" duration=66.690546ms 2023-09-23T22:47:45.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:45 smithi070 conmon[410182]: 2023-09-23T22:47:45.622+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:45.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:45 smithi070 conmon[122257]: 2023-09-23T22:47:45.545+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:45.985 INFO:journalctl@ceph.osd.3.smithi026.stdout:Sep 23 22:47:45 smithi026 bash[726124]: 4ea5b94d55508b1fa5eccefdfc5080a42c5577bf2a7553119010f66b80218f87 2023-09-23T22:47:46.374 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:45 smithi070 conmon[341485]: 2023-09-23T22:47:45.913+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.089228+0000 front 2023-09-23T22:47:20.089110+0000 (oldest deadline 2023-09-23T22:47:43.589277+0000) 2023-09-23T22:47:46.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:46 smithi070 conmon[140233]: 2023-09-23T22:47:46.140+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:46.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:46 smithi070 conmon[410182]: 2023-09-23T22:47:46.615+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:46.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:46 smithi070 conmon[122257]: 2023-09-23T22:47:46.511+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:47.184 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:46 smithi070 conmon[341485]: 2023-09-23T22:47:46.906+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.089228+0000 front 2023-09-23T22:47:20.089110+0000 (oldest deadline 2023-09-23T22:47:43.589277+0000) 2023-09-23T22:47:47.316 DEBUG:teuthology.orchestra.run.smithi026:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T22:47:47.358 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:47.359 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T22:47:47.359 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T22:47:47.359 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T22:47:47.493 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:47 smithi070 conmon[140233]: 2023-09-23T22:47:47.184+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:47.493 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:47 smithi070 systemd[1]: Stopping Ceph osd.4 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:47.864 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:47 smithi070 conmon[410182]: 2023-09-23T22:47:47.656+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:47.864 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:47 smithi070 conmon[122257]: 2023-09-23T22:47:47.492+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:47.864 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:47 smithi070 conmon[341485]: 2023-09-23T22:47:47.655+0000 7f50bd16c700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:47.865 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:47 smithi070 conmon[341485]: 2023-09-23T22:47:47.655+0000 7f50bd16c700 -1 osd.4 683 *** Got signal Terminated *** 2023-09-23T22:47:47.865 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:47 smithi070 conmon[341485]: 2023-09-23T22:47:47.655+0000 7f50bd16c700 -1 osd.4 683 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:47:47.865 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:47 smithi070 conmon[341485]: 2023-09-23T22:47:47.863+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.089228+0000 front 2023-09-23T22:47:20.089110+0000 (oldest deadline 2023-09-23T22:47:43.589277+0000) 2023-09-23T22:47:48.496 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:48 smithi070 conmon[140233]: 2023-09-23T22:47:48.207+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:48.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:48 smithi070 conmon[410182]: 2023-09-23T22:47:48.626+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:48.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:48 smithi070 conmon[122257]: 2023-09-23T22:47:48.495+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:49.173 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:48 smithi070 conmon[341485]: 2023-09-23T22:47:48.904+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.089228+0000 front 2023-09-23T22:47:20.089110+0000 (oldest deadline 2023-09-23T22:47:43.589277+0000) 2023-09-23T22:47:49.451 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:49 smithi070 conmon[140233]: 2023-09-23T22:47:49.172+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:49.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:49 smithi070 conmon[410182]: 2023-09-23T22:47:49.637+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:49.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:49 smithi070 conmon[122257]: 2023-09-23T22:47:49.450+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:50.189 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:49 smithi070 conmon[341485]: 2023-09-23T22:47:49.935+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.089228+0000 front 2023-09-23T22:47:20.089110+0000 (oldest deadline 2023-09-23T22:47:43.589277+0000) 2023-09-23T22:47:50.491 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:50 smithi070 conmon[140233]: 2023-09-23T22:47:50.188+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:50.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:50 smithi070 conmon[410182]: 2023-09-23T22:47:50.682+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:50.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:50 smithi070 conmon[122257]: 2023-09-23T22:47:50.490+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:51.171 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:50 smithi070 conmon[341485]: 2023-09-23T22:47:50.911+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.089228+0000 front 2023-09-23T22:47:20.089110+0000 (oldest deadline 2023-09-23T22:47:43.589277+0000) 2023-09-23T22:47:51.474 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:51 smithi070 conmon[140233]: 2023-09-23T22:47:51.170+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:51.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:51 smithi070 conmon[122257]: 2023-09-23T22:47:51.474+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:51.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:51 smithi070 conmon[410182]: 2023-09-23T22:47:51.666+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:51.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:51 smithi070 conmon[410182]: 2023-09-23T22:47:51.666+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:47:52.214 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:51 smithi070 conmon[341485]: 2023-09-23T22:47:51.942+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.089228+0000 front 2023-09-23T22:47:20.089110+0000 (oldest deadline 2023-09-23T22:47:43.589277+0000) 2023-09-23T22:47:52.215 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:51 smithi070 conmon[341485]: 2023-09-23T22:47:51.942+0000 7f50baf6d700 -1 osd.4 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.090512+0000 front 2023-09-23T22:47:27.090542+0000 (oldest deadline 2023-09-23T22:47:51.790349+0000) 2023-09-23T22:47:52.515 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:52 smithi070 conmon[140233]: 2023-09-23T22:47:52.214+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:52.516 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:52 smithi070 conmon[140233]: 2023-09-23T22:47:52.214+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:47:52.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:52 smithi070 conmon[410182]: 2023-09-23T22:47:52.704+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:52.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:52 smithi070 conmon[410182]: 2023-09-23T22:47:52.704+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:47:52.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:52 smithi070 conmon[122257]: 2023-09-23T22:47:52.515+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:52.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:52 smithi070 conmon[122257]: 2023-09-23T22:47:52.515+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.506520+0000 front 2023-09-23T22:47:27.506626+0000 (oldest deadline 2023-09-23T22:47:51.606585+0000) 2023-09-23T22:47:53.556 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:53 smithi070 conmon[140233]: 2023-09-23T22:47:53.187+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:53.557 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:53 smithi070 conmon[140233]: 2023-09-23T22:47:53.187+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:47:53.874 INFO:journalctl@ceph.osd.4.smithi070.stdout:Sep 23 22:47:53 smithi070 bash[432119]: 3e1513b12ef20402fede1fdd37cd0dcd0fa7a89202d4e6332293cc37fb6159ce 2023-09-23T22:47:53.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:53 smithi070 conmon[410182]: 2023-09-23T22:47:53.694+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:53.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:53 smithi070 conmon[410182]: 2023-09-23T22:47:53.694+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:47:53.875 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:53 smithi070 conmon[122257]: 2023-09-23T22:47:53.555+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:53.875 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:53 smithi070 conmon[122257]: 2023-09-23T22:47:53.555+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.506520+0000 front 2023-09-23T22:47:27.506626+0000 (oldest deadline 2023-09-23T22:47:51.606585+0000) 2023-09-23T22:47:54.520 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:54 smithi070 conmon[140233]: 2023-09-23T22:47:54.165+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:54.520 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:54 smithi070 conmon[140233]: 2023-09-23T22:47:54.165+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:47:54.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:54 smithi070 conmon[410182]: 2023-09-23T22:47:54.683+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:54.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:54 smithi070 conmon[410182]: 2023-09-23T22:47:54.683+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:47:54.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:54 smithi070 conmon[122257]: 2023-09-23T22:47:54.518+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:54.875 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:54 smithi070 conmon[122257]: 2023-09-23T22:47:54.518+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.506520+0000 front 2023-09-23T22:47:27.506626+0000 (oldest deadline 2023-09-23T22:47:51.606585+0000) 2023-09-23T22:47:55.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:55 smithi070 conmon[140233]: 2023-09-23T22:47:55.128+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:55.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:55 smithi070 conmon[140233]: 2023-09-23T22:47:55.128+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:47:55.536 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T22:47:55.576 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:47:55.577 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T22:47:55.577 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T22:47:55.577 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T22:47:55.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:55 smithi070 conmon[410182]: 2023-09-23T22:47:55.675+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:55.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:55 smithi070 conmon[410182]: 2023-09-23T22:47:55.675+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:47:55.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:55 smithi070 conmon[122257]: 2023-09-23T22:47:55.482+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:55.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:55 smithi070 conmon[122257]: 2023-09-23T22:47:55.482+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.506520+0000 front 2023-09-23T22:47:27.506626+0000 (oldest deadline 2023-09-23T22:47:51.606585+0000) 2023-09-23T22:47:55.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:55 smithi070 systemd[1]: Stopping Ceph osd.5 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:47:56.373 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:55 smithi070 conmon[122257]: 2023-09-23T22:47:55.877+0000 7fe386e73700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:47:56.374 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:55 smithi070 conmon[122257]: 2023-09-23T22:47:55.877+0000 7fe386e73700 -1 osd.5 683 *** Got signal Terminated *** 2023-09-23T22:47:56.374 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:55 smithi070 conmon[122257]: 2023-09-23T22:47:55.877+0000 7fe386e73700 -1 osd.5 683 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:47:56.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:56 smithi070 conmon[140233]: 2023-09-23T22:47:56.106+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:56.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:56 smithi070 conmon[140233]: 2023-09-23T22:47:56.106+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:47:56.726 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:56 smithi070 conmon[122257]: 2023-09-23T22:47:56.451+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:56.726 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:56 smithi070 conmon[122257]: 2023-09-23T22:47:56.451+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.506520+0000 front 2023-09-23T22:47:27.506626+0000 (oldest deadline 2023-09-23T22:47:51.606585+0000) 2023-09-23T22:47:57.096 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:56 smithi070 conmon[410182]: 2023-09-23T22:47:56.725+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:57.096 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:56 smithi070 conmon[410182]: 2023-09-23T22:47:56.725+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:47:57.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:57 smithi070 conmon[140233]: 2023-09-23T22:47:57.095+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:57.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:57 smithi070 conmon[140233]: 2023-09-23T22:47:57.095+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:47:57.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:57 smithi070 conmon[122257]: 2023-09-23T22:47:57.500+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:57.874 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:57 smithi070 conmon[122257]: 2023-09-23T22:47:57.500+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.506520+0000 front 2023-09-23T22:47:27.506626+0000 (oldest deadline 2023-09-23T22:47:51.606585+0000) 2023-09-23T22:47:57.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:57 smithi070 conmon[410182]: 2023-09-23T22:47:57.677+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:57.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:57 smithi070 conmon[410182]: 2023-09-23T22:47:57.677+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:47:58.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:58 smithi070 conmon[140233]: 2023-09-23T22:47:58.121+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:58.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:58 smithi070 conmon[140233]: 2023-09-23T22:47:58.121+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:47:58.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:58 smithi070 conmon[140233]: 2023-09-23T22:47:58.121+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:47:58.720 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:58 smithi070 conmon[122257]: 2023-09-23T22:47:58.462+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:58.720 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:58 smithi070 conmon[122257]: 2023-09-23T22:47:58.462+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.506520+0000 front 2023-09-23T22:47:27.506626+0000 (oldest deadline 2023-09-23T22:47:51.606585+0000) 2023-09-23T22:47:59.124 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:58 smithi070 conmon[410182]: 2023-09-23T22:47:58.720+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:47:59.124 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:58 smithi070 conmon[410182]: 2023-09-23T22:47:58.720+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:47:59.429 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:59 smithi070 conmon[140233]: 2023-09-23T22:47:59.122+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:47:59.429 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:59 smithi070 conmon[140233]: 2023-09-23T22:47:59.122+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:47:59.430 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:47:59 smithi070 conmon[140233]: 2023-09-23T22:47:59.122+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:47:59.688 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:59 smithi070 conmon[122257]: 2023-09-23T22:47:59.429+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:47:59.688 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:47:59 smithi070 conmon[122257]: 2023-09-23T22:47:59.429+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.506520+0000 front 2023-09-23T22:47:27.506626+0000 (oldest deadline 2023-09-23T22:47:51.606585+0000) 2023-09-23T22:48:00.123 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:59 smithi070 conmon[410182]: 2023-09-23T22:47:59.687+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:00.124 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:47:59 smithi070 conmon[410182]: 2023-09-23T22:47:59.687+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:00.416 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:00 smithi070 conmon[140233]: 2023-09-23T22:48:00.127+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:00.416 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:00 smithi070 conmon[140233]: 2023-09-23T22:48:00.127+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:00.416 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:00 smithi070 conmon[140233]: 2023-09-23T22:48:00.127+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:00.695 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:48:00 smithi070 conmon[122257]: 2023-09-23T22:48:00.414+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:19.905670+0000 front 2023-09-23T22:47:19.905746+0000 (oldest deadline 2023-09-23T22:47:44.605636+0000) 2023-09-23T22:48:00.696 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:48:00 smithi070 conmon[122257]: 2023-09-23T22:48:00.414+0000 7fe38046b700 -1 osd.5 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:27.506520+0000 front 2023-09-23T22:47:27.506626+0000 (oldest deadline 2023-09-23T22:47:51.606585+0000) 2023-09-23T22:48:01.124 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:00 smithi070 conmon[410182]: 2023-09-23T22:48:00.695+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:01.124 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:00 smithi070 conmon[410182]: 2023-09-23T22:48:00.695+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:01.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:01 smithi070 conmon[140233]: 2023-09-23T22:48:01.174+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:01.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:01 smithi070 conmon[140233]: 2023-09-23T22:48:01.174+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:01.625 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:01 smithi070 conmon[140233]: 2023-09-23T22:48:01.174+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:02.071 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:01 smithi070 conmon[410182]: 2023-09-23T22:48:01.646+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:02.071 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:01 smithi070 conmon[410182]: 2023-09-23T22:48:01.646+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:02.072 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:01 smithi070 conmon[410182]: 2023-09-23T22:48:01.646+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:35.810783+0000 front 2023-09-23T22:47:35.810870+0000 (oldest deadline 2023-09-23T22:48:01.110788+0000) 2023-09-23T22:48:02.373 INFO:journalctl@ceph.osd.5.smithi070.stdout:Sep 23 22:48:02 smithi070 bash[432705]: 628c1ee2ccfc030e8ecc4d8fedc8fb73571cbf5e22926d6d9c906e2553593b28 2023-09-23T22:48:02.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:02 smithi070 conmon[140233]: 2023-09-23T22:48:02.168+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:02.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:02 smithi070 conmon[140233]: 2023-09-23T22:48:02.168+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:02.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:02 smithi070 conmon[140233]: 2023-09-23T22:48:02.168+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:03.007 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:02 smithi070 conmon[410182]: 2023-09-23T22:48:02.664+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:03.007 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:02 smithi070 conmon[410182]: 2023-09-23T22:48:02.664+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:03.007 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:02 smithi070 conmon[410182]: 2023-09-23T22:48:02.664+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:35.810783+0000 front 2023-09-23T22:47:35.810870+0000 (oldest deadline 2023-09-23T22:48:01.110788+0000) 2023-09-23T22:48:03.373 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:03 smithi070 conmon[140233]: 2023-09-23T22:48:03.120+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:03.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:03 smithi070 conmon[140233]: 2023-09-23T22:48:03.120+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:03.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:03 smithi070 conmon[140233]: 2023-09-23T22:48:03.120+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:03.585 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T22:48:03.626 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:48:03.626 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T22:48:03.626 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T22:48:03.626 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T22:48:03.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:03 smithi070 conmon[410182]: 2023-09-23T22:48:03.614+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:03.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:03 smithi070 conmon[410182]: 2023-09-23T22:48:03.614+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:03.875 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:03 smithi070 conmon[410182]: 2023-09-23T22:48:03.614+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:35.810783+0000 front 2023-09-23T22:47:35.810870+0000 (oldest deadline 2023-09-23T22:48:01.110788+0000) 2023-09-23T22:48:03.875 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:03 smithi070 systemd[1]: Stopping Ceph osd.6 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:04.374 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:03 smithi070 conmon[410182]: 2023-09-23T22:48:03.935+0000 7fe38a663700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:48:04.374 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:03 smithi070 conmon[410182]: 2023-09-23T22:48:03.935+0000 7fe38a663700 -1 osd.6 683 *** Got signal Terminated *** 2023-09-23T22:48:04.374 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:03 smithi070 conmon[410182]: 2023-09-23T22:48:03.935+0000 7fe38a663700 -1 osd.6 683 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:48:04.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:04 smithi070 conmon[140233]: 2023-09-23T22:48:04.096+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:04.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:04 smithi070 conmon[140233]: 2023-09-23T22:48:04.096+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:04.375 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:04 smithi070 conmon[140233]: 2023-09-23T22:48:04.096+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:04.873 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:04 smithi070 conmon[410182]: 2023-09-23T22:48:04.576+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:04.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:04 smithi070 conmon[410182]: 2023-09-23T22:48:04.576+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:04.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:04 smithi070 conmon[410182]: 2023-09-23T22:48:04.576+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:35.810783+0000 front 2023-09-23T22:47:35.810870+0000 (oldest deadline 2023-09-23T22:48:01.110788+0000) 2023-09-23T22:48:05.550 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:05 smithi070 conmon[140233]: 2023-09-23T22:48:05.128+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:05.551 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:05 smithi070 conmon[140233]: 2023-09-23T22:48:05.128+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:05.551 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:05 smithi070 conmon[140233]: 2023-09-23T22:48:05.128+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:05.873 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:05 smithi070 conmon[410182]: 2023-09-23T22:48:05.549+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:05.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:05 smithi070 conmon[410182]: 2023-09-23T22:48:05.549+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:05.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:05 smithi070 conmon[410182]: 2023-09-23T22:48:05.549+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:35.810783+0000 front 2023-09-23T22:47:35.810870+0000 (oldest deadline 2023-09-23T22:48:01.110788+0000) 2023-09-23T22:48:06.589 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:06 smithi070 conmon[140233]: 2023-09-23T22:48:06.138+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:06.589 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:06 smithi070 conmon[140233]: 2023-09-23T22:48:06.138+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:06.589 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:06 smithi070 conmon[140233]: 2023-09-23T22:48:06.138+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:06.873 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:06 smithi070 conmon[410182]: 2023-09-23T22:48:06.587+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:06.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:06 smithi070 conmon[410182]: 2023-09-23T22:48:06.587+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:06.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:06 smithi070 conmon[410182]: 2023-09-23T22:48:06.587+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:35.810783+0000 front 2023-09-23T22:47:35.810870+0000 (oldest deadline 2023-09-23T22:48:01.110788+0000) 2023-09-23T22:48:06.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:06 smithi070 conmon[410182]: 2023-09-23T22:48:06.587+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:41.111478+0000 front 2023-09-23T22:47:41.111459+0000 (oldest deadline 2023-09-23T22:48:05.811360+0000) 2023-09-23T22:48:07.586 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:07 smithi070 conmon[140233]: 2023-09-23T22:48:07.143+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:07.586 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:07 smithi070 conmon[140233]: 2023-09-23T22:48:07.143+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:07.586 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:07 smithi070 conmon[140233]: 2023-09-23T22:48:07.143+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:07.873 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:07 smithi070 conmon[410182]: 2023-09-23T22:48:07.585+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:07.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:07 smithi070 conmon[410182]: 2023-09-23T22:48:07.585+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:07.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:07 smithi070 conmon[410182]: 2023-09-23T22:48:07.585+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:35.810783+0000 front 2023-09-23T22:47:35.810870+0000 (oldest deadline 2023-09-23T22:48:01.110788+0000) 2023-09-23T22:48:07.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:07 smithi070 conmon[410182]: 2023-09-23T22:48:07.585+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:41.111478+0000 front 2023-09-23T22:47:41.111459+0000 (oldest deadline 2023-09-23T22:48:05.811360+0000) 2023-09-23T22:48:08.561 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:08 smithi070 conmon[140233]: 2023-09-23T22:48:08.167+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:08.561 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:08 smithi070 conmon[140233]: 2023-09-23T22:48:08.167+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:08.561 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:08 smithi070 conmon[140233]: 2023-09-23T22:48:08.167+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:08.561 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:08 smithi070 conmon[140233]: 2023-09-23T22:48:08.167+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:43.116057+0000 front 2023-09-23T22:47:43.115996+0000 (oldest deadline 2023-09-23T22:48:07.215832+0000) 2023-09-23T22:48:08.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:08 smithi070 conmon[410182]: 2023-09-23T22:48:08.560+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:20.108535+0000 front 2023-09-23T22:47:20.108618+0000 (oldest deadline 2023-09-23T22:47:42.408335+0000) 2023-09-23T22:48:08.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:08 smithi070 conmon[410182]: 2023-09-23T22:48:08.560+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:26.409975+0000 front 2023-09-23T22:47:26.410040+0000 (oldest deadline 2023-09-23T22:47:51.109843+0000) 2023-09-23T22:48:08.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:08 smithi070 conmon[410182]: 2023-09-23T22:48:08.560+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:35.810783+0000 front 2023-09-23T22:47:35.810870+0000 (oldest deadline 2023-09-23T22:48:01.110788+0000) 2023-09-23T22:48:08.874 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:08 smithi070 conmon[410182]: 2023-09-23T22:48:08.560+0000 7fe387462700 -1 osd.6 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:41.111478+0000 front 2023-09-23T22:47:41.111459+0000 (oldest deadline 2023-09-23T22:48:05.811360+0000) 2023-09-23T22:48:09.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:09 smithi070 conmon[140233]: 2023-09-23T22:48:09.120+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:09.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:09 smithi070 conmon[140233]: 2023-09-23T22:48:09.120+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:09.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:09 smithi070 conmon[140233]: 2023-09-23T22:48:09.120+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:09.375 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:09 smithi070 conmon[140233]: 2023-09-23T22:48:09.120+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:43.116057+0000 front 2023-09-23T22:47:43.115996+0000 (oldest deadline 2023-09-23T22:48:07.215832+0000) 2023-09-23T22:48:09.873 INFO:journalctl@ceph.osd.6.smithi070.stdout:Sep 23 22:48:09 smithi070 bash[433278]: 8e7c9aa0f23e8eda39b15167307ee7af79ef52bb843450474a377ea585bb5188 2023-09-23T22:48:10.370 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:10 smithi070 conmon[140233]: 2023-09-23T22:48:10.102+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:10.371 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:10 smithi070 conmon[140233]: 2023-09-23T22:48:10.102+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:10.371 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:10 smithi070 conmon[140233]: 2023-09-23T22:48:10.102+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:10.371 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:10 smithi070 conmon[140233]: 2023-09-23T22:48:10.102+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:43.116057+0000 front 2023-09-23T22:47:43.115996+0000 (oldest deadline 2023-09-23T22:48:07.215832+0000) 2023-09-23T22:48:10.371 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:10 smithi070 conmon[176728]: ts=2023-09-23T22:48:10.269Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2023-09-23T22:48:10.371 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:10 smithi070 conmon[176728]: ts=2023-09-23T22:48:10.269Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.26:8765: connect: connection refused" 2023-09-23T22:48:10.371 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:10 smithi070 conmon[176728]: ts=2023-09-23T22:48:10.269Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.26:8765: connect: connection refused" 2023-09-23T22:48:10.372 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:10 smithi070 conmon[176728]: ts=2023-09-23T22:48:10.271Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.26:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.26:8765: connect: connection refused" 2023-09-23T22:48:11.316 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T22:48:11.355 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:48:11.356 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T22:48:11.356 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T22:48:11.356 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T22:48:11.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:11 smithi070 conmon[140233]: 2023-09-23T22:48:11.111+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:11.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:11 smithi070 conmon[140233]: 2023-09-23T22:48:11.111+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:11.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:11 smithi070 conmon[140233]: 2023-09-23T22:48:11.111+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:11.374 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:11 smithi070 conmon[140233]: 2023-09-23T22:48:11.111+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:43.116057+0000 front 2023-09-23T22:47:43.115996+0000 (oldest deadline 2023-09-23T22:48:07.215832+0000) 2023-09-23T22:48:11.874 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:11 smithi070 systemd[1]: Stopping Ceph osd.7 for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:11.874 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:11 smithi070 conmon[140233]: 2023-09-23T22:48:11.633+0000 7febf20cc700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T22:48:11.874 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:11 smithi070 conmon[140233]: 2023-09-23T22:48:11.633+0000 7febf20cc700 -1 osd.7 683 *** Got signal Terminated *** 2023-09-23T22:48:11.874 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:11 smithi070 conmon[140233]: 2023-09-23T22:48:11.633+0000 7febf20cc700 -1 osd.7 683 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T22:48:12.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:12 smithi070 conmon[140233]: 2023-09-23T22:48:12.133+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:12.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:12 smithi070 conmon[140233]: 2023-09-23T22:48:12.133+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:12.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:12 smithi070 conmon[140233]: 2023-09-23T22:48:12.133+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:12.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:12 smithi070 conmon[140233]: 2023-09-23T22:48:12.133+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:43.116057+0000 front 2023-09-23T22:47:43.115996+0000 (oldest deadline 2023-09-23T22:48:07.215832+0000) 2023-09-23T22:48:13.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:13 smithi070 conmon[140233]: 2023-09-23T22:48:13.180+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:13.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:13 smithi070 conmon[140233]: 2023-09-23T22:48:13.180+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:13.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:13 smithi070 conmon[140233]: 2023-09-23T22:48:13.180+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:13.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:13 smithi070 conmon[140233]: 2023-09-23T22:48:13.180+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:43.116057+0000 front 2023-09-23T22:47:43.115996+0000 (oldest deadline 2023-09-23T22:48:07.215832+0000) 2023-09-23T22:48:14.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:14 smithi070 conmon[140233]: 2023-09-23T22:48:14.172+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:14.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:14 smithi070 conmon[140233]: 2023-09-23T22:48:14.172+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:14.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:14 smithi070 conmon[140233]: 2023-09-23T22:48:14.172+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:14.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:14 smithi070 conmon[140233]: 2023-09-23T22:48:14.172+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:43.116057+0000 front 2023-09-23T22:47:43.115996+0000 (oldest deadline 2023-09-23T22:48:07.215832+0000) 2023-09-23T22:48:15.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:15 smithi070 conmon[140233]: 2023-09-23T22:48:15.154+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:15.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:15 smithi070 conmon[140233]: 2023-09-23T22:48:15.154+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:15.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:15 smithi070 conmon[140233]: 2023-09-23T22:48:15.154+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:15.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:15 smithi070 conmon[140233]: 2023-09-23T22:48:15.154+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:43.116057+0000 front 2023-09-23T22:47:43.115996+0000 (oldest deadline 2023-09-23T22:48:07.215832+0000) 2023-09-23T22:48:16.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:16 smithi070 conmon[140233]: 2023-09-23T22:48:16.174+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6806 osd.0 since back 2023-09-23T22:47:18.711430+0000 front 2023-09-23T22:47:18.711411+0000 (oldest deadline 2023-09-23T22:47:44.011483+0000) 2023-09-23T22:48:16.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:16 smithi070 conmon[140233]: 2023-09-23T22:48:16.174+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6814 osd.1 since back 2023-09-23T22:47:28.712596+0000 front 2023-09-23T22:47:28.712707+0000 (oldest deadline 2023-09-23T22:47:51.612522+0000) 2023-09-23T22:48:16.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:16 smithi070 conmon[140233]: 2023-09-23T22:48:16.174+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6822 osd.2 since back 2023-09-23T22:47:36.214286+0000 front 2023-09-23T22:47:36.214296+0000 (oldest deadline 2023-09-23T22:47:57.914290+0000) 2023-09-23T22:48:16.624 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:16 smithi070 conmon[140233]: 2023-09-23T22:48:16.174+0000 7febefecd700 -1 osd.7 683 heartbeat_check: no reply from 172.21.15.26:6830 osd.3 since back 2023-09-23T22:47:43.116057+0000 front 2023-09-23T22:47:43.115996+0000 (oldest deadline 2023-09-23T22:48:07.215832+0000) 2023-09-23T22:48:18.123 INFO:journalctl@ceph.osd.7.smithi070.stdout:Sep 23 22:48:17 smithi070 bash[433860]: 2a3425d84687238ff0c3cc77571aedd81913d3389d1b0e0ad502d652bf3a8328 2023-09-23T22:48:19.431 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T22:48:19.487 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:48:19.487 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T22:48:19.487 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T22:48:19.488 DEBUG:teuthology.orchestra.run.smithi070:> sudo systemctl stop ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T22:48:19.795 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 systemd[1]: Stopping Ceph prometheus.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:20.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.794Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:48:20.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.795Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T22:48:20.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.795Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T22:48:20.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.795Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T22:48:20.124 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.795Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T22:48:20.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.795Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T22:48:20.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.795Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T22:48:20.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.795Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T22:48:20.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.795Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T22:48:20.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.796Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T22:48:20.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.796Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T22:48:20.125 INFO:journalctl@ceph.prometheus.a.smithi070.stdout:Sep 23 22:48:19 smithi070 conmon[176728]: ts=2023-09-23T22:48:19.796Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T22:48:20.502 DEBUG:teuthology.orchestra.run.smithi070:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T22:48:20.538 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T22:48:20.538 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T22:48:20.539 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T22:48:21.986 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:48:21 smithi026 systemd[1]: Stopping Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:22.302 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:48:22 smithi026 conmon[200253]: ts=2023-09-23T22:48:22.001Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T22:48:22.303 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:48:22 smithi026 bash[727008]: 1cbbb123afed59eec2f699cf2bf0b346962c29b842a8f3957a7991c084cca582 2023-09-23T22:48:22.985 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:48:22 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T22:48:22.986 INFO:journalctl@ceph.alertmanager.a.smithi026.stdout:Sep 23 22:48:22 smithi026 systemd[1]: Stopped Ceph alertmanager.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:48:22.986 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:48:22 smithi026 systemd[1]: Stopping Ceph node-exporter.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:23.735 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:48:23 smithi026 bash[727297]: 68b72a59fe0873927ff143decba2fbae87a12feddfafd97eaa08a4e4f32644da 2023-09-23T22:48:23.736 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:48:23 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:48:23.736 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:48:23 smithi026 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T22:48:23.736 INFO:journalctl@ceph.node-exporter.a.smithi026.stdout:Sep 23 22:48:23 smithi026 systemd[1]: Stopped Ceph node-exporter.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:48:29.092 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T22:48:30.624 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:48:30 smithi070 systemd[1]: Stopping Ceph node-exporter.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:31.039 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:48:31 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:48:31.314 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:48:31 smithi070 bash[435036]: 523ab3522b38d07e9103b568c8890cca5acb62a24d03575d0d6602626af2a36b 2023-09-23T22:48:31.315 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:48:31 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T22:48:31.315 INFO:journalctl@ceph.node-exporter.b.smithi070.stdout:Sep 23 22:48:31 smithi070 systemd[1]: Stopped Ceph node-exporter.b for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:48:31.624 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:48:31 smithi070 systemd[1]: Stopping Ceph grafana.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7... 2023-09-23T22:48:32.374 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:48:32 smithi070 bash[435323]: 42940e65ed5c414390e26bf204820b13b4332b52525f37d419b2dfc73104658f 2023-09-23T22:48:32.375 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:48:32 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T22:48:32.375 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:48:32 smithi070 systemd[1]: ceph-2a3049e2-5a5c-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T22:48:32.375 INFO:journalctl@ceph.grafana.a.smithi070.stdout:Sep 23 22:48:32 smithi070 systemd[1]: Stopped Ceph grafana.a for 2a3049e2-5a5c-11ee-8db4-212e2dc638e7. 2023-09-23T22:48:36.774 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:48:36.812 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T22:48:36.842 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T22:48:36.844 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873/remote/smithi026/crash 2023-09-23T22:48:36.844 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T22:48:36.909 INFO:teuthology.orchestra.run.smithi026.stderr:tar: /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T22:48:36.909 INFO:teuthology.orchestra.run.smithi026.stderr:tar: Error is not recoverable: exiting now 2023-09-23T22:48:36.911 DEBUG:teuthology.misc:Transferring archived files from smithi070:/var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873/remote/smithi070/crash 2023-09-23T22:48:36.912 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T22:48:36.975 INFO:teuthology.orchestra.run.smithi070.stderr:tar: /var/lib/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T22:48:36.976 INFO:teuthology.orchestra.run.smithi070.stderr:tar: Error is not recoverable: exiting now 2023-09-23T22:48:36.977 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T22:48:36.977 DEBUG:teuthology.orchestra.run.smithi026:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T22:48:37.012 INFO:teuthology.orchestra.run.smithi026.stderr:grep: /var/log/ceph/2a3049e2-5a5c-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T22:48:37.032 INFO:tasks.cephadm:Compressing logs... 2023-09-23T22:48:37.032 DEBUG:teuthology.orchestra.run.smithi026:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T22:48:37.075 DEBUG:teuthology.orchestra.run.smithi070:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T22:48:37.115 INFO:teuthology.orchestra.run.smithi026.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T22:48:37.132 INFO:teuthology.orchestra.run.smithi070.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T22:58:37.820 INFO:tasks.cephadm:Archiving logs... 2023-09-23T22:58:37.821 DEBUG:teuthology.misc:Transferring archived files from smithi026:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873/remote/smithi026/log 2023-09-23T22:58:37.822 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T22:59:50.173 DEBUG:teuthology.misc:Transferring archived files from smithi070:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873/remote/smithi070/log 2023-09-23T22:59:50.174 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T23:00:56.349 INFO:tasks.cephadm:Removing cluster... 2023-09-23T23:00:56.350 DEBUG:teuthology.orchestra.run.smithi026:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 --force 2023-09-23T23:00:57.648 DEBUG:teuthology.orchestra.run.smithi070:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2a3049e2-5a5c-11ee-8db4-212e2dc638e7 --force 2023-09-23T23:00:58.880 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T23:00:58.880 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T23:00:58.906 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T23:00:58.941 INFO:tasks.cephadm:Teardown complete 2023-09-23T23:00:58.941 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T23:00:58.957 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T23:00:58.958 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T23:00:58.961 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T23:00:59.080 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T23:00:59.099 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T23:00:59.099 DEBUG:teuthology.orchestra.run.smithi026:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T23:00:59.103 DEBUG:teuthology.orchestra.run.smithi070:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T23:00:59.119 INFO:teuthology.orchestra.run.smithi070.stderr:bash: ntpq: command not found 2023-09-23T23:00:59.119 INFO:teuthology.orchestra.run.smithi026.stderr:bash: ntpq: command not found 2023-09-23T23:00:59.140 INFO:teuthology.orchestra.run.smithi026.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T23:00:59.140 INFO:teuthology.orchestra.run.smithi026.stdout:=============================================================================== 2023-09-23T23:00:59.140 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 62 +375us[ +376us] +/- 88ms 2023-09-23T23:00:59.140 INFO:teuthology.orchestra.run.smithi026.stdout:^* hv02.front.sepia.ceph.com 3 6 377 0 -228us[ -226us] +/- 57ms 2023-09-23T23:00:59.141 INFO:teuthology.orchestra.run.smithi026.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 62 +8907ns[ +10us] +/- 84ms 2023-09-23T23:00:59.141 INFO:teuthology.orchestra.run.smithi026.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T23:00:59.147 INFO:teuthology.orchestra.run.smithi070.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T23:00:59.148 INFO:teuthology.orchestra.run.smithi070.stdout:=============================================================================== 2023-09-23T23:00:59.148 INFO:teuthology.orchestra.run.smithi070.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 65 +381us[ +389us] +/- 88ms 2023-09-23T23:00:59.148 INFO:teuthology.orchestra.run.smithi070.stdout:^* hv02.front.sepia.ceph.com 3 6 377 59 -228us[ -220us] +/- 56ms 2023-09-23T23:00:59.148 INFO:teuthology.orchestra.run.smithi070.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 63 +29us[ +37us] +/- 84ms 2023-09-23T23:00:59.148 INFO:teuthology.orchestra.run.smithi070.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T23:00:59.149 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T23:00:59.162 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T23:00:59.163 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T23:00:59.192 DEBUG:teuthology.orchestra.run.smithi026:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T23:00:59.316 DEBUG:teuthology.orchestra.run.smithi070:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T23:00:59.409 DEBUG:teuthology.orchestra.run.smithi026:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T23:00:59.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:00:59.443 DEBUG:teuthology.orchestra.run.smithi070:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T23:00:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:00:59.474 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T23:00:59.487 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T23:00:59.543 INFO:teuthology.task.internal:Duration was 4670.148229 seconds 2023-09-23T23:00:59.543 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T23:00:59.586 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T23:00:59.586 DEBUG:teuthology.orchestra.run.smithi026:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T23:00:59.589 DEBUG:teuthology.orchestra.run.smithi070:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T23:00:59.658 INFO:teuthology.orchestra.run.smithi070.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T23:00:59.664 INFO:teuthology.orchestra.run.smithi026.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T23:01:00.013 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T23:01:00.013 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi026.front.sepia.ceph.com 2023-09-23T23:01:00.013 DEBUG:teuthology.orchestra.run.smithi026:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T23:01:00.037 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi070.front.sepia.ceph.com 2023-09-23T23:01:00.037 DEBUG:teuthology.orchestra.run.smithi070:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T23:01:00.072 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T23:01:00.072 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T23:01:00.081 DEBUG:teuthology.orchestra.run.smithi070:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T23:01:00.264 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T23:01:00.264 DEBUG:teuthology.orchestra.run.smithi026:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T23:01:00.266 DEBUG:teuthology.orchestra.run.smithi070:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T23:01:00.634 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T23:01:00.649 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T23:01:00.649 DEBUG:teuthology.orchestra.run.smithi026:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T23:01:00.688 DEBUG:teuthology.orchestra.run.smithi070:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T23:01:00.725 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T23:01:00.739 DEBUG:teuthology.orchestra.run.smithi026:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T23:01:00.742 DEBUG:teuthology.orchestra.run.smithi070:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T23:01:00.766 INFO:teuthology.orchestra.run.smithi026.stdout:kernel.core_pattern = core 2023-09-23T23:01:00.795 INFO:teuthology.orchestra.run.smithi070.stdout:kernel.core_pattern = core 2023-09-23T23:01:00.817 DEBUG:teuthology.orchestra.run.smithi026:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T23:01:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:01:00.844 DEBUG:teuthology.orchestra.run.smithi070:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T23:01:00.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:01:00.873 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T23:01:00.886 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T23:01:00.887 DEBUG:teuthology.misc:Transferring archived files from smithi026:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873/remote/smithi026 2023-09-23T23:01:00.887 DEBUG:teuthology.orchestra.run.smithi026:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T23:01:00.948 DEBUG:teuthology.misc:Transferring archived files from smithi070:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873/remote/smithi070 2023-09-23T23:01:00.949 DEBUG:teuthology.orchestra.run.smithi070:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T23:01:00.998 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T23:01:00.999 DEBUG:teuthology.orchestra.run.smithi026:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T23:01:01.002 DEBUG:teuthology.orchestra.run.smithi070:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T23:01:01.044 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T23:01:01.058 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T23:01:01.058 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T23:01:01.088 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T23:01:01.088 DEBUG:teuthology.orchestra.run.smithi026:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T23:01:01.090 DEBUG:teuthology.orchestra.run.smithi070:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T23:01:01.105 INFO:teuthology.orchestra.run.smithi026.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 23:01 /home/ubuntu/cephtest 2023-09-23T23:01:01.105 INFO:teuthology.orchestra.run.smithi026.stdout: 4194321 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 22:45 /home/ubuntu/cephtest/mnt.0 2023-09-23T23:01:01.105 INFO:teuthology.orchestra.run.smithi026.stdout: 4194324 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 22:45 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T23:01:01.106 INFO:teuthology.orchestra.run.smithi070.stdout: 262320 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 23:01 /home/ubuntu/cephtest 2023-09-23T23:01:01.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T23:01:01.114 INFO:teuthology.orchestra.run.smithi026.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T23:01:01.115 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi026 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T23:01:01.120 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T23:01:01.163 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T23:01:01.213 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi026 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T23:01:01.214 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T23:01:01.243 DEBUG:teuthology.nuke:shortname: smithi026 2023-09-23T23:01:01.244 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T23:01:01.265 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi026.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 21:37:51.359124', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T23:01:01.270 DEBUG:teuthology.nuke:shortname: smithi070 2023-09-23T23:01:01.270 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T23:01:01.296 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi070.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401873', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 21:37:51.360798', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T23:01:01.317 INFO:teuthology.orchestra.console.smithi026:Power off 2023-09-23T23:01:01.318 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T23:01:01.348 DEBUG:teuthology.orchestra.console.smithi026:power off output: Chassis Power Control: Down/Off 2023-09-23T23:01:01.348 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:01:01.365 INFO:teuthology.orchestra.console.smithi070:Power off 2023-09-23T23:01:01.365 DEBUG:teuthology.orchestra.console.smithi070:pexpect command: ipmitool -H smithi070.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T23:01:01.374 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on 2023-09-23T23:01:01.387 DEBUG:teuthology.orchestra.console.smithi070:power off output: Chassis Power Control: Down/Off 2023-09-23T23:01:01.388 DEBUG:teuthology.orchestra.console.smithi070:pexpect command: ipmitool -H smithi070.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:01:01.412 DEBUG:teuthology.orchestra.console.smithi070:check power output: Chassis Power is on 2023-09-23T23:01:05.376 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:01:05.414 DEBUG:teuthology.orchestra.console.smithi070:pexpect command: ipmitool -H smithi070.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:01:05.487 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is on 2023-09-23T23:01:05.524 DEBUG:teuthology.orchestra.console.smithi070:check power output: Chassis Power is on 2023-09-23T23:01:09.490 DEBUG:teuthology.orchestra.console.smithi026:pexpect command: ipmitool -H smithi026.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:01:09.526 DEBUG:teuthology.orchestra.console.smithi070:pexpect command: ipmitool -H smithi070.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T23:01:09.601 DEBUG:teuthology.orchestra.console.smithi026:check power output: Chassis Power is off 2023-09-23T23:01:09.636 DEBUG:teuthology.orchestra.console.smithi070:check power output: Chassis Power is off 2023-09-23T23:01:09.703 INFO:teuthology.orchestra.console.smithi026:Power off completed 2023-09-23T23:01:09.737 INFO:teuthology.orchestra.console.smithi070:Power off completed 2023-09-23T23:01:09.842 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 4670.148228645325 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=6c4335fce8894e1381218277d33c3589 status: fail success: false 2023-09-23T23:01:09.843 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T23:01:09.922 INFO:teuthology.run:FAIL